
/*******************************************************************************
*
* File time_Awhat.c
*
* Copyright (C) 2007, 2008, 2011-2013 Martin Luescher
*
* This software is distributed under the terms of the GNU General Public
* License (GPL)
*
* Based on devel/little/time1.c of DD-HMC-1.2.2 
*
* Timing of Awhat().
*
*******************************************************************************/

#define TIME_AWHAT_C

#include <stdlib.h>
#include <stdio.h>
#include <math.h>
#include "mpi.h"
#include "su3.h"
#include "random.h"
#include "flags.h"
#include "utils.h"
#include "lattice.h"
#include "uflds.h"
#include "sflds.h"
#include "vflds.h"
#include "linalg.h"
#include "dirac.h"
#include "dfl.h"
#include "little.h"
#include "global.h"
#include "bm.h"

static int bs[4]={4,4,4,4};

static void random_basis(int Ns)
{
   int i;
   spinor **ws;

   ws=reserve_ws(Ns);

   for (i=0;i<Ns;i++)
   {
      random_s(VOLUME,ws[i],1.0f);
      bnd_s2zero(ALL_PTS,ws[i]);
   }

   dfl_subspace(ws);
   release_ws();
}


void time_Awhat(FILE *flog, double *wdt, int *nb)
{
   int my_rank,bc,count,nt;
   int i,nflds,ifail;
   int Ns,nbb,nv;
   double phi[2],phi_prime[2];
   double mu,wt1,wt2;
   double wt;
   complex **wv;

   MPI_Comm_rank(MPI_COMM_WORLD,&my_rank);
   if (my_rank==0)
   {
      printf("\n");
      printf("Timing of Awhat()\n");
      printf("-----------------\n\n");

      printf("%dx%dx%dx%d lattice, ",NPROC0*L0,NPROC1*L1,NPROC2*L2,NPROC3*L3);
      printf("%dx%dx%dx%d process grid, ",NPROC0,NPROC1,NPROC2,NPROC3);
      printf("%dx%dx%dx%d local lattice\n\n",L0,L1,L2,L3);

   }
   
   Ns=NS;
   bc=3;

   set_lat_parms(5.5,1.0,0,NULL,1.978);
   print_lat_parms();

   phi[0]=0.123;
   phi[1]=-0.534;
   phi_prime[0]=0.912;
   phi_prime[1]=0.078;
   set_bc_parms(bc,0.55,0.78,0.9012,1.2034,phi,phi_prime);
   print_bc_parms();

   set_sw_parms(0.125);
   set_dfl_parms(bs,Ns);
   mu=0.0376;

   (*nb)=VOLUME/(bs[0]*bs[1]*bs[2]*bs[3]);
   nbb=2*(FACE0/(bs[1]*bs[2]*bs[3])+
          FACE1/(bs[0]*bs[2]*bs[3])+
          FACE2/(bs[0]*bs[1]*bs[3])+
          FACE3/(bs[0]*bs[1]*bs[2]));
   nv=Ns*(*nb);

   start_ranlux(0,12);
   geometry();

   alloc_ws(Ns);
   alloc_wvd(2);
   random_ud();
   chs_ubnd(-1);
   random_basis(Ns);

   ifail=set_Awhat(mu);
   error(ifail!=0,1,"module/bm [Awhat_time1.c]","Inversion of Aee or Aoo failed");

   if (my_rank==0)
   {
      printf("Number of points = %d\n",VOLUME);
      printf("Number of blocks = %d\n",(*nb));
      printf("Number of points/block = %d\n",bs[0]*bs[1]*bs[2]*bs[3]);
      printf("Vector field size = %.2f KB\n",
             (double)(sizeof(complex)*nv)*1.0e-3);
      printf("Awhat array size = %.2f MB\n\n",
             (double)(sizeof(complex)*8*Ns*nv)*1.0e-6);
      fflush(flog);
   }

   nflds=(int)(1.0e6/(double)(sizeof(complex)*nv));
   if ((nflds%2)!=0)
      nflds+=1;
   if (nflds==0)
      nflds=2;

   alloc_wv(nflds);
   wv=reserve_wv(nflds);

   for (i=0;i<nflds;i++)
      random_v(nv,wv[i],1.0f);

   nt=(int)(1.0e3/(double)(8*Ns));
   if (nt<1)
      nt=1;
   (*wdt)=0.0;

   while ((*wdt)<5.0)
   {
      MPI_Barrier(MPI_COMM_WORLD);
      wt1=MPI_Wtime();
      for (count=0;count<nt;count++)
      {
         for (i=0;i<nflds;i+=2)
            Awhat(wv[i],wv[i+1]);
      }
      MPI_Barrier(MPI_COMM_WORLD);
      wt2=MPI_Wtime();

      (*wdt)=wt2-wt1;
      nt*=2;
   }

   error_chk();
   (*wdt)=4.0e6*(*wdt)/(double)(nt*nflds);

   if (my_rank==0)
   {
      printf("Time per application of Awhat(), including communications:\n");
      printf("Total:     %4.3f msec\n",(*wdt)*1.0e-3);
      printf("Per block: %4.3f usec",(*wdt)/(double)((*nb)));
      printf(" (%d Mflops [%d bit arithmetic])\n",
             (int)(64.0*(double)((*nb)*Ns*Ns)/(*wdt)),(int)(4*sizeof(complex)));
      printf("Per point: %4.3f usec\n\n",(*wdt)/(double)(VOLUME));
      fflush(flog);
   }

   if (NPROC>1)
   {
      nt/=2;
      if (nt==0)
         nt=1;
      wt=0.0;

      while (wt<5.0)
      {
         for (i=0;i<nflds;i+=2)
            set_v2zero((nbb/2)*Ns,wv[i+1]+nv);

         MPI_Barrier(MPI_COMM_WORLD);
         wt1=MPI_Wtime();
         for (count=0;count<nt;count++)
         {
            for (i=0;i<nflds;i+=2)
            {
               cpv_int_bnd(wv[i]);
               cpv_ext_bnd(wv[i+1]);
            }
         }
         MPI_Barrier(MPI_COMM_WORLD);
         wt2=MPI_Wtime();

         wt=wt2-wt1;
         nt*=2;
      }

      wt=4.0e6*wt/(double)(nt*nflds);

      if (my_rank==0)
      {
         printf("There are %d boundary blocks\n",nbb);
         printf("Time per application of Awhat() for the communications:\n");
         printf("Total:     %4.3f msec\n",wt*1.0e-3);
         printf("Per block: %4.3f usec\n",wt/(double)((*nb)));
         printf("Per point: %4.3f usec\n\n",wt/(double)(VOLUME));
      }
   }

   /*if (my_rank==0)
      fclose(flog);*/
   release_ws();
   release_wvd();
   release_wv();
   
   /*MPI_Finalize();
   exit(0);*/
}
