###################################################################
#                          PGI CONFIGURE                          #
#POWER9 RECOMENDED MODULE:                                        #
#module load ompi/3.0.0 pgi/18.4                                  #
###################################################################

F77      = OMPI_FC=pgfortran mpif90
F90      = OMPI_FC=pgfortran mpif90
FCOCC    = cc -c
FCFLAGS  = -c -fast -Minfo=all -acc -ta=tesla:cuda10.1 -Mpreprocess -I./Objects_x/ -Mbackslash  -Mextend -Mnoopenmp -Munroll -Mnoidiom -module $O
FPPFLAGS = 
EXTRALIB = -lc
EXTRAINC = 
fa2p     = pgfortran -c -x f95-cpp-input -DMPI_OFF -J../../Utils/user/alya2pos -I../../Utils/user/alya2pos
fa2plk   = pgfortran

###################################################################
#                          PERFORMANCE FLAGS                      #
###################################################################
#MINUM
#FOPT      = -O1
#MAXIMUM (I have elimated -xHost due to observations by Yacine)
FOPT      = -O3

#Compilation flags applied only to the Source/modules folder
#MODULEFLAGS = -ipo

# Uncomment the following line to enable NDIME as a parameter (OPTIMIZATION FOR 3D PROBLEMS)
CSALYA   := $(CSALYA) -DNDIMEPAR -DOPENACCHHH -DSUPER_FAST -DDETAILED_TIMES

# Uncomment the following line for DEBUG AND CHECKING FLAGS
#CSALYA   := $(CSALYA) -C -Ktrap=fp -Minform=inform 
 
# Vectorization: put vector size (in principle=4 for MN)
CSALYA   := $(CSALYA) -DVECTOR_SIZE=32768

###################################################################
#                          USER SPECIFIC FLAGS                    #
###################################################################
#  HERBERT
#CSALYA   := $(CSALYA) -DDETAILS_ORTHOMIN

###################################################################
#                          PROFILING FLAGS                        #
###################################################################
# Uncomment the following line to generate profiling info files
#CSALYA   := $(CSALYA) -profile-loops=all -profile-loops-report=2

###################################################################
#                          EXTRAE FLAGS                           #
###################################################################
# Uncomment the following line to compile Alya using extrae
# Compiler used to compile extrae module (make extrae)
EXTRAE_FC=pgfortran
# Extrae installation directory (for linking) (not necessary if loading extrae using module load extrae)
#EXTRAE_HOME=
#@Linking with Extrae
#EXTRALIB   := $(EXTRALIB) -L${EXTRAE_HOME}/lib/ -lmpitracef extrae_module.o
#@Enabling Extrae user calls (normal)
#CSALYA   := $(CSALYA) -DEXTRAE

###################################################################
#                          METIS LIBRARY                          #
###################################################################

# Uncomment the following lines for using metis 4.0 (default)
EXTRALIB   := $(EXTRALIB) -L../../Thirdparties/metis-4.0 -lmetis -acc -ta=tesla:cuda10.1
CSALYA     := $(CSALYA) -DMETIS

# Uncomment the following lines for using metis 5.0
#CSALYA   := $(CSALYA) -DV5METIS
# uncoment FOR MAC
#EXTRALIB   := $(EXTRALIB) -L../../Thirdparties/metis-5.0.2_i8/build/Darwin-i386/libmetis -lmetis
# uncoment FOR LINUX
#EXTRALIB   := $(EXTRALIB) -L../../Thirdparties/metis-5.0.2_i8/build/Linux-x86_64/libmetis -lmetis

# Uncomment the following lines for using parmetis
#CSALYA   := $(CSALYA) -DPARMETIS
#EXTRALIB   := $(EXTRALIB) -L../../Thirdparties/

###################################################################
#                           BLAS LIBRARY                          #
###################################################################

# Uncomment the following lines for using blas
#EXTRALIB   := $(EXTRALIB) -lblas
#CSALYA     := $(CSALYA) -DBLAS

###################################################################
#                          MPI/THREADS/TASKS                      #
###################################################################

# Uncomment the following lines for sequential (NOMPI) version
#VERSIONMPI  = nompi
#CSALYA    := $(CSALYA) -DMPI_OFF

# Uncomment the following lines for OPENMP version
#CSALYA    := $(CSALYA) -openmp
#EXTRALIB   := $(EXTRALIB) -openmp

# Uncomment the following line to disable OPENMP coloring
#CSALYA    := $(CSALYA) -DNO_COLORING

# Uncomment the following line to enable OMPSS in order to activate 
# loops with multidependencies whenever we have a race condition 
#CSALYA    := $(CSALYA) -DALYA_OMPSS

###################################################################
#                             DLB                                 #
#  To use DLB with OMPSS: Define -DALYA_DLB -DALYA_OMPSS          #
#  To use DLB barrier:    Define -DALYA_DLB_BARRIER               #
#  If we compile with mercurium, there is no need to link         #
#  with DLB                                                       #
#                                                                 #
###################################################################

#CSALYA   := $(CSALYA)   -DALYA_DLB 
#FCFLAGS  := $(FCFLAGS)  --dlb
#FPPFLAGS := $(FPPFLAGS) --dlb

###################################################################
#                          INTEGER TYPE                           #
###################################################################
# Uncomment the following lines for 8 byte integers
#CSALYA   := $(CSALYA) -DI8 -m64
# Uncomment the following line for 8 byte integers ONLY in Metis
# In this mode Alya can work in I4 and Metis in I8 (mixed mode)
# For this option the use of Metis v5 library is mandatory
#CSALYA   := $(CSALYA) -DMETISI8


###################################################################
#                          HDF5                                   #
#                                                                 #
#MANDATORY MODULES TO LOAD ON MN3 FOR 4 BYTE INTEGERS VERSION     #
#module load HDF5/1.8.14 SZIP                                     #
#                                                                 #
#MANDATORY MODULES TO LOAD ON MN3 FOR 8 BYTE INTEGERS VERSION     #
#module load SZIP/2.1 HDF5/1.8.15p1_static                        #
#                                                                 #
#MANDATORY SERVICES TO COMPILE                                    #
#hdfpos                                                           #
#MORE INFO:bsccase02.bsc.es/alya/tutorial/hdf5_output.html        #
###################################################################

# Uncomment the following lines for using HDF5 IN 4 BYTE INTEGER VERSION
#CSALYA   := $(CSALYA) -I/apps/HDF5/1.8.14/INTEL/IMPI/include
#EXTRALIB   := $(EXTRALIB) /apps/HDF5/1.8.14/INTEL/IMPI/lib/libhdf5_fortran.a /apps/HDF5/1.8.14/INTEL/IMPI/lib/libhdf5.a 
#EXTRALIB   := $(EXTRALIB) /apps/HDF5/1.8.14/INTEL/IMPI/lib/libhdf5hl_fortran.a /apps/HDF5/1.8.14/INTEL/IMPI/lib/libhdf5_hl.a
#EXTRALIB   := $(EXTRALIB) -L/apps/SZIP/2.1/lib -lsz -L/usr/lib64 -lz -lgpfs

# Uncomment the following lines for using HDF5 IN 8 BYTE INTEGER VERSION
#CSALYA   := $(CSALYA) -I/apps/HDF5/1.8.15p1/static/include
#EXTRALIB   := $(EXTRALIB) /gpfs/apps/MN3/HDF5/1.8.15p1/static/lib/libhdf5_fortran.a /gpfs/apps/MN3/HDF5/1.8.15p1/static/lib/libhdf5_f90cstub.a 
#EXTRALIB   := $(EXTRALIB) /gpfs/apps/MN3/HDF5/1.8.15p1/static/lib/libhdf5.a /gpfs/apps/MN3/HDF5/1.8.15p1/static/lib/libhdf5_tools.a
#EXTRALIB   := $(EXTRALIB) -L/apps/SZIP/2.1/lib -lsz -L/usr/lib64 -lz -lgpfs

# latest Hdf5 (1.8.16) to be used with latest impi (5.1.2.150) and intel(16.0.1) 
# there is module load HDF5/1.8.16-mpi for integers 4 but I have seen it is not necesary to call it
# Toni should clean up this  - my reconmendation is to only leave this last version
# moreover there seems to be no need for module load hdf5 (in the i8 case there exists no module load)

# Uncomment the following lines for using HDF5 IN 4 BYTE INTEGER VERSION
#CSALYA   := $(CSALYA) -I/apps/HDF5/1.8.16/INTEL/IMPI/include
#EXTRALIB   := $(EXTRALIB) /apps/HDF5/1.8.16/INTEL/IMPI/lib/libhdf5_fortran.a /apps/HDF5/1.8.16/INTEL/IMPI/lib/libhdf5hl_fortran.a
#EXTRALIB   := $(EXTRALIB) /apps/HDF5/1.8.16/INTEL/IMPI/lib/libhdf5.a /apps/HDF5/1.8.16/INTEL/IMPI/lib/libhdf5_hl.a
#EXTRALIB   := $(EXTRALIB) -L/apps/SZIP/2.1/lib -lsz -L/usr/lib64 -lz -lgpfs

# Uncomment the following lines for using HDF5 IN 8 BYTE INTEGER VERSION
#CSALYA   := $(CSALYA) -I/apps/HDF5/1.8.16/INTEL/IMPI/int8/include
#EXTRALIB   := $(EXTRALIB) /apps/HDF5/1.8.16/INTEL/IMPI/int8/lib/libhdf5_fortran.a /apps/HDF5/1.8.16/INTEL/IMPI/int8/lib/libhdf5hl_fortran.a 
#EXTRALIB   := $(EXTRALIB) /apps/HDF5/1.8.16/INTEL/IMPI/int8/lib/libhdf5.a /apps/HDF5/1.8.16/INTEL/IMPI/int8/lib/libhdf5_hl.a
#EXTRALIB   := $(EXTRALIB) -L/apps/SZIP/2.1/lib -lsz -L/usr/lib64 -lz -lgpfs

###################################################################
#                          VTK                                    #
#                                                                 #
#MANDATORY THIRDPARTY COMPILATION                                 #
#Go to Alya/Thirdparties/VTK                                      #
#                                                                 #
#MORE INFO:bsccase02.bsc.es/alya/tutorial/vtk_output.html         #
###################################################################

# Uncomment the following lines for using VTK as output
#CSALYA   := $(CSALYA) -DVTK
#EXTRALIB   := $(EXTRALIB) ../../Thirdparties/VTK/vtkXMLWriterF.o -L/apps/VTK/6.1.0_patched/lib -lvtkIOXML-6.1 -lvtkIOGeometry-6.1
#EXTRALIB   := $(EXTRALIB) -lvtkIOXMLParser-6.1 -lvtksys-6.1 -lvtkIOCore-6.1 -lvtkCommonExecutionModel-6.1 -lvtkCommonDataModel-6.1 -lvtkCommonMisc-6.1
#EXTRALIB   := $(EXTRALIB) -lvtkCommonSystem-6.1 -lvtkCommonTransforms-6.1 -lvtkCommonMath-6.1 -lvtkCommonCore-6.1 -lvtkzlib-6.1
#EXTRALIB   := $(EXTRALIB) -lvtkjsoncpp-6.1 -lvtkexpat-6.1 -L/gpfs/apps/MN3/INTEL/tbb/lib/intel64/ -ltbb

###################################################################
#                          NINJA                                  #
#GPU based solvers : GMRES,DEFLATED_CG,CG                         #
#Specify solver in configuration as:				  #
#GMRES -------------> GGMR	 				  #
#CG ----------------> GCG					  #
#DEFLATED_CG -------> GDECG					  #
#GPU Multi Grid-----> GAMGX(Requires CONFIGURATION_FILE in solver)#
#export CUDA_HOME to CUDA version to be used			  #
###################################################################
# Uncomment the following lines to enable NINJA
#GPU_HOME := ${CUDA_HOME}/
#CSALYA   := $(CSALYA) -DNINJA
#EXTRALIB   := $(EXTRALIB) -L../../Thirdparties/ninja -L${GPU_HOME}lib64/ -L/lib64 -lninja -lcublas_static -lcublasLt -lcusparse_static -lculibos -lcudart -lpthread -lstdc++ -ldl -lcusolver

# NINJA also support AMGX. Uncomment the following lines and set AMGX_HOME
#AMGX_HOME :=
#CSALYA   := $(CSALYA) -DAMGX
#EXTRALIB   := $(EXTRALIB) -L${AMGX_HOME}/lib -lamgxsh

###################################################################
#                          CATALYST                               #
#                                                                 #
#MANDATORY THIRDPARTY COMPILATION                                 #
#Go to Alya/Thirdparties/Catalyst                                 #
#                                                                 #
#MORE INFO:hadrien.calmet at bsc.es                               #
###################################################################
# Uncomment the following lines for using CATALYST as output
#CSALYA   := $(CSALYA) -DVTK -DCATA -mt_mpi -I../../Thirdparties/Catalyst
#EXTRALIB   := $(EXTRALIB) ../../Thirdparties/VTK/vtkXMLWriterF.o ../../Thirdparties/Catalyst/FEFortranAdaptor.o ../../Thirdparties/Catalyst/FECxxAdaptor.o
#EXTRALIB   := $(EXTRALIB) -L/apps/PARAVIEW/4.2.0/lib/paraview-4.2/ -lvtkIOXML-pv4.2 -lvtkIOGeometry-pv4.2
#EXTRALIB   := $(EXTRALIB) -lvtkIOXMLParser-pv4.2 -lvtksys-pv4.2 -lvtkIOCore-pv4.2 -lvtkCommonExecutionModel-pv4.2 -lvtkCommonDataModel-pv4.2
#EXTRALIB   := $(EXTRALIB) -lvtkCommonMisc-pv4.2 -lvtkCommonSystem-pv4.2 -lvtkCommonTransforms-pv4.2 -lvtkCommonMath-pv4.2 -lvtkCommonCore-pv4.2
#EXTRALIB   := $(EXTRALIB) -lvtkzlib-pv4.2 -lvtkjsoncpp-pv4.2 -lvtkexpat-pv4.2 -lvtkPVPythonCatalyst-pv4.2 -lvtkPVCatalyst-pv4.2 -lvtkPythonInterpreter-pv4.2
#EXTRALIB   := $(EXTRALIB) -lvtkUtilitiesPythonInitializer-pv4.2  -lvtkPVServerManagerCore-pv4.2 -lvtkPVServerImplementationCore-pv4.2 -lvtkPVClientServerCoreCore-pv4.2
#EXTRALIB   := $(EXTRALIB) -lvtkFiltersParallel-pv4.2 -lvtkFiltersModeling-pv4.2 -lvtkRenderingCore-pv4.2 -lvtkFiltersExtraction-pv4.2 -lvtkFiltersStatistics-pv4.2
#EXTRALIB   := $(EXTRALIB) -lvtkImagingFourier-pv4.2 -lvtkImagingCore-pv4.2 -lvtkalglib-pv4.2 -lvtkFiltersGeometry-pv4.2 -lvtkFiltersSources-pv4.2
#EXTRALIB   := $(EXTRALIB) -lvtkFiltersGeneral-pv4.2 -lvtkCommonComputationalGeometry-pv4.2 -lvtkFiltersProgrammable-pv4.2 -lvtkPVVTKExtensionsCore-pv4.2
#EXTRALIB   := $(EXTRALIB) -lvtkPVCommon-pv4.2 -lvtkClientServer-pv4.2 -lvtkFiltersCore-pv4.2 -lvtkParallelMPI-pv4.2 -lvtkParallelCore-pv4.2 -lvtkIOLegacy-pv4.2
#EXTRALIB   := $(EXTRALIB) -lprotobuf -lvtkWrappingPython27Core-pv4.2 -lvtkpugixml-pv4.2 -lvtkPVServerManagerApplication-pv4.2 -L/gpfs/apps/MN3/INTEL/tbb/lib/intel64/ -ltbb

###################################################################
#                   EoCoE Flags                                   #
###################################################################

# Uncomment the following lines to output matrices                #
#CSALYA   := $(CSALYA) -Doutmateocoe
# Uncomment the following lines to solve with AGMG                #
#MANDATORY MODULES TO LOAD ON MN3 
#module load MKL/11.3
# serial version   -- this is obsolete actually for it to work you need to touch the ifdef in dagmg.f90
#CSALYA   := $(CSALYA) -Dsolve_w_agmg -I${MKLROOT}/include
#EXTRALIB := $(EXTRALIB) -L${MKLROOT}/lib/intel64 -lmkl_intel_lp64 -lmkl_core -lmkl_sequential
#
#
# parallel version academic
#CSALYA   := $(CSALYA) -DPARAL_AGMG -DPARAL_AGMG_ACAD -I${MKLROOT}/include -I/gpfs/projects/bsc21/WORK-HERBERT/svnmn3/MUMPS_5.0.1/include
#For Mumps
#EXTRALIB := $(EXTRALIB) -L/gpfs/projects/bsc21/WORK-HERBERT/svnmn3/MUMPS_5.0.1/lib -ldmumps -lmumps_common  -lpord
#For agmg
#EXTRALIB := $(EXTRALIB) -L${MKLROOT}/lib/intel64 -lmkl_intel_lp64 -lmkl_core -lmkl_sequential -lpthread -lm  -lmkl_blacs_intelmpi_ilp64 -lmkl_blacs_intelmpi_lp64 -lmkl_scalapack_ilp64 -lmkl_scalapack_lp64
#
# parallel version prof  (copy dagmg_par.o from Thirdparties/agmg)
#CSALYA   := $(CSALYA) -DPARAL_AGMG -Duse_dagmg_mumps -I${MKLROOT}/include
#For agmg
#EXTRALIB := $(EXTRALIB) -L${MKLROOT}/lib/intel64 -lmkl_intel_lp64 -lmkl_core -lmkl_sequential -lpthread -lm  -lmkl_blacs_intelmpi_ilp64 -lmkl_blacs_intelmpi_lp64 -lmkl_scalapack_ilp64 -lmkl_scalapack_lp64 -L../../Thirdparties/agmg/ -ldagmg_par

###################################################################
#                          SUNDIALS CVODE                         #
###################################################################

# Uncomment the following lines for using Sundials CVode
#EXTRALIB   := $(EXTRALIB) ../../Thirdparties/sundials/sundials-install/lib/*.a ../../Thirdparties/sundials/sundials-install/lib/libsundials_cvode.a



###################################################################
#                          DBParicles LIBRARY                     #
###################################################################

# Uncomment the following lines for using DBParticles library
#CSALYA   := $(CSALYA) -DDBPARTICLES
#--FOR THRIFT connector--
#EXTRALIB   := $(EXTRALIB) -L../../Thirdparties/dbparticles -ldbparticles -I/usr/local/include/thrift/ -I/usr/local/include -L/usr/local/lib -lm -lstdc++
#EXTRALIB   := $(EXTRALIB) -lthrift -lthriftnb -lthriftqt -lthriftz
#--FOR CASSANDRA connector--
#EXTRALIB   := $(EXTRALIB) ../../Thirdparties/dbparticles/MurmurHash3.o -L../../Thirdparties/dbparticles -ldbparticles -I/usr/lib/x86_64-linux-gnu/include -L/usr/lib/x86_64-linux-gnu -luv -lcassandra -I/usr/local/include -L/usr/local/lib -lm -lstdc++

###################################################################
#                     WSMP SOLVER ILLINOIS                        #
###################################################################

# Uncomment the following lines to use WSMP library
#CSALYA   := $(CSALYA) -DWSMP 
#EXTRALIB := $(EXTRALIB) /gpfs/projects/bsc21/bsc21103/wsmpint/wsmp-Linux64-Intel/lib/ -lwsmp64 -lpthread /apps/LAPACK/3.5.0/lib -mkl

###################################################################
#                          DO NOT TOUCH                           #
###################################################################
FCFLAGS    := $(FCFLAGS) $(CSALYA) $(EXTRAINC)

