Newer
Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
###################################################################
# PGI CONFIGURE #
#POWER9 RECOMENDED MODULE: #
#module load ompi/3.0.0 pgi/18.4 #
###################################################################
F77 = OMPI_FC=pgfortran mpif90
F90 = OMPI_FC=pgfortran mpif90
FCOCC = cc -c
FCFLAGS = -c -fast -Minfo=all -acc -ta=tesla:cuda10.1 -Mpreprocess -I./Objects_x/ -Mbackslash -Mextend -Mnoopenmp -Munroll -Mnoidiom -module $O
FPPFLAGS =
EXTRALIB = -lc
EXTRAINC =
fa2p = pgfortran -c -x f95-cpp-input -DMPI_OFF -J../../Utils/user/alya2pos -I../../Utils/user/alya2pos
fa2plk = pgfortran
###################################################################
# PERFORMANCE FLAGS #
###################################################################
#MINUM
#FOPT = -O1
#MAXIMUM (I have elimated -xHost due to observations by Yacine)
FOPT = -O3
#Compilation flags applied only to the Source/modules folder
#MODULEFLAGS = -ipo
# Uncomment the following line to enable NDIME as a parameter (OPTIMIZATION FOR 3D PROBLEMS)
CSALYA := $(CSALYA) -DNDIMEPAR -DOPENACCHHH -DSUPER_FAST -DDETAILED_TIMES
# Uncomment the following line for DEBUG AND CHECKING FLAGS
#CSALYA := $(CSALYA) -C -Ktrap=fp -Minform=inform
# Vectorization: put vector size (in principle=4 for MN)
CSALYA := $(CSALYA) -DVECTOR_SIZE=32768
###################################################################
# USER SPECIFIC FLAGS #
###################################################################
# HERBERT
#CSALYA := $(CSALYA) -DDETAILS_ORTHOMIN
###################################################################
# PROFILING FLAGS #
###################################################################
# Uncomment the following line to generate profiling info files
#CSALYA := $(CSALYA) -profile-loops=all -profile-loops-report=2
###################################################################
# EXTRAE FLAGS #
###################################################################
# Uncomment the following line to compile Alya using extrae
# Compiler used to compile extrae module (make extrae)
EXTRAE_FC=pgfortran
# Extrae installation directory (for linking) (not necessary if loading extrae using module load extrae)
#EXTRAE_HOME=
#@Linking with Extrae
#EXTRALIB := $(EXTRALIB) -L${EXTRAE_HOME}/lib/ -lmpitracef extrae_module.o
#@Enabling Extrae user calls (normal)
#CSALYA := $(CSALYA) -DEXTRAE
###################################################################
# METIS LIBRARY #
###################################################################
# Uncomment the following lines for using metis 4.0 (default)
EXTRALIB := $(EXTRALIB) -L../../Thirdparties/metis-4.0 -lmetis -acc -ta=tesla:cuda10.1
CSALYA := $(CSALYA) -DMETIS
# Uncomment the following lines for using metis 5.0
#CSALYA := $(CSALYA) -DV5METIS
# uncoment FOR MAC
#EXTRALIB := $(EXTRALIB) -L../../Thirdparties/metis-5.0.2_i8/build/Darwin-i386/libmetis -lmetis
# uncoment FOR LINUX
#EXTRALIB := $(EXTRALIB) -L../../Thirdparties/metis-5.0.2_i8/build/Linux-x86_64/libmetis -lmetis
# Uncomment the following lines for using parmetis
#CSALYA := $(CSALYA) -DPARMETIS
#EXTRALIB := $(EXTRALIB) -L../../Thirdparties/
###################################################################
# BLAS LIBRARY #
###################################################################
# Uncomment the following lines for using blas
#EXTRALIB := $(EXTRALIB) -lblas
#CSALYA := $(CSALYA) -DBLAS
###################################################################
# MPI/THREADS/TASKS #
###################################################################
# Uncomment the following lines for sequential (NOMPI) version
#VERSIONMPI = nompi
#CSALYA := $(CSALYA) -DMPI_OFF
# Uncomment the following lines for OPENMP version
#CSALYA := $(CSALYA) -openmp
#EXTRALIB := $(EXTRALIB) -openmp
# Uncomment the following line to disable OPENMP coloring
#CSALYA := $(CSALYA) -DNO_COLORING
# Uncomment the following line to enable OMPSS in order to activate
# loops with multidependencies whenever we have a race condition
#CSALYA := $(CSALYA) -DALYA_OMPSS
###################################################################
# DLB #
# To use DLB with OMPSS: Define -DALYA_DLB -DALYA_OMPSS #
# To use DLB barrier: Define -DALYA_DLB_BARRIER #
# If we compile with mercurium, there is no need to link #
# with DLB #
# #
###################################################################
#CSALYA := $(CSALYA) -DALYA_DLB
#FCFLAGS := $(FCFLAGS) --dlb
#FPPFLAGS := $(FPPFLAGS) --dlb
###################################################################
# INTEGER TYPE #
###################################################################
# Uncomment the following lines for 8 byte integers
#CSALYA := $(CSALYA) -DI8 -m64
# Uncomment the following line for 8 byte integers ONLY in Metis
# In this mode Alya can work in I4 and Metis in I8 (mixed mode)
# For this option the use of Metis v5 library is mandatory
#CSALYA := $(CSALYA) -DMETISI8
###################################################################
# HDF5 #
# #
#MANDATORY MODULES TO LOAD ON MN3 FOR 4 BYTE INTEGERS VERSION #
#module load HDF5/1.8.14 SZIP #
# #
#MANDATORY MODULES TO LOAD ON MN3 FOR 8 BYTE INTEGERS VERSION #
#module load SZIP/2.1 HDF5/1.8.15p1_static #
# #
#MANDATORY SERVICES TO COMPILE #
#hdfpos #
#MORE INFO:bsccase02.bsc.es/alya/tutorial/hdf5_output.html #
###################################################################
# Uncomment the following lines for using HDF5 IN 4 BYTE INTEGER VERSION
#CSALYA := $(CSALYA) -I/apps/HDF5/1.8.14/INTEL/IMPI/include
#EXTRALIB := $(EXTRALIB) /apps/HDF5/1.8.14/INTEL/IMPI/lib/libhdf5_fortran.a /apps/HDF5/1.8.14/INTEL/IMPI/lib/libhdf5.a
#EXTRALIB := $(EXTRALIB) /apps/HDF5/1.8.14/INTEL/IMPI/lib/libhdf5hl_fortran.a /apps/HDF5/1.8.14/INTEL/IMPI/lib/libhdf5_hl.a
#EXTRALIB := $(EXTRALIB) -L/apps/SZIP/2.1/lib -lsz -L/usr/lib64 -lz -lgpfs
# Uncomment the following lines for using HDF5 IN 8 BYTE INTEGER VERSION
#CSALYA := $(CSALYA) -I/apps/HDF5/1.8.15p1/static/include
#EXTRALIB := $(EXTRALIB) /gpfs/apps/MN3/HDF5/1.8.15p1/static/lib/libhdf5_fortran.a /gpfs/apps/MN3/HDF5/1.8.15p1/static/lib/libhdf5_f90cstub.a
#EXTRALIB := $(EXTRALIB) /gpfs/apps/MN3/HDF5/1.8.15p1/static/lib/libhdf5.a /gpfs/apps/MN3/HDF5/1.8.15p1/static/lib/libhdf5_tools.a
#EXTRALIB := $(EXTRALIB) -L/apps/SZIP/2.1/lib -lsz -L/usr/lib64 -lz -lgpfs
# latest Hdf5 (1.8.16) to be used with latest impi (5.1.2.150) and intel(16.0.1)
# there is module load HDF5/1.8.16-mpi for integers 4 but I have seen it is not necesary to call it
# Toni should clean up this - my reconmendation is to only leave this last version
# moreover there seems to be no need for module load hdf5 (in the i8 case there exists no module load)
# Uncomment the following lines for using HDF5 IN 4 BYTE INTEGER VERSION
#CSALYA := $(CSALYA) -I/apps/HDF5/1.8.16/INTEL/IMPI/include
#EXTRALIB := $(EXTRALIB) /apps/HDF5/1.8.16/INTEL/IMPI/lib/libhdf5_fortran.a /apps/HDF5/1.8.16/INTEL/IMPI/lib/libhdf5hl_fortran.a
#EXTRALIB := $(EXTRALIB) /apps/HDF5/1.8.16/INTEL/IMPI/lib/libhdf5.a /apps/HDF5/1.8.16/INTEL/IMPI/lib/libhdf5_hl.a
#EXTRALIB := $(EXTRALIB) -L/apps/SZIP/2.1/lib -lsz -L/usr/lib64 -lz -lgpfs
# Uncomment the following lines for using HDF5 IN 8 BYTE INTEGER VERSION
#CSALYA := $(CSALYA) -I/apps/HDF5/1.8.16/INTEL/IMPI/int8/include
#EXTRALIB := $(EXTRALIB) /apps/HDF5/1.8.16/INTEL/IMPI/int8/lib/libhdf5_fortran.a /apps/HDF5/1.8.16/INTEL/IMPI/int8/lib/libhdf5hl_fortran.a
#EXTRALIB := $(EXTRALIB) /apps/HDF5/1.8.16/INTEL/IMPI/int8/lib/libhdf5.a /apps/HDF5/1.8.16/INTEL/IMPI/int8/lib/libhdf5_hl.a
#EXTRALIB := $(EXTRALIB) -L/apps/SZIP/2.1/lib -lsz -L/usr/lib64 -lz -lgpfs
###################################################################
# VTK #
# #
#MANDATORY THIRDPARTY COMPILATION #
#Go to Alya/Thirdparties/VTK #
# #
#MORE INFO:bsccase02.bsc.es/alya/tutorial/vtk_output.html #
###################################################################
# Uncomment the following lines for using VTK as output
#CSALYA := $(CSALYA) -DVTK
#EXTRALIB := $(EXTRALIB) ../../Thirdparties/VTK/vtkXMLWriterF.o -L/apps/VTK/6.1.0_patched/lib -lvtkIOXML-6.1 -lvtkIOGeometry-6.1
#EXTRALIB := $(EXTRALIB) -lvtkIOXMLParser-6.1 -lvtksys-6.1 -lvtkIOCore-6.1 -lvtkCommonExecutionModel-6.1 -lvtkCommonDataModel-6.1 -lvtkCommonMisc-6.1
#EXTRALIB := $(EXTRALIB) -lvtkCommonSystem-6.1 -lvtkCommonTransforms-6.1 -lvtkCommonMath-6.1 -lvtkCommonCore-6.1 -lvtkzlib-6.1
#EXTRALIB := $(EXTRALIB) -lvtkjsoncpp-6.1 -lvtkexpat-6.1 -L/gpfs/apps/MN3/INTEL/tbb/lib/intel64/ -ltbb
###################################################################
# NINJA #
#GPU based solvers : GMRES,DEFLATED_CG,CG #
#Specify solver in configuration as: #
#GMRES -------------> GGMR #
#CG ----------------> GCG #
#DEFLATED_CG -------> GDECG #
#GPU Multi Grid-----> GAMGX(Requires CONFIGURATION_FILE in solver)#
#export CUDA_HOME to CUDA version to be used #
###################################################################
# Uncomment the following lines to enable NINJA
#GPU_HOME := ${CUDA_HOME}/
#CSALYA := $(CSALYA) -DNINJA
#EXTRALIB := $(EXTRALIB) -L../../Thirdparties/ninja -L${GPU_HOME}lib64/ -L/lib64 -lninja -lcublas_static -lcublasLt -lcusparse_static -lculibos -lcudart -lpthread -lstdc++ -ldl -lcusolver
# NINJA also support AMGX. Uncomment the following lines and set AMGX_HOME
#AMGX_HOME :=
#CSALYA := $(CSALYA) -DAMGX
#EXTRALIB := $(EXTRALIB) -L${AMGX_HOME}/lib -lamgxsh
###################################################################
# CATALYST #
# #
#MANDATORY THIRDPARTY COMPILATION #
#Go to Alya/Thirdparties/Catalyst #
# #
#MORE INFO:hadrien.calmet at bsc.es #
###################################################################
# Uncomment the following lines for using CATALYST as output
#CSALYA := $(CSALYA) -DVTK -DCATA -mt_mpi -I../../Thirdparties/Catalyst
#EXTRALIB := $(EXTRALIB) ../../Thirdparties/VTK/vtkXMLWriterF.o ../../Thirdparties/Catalyst/FEFortranAdaptor.o ../../Thirdparties/Catalyst/FECxxAdaptor.o
#EXTRALIB := $(EXTRALIB) -L/apps/PARAVIEW/4.2.0/lib/paraview-4.2/ -lvtkIOXML-pv4.2 -lvtkIOGeometry-pv4.2
#EXTRALIB := $(EXTRALIB) -lvtkIOXMLParser-pv4.2 -lvtksys-pv4.2 -lvtkIOCore-pv4.2 -lvtkCommonExecutionModel-pv4.2 -lvtkCommonDataModel-pv4.2
#EXTRALIB := $(EXTRALIB) -lvtkCommonMisc-pv4.2 -lvtkCommonSystem-pv4.2 -lvtkCommonTransforms-pv4.2 -lvtkCommonMath-pv4.2 -lvtkCommonCore-pv4.2
#EXTRALIB := $(EXTRALIB) -lvtkzlib-pv4.2 -lvtkjsoncpp-pv4.2 -lvtkexpat-pv4.2 -lvtkPVPythonCatalyst-pv4.2 -lvtkPVCatalyst-pv4.2 -lvtkPythonInterpreter-pv4.2
#EXTRALIB := $(EXTRALIB) -lvtkUtilitiesPythonInitializer-pv4.2 -lvtkPVServerManagerCore-pv4.2 -lvtkPVServerImplementationCore-pv4.2 -lvtkPVClientServerCoreCore-pv4.2
#EXTRALIB := $(EXTRALIB) -lvtkFiltersParallel-pv4.2 -lvtkFiltersModeling-pv4.2 -lvtkRenderingCore-pv4.2 -lvtkFiltersExtraction-pv4.2 -lvtkFiltersStatistics-pv4.2
#EXTRALIB := $(EXTRALIB) -lvtkImagingFourier-pv4.2 -lvtkImagingCore-pv4.2 -lvtkalglib-pv4.2 -lvtkFiltersGeometry-pv4.2 -lvtkFiltersSources-pv4.2
#EXTRALIB := $(EXTRALIB) -lvtkFiltersGeneral-pv4.2 -lvtkCommonComputationalGeometry-pv4.2 -lvtkFiltersProgrammable-pv4.2 -lvtkPVVTKExtensionsCore-pv4.2
#EXTRALIB := $(EXTRALIB) -lvtkPVCommon-pv4.2 -lvtkClientServer-pv4.2 -lvtkFiltersCore-pv4.2 -lvtkParallelMPI-pv4.2 -lvtkParallelCore-pv4.2 -lvtkIOLegacy-pv4.2
#EXTRALIB := $(EXTRALIB) -lprotobuf -lvtkWrappingPython27Core-pv4.2 -lvtkpugixml-pv4.2 -lvtkPVServerManagerApplication-pv4.2 -L/gpfs/apps/MN3/INTEL/tbb/lib/intel64/ -ltbb
###################################################################
# EoCoE Flags #
###################################################################
# Uncomment the following lines to output matrices #
#CSALYA := $(CSALYA) -Doutmateocoe
# Uncomment the following lines to solve with AGMG #
#MANDATORY MODULES TO LOAD ON MN3
#module load MKL/11.3
# serial version -- this is obsolete actually for it to work you need to touch the ifdef in dagmg.f90
#CSALYA := $(CSALYA) -Dsolve_w_agmg -I${MKLROOT}/include
#EXTRALIB := $(EXTRALIB) -L${MKLROOT}/lib/intel64 -lmkl_intel_lp64 -lmkl_core -lmkl_sequential
#
#
# parallel version academic
#CSALYA := $(CSALYA) -DPARAL_AGMG -DPARAL_AGMG_ACAD -I${MKLROOT}/include -I/gpfs/projects/bsc21/WORK-HERBERT/svnmn3/MUMPS_5.0.1/include
#For Mumps
#EXTRALIB := $(EXTRALIB) -L/gpfs/projects/bsc21/WORK-HERBERT/svnmn3/MUMPS_5.0.1/lib -ldmumps -lmumps_common -lpord
#For agmg
#EXTRALIB := $(EXTRALIB) -L${MKLROOT}/lib/intel64 -lmkl_intel_lp64 -lmkl_core -lmkl_sequential -lpthread -lm -lmkl_blacs_intelmpi_ilp64 -lmkl_blacs_intelmpi_lp64 -lmkl_scalapack_ilp64 -lmkl_scalapack_lp64
#
# parallel version prof (copy dagmg_par.o from Thirdparties/agmg)
#CSALYA := $(CSALYA) -DPARAL_AGMG -Duse_dagmg_mumps -I${MKLROOT}/include
#For agmg
#EXTRALIB := $(EXTRALIB) -L${MKLROOT}/lib/intel64 -lmkl_intel_lp64 -lmkl_core -lmkl_sequential -lpthread -lm -lmkl_blacs_intelmpi_ilp64 -lmkl_blacs_intelmpi_lp64 -lmkl_scalapack_ilp64 -lmkl_scalapack_lp64 -L../../Thirdparties/agmg/ -ldagmg_par
###################################################################
# SUNDIALS CVODE #
###################################################################
# Uncomment the following lines for using Sundials CVode
#EXTRALIB := $(EXTRALIB) ../../Thirdparties/sundials/sundials-install/lib/*.a ../../Thirdparties/sundials/sundials-install/lib/libsundials_cvode.a
###################################################################
# DBParicles LIBRARY #
###################################################################
# Uncomment the following lines for using DBParticles library
#CSALYA := $(CSALYA) -DDBPARTICLES
#--FOR THRIFT connector--
#EXTRALIB := $(EXTRALIB) -L../../Thirdparties/dbparticles -ldbparticles -I/usr/local/include/thrift/ -I/usr/local/include -L/usr/local/lib -lm -lstdc++
#EXTRALIB := $(EXTRALIB) -lthrift -lthriftnb -lthriftqt -lthriftz
#--FOR CASSANDRA connector--
#EXTRALIB := $(EXTRALIB) ../../Thirdparties/dbparticles/MurmurHash3.o -L../../Thirdparties/dbparticles -ldbparticles -I/usr/lib/x86_64-linux-gnu/include -L/usr/lib/x86_64-linux-gnu -luv -lcassandra -I/usr/local/include -L/usr/local/lib -lm -lstdc++
###################################################################
# WSMP SOLVER ILLINOIS #
###################################################################
# Uncomment the following lines to use WSMP library
#CSALYA := $(CSALYA) -DWSMP
#EXTRALIB := $(EXTRALIB) /gpfs/projects/bsc21/bsc21103/wsmpint/wsmp-Linux64-Intel/lib/ -lwsmp64 -lpthread /apps/LAPACK/3.5.0/lib -mkl
###################################################################
# DO NOT TOUCH #
###################################################################
FCFLAGS := $(FCFLAGS) $(CSALYA) $(EXTRAINC)