Newer
Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
#-----------------------------------------------------------
#
# Simulation input parameters
#
#-----------------------------------------------------------
# forward or adjoint simulation
SIMULATION_TYPE = 1 # set to 1 for forward simulations, 2 for adjoint simulations for sources, and 3 for kernel simulations
NOISE_TOMOGRAPHY = 0 # flag of noise tomography, three steps (1,2,3). If earthquake simulation, set it to 0.
SAVE_FORWARD = .false. # save last frame of forward simulation or not
# number of chunks (1,2,3 or 6)
NCHUNKS = 6
# angular width of the first chunk (not used if full sphere with six chunks)
ANGULAR_WIDTH_XI_IN_DEGREES = 90.d0 # angular size of a chunk
ANGULAR_WIDTH_ETA_IN_DEGREES = 90.d0
CENTER_LATITUDE_IN_DEGREES = 90.d0
CENTER_LONGITUDE_IN_DEGREES = 0.d0
GAMMA_ROTATION_AZIMUTH = 0.d0
# number of elements at the surface along the two sides of the first chunk
# (must be multiple of 16 and 8 * multiple of NPROC below)
NEX_XI = 64
NEX_ETA = 64
# number of MPI processors along the two sides of the first chunk
NPROC_XI = 1
NPROC_ETA = 1
#-----------------------------------------------------------
#
# Model
#
#-----------------------------------------------------------
# 1D models with real structure:
# 1D_isotropic_prem, 1D_transversely_isotropic_prem, 1D_iasp91, 1D_1066a, 1D_ak135f_no_mud, 1D_ref, 1D_ref_iso, 1D_jp3d,1D_sea99
#
# 1D models with only one fictitious averaged crustal layer:
# 1D_isotropic_prem_onecrust, 1D_transversely_isotropic_prem_onecrust, 1D_iasp91_onecrust, 1D_1066a_onecrust, 1D_ak135f_no_mud_onecrust
#
# fully 3D models:
# transversely_isotropic_prem_plus_3D_crust_2.0, 3D_anisotropic, 3D_attenuation,
# s20rts, s40rts, s362ani, s362iso, s362wmani, s362ani_prem, s362ani_3DQ, s362iso_3DQ,
# s29ea, s29ea,sea99_jp3d1994,sea99,jp3d1994,heterogen,full_sh
#
# 3D models with 1D crust: append "_1Dcrust" the the 3D model name
# to take the 1D crustal model from the
# associated reference model rather than the default 3D crustal model
# e.g. s20rts_1Dcrust, s362ani_1Dcrust, etc.
MODEL = s362ani
# parameters describing the Earth model
OCEANS = .true.
ELLIPTICITY = .true.
TOPOGRAPHY = .true.
GRAVITY = .true.
ROTATION = .true.
ATTENUATION = .true.
# absorbing boundary conditions for a regional simulation
ABSORBING_CONDITIONS = .false.
# record length in minutes
RECORD_LENGTH_IN_MINUTES = 1d0
# to undo attenuation for sensitivity kernel calculations or forward runs with SAVE_FORWARD
# use one (and only one) of the two flags below. UNDO_ATTENUATION is much better (it is exact)
# but requires a significant amount of disk space for temporary storage.
PARTIAL_PHYS_DISPERSION_ONLY = .false.
UNDO_ATTENUATION = .false.
# How much memory (in GB) is installed on your machine per CPU core (only used for UNDO_ATTENUATION, can be ignored otherwise)
# (or per GPU card or per INTEL MIC Phi board)
# Beware, this value MUST be given per core, i.e. per MPI thread, i.e. per MPI rank, NOT per node.
# This value is for instance:
# - 4 GB on Tiger at Princeton
# - 4 GB on TGCC Curie in Paris
# - 4 GB on Titan at ORNL when using CPUs only (no GPUs); start your run with "aprun -n$NPROC -N8 -S4 -j1"
# - 2 GB on the machine used by Christina Morency
# - 2 GB on the TACC machine used by Min Chen
# - 1.5 GB on the GPU cluster in Marseille
# When running on GPU machines, it is simpler to set PERCENT_OF_MEM_TO_USE_PER_CORE = 100.d0
# and then set MEMORY_INSTALLED_PER_CORE_IN_GB to the amount of memory that you estimate is free (rather than installed)
# on the host of the GPU card while running your GPU job.
# For GPU runs on Titan at ORNL, use PERCENT_OF_MEM_TO_USE_PER_CORE = 100.d0 and MEMORY_INSTALLED_PER_CORE_IN_GB = 25.d0
# and run your job with "aprun -n$NPROC -N1 -S1 -j1"
# (each host has 32 GB on Titan, each GPU has 6 GB, thus even if all the GPU arrays are duplicated on the host
# this leaves 32 - 6 = 26 GB free on the host; leaving 1 GB for the Linux system, we can safely use 100% of 25 GB)
MEMORY_INSTALLED_PER_CORE_IN_GB = 16.0d0
# What percentage of this total do you allow us to use for arrays to undo attenuation, keeping in mind that you
# need to leave some memory available for the GNU/Linux system to run
# (a typical value is 85%; any value below is fine but the code will then save a lot of data to disk;
# values above, say 90% or 92%, can be OK on some systems but can make the adjoint code run out of memory
# on other systems, depending on how much memory per node the GNU/Linux system needs for itself; thus you can try
# a higher value and if the adjoint crashes then try again with a lower value)
PERCENT_OF_MEM_TO_USE_PER_CORE = 85.d0
# three mass matrices instead of one are needed to handle rotation very accurately;
# otherwise rotation is handled slightly less accurately (but still reasonably well);
# set to .true. if you are interested in precise effects related to rotation;
# set to .false. if you are solving very large inverse problems at high frequency and also undoing attenuation exactly
# using the UNDO_ATTENUATION flag above, in which case saving as much memory as possible can be a good idea.
# You can also safely set it to .false. if you are not in a period range in which rotation matters, e.g. if you are targetting very short-period body waves.
# if in doubt, set to .true.
# Set it to .true. if you have ABSORBING_CONDITIONS above, because in that case the code will use the three mass matrices anyway
# and thus there is no additional cost.
# this flag is of course unused if ROTATION above is set to .false.
EXACT_MASS_MATRIX_FOR_ROTATION = .false.
#-----------------------------------------------------------
# this for LDDRK high-order time scheme instead of Newmark
USE_LDDRK = .false.
# the maximum CFL of LDDRK is significantly higher than that of the Newmark scheme,
# in a ratio that is theoretically 1.327 / 0.697 = 1.15 / 0.604 = 1.903 for a solid with Poisson's ratio = 0.25
# and for a fluid (see the manual of the 2D code, SPECFEM2D, Tables 4.1 and 4.2, and that ratio does not
# depend on whether we are in 2D or in 3D). However in practice a ratio of about 1.5 to 1.7 is often safer
# (for instance for models with a large range of Poisson's ratio values).
# Since the code computes the time step using the Newmark scheme, for LDDRK we will simply
# multiply that time step by this ratio when LDDRK is on and when flag INCREASE_CFL_FOR_LDDRK is true.
INCREASE_CFL_FOR_LDDRK = .true.
RATIO_BY_WHICH_TO_INCREASE_IT = 1.5d0
#-----------------------------------------------------------
#
# Visualization
#
#-----------------------------------------------------------
# save AVS or OpenDX movies
#MOVIE_COARSE saves movie only at corners of elements (SURFACE OR VOLUME)
#MOVIE_COARSE does not work with create_movie_AVS_DX
MOVIE_SURFACE = .false.
MOVIE_VOLUME = .false.
MOVIE_COARSE = .false.
NTSTEP_BETWEEN_FRAMES = 100
HDUR_MOVIE = 0.d0
# save movie in volume. Will save element if center of element is in prescribed volume
# top/bottom: depth in KM, use MOVIE_TOP = -100 to make sure the surface is stored.
# west/east: longitude, degrees East [-180/180] top/bottom: latitute, degrees North [-90/90]
# start/stop: frames will be stored at MOVIE_START + i*NSTEP_BETWEEN_FRAMES, where i=(0,1,2..) and iNSTEP_BETWEEN_FRAMES <= MOVIE_STOP
# movie_volume_type: 1=strain, 2=time integral of strain, 3=\mu*time integral of strain
# type 4 saves the trace and deviatoric stress in the whole volume, 5=displacement, 6=velocity
MOVIE_VOLUME_TYPE = 2
MOVIE_TOP_KM = -100.0
MOVIE_BOTTOM_KM = 1000.0
MOVIE_WEST_DEG = -90.0
MOVIE_EAST_DEG = 90.0
MOVIE_NORTH_DEG = 90.0
MOVIE_SOUTH_DEG = -90.0
MOVIE_START = 0
MOVIE_STOP = 40000
# save mesh files to check the mesh
SAVE_MESH_FILES = .false.
# restart files (number of runs can be 1 or higher, choose 1 for no restart files)
NUMBER_OF_RUNS = 1
NUMBER_OF_THIS_RUN = 1
# path to store the local database files on each node
LOCAL_PATH = ./DATABASES_MPI
# temporary wavefield/kernel/movie files
LOCAL_TMP_PATH = ./DATABASES_MPI
# interval at which we output time step info and max of norm of displacement
NTSTEP_BETWEEN_OUTPUT_INFO = 50
#-----------------------------------------------------------
#
# Sources & seismograms
#
#-----------------------------------------------------------
# interval in time steps for temporary writing of seismograms
NTSTEP_BETWEEN_OUTPUT_SEISMOS = 2000
NTSTEP_BETWEEN_READ_ADJSRC = 1000
# use a (tilted) FORCESOLUTION force point source (or several) instead of a CMTSOLUTION moment-tensor source.
# # This can be useful e.g. for asteroid simulations
# # in which the source is a vertical force, normal force, tilted force, impact etc.
# # If this flag is turned on, the FORCESOLUTION file must be edited by giving:
# # - the corresponding time-shift parameter,
# # - the half duration parameter of the source,
# # - the coordinates of the source,
# # - the source time function of the source,
# # - the magnitude of the force source,
# # - the components of a (non necessarily unitary) direction vector for the force source in the E/N/Z_UP basis.
# # The direction vector is made unitary internally in the code and thus only its direction matters here;
# # its norm is ignored and the norm of the force used is the factor force source times the source time function.
USE_FORCE_POINT_SOURCE = .false.
# option to save strain seismograms
# this option is useful for strain Green's tensor
# this feature is currently under development
SAVE_SEISMOGRAMS_STRAIN = .false.
# save seismograms also when running the adjoint runs for an inverse problem
# (usually they are unused and not very meaningful, leave this off in almost all cases)
SAVE_SEISMOGRAMS_IN_ADJOINT_RUN = .false.
# output format for the seismograms (one can use either or all of the three formats)
OUTPUT_SEISMOS_ASCII_TEXT = .true.
OUTPUT_SEISMOS_SAC_ALPHANUM = .false.
OUTPUT_SEISMOS_SAC_BINARY = .false.
OUTPUT_SEISMOS_ASDF = .false.
# rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame
ROTATE_SEISMOGRAMS_RT = .false.
# decide if master process writes all the seismograms or if all processes do it in parallel
WRITE_SEISMOGRAMS_BY_MASTER = .true.
# save all seismograms in one large combined file instead of one file per seismogram
# to avoid overloading shared non-local file systems such as LUSTRE or GPFS for instance
SAVE_ALL_SEISMOS_IN_ONE_FILE = .false.
USE_BINARY_FOR_LARGE_FILE = .false.
# flag to impose receivers at the surface or allow them to be buried
RECEIVERS_CAN_BE_BURIED = .true.
# print source time function
PRINT_SOURCE_TIME_FUNCTION = .false.
#-----------------------------------------------------------
#
# Adjoint kernel outputs
#
#-----------------------------------------------------------
# use ASDF format for reading the adjoint sources
READ_ADJSRC_ASDF = .false.
# this parameter must be set to .true. to compute anisotropic kernels
# in crust and mantle (related to the 21 Cij in geographical coordinates)
# default is .false. to compute isotropic kernels (related to alpha and beta)
ANISOTROPIC_KL = .false.
# output only transverse isotropic kernels (alpha_v,alpha_h,beta_v,beta_h,eta,rho)
# rather than fully anisotropic kernels when ANISOTROPIC_KL above is set to .true.
# means to save radial anisotropic kernels, i.e., sensitivity kernels for beta_v, beta_h, etc.
SAVE_TRANSVERSE_KL_ONLY = .false.
# output approximate Hessian in crust mantle region.
# means to save the preconditioning for gradients, they are cross correlations between forward and adjoint accelerations.
APPROXIMATE_HESS_KL = .false.
# forces transverse isotropy for all mantle elements
# (default is to use transverse isotropy only between MOHO and 220)
# means we allow radial anisotropy between the bottom of the crust to the bottom of the transition zone, i.e., 660~km depth.
USE_FULL_TISO_MANTLE = .false.
# output kernel mask to zero out source region
# to remove large values near the sources in the sensitivity kernels
SAVE_SOURCE_MASK = .false.
# output kernels on a regular grid instead of on the GLL mesh points (a bit expensive)
SAVE_REGULAR_KL = .false.
#-----------------------------------------------------------
# Dimitri Komatitsch, July 2014, CNRS Marseille, France:
# added the ability to run several calculations (several earthquakes)
# in an embarrassingly-parallel fashion from within the same run;
# this can be useful when using a very large supercomputer to compute
# many earthquakes in a catalog, in which case it can be better from
# a batch job submission point of view to start fewer and much larger jobs,
# each of them computing several earthquakes in parallel.
# To turn that option on, set parameter NUMBER_OF_SIMULTANEOUS_RUNS to a value greater than 1.
# To implement that, we create NUMBER_OF_SIMULTANEOUS_RUNS MPI sub-communicators,
# each of them being labeled "my_local_mpi_comm_world", and we use them
# in all the routines in "src/shared/parallel.f90", except in MPI_ABORT() because in that case
# we need to kill the entire run.
# When that option is on, of course the number of processor cores used to start
# the code in the batch system must be a multiple of NUMBER_OF_SIMULTANEOUS_RUNS,
# all the individual runs must use the same number of processor cores,
# which as usual is NPROC in the Par_file,
# and thus the total number of processor cores to request from the batch system
# should be NUMBER_OF_SIMULTANEOUS_RUNS * NPROC.
# All the runs to perform must be placed in directories called run0001, run0002, run0003 and so on
# (with exactly four digits).
#
# Imagine you have 10 independent calculations to do, each of them on 100 cores; you have three options:
#
# 1/ submit 10 jobs to the batch system
#
# 2/ submit a single job on 1000 cores to the batch, and in that script create a sub-array of jobs to start 10 jobs,
# each running on 100 cores (see e.g. http://www.schedmd.com/slurmdocs/job_array.html )
#
# 3/ submit a single job on 1000 cores to the batch, start SPECFEM3D on 1000 cores, create 10 sub-communicators,
# cd into one of 10 subdirectories (called e.g. run0001, run0002,... run0010) depending on the sub-communicator
# your MPI rank belongs to, and run normally on 100 cores using that sub-communicator.
#
# The option below implements 3/.
#
NUMBER_OF_SIMULTANEOUS_RUNS = 1
# if we perform simultaneous runs in parallel, if only the source and receivers vary between these runs
# but not the mesh nor the model (velocity and density) then we can also read the mesh and model files
# from a single run in the beginning and broadcast them to all the others; for a large number of simultaneous
# runs for instance when solving inverse problems iteratively this can DRASTICALLY reduce I/Os to disk in the solver
# (by a factor equal to NUMBER_OF_SIMULTANEOUS_RUNS), and reducing I/Os is crucial in the case of huge runs.
# Thus, always set this option to .true. if the mesh and the model are the same for all simultaneous runs.
# In that case there is no need to duplicate the mesh and model file database (the content of the DATABASES_MPI
# directories) in each of the run0001, run0002,... directories, it is sufficient to have one in run0001
# and the code will broadcast it to the others)
BROADCAST_SAME_MESH_AND_MODEL = .false.
# if one or a few of these simultaneous runs fail, kill all the runs or let the others finish using a fail-safe mechanism
# (in most cases, should be set to false)
USE_FAILSAFE_MECHANISM = .false.
#-----------------------------------------------------------
# set to true to use GPUs
GPU_MODE = .false.
# Only used if GPU_MODE = .true. :
GPU_RUNTIME = 1
# 2 (OpenCL), 1 (Cuda) ou 0 (Compile-time -- does not work if configured with --with-cuda *AND* --with-opencl)
GPU_PLATFORM = NVIDIA
GPU_DEVICE = Tesla
# set to true to use the ADIOS library for I/Os
ADIOS_ENABLED = .false.
ADIOS_FOR_FORWARD_ARRAYS = .true.
ADIOS_FOR_MPI_ARRAYS = .true.
ADIOS_FOR_ARRAYS_SOLVER = .true.
ADIOS_FOR_SOLVER_MESHFILES = .true.
ADIOS_FOR_AVS_DX = .true.
ADIOS_FOR_KERNELS = .true.
ADIOS_FOR_MODELS = .true.
ADIOS_FOR_UNDO_ATTENUATION = .true.