job_irene-skl_test_case_A.msub 1.21 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
#!/bin/bash
#MSUB -A pa4674
#MSUB -q skylake
#MSUB -r specfem3D_test_case_A
#MSUB -N 24
#MSUB -n 96                     # Number of tasks to use
#MSUB -c 12                     # Number of threads per task to use
#MSUB -o specfem3D_test_case_A_irene-skl-%I.output
#MSUB -T 5400
#set -e

source ../env/env_irene-skl
cd $install_dir/TestCaseA/specfem3d_globe

export I_MPI_DOMAIN=auto
export I_MPI_PIN_RESPECT_CPUSET=0
export I_MPI_DEBUG=4

#Make sure that OMP_NUM_THREADS / KMP_HW_SUBSET = cpus-per-task
export KMP_HW_SUBSET=1T
export OMP_NUM_THREADS=12

ulimit -s unlimited

MESHER_EXE=./bin/xmeshfem3D
SOLVER_EXE=./bin/xspecfem3D

# backup files used for this simulation
cp DATA/Par_file OUTPUT_FILES/
cp DATA/STATIONS OUTPUT_FILES/
cp DATA/CMTSOLUTION OUTPUT_FILES/

##
## mesh generation
##
sleep 2

echo
echo `date`
echo "starting MPI mesher"
echo
MPI_PROCESS=96 #` echo "$SLURM_NNODES*$SLURM_NTASKS_PER_NODE" | bc -l`
echo "OMP_NUM_THREADS=$OMP_NUM_THREADS"

time ccc_mprun -n ${MPI_PROCESS} ${MESHER_EXE}
echo "  mesher done: `date`"
echo

##
## forward simulation
##
sleep 2

echo
echo `date`
echo starting run in current directory $PWD
echo
time ccc_mprun -n ${MPI_PROCESS}  ${SOLVER_EXE}

echo "finished successfully"
echo `date`