Add env, scripts for juwels

specfem3d/compile.sh

@@ -7,6 +7,7 @@ echo "     - marconi-knl "
 echo "     - daint-gpu "
 echo "     - daint-cpu-only "
 echo "     - davide "
+echo "     - juwels"
 read machine
 
 source ./env/env_${machine}
@@ -94,12 +95,12 @@ Clean(){
 
 Deploy(){
     echo "install_dir ="$install_dir
-    if [ $machine = "occigen" ] || [ $machine = "marenostrum" ] || [ $machine = "marconi-knl" ] || [ $machine = "daint-cpu-only" ] || [ $machine = "daint-gpu" ]  || [ $machine = "davide" ];then
+    if [ $machine = "occigen" ] || [ $machine = "marenostrum" ] || [ $machine = "marconi-knl" ] || [ $machine = "daint-cpu-only" ] || [ $machine = "daint-gpu" ]  || [ $machine = "davide" ] || [ $machine = "juwels" ];then
         echo "==> Install on $machine :"
 	#Clean
 	mkdir -p $install_dir
         export ueabs_dir=`pwd`
-        #Untar
+        Untar
         Install
     else
         echo "Wrong machine !"

specfem3d/env/env_juwels

+#!/bin/bash
+ 
+module purge
+module load Intel/2019.0.117-GCC-7.3.0 ParaStationMPI/5.2.1-1 #IntelMPI/2019.0.117
+ 
+export machine=juwels
+export software=specfem3d_globe
+export version=31octobre
+
+#export install_dir=$HOME/benchmarks/$machine/$software/$version/
+export install_dir=$SCRATCH_cprpb66/benchmarks/$machine/$software/$version/
+export CC="mpicc"
+export FC="mpifort"
+#export CC="mpiicc"
+#export FC="mpiifort"
+export MPIFC=$FC
+#export FCFLAGS=" -g -O3 -qopenmp -xhost -DUSE_FP32 -DOPT_STREAMS -fp-model fast=2 -traceback -mcmodel=large"
+#export CFLAGS=" -g -O3  -xhost "
+export FCFLAGS=" -g -O3 -qopenmp -xCORE-AVX512 -mtune=skylake -ipo -DUSE_FP32 -DOPT_STREAMS -fp-model fast=2 -traceback -mcmodel=large"
+export CFLAGS=" -g -O3  -xCORE-AVX512 -mtune=skylake -ipo"

specfem3d/job_script/job_juwels_test_case_A.slurm

+#!/bin/bash -x
+#SBATCH -J Test_case_A
+#SBATCH --account=prpb66
+#SBATCH --nodes=24
+#SBATCH --ntasks-per-node=4
+#SBATCH --cpus-per-task=12
+#SBATCH --time=00:30:00
+#SBATCH --partition=batch
+#SBATCH --output=specfem_small_juwels_96MPI_12OMP_srun_AVX512_mtune_skl_ParaStationMPI-%j.output
+#SBATCH --acctg-freq=task=1
+set -e
+
+source ../env/env_juwels
+cd $install_dir/TestCaseA/specfem3d_globe
+
+export I_MPI_DOMAIN=auto
+export I_MPI_PIN_RESPECT_CPUSET=0
+export I_MPI_DEBUG=4
+
+#Make sure that OMP_NUM_THREADS / KMP_HW_SUBSET = cpus-per-task
+export KMP_HW_SUBSET=1T
+export OMP_NUM_THREADS=12
+#export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK}
+
+ulimit -s unlimited
+
+MESHER_EXE=./bin/xmeshfem3D
+SOLVER_EXE=./bin/xspecfem3D
+
+# backup files used for this simulation
+cp DATA/Par_file OUTPUT_FILES/
+cp DATA/STATIONS OUTPUT_FILES/
+cp DATA/CMTSOLUTION OUTPUT_FILES/
+
+##
+## mesh generation
+##
+sleep 2
+
+echo
+echo `date`
+echo "starting MPI mesher"
+echo
+
+MPI_PROCESS=` echo "$SLURM_NNODES*$SLURM_NTASKS_PER_NODE" | bc -l` 
+echo "SLURM_NTASKS_PER_NODE = " $SLURM_NTASKS_PER_NODE
+echo "SLURM_CPUS_PER_TASKS = " $SLURM_CPUS_PER_TASK
+echo "SLURM_NNODES=" $SLURM_NNODES
+echo "MPI_PROCESS $MPI_PROCESS"
+echo "OMP_NUM_THREADS=$OMP_NUM_THREADS"
+
+#time mpirun -n ${MPI_PROCESS} ${MESHER_EXE}
+time srun -n ${MPI_PROCESS} ${MESHER_EXE}
+echo "  mesher done: `date`"
+echo
+
+##
+## forward simulation
+##
+sleep 2
+
+echo
+echo `date`
+echo starting run in current directory $PWD
+echo
+#time mpirun -n ${MPI_PROCESS}  ${SOLVER_EXE}
+time srun -n ${MPI_PROCESS}  ${SOLVER_EXE}
+
+echo "finished successfully"
+echo `date`
+

specfem3d/job_script/job_juwels_test_case_B.slurm

+#!/bin/bash -x
+#SBATCH -J Test_case_B
+#SBATCH --account=prpb66
+#SBATCH --nodes=384
+#SBATCH --ntasks-per-node=4
+#SBATCH --cpus-per-task=6
+#SBATCH --time=00:30:00
+#SBATCH --partition=batch
+#SBATCH --output=specfem_TestCaseB_juwels_12OMP-HT-ParaStationMPI-%j.output
+set -e
+
+source ../env/env_juwels
+cd $install_dir/TestCaseB/specfem3d_globe
+
+#export I_MPI_DOMAIN=auto
+#export I_MPI_PIN_RESPECT_CPUSET=0
+#export I_MPI_DEBUG=4
+
+#Make sure that OMP_NUM_THREADS / KMP_HW_SUBSET = cpus-per-task
+export KMP_HW_SUBSET=2T
+export OMP_NUM_THREADS=12
+#export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK}
+
+#ulimit -s unlimited
+
+MESHER_EXE=./bin/xmeshfem3D
+SOLVER_EXE=./bin/xspecfem3D
+
+# backup files used for this simulation
+cp DATA/Par_file OUTPUT_FILES/
+cp DATA/STATIONS OUTPUT_FILES/
+cp DATA/CMTSOLUTION OUTPUT_FILES/
+
+##
+## mesh generation
+##
+sleep 2
+
+echo
+echo `date`
+echo "starting MPI mesher"
+echo
+
+MPI_PROCESS=` echo "$SLURM_NNODES*$SLURM_NTASKS_PER_NODE" | bc -l` 
+echo "SLURM_NTASKS_PER_NODE = " $SLURM_NTASKS_PER_NODE
+echo "SLURM_CPUS_PER_TASKS = " $SLURM_CPUS_PER_TASK
+echo "SLURM_NNODES=" $SLURM_NNODES
+echo "MPI_PROCESS $MPI_PROCESS"
+echo "OMP_NUM_THREADS=$OMP_NUM_THREADS"
+
+#time mpirun -n ${MPI_PROCESS} ${MESHER_EXE}
+time srun -n ${MPI_PROCESS} ${MESHER_EXE}
+echo "  mesher done: `date`"
+echo
+
+##
+## forward simulation
+##
+sleep 2
+
+echo
+echo `date`
+echo starting run in current directory $PWD
+echo
+#unset FORT_BUFFERED
+#time mpirun -n ${MPI_PROCESS} ${SOLVER_EXE}
+time srun -n ${MPI_PROCESS}  ${SOLVER_EXE}
+
+echo "finished successfully"
+echo `date`
+