From ec524c7fb88c0cfcad37a1015bc866fc295160a7 Mon Sep 17 00:00:00 2001 From: Cedric Jourdain Date: Thu, 25 Nov 2021 15:37:23 +0000 Subject: [PATCH] Add validation case for marconi100 --- ...hmark_run_to_test_more_complex_Earth.slurm | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 specfem3d/job_script/job_marconi100_small_benchmark_run_to_test_more_complex_Earth.slurm diff --git a/specfem3d/job_script/job_marconi100_small_benchmark_run_to_test_more_complex_Earth.slurm b/specfem3d/job_script/job_marconi100_small_benchmark_run_to_test_more_complex_Earth.slurm new file mode 100644 index 0000000..87f600f --- /dev/null +++ b/specfem3d/job_script/job_marconi100_small_benchmark_run_to_test_more_complex_Earth.slurm @@ -0,0 +1,75 @@ +#!/bin/bash +#SBATCH -J Validation_case_specfem-Marconi100_small_benchmark_run_to_test_more_complex_Earth +#SBATCH -A Ppp4x_5850 +#SBATCH -p m100_usr_prod +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=24 # 24 tasks out of 128 +#SBATCH --cpus-per-task=4 +#SBATCH --time=01:59:59 +#SBATCH --output Validation_case_specfem-Marconi100_small_benchmark_run_to_test_more_complex_Earth-xl-spectrumpi-mpirun-GPU_DEVICE-star-%j.out +#SBATCH --gres=gpu:4 # 1 gpus per node out of 4 +##SBATCH --hint=nomultithread +#SBATCH --exclusive +#SBATCH --mem=246000 # memory per node out of 246000MB + +#set -e +source ../env/env_marconi100 +echo "Environment used:" +echo "=================" +grep -E -v '^(#|$)' ../env/env_marconi100 +cat job_marconi100_small_benchmark_run_to_test_more_complex_Earth.slurm +cd $install_dir/specfem3d_globe/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth +export CUDA_VISIBLE_DEVICES=0,1,2,3 + +# Uncomment the 3-5 following lines if it's first time launched after the compilation: +sed -i s/"GPU_MODE = .false."/"GPU_MODE = .true."/g $install_dir/specfem3d_globe/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/DATA/Par_file +sed -i s/"GPU_DEVICE = Tesla"/"GPU_DEVICE = *"/g $install_dir/specfem3d_globe/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/DATA/Par_file +sed -i s/"configure --enable-openmp"/"configure --build=ppc64 --with-cuda=cuda8"/g $install_dir/specfem3d_globe/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/run_this_example.sh +#sed -i s/"mpirun -np"/"srun -n"/g $install_dir/specfem3d_globe/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/run_mesher_solver.bash +sed -i s/"mpirun -np"/"mpirun -gpu -np"/g $install_dir/specfem3d_globe/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/run_mesher_solver.bash +sed -i '38d' $install_dir/specfem3d_globe/Makefile +grep configure $install_dir/specfem3d_globe/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/run_this_example.sh + +#export OMP_NUM_THREADS=1 # $SLURM_CPUS_PER_TASK +ulimit -s unlimited +sed -i '40 i sed -i s/"O4"/"O3"/g Makefile' $install_dir/specfem3d_globe/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/run_this_example.sh +time ./run_this_example.sh +grep GPU $install_dir/specfem3d_globe/DATA/Par_file + +echo "==========" +echo "config.log" +echo "==========" +cat $install_dir/specfem3d_globe/config.log +echo "========" +echo "make.log" +echo "========" +cat $install_dir/specfem3d_globe/make.log + +echo "running seismogram comparisons:" +echo + +cd $install_dir/specfem3d_globe/ +# uncompress seismograms +if [ -e EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/OUTPUT_FILES_reference_OK/II.AAK.MXE.sem.ascii.bz2 ]; then + echo + echo "unzipping references..." + echo + mkdir OUTPUT_FILES_reference_OK/ + bunzip2 EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/OUTPUT_FILES_reference_OK/*.bz2 + echo + echo +fi + +# compares seismograms by plotting correlations +# Python2 (2.7 -virtual env - numpy) +source /m100/home/userexternal/cjourdai/numpy-test/bin/activate +which python +./utils/compare_seismogram_correlations.py EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/OUTPUT_FILES/ EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/OUTPUT_FILES_reference_OK/ + +echo +echo "done" +echo +echo "========" +cat $install_dir//specfem3d_globe/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/OUTPUT_FILES/output_solver.txt +echo "========" +cat $install_dir//specfem3d_globe/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/OUTPUT_FILES/output_mesher.txt -- GitLab