Newer
Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
#!/bin/bash
#SBATCH -J specfem_strong_scaling_gpu_small_benchmark_run_to_test_more_complex_Earth
#SBATCH --nodes=24
#SBATCH --ntasks-per-node=1
#SBATCH --cpus-per-task=128
#SBATCH --time=00:59:59
#SBATCH --output specfem_gpu_small_benchmark_run_to_test_more_complex_Earth-24Nodes-%j.out
#SBATCH --exclusive
#SBATCH -p gpu
#SBATCH --gres=gpu:4
#set -e
source ../env/env_vega-gpu
grep "^[^#;]" ../env/env_vega-gpu
cat job_strong_scaling_vega-gpu_small_benchmark_run_to_test_more_complex_Earth.slurm
cd $install_dir/specfem3d_globe/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth
export OMPI_MCA_pml=ucx
export OMPI_MCA_btl="^uct,tcp,openib,vader"
export CUDA_VISIBLE_DEVICES=0,1,2,3
#export OMP_NUM_THREADS=1
sed -i s/"GPU_MODE = .false."/"GPU_MODE = .true."/g $install_dir/specfem3d_globe/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/DATA/Par_file
sed -i s/"GPU_DEVICE = Tesla"/"GPU_DEVICE = *"/g $install_dir/specfem3d_globe/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/DATA/Par_file
sed -i s/"configure --enable-openmp"/"configure --build=ppc64 --with-cuda=cuda8 "/g $install_dir/specfem3d_globe/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/run_this_example.sh
#sed -i s/"mpirun -np"/"srun -n"/g $install_dir/specfem3d_globe/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/run_mesher_solver.bash
#taskset -a -p $PPID
time ./run_this_example.sh
echo "=========="
echo "config.log"
echo "=========="
cat $install_dir/specfem3d_globe/config.log
echo "========"
echo "make.log"
echo "========"
cat $install_dir/specfem3d_globe/make.log
echo
echo "running seismogram comparisons:"
echo
cd $install_dir/specfem3d_globe/
echo "================================="
echo "done"
ls -lrth $install_dir/specfem3d_globe/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/OUTPUT_FILES/output_*.txt
cat $install_dir/specfem3d_globe/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/OUTPUT_FILES/output_mesher.txt
cat $install_dir/specfem3d_globe/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/OUTPUT_FILES/output_solver.txt