#!/bin/bash
#SBATCH -J Validation_case_specfem-Marconi100_small_benchmark_run_to_test_more_complex_Earth
#SBATCH -A Ppp4x_5850
#SBATCH -p m100_usr_prod
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=24 # 24 tasks out of 128
#SBATCH --cpus-per-task=4
#SBATCH --time=01:59:59
#SBATCH --output Validation_case_specfem-Marconi100_small_benchmark_run_to_test_more_complex_Earth-xl-spectrumpi-mpirun-GPU_DEVICE-star-%j.out
#SBATCH --gres=gpu:4        # 1 gpus per node out of 4
##SBATCH --hint=nomultithread
#SBATCH --exclusive
#SBATCH --mem=246000          # memory per node out of 246000MB

#set -e
source ../env/env_marconi100
echo "Environment used:"
echo "================="
grep -E -v '^(#|$)' ../env/env_marconi100
cat job_marconi100_small_benchmark_run_to_test_more_complex_Earth.slurm
cd $install_dir/specfem3d_globe/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth
export CUDA_VISIBLE_DEVICES=0,1,2,3

# Uncomment the 3-5 following lines if it's first time launched after the compilation:
sed -i s/"GPU_MODE                        = .false."/"GPU_MODE                        = .true."/g  $install_dir/specfem3d_globe/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/DATA/Par_file
sed -i s/"GPU_DEVICE                      = Tesla"/"GPU_DEVICE                      = *"/g  $install_dir/specfem3d_globe/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/DATA/Par_file
sed -i s/"configure  --enable-openmp"/"configure  --build=ppc64 --with-cuda=cuda8"/g $install_dir/specfem3d_globe/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/run_this_example.sh
#sed -i s/"mpirun -np"/"srun -n"/g $install_dir/specfem3d_globe/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/run_mesher_solver.bash
sed -i s/"mpirun -np"/"mpirun -gpu -np"/g $install_dir/specfem3d_globe/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/run_mesher_solver.bash
sed -i '38d' $install_dir/specfem3d_globe/Makefile
grep configure $install_dir/specfem3d_globe/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/run_this_example.sh

#export OMP_NUM_THREADS=1 # $SLURM_CPUS_PER_TASK
ulimit -s unlimited
sed -i '40 i sed -i s/"O4"/"O3"/g Makefile' $install_dir/specfem3d_globe/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/run_this_example.sh
time ./run_this_example.sh
grep GPU $install_dir/specfem3d_globe/DATA/Par_file

echo "=========="
echo "config.log"
echo "=========="
cat $install_dir/specfem3d_globe/config.log
echo "========"
echo "make.log"
echo "========"
cat $install_dir/specfem3d_globe/make.log

echo "running seismogram comparisons:"
echo

cd $install_dir/specfem3d_globe/
# uncompress seismograms
if [ -e EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/OUTPUT_FILES_reference_OK/II.AAK.MXE.sem.ascii.bz2 ]; then
  echo
  echo "unzipping references..."
  echo
  mkdir OUTPUT_FILES_reference_OK/
  bunzip2 EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/OUTPUT_FILES_reference_OK/*.bz2
  echo
  echo
fi

# compares seismograms by plotting correlations
# Python2 (2.7 -virtual env - numpy)
source /m100/home/userexternal/cjourdai/numpy-test/bin/activate
which python
./utils/compare_seismogram_correlations.py EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/OUTPUT_FILES/ EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/OUTPUT_FILES_reference_OK/

echo
echo "done"
echo
echo "========"
cat $install_dir//specfem3d_globe/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/OUTPUT_FILES/output_solver.txt
echo "========"
cat $install_dir//specfem3d_globe/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/OUTPUT_FILES/output_mesher.txt