#! /bin/bash -l
#SBATCH -A prpb101
#SBARCH -N 21
#SBATCH -n 1000 -c 1
#SBATCH --time 1:00:00
#SBATCH -J GPAWbench_large
#SBATCH -p batch
#SBATCH --hint=nomultithread
#SBATCH -o %x-%j.out
#

inputfile=../input.py
benchmark_size='large'
csv_summary="JSC_${benchmark_size}_${SLURM_JOB_ID}_${SLURM_NTASKS}.csv"
module_20_1=GPAW-UEABS/20.1.0-Python38-FFTW-icc
module_20_10=GPAW-UEABS/20.10.0-Python39-FFTW-icc

module load Intel/2021.2.0-GCC-10.3.0
module load IntelMPI/2021.2.0

echo -e "\nWorking in: $(pwd)\n"
echo -e "Modules loaded:\n"
module list
echo -e "Slurm environment:\n$(env | grep SLURM_)\n"
echo -e "\nJob script:\n"
cat $0
echo -e "\n\n"


#
# Check the results
#
function print_header {

    echo '"Module", "python/gpaw/ase/numpy/scipy", "tasks", "time", "iterations", "dipole", "fermi", "energy", "check", "Job ID"' >$1

}

function print_results {

	output=$1
	summary=$2
	module=$3

    . ../bounds.sh

    python_version=$(python -V | awk '{print $2}')
    gpaw_version=$(srun -n 1 -c 1 python -c "import gpaw ; print( gpaw.__version__ )")
    ase_version=$(python -c "import ase ; print( ase.__version__ )")
    numpy_version=$(python -c "import numpy ; print( numpy.__version__ )")
    scipy_version=$(python -c "import scipy ; print( scipy.__version__ )")

      # Extract some data to report form the output file.
    bmtime=$(grep "Total:" $output | sed -e 's/Total: *//' | cut -d " " -f 1)
    iterations=$(grep "Converged after" $output | cut -d " " -f 3)
    dipole=$(grep "Dipole" $output | cut -d " " -f 5 | sed -e 's/)//')
    fermi=$(grep "Fermi level:" $output | cut -d ":" -f 2 | sed -e 's/ //g')
    energy=$(grep "Extrapolated: " $output | cut -d ":" -f 2 | sed -e 's/ //g')
    # Check the bounds
    if (( $(bc -l <<< "(($iterations-0) >= $lower_iterations) && (($iterations-0) <= $upper_iterations)") == 1 )); then iterations_ok="OK"; else iterations_ok="not OK"; fi
    if (( $(bc -l <<< "(($dipole-0)     >= $lower_dipole)     && (($dipole-0)     <= $upper_dipole)") == 1     )); then dipole_ok="OK";     else dipole_ok="not OK";     fi
    if (( $(bc -l <<< "(($fermi-0)      >= $lower_fermi)      && (($fermi-0)      <= $upper_fermi)") == 1      )); then fermi_ok="OK";      else fermi_ok="not OK";      fi
    if (( $(bc -l <<< "(($energy-0)     >= $lower_energy)     && (($energy-0)     <= $upper_energy)") == 1     )); then energy_ok="OK";     else energy_ok="not OK";     fi
    compare=""
    compare+="(($iterations-0) >= $lower_iterations) && (($iterations-0) <= $upper_iterations) && "
    compare+="(($dipole-0)     >= $lower_dipole)     && (($dipole-0)     <= $upper_dipole) && "
    compare+="(($fermi-0)      >= $lower_fermi)      && (($fermi-0)      <= $upper_fermi) && "
    compare+="(($energy-0)     >= $lower_energy)     && (($energy-0)     <= $upper_energy)"
    if (( $(bc -l <<< "$compare") ));
    then
    	bounds_check="OK";
    else
    	bounds_check="not OK"
    fi
    # Output to the slurm.out file
    echo -e "\nResult information:\n" \
        " * Time:                   $bmtime s\n" \
        " * Number of iterations:   $iterations (lower: $lower_iterations, upper: $upper_iterations, $iterations_ok)\n" \
        " * Dipole (3rd component): $dipole (lower: $lower_dipole, upper: $upper_dipole, $dipole_ok)\n" \
        " * Fermi level:            $fermi (lower: $lower_fermi, upper: $upper_fermi, $fermi_ok)\n" \
        " * Extrapolated energy:    $energy (lower: $lower_energy, upper: $upper_energy, $energy_ok)\n" \
        " * Boundary check:         $bounds_check"
    # Output to the summary spreadsheet
    echo "\"$module\", \"$python_version/$gpaw_version/$ase_version/$numpy_version/$scipy_version\"," \
         "\"$SLURM_NTASKS\", \"$bmtime\", \"$iterations\", \"$dipole\", \"$fermi\", \"$energy\", \"$bounds_check\", \"$SLURM_JOB_ID\"" >> $summary

}

#
# Running with GPAW 20.1.0
#

print_header $csv_summary

module purge
# UEABS/2.2 is not a system module but one of our own to basically do some settings
# needed for the project (and to adjust the MODULEPATH). It is not needed if
# $module_20_1 (generated by the build scripts) is in the MODULEPATH
module load UEABS/2.2
module load $module_20_1

echo -e "\n\n"

srun gpaw python $inputfile

print_results output.txt $csv_summary $module_20_1

mv output.txt JSC_${benchmark_size}_${SLURM_JOB_ID}_${SLURM_NTASKS}_20.1.0.txt

#
# Running with GPAW 20.10.0
#

module purge
module load UEABS/2.2
module load $module_20_10

echo -e "\n\n"

srun gpaw python $inputfile

print_results output.txt $csv_summary $module_20_10

mv output.txt JSC_${benchmark_size}_${SLURM_JOB_ID}_${SLURM_NTASKS}_20.10.0.txt