Newer
Older
Kurt Lust
committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
#! /bin/bash -l
#SBATCH -A pn73ye
#SBARCH -N 21
#SBATCH -n 1000 -c 1
#SBATCH --time 40:00
#SBATCH -J GPAWbench
#SBATCH -p general
#SBATCH --hint=nomultithread
#SBATCH -o %x-%j.out
#
inputfile=../input.py
benchmark_size='large'
csv_summary="LRZ_${benchmark_size}_${SLURM_JOB_ID}_${SLURM_NTASKS}.csv"
module_20_1=GPAW-UEABS/20.1.0-Python38-FFTW-icc
module_20_10=GPAW-UEABS/20.10.0-Python39-FFTW-icc
module unload intel-mkl
module unload intel-mpi
module unload intel
# UEABS/2.2 is not a system module but one of our own to basically do some settings
# needed for the project (and to adjust the MODULEPATH). It is not needed if
# $module_20_1 (generated by the build scripts) is in the MODULEPATH
module load UEABS/2.2
echo -e "\nWorking in: $(pwd)\n"
echo -e "Modules loaded:\n"
module list
echo -e "Slurm environment:\n$(env | grep SLURM_)\n"
echo -e "\nJob script:\n"
cat $0
echo -e "\n\n"
echo -e "\nCalled script:\n"
cat ../../LRZ_run_GPAW.slurm
echo -e "\n\n"
if (( $SLURM_NTASKS <= 512 ))
then
#srun --distribution=block:block /hppfs/work/pn73ye/di46ras/UEABS/Run/TEST-JOB/mpi_hello.exe
srun /hppfs/work/pn73ye/di46ras/UEABS/Run/TEST-JOB/mpi_hello.exe
fi
#
# Check the results
#
function print_header {
echo '"Module", "python/gpaw/ase/numpy/scipy", "tasks", "time", "iterations", "dipole", "fermi", "energy", "check", "Job ID"' >$1
}
function print_results {
output=$1
summary=$2
module=$3
. ../bounds.sh
python_version=$(python -V | awk '{print $2}')
gpaw_version=$(srun -n 1 -c 1 python -c "import gpaw ; print( gpaw.__version__ )")
ase_version=$(python -c "import ase ; print( ase.__version__ )")
numpy_version=$(python -c "import numpy ; print( numpy.__version__ )")
scipy_version=$(python -c "import scipy ; print( scipy.__version__ )")
# Extract some data to report form the output file.
bmtime=$(grep "Total:" $output | sed -e 's/Total: *//' | cut -d " " -f 1)
iterations=$(grep "Converged after" $output | cut -d " " -f 3)
dipole=$(grep "Dipole" $output | cut -d " " -f 5 | sed -e 's/)//')
fermi=$(grep "Fermi level:" $output | cut -d ":" -f 2 | sed -e 's/ //g')
energy=$(grep "Extrapolated: " $output | cut -d ":" -f 2 | sed -e 's/ //g')
# Check the bounds
if (( $(bc -l <<< "(($iterations-0) >= $lower_iterations) && (($iterations-0) <= $upper_iterations)") == 1 )); then iterations_ok="OK"; else iterations_ok="not OK"; fi
if (( $(bc -l <<< "(($dipole-0) >= $lower_dipole) && (($dipole-0) <= $upper_dipole)") == 1 )); then dipole_ok="OK"; else dipole_ok="not OK"; fi
if (( $(bc -l <<< "(($fermi-0) >= $lower_fermi) && (($fermi-0) <= $upper_fermi)") == 1 )); then fermi_ok="OK"; else fermi_ok="not OK"; fi
if (( $(bc -l <<< "(($energy-0) >= $lower_energy) && (($energy-0) <= $upper_energy)") == 1 )); then energy_ok="OK"; else energy_ok="not OK"; fi
compare=""
compare+="(($iterations-0) >= $lower_iterations) && (($iterations-0) <= $upper_iterations) && "
compare+="(($dipole-0) >= $lower_dipole) && (($dipole-0) <= $upper_dipole) && "
compare+="(($fermi-0) >= $lower_fermi) && (($fermi-0) <= $upper_fermi) && "
compare+="(($energy-0) >= $lower_energy) && (($energy-0) <= $upper_energy)"
if (( $(bc -l <<< "$compare") ));
then
bounds_check="OK";
else
bounds_check="not OK"
fi
# Output to the slurm.out file
echo -e "\nResult information:\n" \
" * Time: $bmtime s\n" \
" * Number of iterations: $iterations (lower: $lower_iterations, upper: $upper_iterations, $iterations_ok)\n" \
" * Dipole (3rd component): $dipole (lower: $lower_dipole, upper: $upper_dipole, $dipole_ok)\n" \
" * Fermi level: $fermi (lower: $lower_fermi, upper: $upper_fermi, $fermi_ok)\n" \
" * Extrapolated energy: $energy (lower: $lower_energy, upper: $upper_energy, $energy_ok)\n" \
" * Boundary check: $bounds_check"
# Output to the summary spreadsheet
echo "\"$module\", \"$python_version/$gpaw_version/$ase_version/$numpy_version/$scipy_version\"," \
"\"$SLURM_NTASKS\", \"$bmtime\", \"$iterations\", \"$dipole\", \"$fermi\", \"$energy\", \"$bounds_check\", \"$SLURM_JOB_ID\"" >> $summary
}
#
# Running with GPAW 20.1.0
#
print_header $csv_summary
module load $module_20_1
echo -e "\n\nStarting GPAW for $module_20_1\n"
#srun --distribution=block:block gpaw python $inputfile
srun gpaw python $inputfile
echo -e "\n\nGPAW terminated\n"
print_results output.txt $csv_summary $module_20_1
mv output.txt LRZ_${benchmark_size}_${SLURM_JOB_ID}_${SLURM_NTASKS}_20.1.0.txt
#
# Running with GPAW 20.10.0
#
module load $module_20_10
echo -e "\n\n"
echo -e "\n\nStarting GPAW for $module_20_10\n"
#srun --distribution=block:block gpaw python $inputfile
srun gpaw python $inputfile
echo -e "\n\nGPAW terminated\n"
print_results output.txt $csv_summary $module_20_10
mv output.txt LRZ_${benchmark_size}_${SLURM_JOB_ID}_${SLURM_NTASKS}_20.10.0.txt