Newer
Older
Kurt Lust
committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
#! /bin/bash -l
#PBS -l select=8:node_type=rome:mpiprocs=128
#PBS -l walltime=1:00:00
#PBS -N GPPAWbench
numranks=1000
inputfile=../../input.py
benchmark_size='large'
csv_summary="HLRS_${benchmark_size}_${PBS_JOBID}_${numranks}.csv"
module_20_1=GPAW-UEABS/20.1.0-Python38-FFTW-icc
module_20_10=GPAW-UEABS/20.10.0-Python39-FFTW-icc
cd $PBS_O_WORKDIR
compiler_module='intel/19.1.3'
mpi_module='mpt/2.23'
math_module='mkl/19.1.0'
bounds="$(dirname ${inputfile})/bounds.sh"
# The first module is not a system module but one that was used to do the settings
# for the project.
module load UEABS
module load $compiler_module
module load $mpi_module
module load $math_module
export MKL_DEBUG_CPU_TYPE=5
export OMP_NUM_THREADS=1
echo -e "\nWorking in: $(pwd)\n"
echo -e "Modules loaded:\n"
module list
echo -e "PBS environment:\n$(env | grep PBS_)\n"
echo -e "\nJob script:\n"
cat $0
echo -e "\n\n"
#
# Check the results
#
function print_header {
echo '"Module", "python/gpaw/ase/numpy/scipy", "tasks", "time", "iterations", "dipole", "fermi", "energy", "check", "Job ID"' >$1
}
function print_results {
output=$1
summary=$2
module=$3
bounds=$4
#source ${bounds}
source ../../bounds.sh
python_version=$(python -V | awk '{print $2}')
gpaw_version=$(python -c "import gpaw ; print( gpaw.__version__ )")
ase_version=$(python -c "import ase ; print( ase.__version__ )")
numpy_version=$(python -c "import numpy ; print( numpy.__version__ )")
scipy_version=$(python -c "import scipy ; print( scipy.__version__ )")
# Extract some data to report form the output file.
bmtime=$(grep "Total:" $output | sed -e 's/Total: *//' | cut -d " " -f 1)
iterations=$(grep "Converged after" $output | cut -d " " -f 3)
dipole=$(grep "Dipole" $output | cut -d " " -f 5 | sed -e 's/)//')
fermi=$(grep "Fermi level:" $output | cut -d ":" -f 2 | sed -e 's/ //g')
energy=$(grep "Extrapolated: " $output | cut -d ":" -f 2 | sed -e 's/ //g')
# Check the bounds
if (( $(bc -l <<< "(($iterations-0) >= $lower_iterations) && (($iterations-0) <= $upper_iterations)") == 1 )); then iterations_ok="OK"; else iterations_ok="not OK"; fi
if (( $(bc -l <<< "(($dipole-0) >= $lower_dipole) && (($dipole-0) <= $upper_dipole)") == 1 )); then dipole_ok="OK"; else dipole_ok="not OK"; fi
if (( $(bc -l <<< "(($fermi-0) >= $lower_fermi) && (($fermi-0) <= $upper_fermi)") == 1 )); then fermi_ok="OK"; else fermi_ok="not OK"; fi
if (( $(bc -l <<< "(($energy-0) >= $lower_energy) && (($energy-0) <= $upper_energy)") == 1 )); then energy_ok="OK"; else energy_ok="not OK"; fi
compare=""
compare+="(($iterations-0) >= $lower_iterations) && (($iterations-0) <= $upper_iterations) && "
compare+="(($dipole-0) >= $lower_dipole) && (($dipole-0) <= $upper_dipole) && "
compare+="(($fermi-0) >= $lower_fermi) && (($fermi-0) <= $upper_fermi) && "
compare+="(($energy-0) >= $lower_energy) && (($energy-0) <= $upper_energy)"
if (( $(bc -l <<< "$compare") ));
then
bounds_check="OK";
else
bounds_check="not OK"
fi
# Output to the PBS output file
echo -e "\nResult information:\n" \
" * Time: $bmtime s\n" \
" * Number of iterations: $iterations (lower: $lower_iterations, upper: $upper_iterations, $iterations_ok)\n" \
" * Dipole (3rd component): $dipole (lower: $lower_dipole, upper: $upper_dipole, $dipole_ok)\n" \
" * Fermi level: $fermi (lower: $lower_fermi, upper: $upper_fermi, $fermi_ok)\n" \
" * Extrapolated energy: $energy (lower: $lower_energy, upper: $upper_energy, $energy_ok)\n" \
" * Boundary check: $bounds_check"
# Output to the summary spreadsheet
echo "\"$module\", \"$python_version/$gpaw_version/$ase_version/$numpy_version/$scipy_version\"," \
"\"$numranks\", \"$bmtime\", \"$iterations\", \"$dipole\", \"$fermi\", \"$energy\", \"$bounds_check\", \"$PBS_JOBID\"" >> $summary
}
#
# Running with GPAW 20.1.0
#
print_header $csv_summary
module purge
module load UEABS/2.2
module load $module_20_1
echo -e "\n\nGPAW run with $module_20_1\nModules loaded:\n"
module list 2>&1
echo -e "\n\nStarting GPAW...\n"
mpirun -n $numranks gpaw python $inputfile
echo -e "\nGPAW ended, checking results...\n"
print_results output.txt $csv_summary $module_20_1
mv output.txt HLRS_${benchmark_size}_${PBS_JOBID}_${numranks}_20.1.0.txt
#
# Running with GPAW 20.10.0
#
module purge
module load UEABS/2.2
module load $module_20_10
echo -e "\n\nGPAW run with $module_20_10\nModules loaded:\n"
module list 2>&1
echo -e "\n\nStarting GPAW...\n"
echo -e "\n\n"
mpirun -n $numranks gpaw python $inputfile $bounds
echo -e "\nGPAW ended, checking results...\n"
print_results output.txt $csv_summary $module_20_10 $bounds
mv output.txt HLRS_${benchmark_size}_${PBS_JOBID}_${numranks}_20.10.0.txt