build_20.10.0_IntelPython3_icc.sh

#!/bin/bash
#
# Installation script for GPAW 20.10.0:
# * Using the existing IntelPython3 module on the system which has an optimized
#   NumPy and SciPy included.
# * Using the matching version of ase, 3.20.1
# * Compiling with the Intel compilers
#
# The FFT library is discovered at runtime. With the settings used in this script
# this should be MKL FFT, but it is possible to change this at runtime to either
# MKL, FFTW or the built-in NumPy FFT routines, see the installation instructions
# (link below).
#
# The original installation instructions for GPAW can be found at
# https://gitlab.com/gpaw/gpaw/-/blob/20.10.0/doc/install.rst
#

packageID='20.10.0-IntelPython3-icc'

echo -e "\n### Building GPAW-UEABS/$packageID from $0\n\n"

install_root=$VSC_SCRATCH/UEABS
systemID=CalcUA-leibniz-broadwell
UEABS_version='2.2'

#
# NOTE: For easy maintanance we load many settings from the UEABS module.
# Since we want to release the module again and only depend on system modules for
# this build, we copy the settings to other variables and unload the module
# again. It is of course possible to those variables directly by hand without
# first creating a module file.
#

module purge
module use $install_root/$systemID/Modules
module load UEABS/$UEABS_version

# Directory in which downloaded files will be stored.
download_dir=$UEABS_DOWNLOADS
# Directory in which the software package will be installed.
install_dir=$UEABS_PACKAGES/GPAW-UEABS/$packageID
# Subdirectory containing the module file.
modules_dir=$UEABS_MODULES/GPAW-UEABS
# Temporary directory for the build.
build_dir="/dev/shm/$USER/GPAW-UEABS/$packageID"
# Subdirectory in the UEABS repo containing the patch file that is needed.
patch_dir=$UEABS_REPOSITORY/gpaw/build/patches

#
# Software versions
#
IntelPython3_version='2020.02'
libxc_version='4.3.4'
ase_version='3.20.1'
GPAW_version='20.10.0'
GPAWsetups_version='0.9.20000' # Check version on https://wiki.fysik.dtu.dk/gpaw/setups/setups.html

# Compiler settings
compiler_module='intel/2020a'
opt_level='-O2'
proc_opt_flags='-march=core-avx2 -mtune=core-avx2'
fp_opt_flags='-ftz -fp-speculation=safe -fp-model source'
parallel=32

py_maj_min='3.7'

################################################################################
#
# Prepare the system
#
# We do this through modules, but it could be done by running scripts from the
# Intel compiler suite as well.
#

#
# Load modules:
# - Intel compiler module
# - Intel Python
# - buildtools is a module with a number of build tools that are more up-to-date
#   then those on the system. It is mostly used for consistent builds across
#   systems.
#
module purge
MODULEPATH=/apps/antwerpen/modules/centos7/calcua-admin-broadwell/
module load calcua/admin
module load $compiler_module
module load buildtools/2020a
module load IntelPython3/$IntelPython3_version

#
# Create the directories and make sure they are clean if that matters
#
/usr/bin/mkdir -p $download_dir

/usr/bin/mkdir -p $install_dir
/usr/bin/rm -rf   $install_dir
/usr/bin/mkdir -p $install_dir

/usr/bin/mkdir -p $modules_dir

/usr/bin/mkdir -p $build_dir
/usr/bin/rm -rf   $build_dir
/usr/bin/mkdir -p $build_dir


################################################################################
#
# Download components
#

echo -e "\nDownloading files...\n"

cd $download_dir

# https://gitlab.com/libxc/libxc/-/archive/4.3.4/libxc-4.3.4.tar.bz2
libxc_file="libxc-$libxc_version.tar.bz2"
libxc_url="https://gitlab.com/libxc/libxc/-/archive/$libxc_version"
[[ -f $libxc_file ]] || wget "$libxc_url/$libxc_file"

# Downloading ase so that we can gather all sources for reproducibility
ase_file="ase-$ase_version.tar.gz"
ase_url="https://pypi.python.org/packages/source/a/ase"
[[ -f $ase_file ]] || wget "$ase_url/$ase_file"

# GPAW needs customization, so we need to download and unpack the sources.
# https://files.pythonhosted.org/packages/49/a1/cf54c399f5489cfdda1e8da02cae8bfb4b39d7cb7a895ce86608fcd0e1c9/gpaw-1.5.2.tar.gz
GPAW_file="gpaw-$GPAW_version.tar.gz"
GPAW_url="https://pypi.python.org/packages/source/g/gpaw"
[[ -f $GPAW_file ]] || wget "$GPAW_url/$GPAW_file"

# Download GPAW-setup, a number of setup files for GPAW.
# https://wiki.fysik.dtu.dk/gpaw-files/gpaw-setups-0.9.20000.tar.gz
GPAWsetups_file="gpaw-setups-$GPAWsetups_version.tar.gz"
GPAWsetups_url="https://wiki.fysik.dtu.dk/gpaw-files"
[[ -f $GPAWsetups_file ]] || wget "$GPAWsetups_url/$GPAWsetups_file"


################################################################################
#
# Install libxc
#

echo -e "\nInstalling libxc...\n"

cd $build_dir

# Uncompress
tar -xf $download_dir/$libxc_file

cd libxc-$libxc_version

# Configure
autoreconf -i

export CC=icc
export CFLAGS="$opt_level $proc_opt_flags $fp_opt_flags -fPIC"
./configure --prefix="$install_dir" \
            --disable-static --enable-shared --disable-fortran

# Build
make -j $parallel

# Install
make install

# Add bin, lib and include to the PATH variables
PATH=$install_dir/bin:$PATH
LIBRARY_PATH=$install_dir/lib:$LIBRARY_PATH
LD_LIBRARY_PATH=$install_dir/lib:$LD_LIBRARY_PATH
CPATH=$install_dir/include:$CPATH

# Clean-up
unset CC
unset CFLAGS


################################################################################
#
# Prepare for installing Python packages
#

/usr/bin/mkdir -p "$install_dir/lib/python$py_maj_min/site-packages"
PYTHONPATH="$install_dir/lib/python$py_maj_min/site-packages:$PYTHONPATH"


################################################################################
#
# Optional: Install pytest and its dependencies to test NumPy and SciPy with
# import numpy
# numpy.test()
# import scipy
# scipy.text()
# We don't care about version numbers here as it is not important for the
# reproducibility of the benchmarks.
#

echo -e "\nInstalling pytest...\n"

cd $build_dir
# IntelPython3 does not define pip3 or pip3.7.
pip install --prefix=$install_dir pytest


################################################################################
#
# Install ase
#

echo -e "\nInstalling ase...\n"

# IntelPython3 does not define pip3 or pip3.7.
pip install --prefix=$install_dir --no-deps $download_dir/$ase_file

# Brief test
cd $build_dir
python$py_maj_min -c "import ase"


################################################################################
#
# Install GPAW-setups
#

echo -e "\nInstalling gpaw-setups...\n"

mkdir -p $install_dir/share/gpaw-setups
cd $install_dir/share/gpaw-setups
tar -xf $download_dir/$GPAWsetups_file --strip-components=1


################################################################################
#
# Install GPAW
#

echo -e "\nInstalling GPAW...\n"

cd $build_dir

# Uncompress
tar -xf $download_dir/$GPAW_file

cd gpaw-$GPAW_version

# Make the customize.py script
cat >siteconfig.py <<EOF
print( 'GPAW EasyBuild INFO: Starting execution of the customization script' )
print( 'GPAW EasyBuild INFO: Variables at the start of the customization script' )
print( 'GPAW EasyBuild INFO: libraries =               ', libraries )
print( 'GPAW EasyBuild INFO: mpi_libaries =            ', mpi_libraries )
print( 'GPAW EasyBuild INFO: library_dirs =            ', library_dirs )
print( 'GPAW EasyBuild INFO: mpi_libary_dirs =         ', mpi_library_dirs )
print( 'GPAW EasyBuild INFO: runtime_library_dirs =    ', runtime_library_dirs )
print( 'GPAW EasyBuild INFO: mpi_runtime_libary_dirs = ', mpi_runtime_library_dirs )
print( 'GPAW EasyBuild INFO: include_dirs =            ', include_dirs )
print( 'GPAW EasyBuild INFO: mpi_include_dirs =        ', mpi_include_dirs )
print( 'GPAW EasyBuild INFO: compiler =                ', compiler )
print( 'GPAW EasyBuild INFO: mpicompiler =             ', mpicompiler )
print( 'GPAW EasyBuild INFO: mpilinker =               ', mpilinker )
print( 'GPAW EasyBuild INFO: extra_compile_args =      ', extra_compile_args )
print( 'GPAW EasyBuild INFO: extra_link_args =         ', extra_link_args )
print( 'GPAW EasyBuild INFO: define_macros =           ', define_macros )
print( 'GPAW EasyBuild INFO: mpi_define_macros =       ', mpi_define_macros )
print( 'GPAW EasyBuild INFO: undef_macros =            ', undef_macros )
print( 'GPAW EasyBuild INFO: fftw =                    ', fftw )
print( 'GPAW EasyBuild INFO: scalapack =               ', scalapack )
print( 'GPAW EasyBuild INFO: libvdwxc =                ', libvdwxc )
print( 'GPAW EasyBuild INFO: elpa =                    ', elpa )
print( 'GPAW EasyBuild INFO: noblas =                      ', noblas )
print( 'GPAW EasyBuild INFO: parallel_python_interpreter = ', parallel_python_interpreter )

# LibXC
include_dirs.append('$install_dir/include')
#libraries.append('xc')

# libvdwxc
libvdwxc = False

# ELPA
elpa = False

# Use NumPy FFTW
fftw = False

# ScaLAPACK
scalapack = True
libraries += ['mkl_scalapack_lp64', 'mkl_blacs_intelmpi_lp64']

# MKL BLAS
libraries += ['mkl_sequential','mkl_core', 'mkl_rt', ]

# Add other EasyBuild library directoryes.
library_dirs = os.environ['LIBRARY_PATH'].split(':')

# Set the compilers
compiler =    os.environ['CC']
mpicompiler = os.environ['MPICC']
mpilinker =   os.environ['MPICC']

print( 'GPAW EasyBuild INFO: Variables at the end of the customization script' )
print( 'GPAW EasyBuild INFO: libraries =               ', libraries )
print( 'GPAW EasyBuild INFO: mpi_libaries =            ', mpi_libraries )
print( 'GPAW EasyBuild INFO: library_dirs =            ', library_dirs )
print( 'GPAW EasyBuild INFO: mpi_libary_dirs =         ', mpi_library_dirs )
print( 'GPAW EasyBuild INFO: runtime_library_dirs =    ', runtime_library_dirs )
print( 'GPAW EasyBuild INFO: mpi_runtime_libary_dirs = ', mpi_runtime_library_dirs )
print( 'GPAW EasyBuild INFO: include_dirs =            ', include_dirs )
print( 'GPAW EasyBuild INFO: mpi_include_dirs =        ', mpi_include_dirs )
print( 'GPAW EasyBuild INFO: compiler =                ', compiler )
print( 'GPAW EasyBuild INFO: mpicompiler =             ', mpicompiler )
print( 'GPAW EasyBuild INFO: mpilinker =               ', mpilinker )
print( 'GPAW EasyBuild INFO: extra_compile_args =      ', extra_compile_args )
print( 'GPAW EasyBuild INFO: extra_link_args =         ', extra_link_args )
print( 'GPAW EasyBuild INFO: define_macros =           ', define_macros )
print( 'GPAW EasyBuild INFO: mpi_define_macros =       ', mpi_define_macros )
print( 'GPAW EasyBuild INFO: undef_macros =            ', undef_macros )
print( 'GPAW EasyBuild INFO: fftw =                    ', fftw )
print( 'GPAW EasyBuild INFO: scalapack =               ', scalapack )
print( 'GPAW EasyBuild INFO: libvdwxc =                ', libvdwxc )
print( 'GPAW EasyBuild INFO: elpa =                    ', elpa )
print( 'GPAW EasyBuild INFO: Ending execution of the customization script' )
print( 'GPAW EasyBuild INFO: noblas =                      ', noblas )
print( 'GPAW EasyBuild INFO: parallel_python_interpreter = ', parallel_python_interpreter )
EOF

# Now install gpaw
export CC=icc
export MPICC=mpiicc
export CFLAGS="-std=c99 $opt_level $proc_opt_flags $fp_opt_flags -qno-openmp-simd"
python$py_maj_min setup.py build -j $parallel

# Install GPAW
pip install --prefix=$install_dir --no-deps --ignore-installed --no-build-isolation .

# Brief test
cd $build_dir
python$py_maj_min -c "import gpaw"

# Clean-up
unset CC
unset MPICC
unset CFLAGS


################################################################################
#
# Finish the install
#

echo -e "\nCleaning up and making the LUA-module GPAW-UEABS/$packageID...\n"

# Go to a different directory before cleaning up the build directory
cd $modules_dir
/bin/rm -rf $build_dir

# Create a module file
python_version=$(python -V | cut -d ' ' -f 2)
numpy_version=$(python3 -c "import numpy ; print(numpy.__version__)")
scipy_version=$(python3 -c "import scipy ; print(scipy.__version__)")

cat >$packageID.lua <<EOF
help([==[

Description
===========

GPAW $GPAW_version for the UEABS benchmark.

Configuration:
  * Parallel GPAW $GPAW_version with ase $ase_version
  * On top of the system IntelPython3 $IntelPython3_version module, which provides
    Python $python_version, NumPy $numpy_version and SciPy $scipy_version.
  * NumPy is used for FFT, so the performance of the code
    will depend a lot on a proper configuration of NumPy for FFT.
  * libxc and GPAW compiled with the Intel compilers

Detailed configuration:
  * Compiler module: $compiler_module (a module file written
    at UAntwerp, not generated via EasyBuild)
  * Modules from the system
      * Intel Python3 $IntelPython3_version (using a module file written
        at UAntwerp)
  * libxc $libxc_version
  * Python packages: ase-$ase_version, gpaw-$GPAW_version
  * GPAW setups $GPAWsetups_version


More information
================
 - Homepage: http://wiki.fysik.dtu.dk/gpaw
 - Documentation:
    - GPAW web-based documentation: https://wiki.fysik.dtu.dk/gpaw/
    - Version information at https://gitlab.com/gpaw/gpaw/-/blob/$GPAW_version/doc/
    - ASE web-based documentation: https://wiki.fysik.dtu.dk/ase/
]==])

whatis([==[Description: GPAW $GPAW_version with ase $ase_version, Intel Python3 $IntelPython3_version with numpy $numpy_version and scipy $scipy_version: UEABS benchmark configuration.]==])

family("GPAW")

prepend_path('MODULEPATH','/apps/antwerpen/modules/centos8/calcua-admin-rome')

if not ( isloaded("calcua/admin") ) then
    load("calcua/admin")
end

if not ( isloaded("$compiler_module") ) then
    load("$compiler_module")
end

if not ( isloaded("IntelPython3/$IntelPython3_version") ) then
    load("IntelPython3/$IntelPython3_version")
end

prepend_path("PATH",            "$install_dir/bin")
prepend_path("LD_LIBRARY_PATH", "$install_dir/lib")
prepend_path("LIBRARY_PATH",    "$install_dir/lib")
prepend_path("PYTHONPATH",      "$install_dir/lib/python$py_maj_min/site-packages")

setenv("GPAW_SETUP_PATH", "$install_dir/share/gpaw-setups")
EOF