build_20.1.0_Python38icc_FFTW_icc.sh

#!/bin/bash
#
# Installation script for GPAW 20.1.0:
# * Using a Python and FFTW module compiled with EasyBuild, but libxc and
#   NumPy/SciPy compiled by hand.
# * Using the matching version of ase, 3.19.3
# * Compiling with the Intel compilers
#
# The FFT library is discovered at runtime. With the settings used in this script
# this should be MKL FFT, but it is possible to change this at runtime to either
# MKL, FFTW or the built-in NumPy FFT routines, see the installation instructions
# (link below).
#
# The original installation instructions for GPAW can be found at
# https://gitlab.com/gpaw/gpaw/-/blob/20.1.0/doc/install.rst
#

#set -e

packageID='20.1.0-Python38icc-FFTW-icc'
packageName='GPAW-UEABS'

echo -e "\n### Building $packageName/$packageID from $0\n\n"

#
# NOTE: For easy maintanance we load many settings from the UEABS module.
# Of course these variables can be set by hand also.
# Note that the UEABS module also completes the MODULEPATH set below.
#

# The next three variables are only used to load the right UEABS module
# and to give example values for variable settings in comments.
install_root=$VSC_SCRATCH/UEABS
systemID=CalcUA-vaughan-rome
UEABS_version='2.2'

module purge
MODULEPATH=$install_root/$systemID/Modules
module load UEABS/$UEABS_version

#
# The following UEABS_ variables are needed:
#
# Directory to put the downloaded sources of the packages.
# UEABS_DOWNLOADS=$install_root/Downloads
# Directory where packages should be installed.
# UEABS_PACKAGES=$install_root/$systemID/Packages
# Directory where modules are installed
# UEABS_MODULES=$install_root/$systemID/Modules

install_dir=$UEABS_PACKAGES/$packageName/$packageID
modules_dir=$UEABS_MODULES/$packageName
build_dir="/dev/shm/$USER/$packageName/$packageID"

# Software versions
python_version='3.8.7'
fftw_version='3.3.8'
libxc_version='4.3.4'
#wheel_version='0.35.1'
cython_version='0.29.21'

numpy_version='1.18.5'
scipy_version='1.5.4'
ase_version='3.19.3'
GPAW_version='20.1.0'

GPAWsetups_version='0.9.20000' # Check version on https://wiki.fysik.dtu.dk/gpaw/setups/setups.html

# Compiler settings
compiler_module='intel/2020a'
opt_level='-O2'
proc_opt_flags='-march=core-avx2 -mtune=core-avx2'
fp_opt_flags='-ftz -fp-speculation=safe -fp-model source'
parallel=32

py_maj_min='3.8'

################################################################################
#
# Prepare the system
#

#
# Load modules
# - Compiler module: This is a system-provide module
# - buildtools module: Contains a number of updated build tools already available
#   on most systems. We use it to get more consistent builds across systems, but
#   the module is not currently included in the UEABS repository. In most cases,
#   it will not be needed at all except for more exactly reproducing our test
#   results.
# - An EasyBuild-generated module containing a suitable FFTW implementation.
# - An EasyBuild-generated module with Python 3.9 and the basic installation tools
#   (pip, setuptools, wheel)
# In some cases we used modules build specifically for this project, but this was
# to determine the minimum set of libraries etc. that are needed. Modules provided
# on the system would have done the job as well.
#
module load $compiler_module
module load buildtools/$UEABS_version
module load FFTW/$fftw_version-intel-2020a-Cfg5-UEABS-GPAW
module load Python/$python_version-intel-2020a-Cfg5-UEABS-GPAW

#
# Create the directories and make sure they are clean if that matters
#
/usr/bin/mkdir -p $UEABS_DOWNLOADS

/usr/bin/mkdir -p $install_dir
/usr/bin/rm -rf   $install_dir
/usr/bin/mkdir -p $install_dir

/usr/bin/mkdir -p $modules_dir

/usr/bin/mkdir -p $build_dir
/usr/bin/rm -rf   $build_dir
/usr/bin/mkdir -p $build_dir


################################################################################
#
# Download components
#

echo -e "\nDownloading files...\n"

cd $UEABS_DOWNLOADS

# https://gitlab.com/libxc/libxc/-/archive/4.3.4/libxc-4.3.4.tar.bz2
libxc_file="libxc-$libxc_version.tar.bz2"
libxc_url="https://gitlab.com/libxc/libxc/-/archive/$libxc_version"
[[ -f $libxc_file ]] || wget "$libxc_url/$libxc_file"

# As wheel is already present in the Python module we don't need to download it again.
# Downloading wheel so that we can gather all sources for reproducibility.
# wheel_file="wheel-$wheel_version.tar.gz"
# wheel_url="https://pypi.python.org/packages/source/w/wheel"
# [[ -f $wheel_file ]] || wget "$wheel_url/$wheel_file"

# Downloading Cython so that we can gather all sources for reproducibility.
cython_file="Cython-$cython_version.tar.gz"
cython_url="https://pypi.python.org/packages/source/c/cython"
[[ -f $cython_file ]] || wget "$cython_url/$cython_file"

# NumPy needs customizations, so we need to download and unpack the sources
numpy_file="numpy-$numpy_version.zip"
numpy_url="https://pypi.python.org/packages/source/n/numpy"
[[ -f $numpy_file ]] || wget "$numpy_url/$numpy_file"

# SciPy
scipy_file="scipy-$scipy_version.tar.gz"
scipy_url="https://pypi.python.org/packages/source/s/scipy"
[[ -f $scipy_file ]] || wget "$scipy_url/$scipy_file"

# Downloading ase so that we can gather all sources for reproducibility
ase_file="ase-$ase_version.tar.gz"
ase_url="https://pypi.python.org/packages/source/a/ase"
[[ -f $ase_file ]] || wget "$ase_url/$ase_file"

# GPAW needs customization, so we need to download and unpack the sources.
# https://files.pythonhosted.org/packages/49/a1/cf54c399f5489cfdda1e8da02cae8bfb4b39d7cb7a895ce86608fcd0e1c9/gpaw-1.5.2.tar.gz
GPAW_file="gpaw-$GPAW_version.tar.gz"
GPAW_url="https://pypi.python.org/packages/source/g/gpaw"
[[ -f $GPAW_file ]] || wget "$GPAW_url/$GPAW_file"

# Download GPAW-setup, a number of setup files for GPAW.
# https://wiki.fysik.dtu.dk/gpaw-files/gpaw-setups-0.9.20000.tar.gz
GPAWsetups_file="gpaw-setups-$GPAWsetups_version.tar.gz"
GPAWsetups_url="https://wiki.fysik.dtu.dk/gpaw-files"
[[ -f $GPAWsetups_file ]] || wget "$GPAWsetups_url/$GPAWsetups_file"


################################################################################
#
# Install libxc
#

echo -e "\nInstalling libxc...\n"

cd $build_dir

# Uncompress
tar -xf $UEABS_DOWNLOADS/$libxc_file

cd libxc-$libxc_version

# Configure
autoreconf -i

export CC=icc
export CFLAGS="$opt_level $proc_opt_flags $fp_opt_flags -fPIC"
./configure --prefix="$install_dir" \
            --disable-static --enable-shared --disable-fortran
# Build
make -j $parallel

# Install
make install

# Add bin, lib and include to the PATH variables
PATH=$install_dir/bin:$PATH
LIBRARY_PATH=$install_dir/lib:$LIBRARY_PATH
LD_LIBRARY_PATH=$install_dir/lib:$LD_LIBRARY_PATH
CPATH=$install_dir/include:$CPATH

# Clean-up
unset CC
unset CFLAGS


################################################################################
#
# Initialising for installing Python packages
#

echo -e "\nInitialising for installing Python packages...\n"

/usr/bin/mkdir -p "$install_dir/lib/python$py_maj_min/site-packages"
cd $install_dir
/usr/bin/ln -s lib lib64
PYTHONPATH="$install_dir/lib/python$py_maj_min/site-packages"


################################################################################
#
# No need to install wheel as it would erase the wheel version already present in
# the Python module.
#

echo -e "\nNot installing wheel...\n"

#cd $build_dir
#pip$py_maj_min install --prefix=$install_dir --no-deps $UEABS_DOWNLOADS/$wheel_file


################################################################################
#
# Optional: Install pytest and its dependencies to test NumPy and SciPy with
# import numpy
# numpy.test()
# import scipy
# scipy.text()
# We don't care about version numbers here as it is not important for the
# reproducibility of the benchmarks.
#

echo -e "\nInstalling pytest...\n"

cd $build_dir
pip$py_maj_min install --prefix=$install_dir pytest


################################################################################
#
# Install Cython
#

echo -e "\nInstalling Cython...\n"

cd $build_dir
pip$py_maj_min install --prefix=$install_dir --no-deps $UEABS_DOWNLOADS/$cython_file


################################################################################
#
# Install NumPy
#

echo -e "\nInstalling NumPy...\n"

cd $build_dir

# Uncompress
unzip $UEABS_DOWNLOADS/$numpy_file

cd numpy-$numpy_version

cat >site.cfg <<EOF
[DEFAULT]
library_dirs = $EBROOTFFTW/lib:$MKLROOT/lib/intel64_lin
include_dirs = $EBROOTFFTW/include:$MKLROOT/include
search_static_first=True
[mkl]
lapack_libs = mkl_rt
mkl_libs = mkl_rt
[fftw]
libraries = fftw3
EOF

export CC=icc
export CFLAGS="$opt_level $proc_opt_flags $fp_opt_flags -fwrapv -fPIC -diag-disable=1678,10148,111,188,3438,2650,3175,1890"
export CXX=icpc
export FC=ifort
export F90=$FC
export FFLAGS="$opt_level $proc_opt_flags $fp_opt_flags -fPIC"
export LD=icc

# Build NumPy.
# Note: Try python setup.py build --help-compiler and python setup.py build --help-fcompiler to list the
# choice of compilers. We did experience though that if the Intel compilers are selected explicitly,
# the wrong linker was used leading to problems there.
# Parallel build seems dangereous? We got some random failures.
python$py_maj_min setup.py config

python$py_maj_min setup.py build --compiler=intel --fcompiler=intelem

# Install NumPy
pip$py_maj_min install --prefix $install_dir --no-deps  --ignore-installed  --no-build-isolation .

# Brief test: Should not be run in the NumPy build directory...
cd $build_dir
python$py_maj_min -c "import numpy"

# Clean-up
unset CC
unset CFLAGS
unset CXX
unset FC
unset F90
unset FFLAGS
unset FOPT
unset LD


################################################################################
#
# Install SciPy
#

echo -e "\nInstalling SciPy...\n"

cd $build_dir

# Uncompress
tar -xf $UEABS_DOWNLOADS/$scipy_file

cd scipy-$scipy_version

export CC=icc
export CFLAGS="$opt_level $proc_opt_flags $fp_opt_flags -fwrapv -fPIC -diag-disable=1678,10148,111,188,3438"
export CXX=icpc
export FC=ifort
export F90=$FC
# FFLAGS is used in the fitpack Makefile
export FFLAGS="$opt_level $proc_opt_flags $fp_opt_flags -fPIC"
# FOPT is used in the odrpack Makefile
export FOPT="$opt_level $proc_opt_flags $fp_opt_flags -fPIC"
export LD=icc

# Build scipy
# SciPy contains Fortran code so a parallel build may be dangerous.
python$py_maj_min setup.py build --compiler=intel --fcompiler=intelem

# Install scipy
pip$py_maj_min install --prefix $install_dir --no-deps  --ignore-installed  --no-build-isolation .

# Brief test: Should not be run in the NumPy build directory...
cd $build_dir
python$py_maj_min -c "import scipy"

# Clean-up
unset CC
unset CFLAGS
unset CXX
unset FC
unset F90
unset FFLAGS
unset FOPT
unset LD


################################################################################
#
# Install ase
#

echo -e "\nInstalling ase...\n"

cd $build_dir
pip$py_maj_min install --prefix=$install_dir --no-deps $UEABS_DOWNLOADS/$ase_file
#pip install --prefix=$install_dir --no-deps ase==$ase_version

# Brief test
cd $build_dir
python$py_maj_min -c "import ase"


################################################################################
#
# Install GPAW-setups
#

echo -e "\nInstalling gpaw-setups...\n"

mkdir -p $install_dir/share/gpaw-setups
cd $install_dir/share/gpaw-setups
tar -xf $UEABS_DOWNLOADS/$GPAWsetups_file --strip-components=1


################################################################################
#
# Install GPAW
#

echo -e "\nInstalling GPAW...\n"

cd $build_dir

# Uncompress
tar -xf $UEABS_DOWNLOADS/$GPAW_file

cd gpaw-$GPAW_version

# Make the siteconfig.py script
cat >siteconfig.py <<EOF
print( 'GPAW EasyBuild INFO: Starting execution of the customization script' )
print( 'GPAW EasyBuild INFO: Variables at the start of the customization script' )
print( 'GPAW EasyBuild INFO: libraries =               ', libraries )
print( 'GPAW EasyBuild INFO: mpi_libaries =            ', mpi_libraries )
print( 'GPAW EasyBuild INFO: library_dirs =            ', library_dirs )
print( 'GPAW EasyBuild INFO: mpi_libary_dirs =         ', mpi_library_dirs )
print( 'GPAW EasyBuild INFO: runtime_library_dirs =    ', runtime_library_dirs )
print( 'GPAW EasyBuild INFO: mpi_runtime_libary_dirs = ', mpi_runtime_library_dirs )
print( 'GPAW EasyBuild INFO: include_dirs =            ', include_dirs )
print( 'GPAW EasyBuild INFO: mpi_include_dirs =        ', mpi_include_dirs )
print( 'GPAW EasyBuild INFO: compiler =                ', compiler )
print( 'GPAW EasyBuild INFO: mpicompiler =             ', mpicompiler )
print( 'GPAW EasyBuild INFO: mpilinker =               ', mpilinker )
print( 'GPAW EasyBuild INFO: extra_compile_args =      ', extra_compile_args )
print( 'GPAW EasyBuild INFO: extra_link_args =         ', extra_link_args )
print( 'GPAW EasyBuild INFO: define_macros =           ', define_macros )
print( 'GPAW EasyBuild INFO: mpi_define_macros =       ', mpi_define_macros )
print( 'GPAW EasyBuild INFO: undef_macros =            ', undef_macros )
print( 'GPAW EasyBuild INFO: fftw =                    ', fftw )
print( 'GPAW EasyBuild INFO: scalapack =               ', scalapack )
print( 'GPAW EasyBuild INFO: libvdwxc =                ', libvdwxc )
print( 'GPAW EasyBuild INFO: elpa =                    ', elpa )
print( 'GPAW EasyBuild INFO: noblas =                      ', noblas )
print( 'GPAW EasyBuild INFO: parallel_python_interpreter = ', parallel_python_interpreter )

# LibXC
include_dirs.append('$install_dir/include')
#libraries.append('xc')

# libvdwxc
libvdwxc = False

# ELPA
elpa = False

# Use regular FFTW
fftw = True
libraries += ['fftw3']

# ScaLAPACK
scalapack = True
libraries += ['mkl_scalapack_lp64', 'mkl_blacs_intelmpi_lp64']

# MKL BLAS
libraries += ['mkl_sequential','mkl_core', 'mkl_rt', ]

# Add other EasyBuild library directoryes.
library_dirs = os.environ['LIBRARY_PATH'].split(':')

# Set the compilers
compiler =    os.environ['CC']
mpicompiler = os.environ['MPICC']
mpilinker =   os.environ['MPICC']

print( 'GPAW EasyBuild INFO: Variables at the end of the customization script' )
print( 'GPAW EasyBuild INFO: libraries =               ', libraries )
print( 'GPAW EasyBuild INFO: mpi_libaries =            ', mpi_libraries )
print( 'GPAW EasyBuild INFO: library_dirs =            ', library_dirs )
print( 'GPAW EasyBuild INFO: mpi_libary_dirs =         ', mpi_library_dirs )
print( 'GPAW EasyBuild INFO: runtime_library_dirs =    ', runtime_library_dirs )
print( 'GPAW EasyBuild INFO: mpi_runtime_libary_dirs = ', mpi_runtime_library_dirs )
print( 'GPAW EasyBuild INFO: include_dirs =            ', include_dirs )
print( 'GPAW EasyBuild INFO: mpi_include_dirs =        ', mpi_include_dirs )
print( 'GPAW EasyBuild INFO: compiler =                ', compiler )
print( 'GPAW EasyBuild INFO: mpicompiler =             ', mpicompiler )
print( 'GPAW EasyBuild INFO: mpilinker =               ', mpilinker )
print( 'GPAW EasyBuild INFO: extra_compile_args =      ', extra_compile_args )
print( 'GPAW EasyBuild INFO: extra_link_args =         ', extra_link_args )
print( 'GPAW EasyBuild INFO: define_macros =           ', define_macros )
print( 'GPAW EasyBuild INFO: mpi_define_macros =       ', mpi_define_macros )
print( 'GPAW EasyBuild INFO: undef_macros =            ', undef_macros )
print( 'GPAW EasyBuild INFO: fftw =                    ', fftw )
print( 'GPAW EasyBuild INFO: scalapack =               ', scalapack )
print( 'GPAW EasyBuild INFO: libvdwxc =                ', libvdwxc )
print( 'GPAW EasyBuild INFO: elpa =                    ', elpa )
print( 'GPAW EasyBuild INFO: Ending execution of the customization script' )
print( 'GPAW EasyBuild INFO: noblas =                      ', noblas )
print( 'GPAW EasyBuild INFO: parallel_python_interpreter = ', parallel_python_interpreter )
EOF

export CC=icc
export MPICC=mpiicc
export CFLAGS="-std=c99 $opt_level $proc_opt_flags $fp_opt_flags -qno-openmp-simd"
python$py_maj_min setup.py build -j $parallel

# Install GPAW
#python$py_maj_min setup.py install --prefix="$install_dir"
pip$py_maj_min install --prefix $install_dir --no-deps  --ignore-installed  --no-build-isolation .

# Brief test
cd $build_dir
python$py_maj_min -c "import gpaw"

# Clean-up
unset CC
unset MPICC
unset CFLAGS


################################################################################
#
# Finish the install
#

echo -e "\nCleaning up and making the LUA-module $packageName/$packageID...\n"


# Go to a different directory before cleaning up the build directory
cd $modules_dir
/bin/rm -rf $build_dir

# Create a module file
cat >$packageID.lua <<EOF
help([==[

Description
===========

GPAW $GPAW_version for the UEABS benchmark.

Configuration:
  * Parallel GPAW $GPAW_version with ase $ase_version
  * Using FFTW for the FFT computations and MKL for the BLAS/Lapack
  * Python, FFTW and the dependencies of Python build with EasyBuild.

Detailed configuration:
  * Compiler module: $compiler_module, used for all packages
  * EasyBuild modules (special build for the benchmarks, not taken from the system)
      * FFTW $fftw_version
      * Python $python_version
  * libxc $libxc_version
  * Important Python packages: numpy-$numpy_version, scipy-$scipy_version, ase-$ase_version, gpaw-$GPAW_version
  * Additional Python packages: wheel-$wheel_version, cython-$cython_version
  * GPAW setups $GPAWsetups_version


More information
================
 - Homepage: http://wiki.fysik.dtu.dk/gpaw
 - Documentation:
    - GPAW web-based documentation: https://wiki.fysik.dtu.dk/gpaw/
    - Version information at https://gitlab.com/gpaw/gpaw/-/blob/$GPAW_version/doc/
    - ASE web-based documentation: https://wiki.fysik.dtu.dk/ase/


Included extensions
===================
ase-$ase_version, gpaw-$GPAW_version, numpy-$numpy_version, scipy-$scipy_version, cython-$cython_version
]==])

whatis([==[Description: GPAW $GPAW_version with ase $ase_version, numpy $numpy_version and scipty $scipy_version: UEABS benchmark configuration.]==])

conflict("GPAW")
conflict("$packageName")

if not ( isloaded("$compiler_module") ) then
    load("$compiler_module")
end

if not ( isloaded("FFTW/$fftw_version-intel-2020a-Cfg5-UEABS-GPAW") ) then
    load("FFTW/$fftw_version-intel-2020a-Cfg5-UEABS-GPAW")
end

if not ( isloaded("Python/$python_version-intel-2020a-Cfg5-UEABS-GPAW") ) then
    load("Python/$python_version-intel-2020a-Cfg5-UEABS-GPAW")
end

prepend_path("PATH",            "$install_dir/bin")
prepend_path("LD_LIBRARY_PATH", "$install_dir/lib")
prepend_path("LIBRARY_PATH",    "$install_dir/lib")
prepend_path("PYTHONPATH",      "$install_dir/lib/python$py_maj_min/site-packages")

setenv("GPAW_SETUP_PATH", "$install_dir/share/gpaw-setups")
EOF