#!/bin/bash
#
# Installation script for GPAW 20.1.0:
# * We compile our own Python as this is the best guarantee to not have to
#   struggle with compatibility problems between various compilers used for
#   various components
# * Using the matching version of ase, 3.19.3
# * Compiling with the Intel compilers
#
# The FFT library is discovered at runtime. With the settings used in this script
# this should be MKL FFT, but it is possible to change this at runtime to either
# MKL, FFTW or the built-in NumPy FFT routines, see the installation instructions
# (link below).
#
# The original installation instructions for GPAW can be found at
# https://gitlab.com/gpaw/gpaw/-/blob/20.1.0/doc/install.rst
#

packageID='20.1.0-Python38-FFTW-icc'
packageName='GPAW-UEABS'

echo -e "\n###### Building $packageName/$packageID from $0\n\n"

# The next three variables are only used to load the right UEABS module
# and to give example values for variable settings in comments.
install_root=$HOME/UEABS
systemID=HLRS-hawk-rome

module purge
MODULEPATH=$install_root/$systemID/Modules:$MODULEPATH

#
# The following UEABS_ variables are needed:
# We set them manually as we have no UEABS module for this system at the moment.
#
# Directory to put the downloaded sources of the packages.
UEABS_DOWNLOADS=$install_root/SOURCES
# Directory where packages should be installed.
UEABS_PACKAGES=$install_root/$systemID/Packages
# Directory where modules are installed
UEABS_MODULES=$install_root/$systemID/Modules

install_dir=$UEABS_PACKAGES/$packageName/$packageID
modules_dir=$UEABS_MODULES/$packageName
#build_dir="/dev/shm/$USER/$packageName/$packageID"
build_dir="$XDG_RUNTIME_DIR/UEABS-compile/$packageName/$packageID"

# Software versions
python_version='3.8.7'
zlib_version='1.2.11'
ncurses_version='6.2'
readline_version='8.0'
sqlite_version='3.33.0'
sqlite_download='3330000'
libffi_version='3.3'
fftw_version='3.3.8'
libxc_version='4.3.4'

setuptools_version='56.0.0'
setuptoolsscm_version='6.0.1'
wheel_version='0.35.1'

attrs_version='20.3.0'
pybind11_version='2.6.2'
cython_version='0.29.21'

py_version='1.10.0'
pyparsing_version='2.4.7'
toml_version='0.10.2'
iniconfig_version='1.1.1'
packaging_version='20.9'
#pluggy_version='0.13.1'
pytest_version='6.2.3'

numpy_version='1.18.5'
scipy_version='1.5.4'
ase_version='3.19.3'
GPAW_version='20.1.0'

GPAWsetups_version='0.9.20000' # Check version on https://wiki.fysik.dtu.dk/gpaw/setups/setups.html

# Compiler settings
compiler_module='intel/19.1.3'
mpi_module='mpt/2.23'
math_module='mkl/19.1.0'
opt_level='-O2'
proc_opt_flags='-march=core-avx2 -mtune=core-avx2'
fp_opt_flags='-ftz -fp-speculation=safe -fp-model source'
parallel=16

py_maj_min='3.8'

################################################################################
#
# Prepare the system
#

#
# Load modules
#
mkdir -p $modules_dir
module load $compiler_module
module load $mpi_module
module load $math_module

#
# Create the directories and make sure they are clean if that matters
#
/usr/bin/mkdir -p $UEABS_DOWNLOADS

/usr/bin/mkdir -p $install_dir
/usr/bin/rm -rf   $install_dir
/usr/bin/mkdir -p $install_dir

/usr/bin/mkdir -p $modules_dir

/usr/bin/mkdir -p $build_dir
/usr/bin/rm -rf   $build_dir
/usr/bin/mkdir -p $build_dir


################################################################################
#
# Download components
#

echo -e "\n### Downloading files...\n"

function wget() {

    echo "Please download $1 to $UEABS_DOWNLOADS"

}

cd $UEABS_DOWNLOADS

downloads_OK=1

# zlib: https://www.zlib.net/zlib-1.2.11.tar.gz
zlib_file="zlib-$zlib_version.tar.gz"
zlib_url="https://www.zlib.net"
[[ -f $zlib_file ]] || wget "$zlib_url/$zlib_file"
[[ -f $zlib_file ]] || downloads_OK=0

# ncurses: https://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.2.tar.gz
ncurses_file="ncurses-$ncurses_version.tar.gz"
ncurses_url="https://ftp.gnu.org/pub/gnu/ncurses"
[[ -f $ncurses_file ]] || wget "$ncurses_url/$ncurses_file"
[[ -f $ncurses_file ]] || downloads_OK=0

# readline: https://ftp.gnu.org/pub/gnu/readline/readline-8.0.tar.gz
readline_file="readline-$readline_version.tar.gz"
readline_url="https://ftp.gnu.org/pub/gnu/readline"
[[ -f $readline_file ]] || wget "$readline_url/$readline_file"
[[ -f $readline_file ]] || downloads_OK=0

# sqlite: https://www.sqlite.org/2020/sqlite-autoconf-3330000.tar.gz
sqlite_file="sqlite-autoconf-$sqlite_download.tar.gz"
sqlite_url="https://www.sqlite.org/2020"
[[ -f $sqlite_file ]] || wget "$sqlite_url/$sqlite_file"
[[ -f $sqlite_file ]] || downloads_OK=0

# libffi: https://github.com/libffi/libffi/releases/download/v3.3/libffi-3.3.tar.gz
libffi_file="libffi-$libffi_version.tar.gz"
libffi_url="https://github.com/libffi/libffi/releases/download/v$libffi_version"
[[ -f $libffi_file ]] || wget "$libffi_url/$libffi_file"
[[ -f $libffi_file ]] || downloads_OK=0

# FFTW: http://www.fftw.org/fftw-3.3.8.tar.gz
fftw_file="fftw-$fftw_version.tar.gz"
fftw_url="http://www.fftw.org"
[[ -f $fftw_file ]] || wget "$fftw_url/$fftw_file"
[[ -f $fftw_file ]] || downloads_OK=0

# https://gitlab.com/libxc/libxc/-/archive/4.3.4/libxc-4.3.4.tar.bz2
libxc_file="libxc-$libxc_version.tar.bz2"
libxc_url="https://gitlab.com/libxc/libxc/-/archive/$libxc_version"
[[ -f $libxc_file ]] || wget "$libxc_url/$libxc_file"
[[ -f $libxc_file ]] || downloads_OK=0

# Python: https://www.python.org/ftp/python/3.7.9/Python-3.7.9.tar.xz
python_file="Python-$python_version.tar.xz"
python_url="https://www.python.org/ftp/python/$python_version"
[[ -f $python_file ]] || wget "$python_url/$python_file"
[[ -f $python_file ]] || downloads_OK=0

# Downloading setuptools.
setuptools_file="setuptools-$setuptools_version.tar.gz"
setuptools_url="https://pypi.python.org/packages/source/s/setuptools"
[[ -f $setuptools_file ]] || wget $setuptools_url/$setuptools_file
[[ -f $setuptools_file ]] || downloads_OK=0

# Downloading setuptoolssscm so that we can gather all sources for reproducibility.
# https://files.pythonhosted.org/packages/c4/d5/e50358c82026f44cd8810c8165002746cd3f8b78865f6bcf5d7f0fe4f652/setuptools_scm-6.0.1-py3-none-any.whl
setuptoolsscm_file="setuptools_scm-$setuptoolsscm_version.tar.gz"
#setuptoolsscm_file="setuptools_scm-$setuptoolsscm_version-py3-none-any.whl"
setuptoolsscm_url="https://pypi.python.org/packages/source/s/setuptools_scm"
[[ -f $setuptoolsscm_file ]] || wget $setuptoolsscm_url/$setuptoolsscm_file
[[ -f $setuptoolsscm_file ]] || downloads_OK=0

# Downloading wheel so that we can gather all sources for reproducibility.
wheel_file="wheel-$wheel_version.tar.gz"
wheel_url="https://pypi.python.org/packages/source/w/wheel"
[[ -f $wheel_file ]] || wget "$wheel_url/$wheel_file"
[[ -f $wheel_file ]] || downloads_OK=0

# Downloading attrs so that we can gather all sources for reproducibility.
attrs_file="attrs-$attrs_version.tar.gz"
attrs_url="https://pypi.python.org/packages/source/a/attrs"
[[ -f $attrs_file ]] || wget $attrs_url/$attrs_file
[[ -f $attrs_file ]] || downloads_OK=0

# Downloading pybind11 so that we can gather all sources for reproducibility.
pybind11_file="pybind11-$pybind11_version.tar.gz"
pybind11_url="https://pypi.python.org/packages/source/p/pybind11"
[[ -f $pybind11_file ]] || wget $pybind11_url/$pybind11_file
[[ -f $pybind11_file ]] || downloads_OK=0

# Downloading Cython so that we can gather all sources for reproducibility.
cython_file="Cython-$cython_version.tar.gz"
cython_url="https://pypi.python.org/packages/source/c/cython"
[[ -f $cython_file ]] || wget "$cython_url/$cython_file"
[[ -f $cython_file ]] || downloads_OK=0

# Downloading py so that we can gather all sources for reproducibility.
py_file="py-$py_version.tar.gz"
py_url="https://pypi.python.org/packages/source/p/py"
[[ -f $py_file ]] || wget $py_url/$py_file
[[ -f $py_file ]] || downloads_OK=0

# Downloading pyparsing so that we can gather all sources for reproducibility.
pyparsing_file="pyparsing-$pyparsing_version.tar.gz"
pyparsing_url="https://pypi.python.org/packages/source/p/pyparsing"
[[ -f $pyparsing_file ]] || wget $pyparsing_url/$pyparsing_file
[[ -f $pyparsing_file ]] || downloads_OK=0

# Downloading toml so that we can gather all sources for reproducibility.
toml_file="toml-$toml_version.tar.gz"
toml_url="https://pypi.python.org/packages/source/t/toml"
[[ -f $toml_file ]] || wget $toml_url/$toml_file
[[ -f $toml_file ]] || downloads_OK=0

# Downloading iniconfig so that we can gather all sources for reproducibility.
iniconfig_file="iniconfig-$iniconfig_version.tar.gz"
iniconfig_url="https://pypi.python.org/packages/source/i/iniconfig"
[[ -f $iniconfig_file ]] || wget $iniconfig_url/$iniconfig_file
[[ -f $iniconfig_file ]] || downloads_OK=0

# Downloading packaging so that we can gather all sources for reproducibility.
packaging_file="packaging-$packaging_version.tar.gz"
packaging_url="https://pypi.python.org/packages/source/p/packaging"
[[ -f $packaging_file ]] || wget $packaging_url/$packaging_file
[[ -f $packaging_file ]] || downloads_OK=0

## Downloading pluggy so that we can gather all sources for reproducibility.
#pluggy_file="pluggy-$pluggy_version.tar.gz"
#pluggy_url="https://pypi.python.org/packages/source/p/pluggy"
#[[ -f $pluggy_file ]] || wget $pluggy_url/$pluggy_file
#[[ -f $pluggy_file ]] || downloads_OK=0

# Downloading pytest so that we can gather all sources for reproducibility.
pytest_file="pytest-$pytest_version.tar.gz"
pytest_url="https://pypi.python.org/packages/source/p/pytest"
[[ -f $pytest_file ]] || wget $pytest_url/$pytest_file
[[ -f $pytest_file ]] || downloads_OK=0

# NumPy needs customizations, so we need to download and unpack the sources
numpy_file="numpy-$numpy_version.zip"
numpy_url="https://pypi.python.org/packages/source/n/numpy"
[[ -f $numpy_file ]] || wget "$numpy_url/$numpy_file"
[[ -f $numpy_file ]] || downloads_OK=0

# SciPy
scipy_file="scipy-$scipy_version.tar.gz"
scipy_url="https://pypi.python.org/packages/source/s/scipy"
[[ -f $scipy_file ]] || wget "$scipy_url/$scipy_file"
[[ -f $scipy_file ]] || downloads_OK=0

# Downloading ase so that we can gather all sources for reproducibility
ase_file="ase-$ase_version.tar.gz"
ase_url="https://pypi.python.org/packages/source/a/ase"
[[ -f $ase_file ]] || wget "$ase_url/$ase_file"
[[ -f $ase_file ]] || downloads_OK=0

# GPAW needs customization, so we need to download and unpack the sources.
GPAW_file="gpaw-$GPAW_version.tar.gz"
GPAW_url="https://pypi.python.org/packages/source/g/gpaw"
[[ -f $GPAW_file ]] || wget "$GPAW_url/$GPAW_file"
[[ -f $GPAW_file ]] || downloads_OK=0

# Download GPAW-setup, a number of setup files for GPAW.
# https://wiki.fysik.dtu.dk/gpaw-files/gpaw-setups-0.9.20000.tar.gz
GPAWsetups_file="gpaw-setups-$GPAWsetups_version.tar.gz"
GPAWsetups_url="https://wiki.fysik.dtu.dk/gpaw-files"
[[ -f $GPAWsetups_file ]] || wget "$GPAWsetups_url/$GPAWsetups_file"
[[ -f $GPAWsetups_file ]] || downloads_OK=0

[[ $downloads_OK ]] || exit


################################################################################
#
# Install ncurses
#
# We mirror the two-step EasyBuild install process. This may be overkill, but
# we know it works.
#

echo -e "\n### Installing ncurses...\n"

cd $build_dir

# Uncompress
tar -xf $UEABS_DOWNLOADS/$ncurses_file

cd ncurses-$ncurses_version

# Configure step 1
export CC=icc
export CFLAGS="$opt_level $proc_opt_flags $fp_opt_flags -fPIC"
export CXX=icpc
export CXXFLAGS="$opt_level $proc_opt_flags $fp_opt_flags -fPIC"
./configure --prefix="$install_dir" \
            --with-shared --enable-overwrite --without-ada --enable-symlinks

# Build step 1
make -j $parallel

# Install step 1
make install

# Add bin, lib and include to the PATH variables
PATH=$install_dir/bin:$PATH
LIBRARY_PATH=$install_dir/lib:$LIBRARY_PATH
LD_LIBRARY_PATH=$install_dir/lib:$LD_LIBRARY_PATH
CPATH=$install_dir/include:$CPATH

# Configure step 2
make distclean
./configure --prefix="$install_dir" \
            --with-shared --enable-overwrite --without-ada --enable-symlinks \
            --enable-ext-colors --enable-widec \
            --includedir=$install_dir/include/ncursesw/

# Build step 2
make -j $parallel

# Install step 2
make install

# Clean-up
unset CC
unset CFLAGS
unset CXX
unset CXXFLAGS

echo -e "\n### Finishing ncurses installation...\n"


################################################################################
#
# Install readline
#

echo -e "\n### Installing readline...\n"

cd $build_dir

# Uncompress
tar -xf $UEABS_DOWNLOADS/$readline_file

cd readline-$readline_version

# Configure
export CC=icc
export CFLAGS="$opt_level $proc_opt_flags $fp_opt_flags -fPIC"
export CPPFLAGS="-I$install_dir/include"
export LDFLAGS="-L$install_dir/lib -lncurses"
./configure --prefix="$install_dir"

# Build
make -j $parallel

# Install
make install

# Clean-up
unset CC
unset CFLAGS
unset CPPFLAGS
unset LDFLAGS

echo -e "\n### Finishing readline installation...\n"


################################################################################
#
# Install zlib
#

echo -e "\n### Installing zlib...\n"

cd $build_dir

# Uncompress
tar -xf $UEABS_DOWNLOADS/$zlib_file

cd zlib-$zlib_version

# Configure
export CC=icc
export CFLAGS="$opt_level $proc_opt_flags $fp_opt_flags -fPIC"
./configure --prefix="$install_dir"

# Build
make -j $parallel

# Install
make install

# Clean-up
unset CC
unset CFLAGS

echo -e "\n### Finishing zlib installation...\n"


################################################################################
#
# Install libffi
#

echo -e "\n### Installing libffi...\n"

cd $build_dir

# Uncompress
tar -xf $UEABS_DOWNLOADS/$libffi_file

cd libffi-$libffi_version

# Configure
export CC=icc
export CFLAGS="$opt_level $proc_opt_flags $fp_opt_flags -fPIC"
./configure --prefix="$install_dir" \
            --disable-multi-os-directory

# Build
make -j $parallel

# Install
make install

# Clean-up
unset CC
unset CFLAGS

echo -e "\n### Finishing libffi installation...\n"


################################################################################
#
# Install SQLite
#

echo -e "\n### Installing SQLite...\n"

cd $build_dir

# Uncompress
tar -xf $UEABS_DOWNLOADS/$sqlite_file

cd sqlite-autoconf-$sqlite_download

# Configure
export CC=icc
export CFLAGS="$opt_level $proc_opt_flags $fp_opt_flags -fPIC -DSQLITE_DISABLE_INTRINSIC"
export CPPFLAGS="-I$install_dir/include"
./configure --prefix="$install_dir"

# Build
make -j $parallel

# Install
make install

# Clean-up
unset CC
unset CFLAGS
unset CPPFLAGS

echo -e "\n### Finishing SQLite installation...\n"


################################################################################
#
# Install FFTW
#

echo -e "\n### Installing FFTW...\n"

cd $build_dir

# Uncompress
tar -xf $UEABS_DOWNLOADS/$fftw_file

# Patch the sources to compile with icc
cat >fftw.patch <<EOF
avoid using -no-gcc when compiling FFTW with Intel compilers,
since that fails on CentOS 8
see https://github.com/easybuilders/easybuild-easyconfigs/issues/10932
and https://github.com/FFTW/fftw3/issues/184
--- fftw-3.3.8/configure.orig
+++ fftw-3.3.8/configure
@@ -14861,6 +14861,9 @@
    intel) # Stop icc from defining __GNUC__, except on MacOS where this fails
         case "\${host_os}" in
             *darwin*) ;; # icc -no-gcc fails to compile some system headers
+            # using -no-gcc with recent Intel compilers fails on CentOS 8,
+            # and was only needed as a workaround for old Intel compilers anyway
+            *linux*) ;;
             *)
 	        { \$as_echo "\$as_me:\${as_lineno-\$LINENO}: checking whether C compiler accepts -no-gcc" >&5
 $as_echo_n "checking whether C compiler accepts -no-gcc... " >&6; }

EOF

cd fftw-$fftw_version
patch -p1 <../fftw.patch

# Configure
export CC=icc
export CFLAGS="$opt_level $proc_opt_flags $fp_opt_flags -fPIC"
export MPICC=mpicc
export F77=ifort
export F77FLAGS="$opt_level $proc_opt_flags $fp_opt_flags -fPIC"
./configure --prefix="$install_dir" \
            --disable-sse --disable-sse2 --disable-avx --enable-avx2 --disable-avx512 --disable-avx-128-fma --disable-kcvi \
            --enable-fma \
            --disable-openmp --disable-threads \
            --enable-mpi \
            -disable-static --enable-shared --disable-fortran

# Build
make -j $parallel

# Install
make install

# Clean-up
unset CC
unset CFLAGS
unset MPICC
unset F77
unset F77FLAGS

echo -e "\n### Finishing FFTW installation...\n"


################################################################################
#
# Install libxc
#

echo -e "\n### Installing libxc...\n"

cd $build_dir

# Uncompress
tar -xf $UEABS_DOWNLOADS/$libxc_file

cd libxc-$libxc_version

# Configure
autoreconf -i

export CC=icc
export CFLAGS="$opt_level $proc_opt_flags $fp_opt_flags -fPIC"
./configure --prefix="$install_dir" \
            --disable-static --enable-shared --disable-fortran
# Build
make -j $parallel

# Install
make -j install

# Clean-up
unset CC
unset CFLAGS

echo -e "\n### Finishing libxc installation...\n"


################################################################################
#
# Install Python
#

echo -e "\n### Installing Python...\n"

cd $build_dir

# Uncompress
tar -xf $UEABS_DOWNLOADS/$python_file

cd Python-$python_version

# Configure
export CC=icc
export CFLAGS="$opt_level $proc_opt_flags $fp_opt_flags -fwrapv -fPIC -diag-disable=1678,10148,111,169,188,3438,2650,3175,1890"
export CXX=icpc
export FC=ifort
export F90=$FC
export FFLAGS="$opt_level $proc_opt_flags $fp_opt_flags -fwrapv -fPIC"
export LD=icc
export LDFLAGS="-L$install_dir/lib"
export CPPFLAGS="-I$install_dir/include"
./configure --prefix="$install_dir" \
            --build=x86_64-pc-linux-gnu  --host=x86_64-pc-linux-gnu \
            --enable-shared --disable-ipv6 \
            --enable-optimizations \
            --with-ensurepip=upgrade
#             --with-icc \

# Build
make -j $parallel

# Install
make install
cd $install_dir/bin
ln -s python$py_maj_min python

# Clean-up
unset CC
unset CFLAGS
unset CXX
unset FC
unset F90
unset FFLAGS
unset LD
unset LDFLAGS
unset CPPFLAGS

echo -e "\n### Finishing Python installation...\n"


################################################################################
#
# Initialising for installing Python packages
#

echo -e "\n### Initialising for installing Python packages...\n"

/usr/bin/mkdir -p "$install_dir/lib/python$py_maj_min/site-packages"
cd $install_dir
/usr/bin/ln -s lib lib64
PYTHONPATH="$install_dir/lib/python$py_maj_min/site-packages"


################################################################################
#
# Install wheel
#

echo -e "\n### Installing wheel...\n"

cd $build_dir
pip$py_maj_min install --prefix="$install_dir" --no-deps --ignore-installed --find-links=$UEABS_DOWNLOADS $UEABS_DOWNLOADS/$setuptools_file
pip$py_maj_min install --prefix="$install_dir" --no-deps --ignore-installed --find-links=$UEABS_DOWNLOADS $UEABS_DOWNLOADS/$wheel_file
pip$py_maj_min install --prefix="$install_dir" --no-deps --ignore-installed --find-links=$UEABS_DOWNLOADS $UEABS_DOWNLOADS/$setuptoolsscm_file

echo -e "\n### Finishing wheel installation...\n"


################################################################################
#
# Some other Python packages that are needed
#

echo -e "\n### Installing additional Python packages...\n"

cd $build_dir

# attrs is needed for Cython and the optional pytest.
pip$py_maj_min install --prefix="$install_dir" --no-deps --ignore-installed --find-links=$UEABS_DOWNLOADS $UEABS_DOWNLOADS/$attrs_file
# pybind11 is needed for scipy
pip$py_maj_min install --prefix="$install_dir" --no-deps --ignore-installed --find-links=$UEABS_DOWNLOADS $UEABS_DOWNLOADS/$pybind11_file

echo -e "\n### Finishing additional Python packages installation...\n"


################################################################################
#
# Optional: Install pytest and its dependencies to test NumPy and SciPy with
# import numpy
# numpy.test()
# import scipy
# scipy.text()
# We don't care about version numbers here as it is not important for the
# reproducibility of the benchmarks.
#

echo -e "\n### Installing pytest...\n"

cd $build_dir
pip$py_maj_min install --prefix="$install_dir" --no-deps --ignore-installed --find-links=$UEABS_DOWNLOADS $UEABS_DOWNLOADS/$toml_file
pip$py_maj_min install --prefix="$install_dir" --no-deps --ignore-installed --find-links=$UEABS_DOWNLOADS $UEABS_DOWNLOADS/$pyparsing_file
pip$py_maj_min install --prefix="$install_dir" --no-deps --ignore-installed --find-links=$UEABS_DOWNLOADS $UEABS_DOWNLOADS/$py_file
pip$py_maj_min install --prefix="$install_dir" --no-deps --ignore-installed --find-links=$UEABS_DOWNLOADS $UEABS_DOWNLOADS/$iniconfig_file
pip$py_maj_min install --prefix="$install_dir" --no-deps --ignore-installed --find-links=$UEABS_DOWNLOADS $UEABS_DOWNLOADS/$packaging_file
#pip$py_maj_min install --prefix="$install_dir" --no-deps --ignore-installed --find-links=$UEABS_DOWNLOADS $UEABS_DOWNLOADS/$pluggy_file
pip$py_maj_min install --prefix="$install_dir" --no-deps --ignore-installed --find-links=$UEABS_DOWNLOADS $UEABS_DOWNLOADS/$pytest_file

echo -e "\n### Finishing pytest installation...\n"


################################################################################
#
# Install Cython
#

echo -e "\n### Installing Cython...\n"

cd $build_dir
pip$py_maj_min install --prefix="$install_dir" --no-deps --ignore-installed --find-links=$UEABS_DOWNLOADS $UEABS_DOWNLOADS/$cython_file

echo -e "\n### Finishing Cython installation...\n"


################################################################################
#
# Install NumPy
#

echo -e "\n### Installing NumPy...\n"

cd $build_dir

# Uncompress
unzip $UEABS_DOWNLOADS/$numpy_file

cd numpy-$numpy_version

cat >site.cfg <<EOF
[DEFAULT]
library_dirs = $install_dir/lib:$MKLROOT/lib/intel64
include_dirs = $install_dir/include:$MKLROOT/include
search_static_first=True
[mkl]
lapack_libs = mkl_rt
mkl_libs = mkl_rt
[fftw]
libraries = fftw3
EOF

export CC=icc
export CFLAGS="$opt_level $proc_opt_flags $fp_opt_flags -fwrapv -fPIC -diag-disable=1678,10148,111,169,188,3438,2650,3175,1890"
export CXX=icpc
export FC=ifort
export F90=$FC
export FFLAGS="$opt_level $proc_opt_flags $fp_opt_flags -fPIC"
export LD=icc

# Build NumPy.
# Note: Try python setup.py build --help-compiler and python setup.py build --help-fcompiler to list the
# choice of compilers. We did experience though that if the Intel compilers are selected explicitly,
# the wrong linker was used leading to problems there.
# Parallel build seems dangereous? We got some random failures.
# TODO: CFLAGS doesn't seem to be picked up.
python$py_maj_min setup.py config

python$py_maj_min setup.py build --compiler=intel --fcompiler=intelem

# Install NumPy
pip$py_maj_min install --prefix $install_dir --no-deps  --ignore-installed  --no-build-isolation .

# Brief test: Should not be run in the NumPy build directory...
cd $build_dir
python$py_maj_min -c "import numpy"

# Clean-up
unset CC
unset CFLAGS
unset CXX
unset FC
unset F90
unset FFLAGS
unset FOPT
unset LD

echo -e "\n### Finishing NumPy installation...\n"


################################################################################
#
# Install SciPy
#

echo -e "\n### Installing SciPy...\n"

cd $build_dir

# Uncompress
tar -xf $UEABS_DOWNLOADS/$scipy_file

cd scipy-$scipy_version

export CC=icc
export CFLAGS="$opt_level $proc_opt_flags $fp_opt_flags -fwrapv -fPIC -diag-disable=1678,10148,111,188,3438"
export CXX=icpc
export FC=ifort
export F90=$FC
# FFLAGS is used in the fitpack Makefile
export FFLAGS="$opt_level $proc_opt_flags $fp_opt_flags -fPIC"
# FOPT is used in the odrpack Makefile
export FOPT="$opt_level $proc_opt_flags $fp_opt_flags -fPIC"
export LD=icc

# Build scipy
# SciPy contains Fortran code so a parallel build may be dangerous.
python$py_maj_min setup.py build --compiler=intel --fcompiler=intelem

# Install scipy
pip$py_maj_min install --prefix $install_dir --no-deps  --ignore-installed  --no-build-isolation .

# Brief test: Should not be run in the NumPy build directory...
cd $build_dir
python$py_maj_min -c "import scipy"

# Clean-up
unset CC
unset CFLAGS
unset CXX
unset FC
unset F90
unset FFLAGS
unset FOPT
unset LD

echo -e "\n### Finishing SciPy installation...\n"


################################################################################
#
# Install ase
#

echo -e "\n### Installing ase...\n"

cd $build_dir
pip$py_maj_min install --prefix="$install_dir" --no-deps $UEABS_DOWNLOADS/$ase_file
#pip install --prefix="$install_dir" --no-deps ase==$ase_version

# Brief test
cd $build_dir
python$py_maj_min -c "import ase"

echo -e "\n### Finishing ase installation...\n"


################################################################################
#
# Install GPAW-setups
#

echo -e "\n### Installing gpaw-setups...\n"

mkdir -p $install_dir/share/gpaw-setups
cd $install_dir/share/gpaw-setups
tar -xf $UEABS_DOWNLOADS/$GPAWsetups_file --strip-components=1

echo -e "\n### Finishing gpaw-setups installation...\n"


################################################################################
#
# Install GPAW
#

echo -e "\n###\n### Installing GPAW...\n###\n"

cd $build_dir

# Uncompress
tar -xf $UEABS_DOWNLOADS/$GPAW_file

cd gpaw-$GPAW_version

# Make the siteconfig.py script
cat >siteconfig.py <<EOF
print( 'GPAW EasyBuild INFO: Starting execution of the customization script' )
print( 'GPAW EasyBuild INFO: Variables at the start of the customization script' )
print( 'GPAW EasyBuild INFO: libraries =               ', libraries )
print( 'GPAW EasyBuild INFO: mpi_libaries =            ', mpi_libraries )
print( 'GPAW EasyBuild INFO: library_dirs =            ', library_dirs )
print( 'GPAW EasyBuild INFO: mpi_libary_dirs =         ', mpi_library_dirs )
print( 'GPAW EasyBuild INFO: runtime_library_dirs =    ', runtime_library_dirs )
print( 'GPAW EasyBuild INFO: mpi_runtime_libary_dirs = ', mpi_runtime_library_dirs )
print( 'GPAW EasyBuild INFO: include_dirs =            ', include_dirs )
print( 'GPAW EasyBuild INFO: mpi_include_dirs =        ', mpi_include_dirs )
print( 'GPAW EasyBuild INFO: compiler =                ', compiler )
print( 'GPAW EasyBuild INFO: mpicompiler =             ', mpicompiler )
print( 'GPAW EasyBuild INFO: mpilinker =               ', mpilinker )
print( 'GPAW EasyBuild INFO: extra_compile_args =      ', extra_compile_args )
print( 'GPAW EasyBuild INFO: extra_link_args =         ', extra_link_args )
print( 'GPAW EasyBuild INFO: define_macros =           ', define_macros )
print( 'GPAW EasyBuild INFO: mpi_define_macros =       ', mpi_define_macros )
print( 'GPAW EasyBuild INFO: undef_macros =            ', undef_macros )
print( 'GPAW EasyBuild INFO: fftw =                    ', fftw )
print( 'GPAW EasyBuild INFO: scalapack =               ', scalapack )
print( 'GPAW EasyBuild INFO: libvdwxc =                ', libvdwxc )
print( 'GPAW EasyBuild INFO: elpa =                    ', elpa )
print( 'GPAW EasyBuild INFO: noblas =                      ', noblas )
print( 'GPAW EasyBuild INFO: parallel_python_interpreter = ', parallel_python_interpreter )

# LibXC
include_dirs.append('$install_dir/include')
#libraries.append('xc')

# libvdwxc
libvdwxc = False

# ELPA
elpa = False

# Use regular FFTW
fftw = True
libraries += ['fftw3']

# ScaLAPACK
scalapack = True
libraries += ['mkl_scalapack_lp64', 'mkl_blacs_sgimpt_lp64']

# MKL BLAS
libraries += ['mkl_sequential','mkl_core', 'mkl_rt', ]

# Add other EasyBuild library directoryes.
library_dirs = os.environ['LIBRARY_PATH'].split(':')

# Set the compilers
compiler =    os.environ['CC']
mpicompiler = os.environ['MPICC']
mpilinker =   os.environ['MPICC']

print( 'GPAW EasyBuild INFO: Variables at the end of the customization script' )
print( 'GPAW EasyBuild INFO: libraries =               ', libraries )
print( 'GPAW EasyBuild INFO: mpi_libaries =            ', mpi_libraries )
print( 'GPAW EasyBuild INFO: library_dirs =            ', library_dirs )
print( 'GPAW EasyBuild INFO: mpi_libary_dirs =         ', mpi_library_dirs )
print( 'GPAW EasyBuild INFO: runtime_library_dirs =    ', runtime_library_dirs )
print( 'GPAW EasyBuild INFO: mpi_runtime_libary_dirs = ', mpi_runtime_library_dirs )
print( 'GPAW EasyBuild INFO: include_dirs =            ', include_dirs )
print( 'GPAW EasyBuild INFO: mpi_include_dirs =        ', mpi_include_dirs )
print( 'GPAW EasyBuild INFO: compiler =                ', compiler )
print( 'GPAW EasyBuild INFO: mpicompiler =             ', mpicompiler )
print( 'GPAW EasyBuild INFO: mpilinker =               ', mpilinker )
print( 'GPAW EasyBuild INFO: extra_compile_args =      ', extra_compile_args )
print( 'GPAW EasyBuild INFO: extra_link_args =         ', extra_link_args )
print( 'GPAW EasyBuild INFO: define_macros =           ', define_macros )
print( 'GPAW EasyBuild INFO: mpi_define_macros =       ', mpi_define_macros )
print( 'GPAW EasyBuild INFO: undef_macros =            ', undef_macros )
print( 'GPAW EasyBuild INFO: fftw =                    ', fftw )
print( 'GPAW EasyBuild INFO: scalapack =               ', scalapack )
print( 'GPAW EasyBuild INFO: libvdwxc =                ', libvdwxc )
print( 'GPAW EasyBuild INFO: elpa =                    ', elpa )
print( 'GPAW EasyBuild INFO: noblas =                      ', noblas )
print( 'GPAW EasyBuild INFO: parallel_python_interpreter = ', parallel_python_interpreter )
print( 'GPAW EasyBuild INFO: Ending execution of the customization script' )
EOF

# Now install gpaw
export CC=icc
export MPICC=mpicc
export CFLAGS="-std=c99 $opt_level $proc_opt_flags $fp_opt_flags -qno-openmp-simd"
echo -e "\n###\n### Calling setup.py build for GPAW...\n###\n"
python$py_maj_min setup.py build -j $parallel

# Install GPAW
#python$py_maj_min setup.py install --prefix="$install_dir"
echo -e "\n###\n### Calling pip$py_maj_min install for GPAW...\n###\n"
pip$py_maj_min install --prefix="$install_dir" --no-deps --ignore-installed --no-build-isolation .

# Brief test
cd $build_dir
echo -e "\n###\n### Testing if GPAW loads...\n###\n"
mpirun -n 1 python$py_maj_min -c "import gpaw"

# Clean-up
unset CC
unset MPICC
unset CFLAGS

echo -e "\n### Finishing gpaw installation...\n"


################################################################################
#
# Finish the install
#

echo -e "\n### Cleaning up and making the LUA-module $packageName/$packageID...\n"


# Go to a different directory before cleaning up the build directory
cd $modules_dir
/bin/rm -rf $build_dir

# Create a module file
cat >$packageID.lua <<EOF
help([==[

Description
===========

GPAW $GPAW_version for the UEABS benchmark.

Configuration:
  * Parallel GPAW $GPAW_version with ase $ase_version
  * Using FFTW for the FFT computations and MKL for the BLAS/Lapack
  * Minimal use of system libraries or libraries that are installed
    through package managers such as EasyBuild or Spack.

Detailed configuration:
  * Compiler module: $compiler_module, used for all packages
  * Python dependencies:
      * zlib $zlib_version
      * ncurses $ncurses_version
      * readline $readline_version
      * SQLite $sqlite_version
      * libffi $libffi_version
  * FFTW $fftw_version
  * libxc $libxc_version
  * Python $python_version
  * Important Python packages: numpy-$numpy_version, scipy-$scipy_version, ase-$ase_version, gpaw-$GPAW_version
  * Additional Python packages: wheel-$wheel_version, cython-$cython_version
  * GPAW setups $GPAWsetups_version


More information
================
 - Homepage: http://wiki.fysik.dtu.dk/gpaw
 - Documentation:
    - GPAW web-based documentation: https://wiki.fysik.dtu.dk/gpaw/
    - Version information at https://gitlab.com/gpaw/gpaw/-/blob/$GPAW_version/doc/
    - ASE web-based documentation: https://wiki.fysik.dtu.dk/ase/
]==])

whatis([==[Description: GPAW $GPAW_version with ase $ase_version: UEABS benchmark configuration with Python $python_version, numpy-$numpy_version, scipy-$scipy_version, libxc $libxc_version and FFTW $fftw_version.]==])

conflict("GPAW")
conflict("$packageName")

if not ( isloaded("$compiler_module") ) then
    load("$compiler_module")
end

if not ( isloaded("$mpi_module") ) then
    load("$mpi_module")
end

if not ( isloaded("$math_module") ) then
    load("$math_module")
end

prepend_path("PATH",            "$install_dir/bin")
prepend_path("LD_LIBRARY_PATH", "$install_dir/lib")
prepend_path("LIBRARY_PATH",    "$install_dir/lib")
prepend_path("PYTHONPATH",      "$install_dir/lib/python$py_maj_min/site-packages")

setenv("GPAW_SETUP_PATH", "$install_dir/share/gpaw-setups")
EOF