#!/bin/bash
#
# Installation script for GPAW 1.5.2:
# * Using the existing IntelPython3 module on the system which has an optimized 
#   NumPy and SciPy included.
# * Using the matching version of ase, 3.17.0
# * Compiling with the Intel compilers
#
# The FFT library is discovered at runtime. With the settings used in this script
# this should be MKL FFT, but it is possible to change this at runtime to either 
# MKL, FFTW or the built-in NumPy FFT routines, see the installation instructions
# (link below).
#
# The original installation instructions for GPAW can be found at
# https://gitlab.com/gpaw/gpaw/-/blob/1.5.2/doc/install.rst
#

packageID='1.5.2-Python37icc-icc'

install_root=$VSC_SCRATCH/UEABS
systemID=CalcUA-vaughan-rome

download_dir=$install_root/Downloads
install_dir=$install_root/$systemID/Packages/GPAW-manual/$packageID
modules_dir=$install_root/$systemID/Modules/GPAW-manual
build_dir="/dev/shm/$USER/GPAW-manual/$packageID"
patch_dir=$VSC_DATA/Projects/PRACE/GPAW-experiments/UEABS/build/patches

libxc_version='4.3.4'
cython_version='0.29.21'
numpy_version='1.18.5'
#numpy_version='1.19.2'
scipy_version='1.4.1'
#scipy_version='1.5.3'
ase_version='3.17.0'
GPAW_version='1.5.2'
GPAWsetups_version='0.9.20000' # Check version on https://wiki.fysik.dtu.dk/gpaw/setups/setups.html

py_maj_min='3.7'

################################################################################
#
# Prepare the system
#

#
# Load modules
#
module purge
module load calcua/2020a
module load personal/UEABS
module load intel/2020a
module load buildtools/2020a
module load FFTW/3.3.8-intel-2020a
module load Python/3.7.9-intel-2020a-Cfg5-UEABS-GPAW

#
# Create the directories and make sure they are clean if that matters
#
/usr/bin/mkdir -p $download_dir

/usr/bin/mkdir -p $install_dir
/usr/bin/rm -rf   $install_dir
/usr/bin/mkdir -p $install_dir

/usr/bin/mkdir -p $modules_dir

/usr/bin/mkdir -p $build_dir
/usr/bin/rm -rf   $build_dir
/usr/bin/mkdir -p $build_dir



################################################################################
#
# Download components
#

echo -e "\nDownloading files...\n"

cd $download_dir

# https://gitlab.com/libxc/libxc/-/archive/4.3.4/libxc-4.3.4.tar.bz2
libxc_file="libxc-$libxc_version.tar.bz2"
libxc_url="https://gitlab.com/libxc/libxc/-/archive/$libxc_version"
[[ -f $libxc_file ]] || wget "$libxc_url/$libxc_file"

# We do not download Cython but simply install it using pip.

# NumPy needs customizations, so we need to download and unpack the sources
numpy_file="numpy-$numpy_version.zip"
numpy_url="https://pypi.python.org/packages/source/n/numpy"
[[ -f $numpy_file ]] || wget "$numpy_url/$numpy_file"

# SciPy
scipy_file="scipy-$scipy_version.tar.gz"
scipy_url="https://pypi.python.org/packages/source/s/scipy"
[[ -f $scipy_file ]] || wget "$scipy_url/$scipy_file"

# We do not download ase in this script. As it is pure python and doesn't need
# customization, we will install it using pip right away.
## https://files.pythonhosted.org/packages/d9/08/35969da23b641d3dfca46ba7559f651fcfdca81dbbc00b9058c934e75769/ase-3.17.0.tar.gz
#ase_file="ase-$ase_version.tar.gz"
#ase_url="https://files.pythonhosted.org/packages/d9/08/35969da23b641d3dfca46ba7559f651fcfdca81dbbc00b9058c934e75769"
#[[ -f $ase_file ]] || wget "$ase_url/$ase_file"

# GPAW needs customization, so we need to download and unpack the sources.
# https://files.pythonhosted.org/packages/49/a1/cf54c399f5489cfdda1e8da02cae8bfb4b39d7cb7a895ce86608fcd0e1c9/gpaw-1.5.2.tar.gz
GPAW_file="gpaw-$GPAW_version.tar.gz"
GPAW_url="https://pypi.python.org/packages/source/g/gpaw"
[[ -f $GPAW_file ]] || wget "$GPAW_url/$GPAW_file"

# Download GPAW-setup, a number of setup files for GPAW.
# https://wiki.fysik.dtu.dk/gpaw-files/gpaw-setups-0.9.20000.tar.gz
GPAWsetups_file="gpaw-setups-$GPAWsetups_version.tar.gz"
GPAWsetups_url="https://wiki.fysik.dtu.dk/gpaw-files"
[[ -f $GPAWsetups_file ]] || wget "$GPAWsetups_url/$GPAWsetups_file"



################################################################################
#
# Install libxc
#

echo -e "\nInstalling libxc...\n"

cd $build_dir

# Uncompress
tar -xf $download_dir/$libxc_file

cd libxc-$libxc_version

# Configure
autoreconf -i

export CC=icc
#export CFLAGS="-O2 -march=core-avx2 -mtune=core-avx2 -fPIC"
export CFLAGS="-O2 -march=core-avx2 -mtune=core-avx2 -ftz -fp-speculation=safe -fp-model source -fPIC"
#export CFLAGS="-O0 -march=core-avx2 -mtune=core-avx2 -ftz -fp-speculation=safe -fp-model source -fPIC"
./configure --prefix="$install_dir" \
            --disable-static --enable-shared --disable-fortran
# Build
make -j 16

# Install
make -j 16 install

# Add bin, lib and include to the PATH variables
PATH=$install_dir/bin:$PATH
LIBRARY_PATH=$install_dir/lib:$LIBRARY_PATH
LD_LIBRARY_PATH=$install_dir/lib:$LD_LIBRARY_PATH
CPATH=$install_dir/include:$CPATH


################################################################################
#
# Install Cython
#

echo -e "\nInstalling Cython...\n"

/usr/bin/mkdir -p "$install_dir/lib/python$py_maj_min/site-packages"
cd $install_dir
/usr/bin/ln -s lib lib64
PYTHONPATH="$install_dir/lib/python$py_maj_min/site-packages"

cd $build_dir

pip3 install --prefix=$install_dir --no-deps cython==$cython_version


################################################################################
#
# Install NumPy
#

echo -e "\nInstalling NumPy...\n"

cd $build_dir

# Uncompress
unzip $download_dir/$numpy_file

cd numpy-$numpy_version

cat >site.cfg <<EOF
[DEFAULT]
library_dirs = $EBROOTFFTW/lib:$MKLROOT/lib/intel64_lin
include_dirs = $EBROOTFFTW/include:$MKLROOT/include
search_static_first=True
[mkl]
lapack_libs = mkl_rt
mkl_libs = mkl_rt
[fftw]
libraries = fftw3
EOF

export CC=icc
export CFLAGS='-O2 -march=core-avx2 -mtune=core-avx2 -ftz -fp-speculation=safe -fp-model source -fwrapv -fPIC -diag-disable=1678,10148,111,188,3438,2650,3175,1890'
export CXX=icpc
export FC=ifort
export F90=$FC
export FFLAGS='-O2 -march=core-avx2 -mtune=core-avx2 -ftz -fp-speculation=safe -fp-model source -fPIC'
export LD=icc

# Build NumPy.
# Note: Try python setup.py build --help-compiler and python setup.py build --help-fcompiler to list the
# choice of compilers. We did experience though that if the Intel compilers are selected explicitly,
# the wrong linker was used leading to problems there.
python setup.py build -j 32

# Install NumPy
python setup.py install --prefix $install_dir

# Brief test: Should not be run in the NumPy build directory...
cd $build_dir
python -c "import numpy"


################################################################################
#
# Install SciPy
#

echo -e "\nInstalling SciPy...\n"

cd $build_dir

# Uncompress
tar -xf $download_dir/$scipy_file

cd scipy-$scipy_version

export CC=icc
export CFLAGS='-O2 -march=core-avx2 -mtune=core-avx2 -ftz -fp-speculation=safe -fp-model source -fwrapv -fPIC -diag-disable=1678,10148,111,188,3438'
export CXX=icpc
export FC=ifort
export F90=$FC
# FFLAGS is used in the fitpack Makefile
export FFLAGS='-O2 -march=core-avx2 -mtune=core-avx2 -ftz -fp-speculation=safe -fp-model source -fPIC'
# FOPT is used in the odrpack Makefile
export FOPT='-O2 -march=core-avx2 -mtune=core-avx2 -ftz -fp-speculation=safe -fp-model source -fPIC'
export LD=icc

# Build scipy
python setup.py build -j 32 

# Install scipy
python setup.py install --prefix $install_dir

# Brief test: Should not be run in the NumPy build directory...
cd $build_dir
python -c "import scipy"


################################################################################
#
# Install ase
#

echo -e "\nInstalling ase...\n"

cd $build_dir

pip3 install --prefix=$install_dir --no-deps ase==$ase_version


################################################################################
#
# Install GPAW-setups
#

echo -e "\nInstalling gpaw-setups...\n"

mkdir -p $install_dir/share/gpaw-setups
cd $install_dir/share/gpaw-setups
tar -xf $download_dir/$GPAWsetups_file --strip-components=1


################################################################################
#
# Install GPAW
#

echo -e "\nInstalling GPAW...\n"

cd $build_dir

# Uncompress
tar -xf $download_dir/$GPAW_file

# Apply patches
patch -p0 <$patch_dir/gpaw-1.5.2.patch

cd gpaw-$GPAW_version

# Make the customize.py script
mv customize.py customize.py.orig
cat >customize.py <<EOF
print( 'GPAW build INFO: Starting execution of the customization script' )
print( 'GPAW build INFO: Variables at the start of the customization script' )
print( 'GPAW build INFO: libraries =               ', libraries )
print( 'GPAW build INFO: mpi_libaries =            ', mpi_libraries )
print( 'GPAW build INFO: library_dirs =            ', library_dirs )
print( 'GPAW build INFO: mpi_libary_dirs =         ', mpi_library_dirs )
print( 'GPAW build INFO: runtime_library_dirs =    ', runtime_library_dirs )
print( 'GPAW build INFO: mpi_runtime_libary_dirs = ', mpi_runtime_library_dirs )
print( 'GPAW build INFO: include_dirs =            ', include_dirs )
print( 'GPAW build INFO: mpi_include_dirs =        ', mpi_include_dirs )
print( 'GPAW build INFO: compiler =                ', compiler )
print( 'GPAW build INFO: mpicompiler =             ', mpicompiler )
print( 'GPAW build INFO: mpilinker =               ', mpilinker )
print( 'GPAW build INFO: extra_compile_args =      ', extra_compile_args )
print( 'GPAW build INFO: extra_link_args =         ', extra_link_args )
print( 'GPAW build INFO: define_macros =           ', define_macros )
print( 'GPAW build INFO: mpi_define_macros =       ', mpi_define_macros )
print( 'GPAW build INFO: undef_macros =            ', undef_macros )
print( 'GPAW build INFO: scalapack =               ', scalapack )
print( 'GPAW build INFO: libvdwxc =                ', libvdwxc )
print( 'GPAW build INFO: elpa =                    ', elpa )

# Reset the lists of libraries as often the wrong BLAS library is picked up.
libraries = []
mpi_libraries = []

# LibXC. Re-add the library (removed by resetting libraries).
# There is no need to add the library directory as we do set library_dirs to the
# content of LIBRARY_PATH further down.
# There should be no need to add the include directory as it is in CPATH which is
# set when we install gpaw. 
#include_dirs.append('$install_dir/include')
libraries.append('xc')

# libvdwxc
libvdwxc = False

# ELPA
elpa = False

# ScaLAPACK
scalapack = True
mpi_libraries += ['mkl_scalapack_lp64', 'mkl_blacs_intelmpi_lp64']
mpi_define_macros += [('GPAW_NO_UNDERSCORE_CBLACS', '1')]
mpi_define_macros += [('GPAW_NO_UNDERSCORE_CSCALAPACK', '1')]

# Add EasyBuild LAPACK/BLAS libs
# This should also enable MKL FFTW according to the documentation of GPAW 1.5.2
libraries += ['mkl_intel_lp64', 'mkl_sequential', 'mkl_core']

# Add other EasyBuild library directories.
library_dirs = os.environ['LIBRARY_PATH'].split(':')

# Set the compilers
compiler =    os.environ['CC']
mpicompiler = os.environ['MPICC']
mpilinker =   os.environ['MPICC']

# We need extra_compile_args to have the right compiler options when re-compiling
# files for gpaw-python. It does imply double compiler options for the other
# compiles though.
extra_compile_args = os.environ['CFLAGS'].split(' ')

print( 'GPAW build INFO: Variables at the end of the customization script' )
print( 'GPAW build INFO: libraries =               ', libraries )
print( 'GPAW build INFO: mpi_libaries =            ', mpi_libraries )
print( 'GPAW build INFO: library_dirs =            ', library_dirs )
print( 'GPAW build INFO: mpi_libary_dirs =         ', mpi_library_dirs )
print( 'GPAW build INFO: runtime_library_dirs =    ', runtime_library_dirs )
print( 'GPAW build INFO: mpi_runtime_libary_dirs = ', mpi_runtime_library_dirs )
print( 'GPAW build INFO: include_dirs =            ', include_dirs )
print( 'GPAW build INFO: mpi_include_dirs =        ', mpi_include_dirs )
print( 'GPAW build INFO: compiler =                ', compiler )
print( 'GPAW build INFO: mpicompiler =             ', mpicompiler )
print( 'GPAW build INFO: mpilinker =               ', mpilinker )
print( 'GPAW build INFO: extra_compile_args =      ', extra_compile_args )
print( 'GPAW build INFO: extra_link_args =         ', extra_link_args )
print( 'GPAW build INFO: define_macros =           ', define_macros )
print( 'GPAW build INFO: mpi_define_macros =       ', mpi_define_macros )
print( 'GPAW build INFO: undef_macros =            ', undef_macros )
print( 'GPAW build INFO: scalapack =               ', scalapack )
print( 'GPAW build INFO: libvdwxc =                ', libvdwxc )
print( 'GPAW build INFO: elpa =                    ', elpa )
print( 'GPAW build INFO: Ending execution of the customization script' )
EOF

# Build gpaw
export CC=icc
export MPICC=mpiicc
#export CFLAGS="-O2 -march=core-avx2 -mtune=core-avx2 -qno-openmp-simd"
export CFLAGS="-O2 -march=core-avx2 -mtune=core-avx2 -ftz -fp-speculation=safe -fp-model source -qno-openmp-simd"
#export CFLAGS="-O0 -march=core-avx2 -mtune=core-avx2 -ftz -fp-speculation=safe -fp-model source -qno-openmp-simd"
python setup.py build -j 32

# Install GPAW
python setup.py install --prefix="$install_dir"


################################################################################
#
# Finish the install
#

echo -e "\nCleaning up and making the LUA-module GPAW-manual/$packageID...\n"


# Go to a different directory before cleaning up the build directory
cd $modules_dir
#/bin/rm -rf $build_dir

# Create a module file
cat >$packageID.lua <<EOF
help([==[

Description
===========

GPAW $GPAW_version for the UEABS benchmark.

Configuration:
  * Python 3 compiled with the Intel compilers pre-installed on the system
  * NumPy/SciPy using MKL for linear algebra and FFTW for FFT.
  * Parallel GPAW $GPAW_version with ase $ase_version
  * FFT library selected at runtime. The default with the path
    as set through this module should be MKL. However, we set GPAW_FFTWSO
    in this module to select the regular FFTW libraries as we have 
    experienced problems with the Intel MKL FFTW routines on some systems.
    Setting GPAW_FFTWSO is documented in the install instructions at
    https://gitlab.com/gpaw/gpaw/-/blob/$GPAW_version/doc/install.rst
  * libxc and GPAW compiled with the Intel compilers


More information
================
 - Homepage: http://wiki.fysik.dtu.dk/gpaw
 - Documentation:
    - GPAW web-based documentation: https://wiki.fysik.dtu.dk/gpaw/
    - Version information at https://gitlab.com/gpaw/gpaw/-/blob/$GPAW_version/doc/
    - ASE web-based documentation: https://wiki.fysik.dtu.dk/ase/


Included extensions
===================
ase-$ase_version, gpaw-$GPAW_version
]==])

whatis([==[Description: GPAW $GPAW_version with ase $ase_version: UEABS benchmark configuration.]==])

conflict("GPAW")
conflict("GPAW-manual")

if not ( isloaded("calcua/2020a") ) then
    load("calcua/2020a")
end

-- if not ( isloaded("personal/UEABS") ) then
--     load("personal/UEABS")
-- end

if not ( isloaded("intel/2020a") ) then
    load("intel/2020a")
end

if not ( isloaded("FFTW/3.3.8-intel-2020a") ) then
    load("FFTW/3.3.8-intel-2020a")
end

if not ( isloaded("Python/3.7.9-intel-2020a-Cfg5-UEABS-GPAW") ) then
    load("Python/3.7.9-intel-2020a-Cfg5-UEABS-GPAW")
end

prepend_path("PATH",            "$install_dir/bin")
prepend_path("LD_LIBRARY_PATH", "$install_dir/lib")
prepend_path("LIBRARY_PATH",    "$install_dir/lib")
prepend_path("PYTHONPATH",      "$install_dir/lib/python$py_maj_min/site-packages")

setenv("GPAW_SETUP_PATH", "$install_dir/share/gpaw-setups")
setenv("GPAW_FFTWSO",     "libfftw3.so")
EOF