gpaw/build/examples/davide/gpaw/setup/customize-cuda.py

-# Setup customisation for gpaw/cuda
-import os
-
-# compiler and linker
-compiler = 'gcc'
-mpicompiler = 'mpicc'
-mpilinker = 'mpicc'
-extra_compile_args = ['-std=c99', '-mcpu=power8']
-
-# libraries
-libraries = ['z']
-
-# cuda
-library_dirs += [os.environ['CUDA_LIB'], './c/cuda']
-include_dirs += [os.environ['CUDA_HOME'] + '/include']
-libraries += ['gpaw-cuda', 'cublas', 'cudart', 'stdc++']
-
-# openblas
-library_dirs += [os.environ['OPENBLAS_ROOT'] + '/lib']
-include_dirs += [os.environ['OPENBLAS_ROOT'] + '/include']
-libraries += ['openblas']
-
-# scalapack
-library_dirs += [os.environ['SCALAPACK_ROOT'] + '/lib']
-libraries += ['scalapack']
-
-# libxc
-library_dirs += [os.environ['LIBXCDIR'] + '/lib']
-include_dirs += [os.environ['LIBXCDIR'] + '/include']
-libraries += ['xc']
-
-# GPAW defines
-define_macros += [('GPAW_NO_UNDERSCORE_CBLACS', '1')]
-define_macros += [('GPAW_NO_UNDERSCORE_CSCALAPACK', '1')]
-define_macros += [("GPAW_ASYNC",1)]
-define_macros += [("GPAW_MPI2",1)]
-define_macros += [('GPAW_CUDA', '1')]
-
-# ScaLAPACK
-scalapack = True
-
-# HDF5
-hdf5 = False
-

gpaw/build/examples/davide/gpaw/setup/load-gpaw.sh

-#!/bin/bash
-module load cuda/9.2.88
-module load cudnn/7.1.4--cuda--9.2.88
-module load gnu/6.4.0
-module load openmpi/3.1.0--gnu--6.4.0
-source $CINECA_SCRATCH/lib/openblas-0.3.4-openmp/load.sh
-source <PYTHONHOME>/load.sh
-source $CINECA_SCRATCH/lib/scalapack-2.0.2/load.sh
-export GPAW_SETUP_PATH=$CINECA_SCRATCH/lib/gpaw-setups-0.9.11271
-export PATH=<BASE>/bin:$PATH
-export PYTHONPATH=<BASE>/lib/python2.7/site-packages:$PYTHONPATH

gpaw/build/examples/davide/gpaw/setup/make.inc

-####################################################################
-#  make include file.                                              #
-####################################################################
-#
-SHELL = /bin/sh
-
-# ----------------------------------------------------------------------
-# - gpaw-cuda Directory Structure / gpaw-cuda library --------------------
-# ----------------------------------------------------------------------
-#
-TOPdir        = .
-INCdir        = $(TOPdir)
-PYTHONINCdir ?= $(PYTHONHOME)/include/python2.7/
-PYTHONLIBdir ?= $(PYTHONHOME)/lib/
-NUMPYINCdir  ?= `python -c "import numpy; print numpy.get_include()"`
-MPIINCdir    ?= $(OPENMPI_HOME)/include
-LIBdir        = $(TOPdir)
-CUGPAWLIB     = $(LIBdir)/libgpaw-cuda.a
-
-#
-# ----------------------------------------------------------------------
-# - NVIDIA CUDA includes / libraries / specifics -----------------------
-# ----------------------------------------------------------------------
-CUDAINCdir    = $(CUDADIR)/include
-CUDALIBdir    = $(CUDADIR)/lib64
-CUDA_OPTS     =
-
-#
-# ----------------------------------------------------------------------
-# - gpaw-cuda includes / libraries / specifics -------------------------------
-# ----------------------------------------------------------------------
-#
-
-CUGPAW_INCLUDES =  -I$(INCdir) -I$(CUDAINCdir) -I$(MPIINCdir) -I$(NUMPYINCdir)  -I$(PYTHONINCdir)
-CUGPAW_OPTS     = -DPARALLEL=1 -DGPAW_CUDA=1
-
-#
-# ----------------------------------------------------------------------
-#
-
-CUGPAW_DEFS     = $(CUGPAW_OPTS) $(CUDA_OPTS)  $(CUGPAW_INCLUDES)
-
-#
-# ----------------------------------------------------------------------
-# - Compilers / linkers - Optimization flags ---------------------------
-# ----------------------------------------------------------------------
-CC           = gcc
-CCNOOPT      = $(CUGPAW_DEFS)
-CCFLAGS      = $(CUGPAW_DEFS) -g -fPIC -std=c99 -m64 -O3
-
-NVCC           = nvcc
-NVCCFLAGS      = $(CUGPAW_DEFS) -O3 -g -gencode arch=compute_60,code=sm_60 -m64 --compiler-options '-O3 -g -std=c99 -fPIC'
-
-ARCH     = ar
-ARCHFLAGS= cr
-RANLIB   = ranlib
-

gpaw/build/examples/davide/gpaw/setup/patch-rmmdiis.diff

-commit 761cba649d58e2d2f24c0a1e2fdad917b5929679
-Author: Martti Louhivuori <martti.louhivuori@csc.fi>
-Date:   Thu May 18 10:56:07 2017 +0300
-
-    Remove obsolete error calculation from DIIS step
-
-diff --git a/gpaw/eigensolvers/rmmdiis.py b/gpaw/eigensolvers/rmmdiis.py
-index 7d60553..d182713 100644
---- a/gpaw/eigensolvers/rmmdiis.py
-+++ b/gpaw/eigensolvers/rmmdiis.py
-@@ -299,23 +299,6 @@ class RMMDIIS(Eigensolver):
-                                              P_axi, kpt.eps_n[n_x], R_xG, n_x,
-                                              calculate_change=True)
-                     self.timer.stop('Calculate residuals')
--                    self.timer.start('Calculate errors')
--                    errors_new_x = np.zeros(B)
--                    # errors_x[:] = 0.0
--                    for n in range(n1, n2):
--                        if kpt.f_n is None:
--                            weight = kpt.weight
--                        else:
--                            weight = kpt.f_n[n]
--                        if self.nbands_converge != 'occupied':
--                            if wfs.bd.global_index(n) < self.nbands_converge:
--                                weight = kpt.weight
--                            else:
--                                weight = 0.0
--                        errors_new_x[n-n1] += weight * integrate(R_xG[n - n1],
--                                                                 R_xG[n - n1])
--                    comm.sum(errors_x)
--                    self.timer.stop('Calculate errors')
- 
-             self.timer.stop('DIIS step')
-             # Final trial step

gpaw/build/examples/davide/openblas/build.sh

-### OpenBLAS installation script for D.A.V.I.D.E
-
-# version numbers (modify if needed)
-openblas_version=0.3.4
-
-# installation directory (modify!)
-tgt=$CINECA_SCRATCH/lib/openblas-${openblas_version}-openmp
-
-# setup build environment
-module load cuda/9.2.88
-module load cudnn/7.1.4--cuda--9.2.88
-module load gnu/6.4.0
-module load openmpi/3.1.0--gnu--6.4.0
-export CC=gcc
-export CFLAGS='-mcpu=power8 -O3'
-export CXX=g++
-export CXXFLAGS='-mcpu=power8 -O3'
-export FC=gfortran
-export FFLAGS='-mcpu=power8 -O3'
-
-# openblas
-git clone https://github.com/xianyi/OpenBLAS
-cd OpenBLAS
-git checkout v$openblas_version
-make TARGET=POWER8 USE_OPENMP=1 2>&1 | tee loki-make
-make install PREFIX=$tgt 2>&1 | tee loki-install
-sed -e "s|<BASE>|$tgt|g" ../setup/load-openblas.sh > $tgt/load.sh
-cd ..
-
-# fix permissions
-chmod -R g+rwX $tgt
-chmod -R o+rX $tgt

gpaw/build/examples/davide/openblas/setup/load-openblas.sh

-#!/bin/bash
-export OPENBLAS_ROOT=<BASE>
-export LD_LIBRARY_PATH=$OPENBLAS_ROOT/lib:$LD_LIBRARY_PATH

gpaw/build/examples/davide/python/build-extra.sh

-### Python (extra) modules installation script for D.A.V.I.D.E
-###   uses PYTHONUSERBASE to bundle all modules into a separate location
-###   away from the base python installation
-
-# load Python
-source $PYTHONHOME/load.sh
-
-# bundle ID (e.g. time of release) (modify if needed)
-bundle=2016-06
-
-# version numbers (modify if needed)
-numpy_version=1.10.4
-scipy_version=0.17.1
-ase_version=3.11.0
-pycuda_version=2017.1.1
-libxc_version=2.1.3
-
-# installation directory (modify!)
-tgt=$PYTHONHOME/bundle/$bundle
-
-# setup build environment
-export CFLAGS="-fPIC $CFLAGS"
-export FFLAGS="-fPIC $FFLAGS"
-
-# use --user to install modules
-export PYTHONUSERBASE=$tgt
-mkdir -p $PYTHONUSERBASE/lib/python2.7/site-packages
-
-# build in a separate directory
-mkdir bundle-$bundle
-cd bundle-$bundle
-
-# cython + mpi4py
-pip install --user cython
-pip install --user mpi4py
-
-# numpy
-git clone git://github.com/numpy/numpy.git numpy-$numpy_version
-cd numpy-$numpy_version
-git checkout v$numpy_version
-sed -e "s|<OPENBLAS_ROOT>|$OPENBLAS_ROOT|g" ../../setup/davide-openblas.cfg > site.cfg
-python setup.py build -j 4 install --user 2>&1 | tee loki-inst
-cd ..
-
-# scipy
-git clone git://github.com/scipy/scipy.git scipy-$scipy_version
-cd scipy-$scipy_version
-git checkout v$scipy_version
-python setup.py build -j 4 install --user 2>&1 | tee loki-inst
-cd ..
-
-# ase
-git clone https://gitlab.com/ase/ase.git ase-$ase_version
-cd ase-$ase_version
-git checkout $ase_version
-python setup.py install --user 2>&1 | tee loki-inst
-cd ..
-
-# libxc
-tar xvfz ~/src/libxc-${libxc_version}.tar.gz
-cd libxc-$libxc_version
-./configure --prefix=$PYTHONUSERBASE --enable-shared | tee loki-conf
-make | tee loki-make
-make install | tee loki-inst
-export LD_LIBRARY_PATH=$PYTHONUSERBASE/lib:$LD_LIBRARY_PATH
-cd ..
-
-# pycuda
-pip install --user pycuda==$pycuda_version
-
-# go back to the main build directory
-cd ..
-
-# if this is the first bundle, use it as default
-if [ ! -e $PYTHONHOME/bundle/default ]
-then
-    cd $PYTHONHOME/bundle
-    ln -s $bundle default
-    cd -
-fi
-
-# fix permissions
-chmod -R g+rwX $tgt
-chmod -R o+rX $tgt

gpaw/build/examples/davide/python/build.sh

-### Python installation script for D.A.V.I.D.E
-###   uses --prefix to set a custom installation directory
-
-# version numbers (modify if needed)
-python_version=2.7.13
-
-# installation directory (modify!)
-tgt=$CINECA_SCRATCH/lib/python-2018-12-cuda
-
-# setup build environment
-module load cuda/9.2.88
-module load cudnn/7.1.4--cuda--9.2.88
-module load gnu/6.4.0
-module load openmpi/3.1.0--gnu--6.4.0
-source $CINECA_SCRATCH/lib/openblas-0.3.4-openmp/load.sh
-export CC=gcc
-export CFLAGS='-mcpu=power8 -O3'
-export CXX=g++
-export CXXFLAGS='-mcpu=power8 -O3'
-export F77=gfortran
-export FFLAGS='-mcpu=power8 -O3'
-
-# python
-git clone https://github.com/python/cpython.git python-$python_version
-cd python-$python_version
-git checkout v$python_version
-./configure --prefix=$tgt --enable-shared --disable-ipv6 --enable-unicode=ucs4 2>&1 | tee loki-conf
-make 2>&1 | tee loki-make
-make install 2>&1 | tee loki-inst
-cd ..
-sed -e "s|<BASE>|$tgt|g" setup/load-python.sh > $tgt/load.sh
-
-# install pip
-source $tgt/load.sh
-python -m ensurepip
-pip install --upgrade pip
-
-# fix permissions
-chmod -R g+rwX $tgt
-chmod -R o+rX $tgt

gpaw/build/examples/davide/python/setup/davide-openblas.cfg

-[openblas]
-libraries = openblas
-library_dirs = <OPENBLAS_ROOT>/lib
-include_dirs = <OPENBLAS_ROOT>/include

gpaw/build/examples/davide/python/setup/load-python.sh

-#!/bin/bash
-export PYTHONHOME=<BASE>
-export PYTHONPATH=$PYTHONHOME/lib
-export PATH=$PYTHONHOME/bin:$PATH
-export MANPATH=$PYTHONHOME/share/man:$MANPATH
-export LD_LIBRARY_PATH=$PYTHONHOME/lib:$LD_LIBRARY_PATH
-
-if [[ $# -gt 0 ]]
-then
-    export PYTHONUSERBASE=$PYTHONHOME/bundle/$1
-    export PATH=$PYTHONUSERBASE/bin:$PATH
-    export LD_LIBRARY_PATH=$PYTHONUSERBASE/lib:$LD_LIBRARY_PATH
-elif [[ -e $PYTHONHOME/bundle/default ]]
-then
-    export PYTHONUSERBASE=$PYTHONHOME/bundle/default
-    export PATH=$PYTHONUSERBASE/bin:$PATH
-    export LD_LIBRARY_PATH=$PYTHONUSERBASE/lib:$LD_LIBRARY_PATH
-fi
-if [[ -e $PYTHONUSERBASE/include/xc.h ]]
-then
-    export LIBXCDIR=$PYTHONUSERBASE
-fi

gpaw/build/examples/davide/scalapack/build.sh

-### ScaLAPACK installation script for D.A.V.I.D.E
-
-# version numbers (modify if needed)
-scalapack_version=2.0.2
-
-# installation directory (modify!)
-tgt=$CINECA_SCRATCH/lib/scalapack-${scalapack_version}
-
-# setup build environment
-module load cuda/9.2.88
-module load cudnn/7.1.4--cuda--9.2.88
-module load gnu/6.4.0
-module load openmpi/3.1.0--gnu--6.4.0
-source $CINECA_SCRATCH/lib/openblas-0.3.4-openmp/load.sh
-export CFLAGS="-mcpu=power8 -O3"
-export FFLAGS="-mcpu=power8 -O3"
-
-# scalapack
-tar xvfz ~/scalapack-${scalapack_version}.tgz
-cd scalapack-${scalapack_version}
-cp ../setup/SLmake.inc .
-mkdir build
-cd build
-cmake -DBLAS_LIBRARIES=$OPENBLAS_ROOT/lib/libopenblas.so -DLAPACK_LIBRARIES=$OPENBLAS_ROOT/lib/libopenblas.so -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=$tgt ..
-make 2>&1 | tee loki-make
-make install 2>&1 | tee loki-install
-sed -e "s|<BASE>|$tgt|g" ../../setup/load-scalapack.sh > $tgt/load.sh
-cd ../..
-
-# fix permissions
-chmod -R g+rwX $tgt
-chmod -R o+rX $tgt

gpaw/build/examples/davide/scalapack/setup/SLmake.inc

-############################################################################
-#
-#  Program:         ScaLAPACK
-#
-#  Module:          SLmake.inc
-#
-#  Purpose:         Top-level Definitions
-#
-#  Creation date:   February 15, 2000
-#
-#  Modified:        October 13, 2011
-#
-#  Send bug reports, comments or suggestions to scalapack@cs.utk.edu
-#
-############################################################################
-#
-#  C preprocessor definitions:  set CDEFS to one of the following:
-#
-#     -DNoChange (fortran subprogram names are lower case without any suffix)
-#     -DUpCase   (fortran subprogram names are upper case without any suffix)
-#     -DAdd_     (fortran subprogram names are lower case with "_" appended)
-
-CDEFS         = -DAdd_
-
-#
-#  The fortran and C compilers, loaders, and their flags
-#
-
-FC            = mpif90
-CC            = mpicc 
-NOOPT         = -mcpu=power8 -O0
-FCFLAGS       = -mcpu=power8 -O3 -shared -fPIC
-CCFLAGS       = -mcpu=power8 -O3 -shared -fPIC
-FCLOADER      = $(FC)
-CCLOADER      = $(CC)
-FCLOADFLAGS   = $(FCFLAGS)
-CCLOADFLAGS   = $(CCFLAGS)
-
-#
-#  The archiver and the flag(s) to use when building archive (library)
-#  Also the ranlib routine.  If your system has no ranlib, set RANLIB = echo
-#
-
-ARCH          = ar
-ARCHFLAGS     = cr
-RANLIB        = ranlib
-
-#
-#  The name of the ScaLAPACK library to be created
-#
-
-SCALAPACKLIB  = libscalapack.so
-
-#
-#  BLAS, LAPACK (and possibly other) libraries needed for linking test programs
-#
-
-BLASLIB       = -lopenblas -L$(OPENBLAS_ROOT)/lib -I$(OPENBLAS_ROOT)/include
-LAPACKLIB     = 
-LIBS          = $(LAPACKLIB) $(BLASLIB)
-

gpaw/build/examples/davide/scalapack/setup/load-scalapack.sh

-#!/bin/bash
-export SCALAPACK_ROOT=<BASE>
-export LD_LIBRARY_PATH=$SCALAPACK_ROOT/lib:$LD_LIBRARY_PATH

gpaw/build/examples/juwels/gpaw/build.sh

-### GPAW installation script for JUWELS
-
-# version numbers (modify if needed)
-gpaw_version=1.1.0
-
-# installation directory (modify!)
-tgt=$HOME/lib/gpaw-${gpaw_version}
-
-# setup build environment
-module load CUDA/9.2.88
-module load Intel/2019.0.117-GCC-7.3.0
-module load IntelMPI/2018.4.274
-module load imkl/2019.0.117
-source $HOME/lib/python-2019-01/load.sh 2016-06
-export GPAW_SETUP_PATH=$HOME/lib/gpaw-setups-0.9.11271
-export CFLAGS=""
-
-# gpaw
-git clone https://gitlab.com/mlouhivu/gpaw.git gpaw-$gpaw_version
-cd gpaw-$gpaw_version
-git checkout $gpaw_version
-patch gpaw/eigensolvers/rmm_diis.py ../setup/patch-rmmdiis.diff
-cp ../setup/customize-juwels.py .
-python setup.py install --customize=customize-juwels.py --prefix=$tgt 2>&1 | tee loki-inst
-cd ..
-sed -e "s|<BASE>|$tgt|g" -e "s|<PYTHONHOME>|$PYTHONHOME|" setup/load-gpaw.sh > $tgt/load.sh
-
-# fix permissions
-chmod -R g+rwX $tgt
-chmod -R o+rX $tgt

gpaw/build/examples/juwels/gpaw/setup/customize-juwels.py

-# Setup customisation for gpaw/cuda
-import os
-
-# compiler and linker
-compiler = 'icc'
-mpicompiler = 'mpicc'
-mpilinker = 'mpicc'
-extra_compile_args = ['-std=c99']
-
-# libraries
-libraries = ['z']
-
-# use MKL
-library_dirs += [os.environ['MKLROOT'] + '/lib/intel64/']
-libraries = ['mkl_intel_lp64', 'mkl_sequential', 'mkl_core']
-mpi_libraries += ['mkl_scalapack_lp64', 'mkl_blacs_intelmpi_lp64']
-
-# libxc
-library_dirs += [os.environ['LIBXCDIR'] + '/lib']
-include_dirs += [os.environ['LIBXCDIR'] + '/include']
-libraries += ['xc']
-
-# GPAW defines
-define_macros += [('GPAW_NO_UNDERSCORE_CBLACS', '1')]
-define_macros += [('GPAW_NO_UNDERSCORE_CSCALAPACK', '1')]
-define_macros += [("GPAW_ASYNC",1)]
-define_macros += [("GPAW_MPI2",1)]
-
-# ScaLAPACK
-scalapack = True
-
-# HDF5
-hdf5 = False
-

gpaw/build/examples/juwels/gpaw/setup/load-gpaw.sh

-#!/bin/bash
-module load CUDA/9.2.88
-module load Intel/2019.0.117-GCC-7.3.0
-module load IntelMPI/2018.4.274
-module load imkl/2019.0.117
-source $HOME/lib/python-2019-01/load.sh 2016-06
-export GPAW_SETUP_PATH=$HOME/lib/gpaw-setups-0.9.11271
-export PATH=<BASE>/bin:$PATH
-export PYTHONPATH=<BASE>/lib/python2.7/site-packages:$PYTHONPATH

gpaw/build/examples/juwels/gpaw/setup/patch-rmmdiis.diff

-commit 761cba649d58e2d2f24c0a1e2fdad917b5929679
-Author: Martti Louhivuori <martti.louhivuori@csc.fi>
-Date:   Thu May 18 10:56:07 2017 +0300
-
-    Remove obsolete error calculation from DIIS step
-
-diff --git a/gpaw/eigensolvers/rmmdiis.py b/gpaw/eigensolvers/rmmdiis.py
-index 7d60553..d182713 100644
---- a/gpaw/eigensolvers/rmmdiis.py
-+++ b/gpaw/eigensolvers/rmmdiis.py
-@@ -299,23 +299,6 @@ class RMMDIIS(Eigensolver):
-                                              P_axi, kpt.eps_n[n_x], R_xG, n_x,
-                                              calculate_change=True)
-                     self.timer.stop('Calculate residuals')
--                    self.timer.start('Calculate errors')
--                    errors_new_x = np.zeros(B)
--                    # errors_x[:] = 0.0
--                    for n in range(n1, n2):
--                        if kpt.f_n is None:
--                            weight = kpt.weight
--                        else:
--                            weight = kpt.f_n[n]
--                        if self.nbands_converge != 'occupied':
--                            if wfs.bd.global_index(n) < self.nbands_converge:
--                                weight = kpt.weight
--                            else:
--                                weight = 0.0
--                        errors_new_x[n-n1] += weight * integrate(R_xG[n - n1],
--                                                                 R_xG[n - n1])
--                    comm.sum(errors_x)
--                    self.timer.stop('Calculate errors')
- 
-             self.timer.stop('DIIS step')
-             # Final trial step

gpaw/build/examples/juwels/python/build-extra.sh

-### Python (extra) modules installation script for JUWELS
-###   uses PYTHONUSERBASE to bundle all modules into a separate location
-###   away from the base python installation
-
-# load Python
-source $PYTHONHOME/load.sh
-
-# bundle ID (e.g. time of release) (modify if needed)
-bundle=2016-06
-
-# version numbers (modify if needed)
-numpy_version=1.10.4
-scipy_version=0.17.1
-ase_version=3.11.0
-pycuda_version=2017.1.1
-libxc_version=2.1.3
-
-# installation directory (modify!)
-tgt=$PYTHONHOME/bundle/$bundle
-
-# setup build environment
-export CFLAGS="-fPIC $CFLAGS"
-export FFLAGS="-fPIC $FFLAGS"
-
-# use --user to install modules
-export PYTHONUSERBASE=$tgt
-mkdir -p $PYTHONUSERBASE/lib/python2.7/site-packages
-
-# build in a separate directory
-mkdir bundle-$bundle
-cd bundle-$bundle
-
-# cython + mpi4py
-pip install --user cython
-pip install --user mpi4py
-
-# numpy
-git clone git://github.com/numpy/numpy.git numpy-$numpy_version
-cd numpy-$numpy_version
-git checkout v$numpy_version
-sed -e "s|<MKLROOT>|$MKLROOT|g" ../../setup/juwels-mkl.cfg > site.cfg
-sed -e "s|<FFLAGS>|$FFLAGS|g" ../../setup/patch-intel-fcompiler.diff > patch-intel-fcompiler.diff
-sed -e "s|<CFLAGS>|$CFLAGS|g" ../../setup/patch-intel-ccompiler.diff > patch-intel-ccompiler.diff
-patch numpy/distutils/fcompiler/intel.py patch-intel-fcompiler.diff
-patch numpy/distutils/intelccompiler.py patch-intel-ccompiler.diff
-python setup.py build -j 4 install --user 2>&1 | tee loki-inst
-cd ..
-
-# scipy
-git clone git://github.com/scipy/scipy.git scipy-$scipy_version
-cd scipy-$scipy_version
-git checkout v$scipy_version
-python setup.py build -j 4 install --user 2>&1 | tee loki-inst
-cd ..
-
-# ase
-git clone https://gitlab.com/ase/ase.git ase-$ase_version
-cd ase-$ase_version
-git checkout $ase_version
-python setup.py install --user 2>&1 | tee loki-inst
-cd ..
-
-# libxc
-tar xvfz ~/src/libxc-${libxc_version}.tar.gz
-cd libxc-$libxc_version
-./configure --prefix=$PYTHONUSERBASE --enable-shared | tee loki-conf
-make | tee loki-make
-make install | tee loki-inst
-export LD_LIBRARY_PATH=$PYTHONUSERBASE/lib:$LD_LIBRARY_PATH
-cd ..
-
-# pycuda
-pip install --user pycuda==$pycuda_version
-
-# go back to the main build directory
-cd ..
-
-# if this is the first bundle, use it as default
-if [ ! -e $PYTHONHOME/bundle/default ]
-then
-    cd $PYTHONHOME/bundle
-    ln -s $bundle default
-    cd -
-fi
-
-# fix permissions
-chmod -R g+rwX $tgt
-chmod -R o+rX $tgt

gpaw/build/examples/juwels/python/build.sh

-### Python installation script for JUWELS
-###   uses --prefix to set a custom installation directory
-
-# version numbers (modify if needed)
-python_version=2.7.13
-
-# installation directory (modify!)
-tgt=$HOME/lib/python-2019-01
-
-# setup build environment
-module load CUDA/9.2.88
-module load Intel/2019.0.117-GCC-7.3.0
-module load IntelMPI/2018.4.274
-module load imkl/2019.0.117
-export CC=icc
-export CFLAGS='-O2 -xAVX2 -axCORE-AVX512'
-export CXX=icpc
-export FC=ifort
-export F90=$FC
-export FFLAGS=$CFLAGS
-
-# python
-git clone https://github.com/python/cpython.git python-$python_version
-cd python-$python_version
-git checkout v$python_version
-./configure --prefix=$tgt --enable-shared --disable-ipv6 --enable-unicode=ucs4 2>&1 | tee loki-conf
-make 2>&1 | tee loki-make
-make install 2>&1 | tee loki-inst
-cd ..
-sed -e "s|<BASE>|$tgt|g" setup/load-python.sh > $tgt/load.sh
-
-# install pip
-source $tgt/load.sh
-python -m ensurepip
-pip install --upgrade pip
-
-# fix permissions
-chmod -R g+rwX $tgt
-chmod -R o+rX $tgt

gpaw/build/examples/juwels/python/setup/juwels-mkl.cfg

-[mkl]
-library_dirs = <MKLROOT>/lib/intel64
-include_dirs = <MKLROOT>/include
-lapack_libs =
-mkl_libs = mkl_rt