Example build scripts on JUWELS and DAVIDE

gpaw/build/examples/davide/gpaw-cpu/build.sh

+### GPAW installation script for D.A.V.I.D.E
+
+# version numbers (modify if needed)
+gpaw_version=1.1.0
+
+# installation directory (modify!)
+tgt=$CINECA_SCRATCH/lib/gpaw-${gpaw_version}
+
+# setup build environment
+module load cuda/9.2.88
+module load cudnn/7.1.4--cuda--9.2.88
+module load gnu/6.4.0
+module load openmpi/3.1.0--gnu--6.4.0
+source $CINECA_SCRATCH/lib/openblas-0.3.4-openmp/load.sh
+source $CINECA_SCRATCH/lib/python-2018-12-cuda/load.sh 2016-06
+source $CINECA_SCRATCH/lib/scalapack-2.0.2/load.sh
+export GPAW_SETUP_PATH=$CINECA_SCRATCH/lib/gpaw-setups-0.9.11271
+export CFLAGS=""
+
+# gpaw
+git clone https://gitlab.com/mlouhivu/gpaw.git gpaw-$gpaw_version
+cd gpaw-$gpaw_version
+git checkout $gpaw_version
+patch gpaw/eigensolvers/rmm_diis.py ../setup/patch-rmmdiis.diff
+cp ../setup/customize-power8.py .
+python setup.py install --customize=customize-power8.py --prefix=$tgt 2>&1 | tee loki-inst
+cd ..
+sed -e "s|<BASE>|$tgt|g" -e "s|<PYTHONHOME>|$PYTHONHOME|" setup/load-gpaw.sh > $tgt/load.sh
+
+# fix permissions
+chmod -R g+rwX $tgt
+chmod -R o+rX $tgt

gpaw/build/examples/davide/gpaw-cpu/setup/customize-power8.py

+# Setup customisation for gpaw/cuda
+import os
+
+# compiler and linker
+compiler = 'gcc'
+mpicompiler = 'mpicc'
+mpilinker = 'mpicc'
+extra_compile_args = ['-std=c99', '-mcpu=power8']
+
+# libraries
+libraries = ['z']
+
+# openblas
+library_dirs += [os.environ['OPENBLAS_ROOT'] + '/lib']
+include_dirs += [os.environ['OPENBLAS_ROOT'] + '/include']
+libraries += ['openblas']
+
+# scalapack
+library_dirs += [os.environ['SCALAPACK_ROOT'] + '/lib']
+libraries += ['scalapack']
+
+# libxc
+library_dirs += [os.environ['LIBXCDIR'] + '/lib']
+include_dirs += [os.environ['LIBXCDIR'] + '/include']
+libraries += ['xc']
+
+# GPAW defines
+define_macros += [('GPAW_NO_UNDERSCORE_CBLACS', '1')]
+define_macros += [('GPAW_NO_UNDERSCORE_CSCALAPACK', '1')]
+define_macros += [("GPAW_ASYNC",1)]
+define_macros += [("GPAW_MPI2",1)]
+define_macros += [('GPAW_CUDA', '1')]
+
+# ScaLAPACK
+scalapack = True
+
+# HDF5
+hdf5 = False
+

gpaw/build/examples/davide/gpaw-cpu/setup/load-gpaw.sh

+#!/bin/bash
+module load cuda/9.2.88
+module load cudnn/7.1.4--cuda--9.2.88
+module load gnu/6.4.0
+module load openmpi/3.1.0--gnu--6.4.0
+source $CINECA_SCRATCH/lib/openblas-0.3.4-openmp/load.sh
+source <PYTHONHOME>/load.sh
+source $CINECA_SCRATCH/lib/scalapack-2.0.2/load.sh
+export GPAW_SETUP_PATH=$CINECA_SCRATCH/lib/gpaw-setups-0.9.11271
+export PATH=<BASE>/bin:$PATH
+export PYTHONPATH=<BASE>/lib/python2.7/site-packages:$PYTHONPATH

gpaw/build/examples/davide/gpaw-cpu/setup/patch-rmmdiis.diff

+commit 761cba649d58e2d2f24c0a1e2fdad917b5929679
+Author: Martti Louhivuori <martti.louhivuori@csc.fi>
+Date:   Thu May 18 10:56:07 2017 +0300
+
+    Remove obsolete error calculation from DIIS step
+
+diff --git a/gpaw/eigensolvers/rmmdiis.py b/gpaw/eigensolvers/rmmdiis.py
+index 7d60553..d182713 100644
+--- a/gpaw/eigensolvers/rmmdiis.py
++++ b/gpaw/eigensolvers/rmmdiis.py
+@@ -299,23 +299,6 @@ class RMMDIIS(Eigensolver):
+                                              P_axi, kpt.eps_n[n_x], R_xG, n_x,
+                                              calculate_change=True)
+                     self.timer.stop('Calculate residuals')
+-                    self.timer.start('Calculate errors')
+-                    errors_new_x = np.zeros(B)
+-                    # errors_x[:] = 0.0
+-                    for n in range(n1, n2):
+-                        if kpt.f_n is None:
+-                            weight = kpt.weight
+-                        else:
+-                            weight = kpt.f_n[n]
+-                        if self.nbands_converge != 'occupied':
+-                            if wfs.bd.global_index(n) < self.nbands_converge:
+-                                weight = kpt.weight
+-                            else:
+-                                weight = 0.0
+-                        errors_new_x[n-n1] += weight * integrate(R_xG[n - n1],
+-                                                                 R_xG[n - n1])
+-                    comm.sum(errors_x)
+-                    self.timer.stop('Calculate errors')
+ 
+             self.timer.stop('DIIS step')
+             # Final trial step

gpaw/build/examples/davide/gpaw/build.sh

+### GPAW installation script for D.A.V.I.D.E
+
+# version numbers (modify if needed)
+gpaw_version=cuda
+
+# installation directory (modify!)
+tgt=$CINECA_SCRATCH/lib/gpaw-${gpaw_version}
+
+# setup build environment
+module load cuda/9.2.88
+module load cudnn/7.1.4--cuda--9.2.88
+module load gnu/6.4.0
+module load openmpi/3.1.0--gnu--6.4.0
+source $CINECA_SCRATCH/lib/openblas-0.3.4-openmp/load.sh
+source $CINECA_SCRATCH/lib/python-2018-12-cuda/load.sh 2016-06
+source $CINECA_SCRATCH/lib/scalapack-2.0.2/load.sh
+export GPAW_SETUP_PATH=$CINECA_SCRATCH/lib/gpaw-setups-0.9.11271
+export CFLAGS=""
+
+# gpaw
+git clone https://gitlab.com/mlouhivu/gpaw.git gpaw-$gpaw_version
+cd gpaw-$gpaw_version
+git checkout $gpaw_version
+patch gpaw/eigensolvers/rmm_diis.py ../setup/patch-rmmdiis.diff
+cp ../setup/customize-cuda.py .
+cd c/cuda
+cp ../../../setup/make.inc .
+make 2>&1 | tee loki-make
+cd -
+python setup.py install --customize=customize-cuda.py --prefix=$tgt 2>&1 | tee loki-inst
+cd ..
+sed -e "s|<BASE>|$tgt|g" -e "s|<PYTHONHOME>|$PYTHONHOME|" setup/load-gpaw.sh > $tgt/load.sh
+
+# fix permissions
+chmod -R g+rwX $tgt
+chmod -R o+rX $tgt

gpaw/build/examples/davide/gpaw/setup/customize-cuda.py

+# Setup customisation for gpaw/cuda
+import os
+
+# compiler and linker
+compiler = 'gcc'
+mpicompiler = 'mpicc'
+mpilinker = 'mpicc'
+extra_compile_args = ['-std=c99', '-mcpu=power8']
+
+# libraries
+libraries = ['z']
+
+# cuda
+library_dirs += [os.environ['CUDA_LIB'], './c/cuda']
+include_dirs += [os.environ['CUDA_HOME'] + '/include']
+libraries += ['gpaw-cuda', 'cublas', 'cudart', 'stdc++']
+
+# openblas
+library_dirs += [os.environ['OPENBLAS_ROOT'] + '/lib']
+include_dirs += [os.environ['OPENBLAS_ROOT'] + '/include']
+libraries += ['openblas']
+
+# scalapack
+library_dirs += [os.environ['SCALAPACK_ROOT'] + '/lib']
+libraries += ['scalapack']
+
+# libxc
+library_dirs += [os.environ['LIBXCDIR'] + '/lib']
+include_dirs += [os.environ['LIBXCDIR'] + '/include']
+libraries += ['xc']
+
+# GPAW defines
+define_macros += [('GPAW_NO_UNDERSCORE_CBLACS', '1')]
+define_macros += [('GPAW_NO_UNDERSCORE_CSCALAPACK', '1')]
+define_macros += [("GPAW_ASYNC",1)]
+define_macros += [("GPAW_MPI2",1)]
+define_macros += [('GPAW_CUDA', '1')]
+
+# ScaLAPACK
+scalapack = True
+
+# HDF5
+hdf5 = False
+

gpaw/build/examples/davide/gpaw/setup/load-gpaw.sh

+#!/bin/bash
+module load cuda/9.2.88
+module load cudnn/7.1.4--cuda--9.2.88
+module load gnu/6.4.0
+module load openmpi/3.1.0--gnu--6.4.0
+source $CINECA_SCRATCH/lib/openblas-0.3.4-openmp/load.sh
+source <PYTHONHOME>/load.sh
+source $CINECA_SCRATCH/lib/scalapack-2.0.2/load.sh
+export GPAW_SETUP_PATH=$CINECA_SCRATCH/lib/gpaw-setups-0.9.11271
+export PATH=<BASE>/bin:$PATH
+export PYTHONPATH=<BASE>/lib/python2.7/site-packages:$PYTHONPATH

gpaw/build/examples/davide/gpaw/setup/make.inc

+####################################################################
+#  make include file.                                              #
+####################################################################
+#
+SHELL = /bin/sh
+
+# ----------------------------------------------------------------------
+# - gpaw-cuda Directory Structure / gpaw-cuda library --------------------
+# ----------------------------------------------------------------------
+#
+TOPdir        = .
+INCdir        = $(TOPdir)
+PYTHONINCdir ?= $(PYTHONHOME)/include/python2.7/
+PYTHONLIBdir ?= $(PYTHONHOME)/lib/
+NUMPYINCdir  ?= `python -c "import numpy; print numpy.get_include()"`
+MPIINCdir    ?= $(OPENMPI_HOME)/include
+LIBdir        = $(TOPdir)
+CUGPAWLIB     = $(LIBdir)/libgpaw-cuda.a
+
+#
+# ----------------------------------------------------------------------
+# - NVIDIA CUDA includes / libraries / specifics -----------------------
+# ----------------------------------------------------------------------
+CUDAINCdir    = $(CUDADIR)/include
+CUDALIBdir    = $(CUDADIR)/lib64
+CUDA_OPTS     =
+
+#
+# ----------------------------------------------------------------------
+# - gpaw-cuda includes / libraries / specifics -------------------------------
+# ----------------------------------------------------------------------
+#
+
+CUGPAW_INCLUDES =  -I$(INCdir) -I$(CUDAINCdir) -I$(MPIINCdir) -I$(NUMPYINCdir)  -I$(PYTHONINCdir)
+CUGPAW_OPTS     = -DPARALLEL=1 -DGPAW_CUDA=1
+
+#
+# ----------------------------------------------------------------------
+#
+
+CUGPAW_DEFS     = $(CUGPAW_OPTS) $(CUDA_OPTS)  $(CUGPAW_INCLUDES)
+
+#
+# ----------------------------------------------------------------------
+# - Compilers / linkers - Optimization flags ---------------------------
+# ----------------------------------------------------------------------
+CC           = gcc
+CCNOOPT      = $(CUGPAW_DEFS)
+CCFLAGS      = $(CUGPAW_DEFS) -g -fPIC -std=c99 -m64 -O3
+
+NVCC           = nvcc
+NVCCFLAGS      = $(CUGPAW_DEFS) -O3 -g -gencode arch=compute_60,code=sm_60 -m64 --compiler-options '-O3 -g -std=c99 -fPIC'
+
+ARCH     = ar
+ARCHFLAGS= cr
+RANLIB   = ranlib
+

gpaw/build/examples/davide/gpaw/setup/patch-rmmdiis.diff

+commit 761cba649d58e2d2f24c0a1e2fdad917b5929679
+Author: Martti Louhivuori <martti.louhivuori@csc.fi>
+Date:   Thu May 18 10:56:07 2017 +0300
+
+    Remove obsolete error calculation from DIIS step
+
+diff --git a/gpaw/eigensolvers/rmmdiis.py b/gpaw/eigensolvers/rmmdiis.py
+index 7d60553..d182713 100644
+--- a/gpaw/eigensolvers/rmmdiis.py
++++ b/gpaw/eigensolvers/rmmdiis.py
+@@ -299,23 +299,6 @@ class RMMDIIS(Eigensolver):
+                                              P_axi, kpt.eps_n[n_x], R_xG, n_x,
+                                              calculate_change=True)
+                     self.timer.stop('Calculate residuals')
+-                    self.timer.start('Calculate errors')
+-                    errors_new_x = np.zeros(B)
+-                    # errors_x[:] = 0.0
+-                    for n in range(n1, n2):
+-                        if kpt.f_n is None:
+-                            weight = kpt.weight
+-                        else:
+-                            weight = kpt.f_n[n]
+-                        if self.nbands_converge != 'occupied':
+-                            if wfs.bd.global_index(n) < self.nbands_converge:
+-                                weight = kpt.weight
+-                            else:
+-                                weight = 0.0
+-                        errors_new_x[n-n1] += weight * integrate(R_xG[n - n1],
+-                                                                 R_xG[n - n1])
+-                    comm.sum(errors_x)
+-                    self.timer.stop('Calculate errors')
+ 
+             self.timer.stop('DIIS step')
+             # Final trial step

gpaw/build/examples/davide/openblas/build.sh

+### OpenBLAS installation script for D.A.V.I.D.E
+
+# version numbers (modify if needed)
+openblas_version=0.3.4
+
+# installation directory (modify!)
+tgt=$CINECA_SCRATCH/lib/openblas-${openblas_version}-openmp
+
+# setup build environment
+module load cuda/9.2.88
+module load cudnn/7.1.4--cuda--9.2.88
+module load gnu/6.4.0
+module load openmpi/3.1.0--gnu--6.4.0
+export CC=gcc
+export CFLAGS='-mcpu=power8 -O3'
+export CXX=g++
+export CXXFLAGS='-mcpu=power8 -O3'
+export FC=gfortran
+export FFLAGS='-mcpu=power8 -O3'
+
+# openblas
+git clone https://github.com/xianyi/OpenBLAS
+cd OpenBLAS
+git checkout v$openblas_version
+make TARGET=POWER8 USE_OPENMP=1 2>&1 | tee loki-make
+make install PREFIX=$tgt 2>&1 | tee loki-install
+sed -e "s|<BASE>|$tgt|g" ../setup/load-openblas.sh > $tgt/load.sh
+cd ..
+
+# fix permissions
+chmod -R g+rwX $tgt
+chmod -R o+rX $tgt

gpaw/build/examples/davide/openblas/setup/load-openblas.sh

+#!/bin/bash
+export OPENBLAS_ROOT=<BASE>
+export LD_LIBRARY_PATH=$OPENBLAS_ROOT/lib:$LD_LIBRARY_PATH

gpaw/build/examples/davide/python/build-extra.sh

+### Python (extra) modules installation script for D.A.V.I.D.E
+###   uses PYTHONUSERBASE to bundle all modules into a separate location
+###   away from the base python installation
+
+# load Python
+source $PYTHONHOME/load.sh
+
+# bundle ID (e.g. time of release) (modify if needed)
+bundle=2016-06
+
+# version numbers (modify if needed)
+numpy_version=1.10.4
+scipy_version=0.17.1
+ase_version=3.11.0
+pycuda_version=2017.1.1
+libxc_version=2.1.3
+
+# installation directory (modify!)
+tgt=$PYTHONHOME/bundle/$bundle
+
+# setup build environment
+export CFLAGS="-fPIC $CFLAGS"
+export FFLAGS="-fPIC $FFLAGS"
+
+# use --user to install modules
+export PYTHONUSERBASE=$tgt
+mkdir -p $PYTHONUSERBASE/lib/python2.7/site-packages
+
+# build in a separate directory
+mkdir bundle-$bundle
+cd bundle-$bundle
+
+# cython + mpi4py
+pip install --user cython
+pip install --user mpi4py
+
+# numpy
+git clone git://github.com/numpy/numpy.git numpy-$numpy_version
+cd numpy-$numpy_version
+git checkout v$numpy_version
+sed -e "s|<OPENBLAS_ROOT>|$OPENBLAS_ROOT|g" ../../setup/davide-openblas.cfg > site.cfg
+python setup.py build -j 4 install --user 2>&1 | tee loki-inst
+cd ..
+
+# scipy
+git clone git://github.com/scipy/scipy.git scipy-$scipy_version
+cd scipy-$scipy_version
+git checkout v$scipy_version
+python setup.py build -j 4 install --user 2>&1 | tee loki-inst
+cd ..
+
+# ase
+git clone https://gitlab.com/ase/ase.git ase-$ase_version
+cd ase-$ase_version
+git checkout $ase_version
+python setup.py install --user 2>&1 | tee loki-inst
+cd ..
+
+# libxc
+tar xvfz ~/src/libxc-${libxc_version}.tar.gz
+cd libxc-$libxc_version
+./configure --prefix=$PYTHONUSERBASE --enable-shared | tee loki-conf
+make | tee loki-make
+make install | tee loki-inst
+export LD_LIBRARY_PATH=$PYTHONUSERBASE/lib:$LD_LIBRARY_PATH
+cd ..
+
+# pycuda
+pip install --user pycuda==$pycuda_version
+
+# go back to the main build directory
+cd ..
+
+# if this is the first bundle, use it as default
+if [ ! -e $PYTHONHOME/bundle/default ]
+then
+    cd $PYTHONHOME/bundle
+    ln -s $bundle default
+    cd -
+fi
+
+# fix permissions
+chmod -R g+rwX $tgt
+chmod -R o+rX $tgt

gpaw/build/examples/davide/python/build.sh

+### Python installation script for D.A.V.I.D.E
+###   uses --prefix to set a custom installation directory
+
+# version numbers (modify if needed)
+python_version=2.7.13
+
+# installation directory (modify!)
+tgt=$CINECA_SCRATCH/lib/python-2018-12-cuda
+
+# setup build environment
+module load cuda/9.2.88
+module load cudnn/7.1.4--cuda--9.2.88
+module load gnu/6.4.0
+module load openmpi/3.1.0--gnu--6.4.0
+source $CINECA_SCRATCH/lib/openblas-0.3.4-openmp/load.sh
+export CC=gcc
+export CFLAGS='-mcpu=power8 -O3'
+export CXX=g++
+export CXXFLAGS='-mcpu=power8 -O3'
+export F77=gfortran
+export FFLAGS='-mcpu=power8 -O3'
+
+# python
+git clone https://github.com/python/cpython.git python-$python_version
+cd python-$python_version
+git checkout v$python_version
+./configure --prefix=$tgt --enable-shared --disable-ipv6 --enable-unicode=ucs4 2>&1 | tee loki-conf
+make 2>&1 | tee loki-make
+make install 2>&1 | tee loki-inst
+cd ..
+sed -e "s|<BASE>|$tgt|g" setup/load-python.sh > $tgt/load.sh
+
+# install pip
+source $tgt/load.sh
+python -m ensurepip
+pip install --upgrade pip
+
+# fix permissions
+chmod -R g+rwX $tgt
+chmod -R o+rX $tgt

gpaw/build/examples/davide/python/setup/davide-openblas.cfg

+[openblas]
+libraries = openblas
+library_dirs = <OPENBLAS_ROOT>/lib
+include_dirs = <OPENBLAS_ROOT>/include

gpaw/build/examples/davide/python/setup/load-python.sh

+#!/bin/bash
+export PYTHONHOME=<BASE>
+export PYTHONPATH=$PYTHONHOME/lib
+export PATH=$PYTHONHOME/bin:$PATH
+export MANPATH=$PYTHONHOME/share/man:$MANPATH
+export LD_LIBRARY_PATH=$PYTHONHOME/lib:$LD_LIBRARY_PATH
+
+if [[ $# -gt 0 ]]
+then
+    export PYTHONUSERBASE=$PYTHONHOME/bundle/$1
+    export PATH=$PYTHONUSERBASE/bin:$PATH
+    export LD_LIBRARY_PATH=$PYTHONUSERBASE/lib:$LD_LIBRARY_PATH
+elif [[ -e $PYTHONHOME/bundle/default ]]
+then
+    export PYTHONUSERBASE=$PYTHONHOME/bundle/default
+    export PATH=$PYTHONUSERBASE/bin:$PATH
+    export LD_LIBRARY_PATH=$PYTHONUSERBASE/lib:$LD_LIBRARY_PATH
+fi
+if [[ -e $PYTHONUSERBASE/include/xc.h ]]
+then
+    export LIBXCDIR=$PYTHONUSERBASE
+fi

gpaw/build/examples/davide/scalapack/build.sh

+### ScaLAPACK installation script for D.A.V.I.D.E
+
+# version numbers (modify if needed)
+scalapack_version=2.0.2
+
+# installation directory (modify!)
+tgt=$CINECA_SCRATCH/lib/scalapack-${scalapack_version}
+
+# setup build environment
+module load cuda/9.2.88
+module load cudnn/7.1.4--cuda--9.2.88
+module load gnu/6.4.0
+module load openmpi/3.1.0--gnu--6.4.0
+source $CINECA_SCRATCH/lib/openblas-0.3.4-openmp/load.sh
+export CFLAGS="-mcpu=power8 -O3"
+export FFLAGS="-mcpu=power8 -O3"
+
+# scalapack
+tar xvfz ~/scalapack-${scalapack_version}.tgz
+cd scalapack-${scalapack_version}
+cp ../setup/SLmake.inc .
+mkdir build
+cd build
+cmake -DBLAS_LIBRARIES=$OPENBLAS_ROOT/lib/libopenblas.so -DLAPACK_LIBRARIES=$OPENBLAS_ROOT/lib/libopenblas.so -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=$tgt ..
+make 2>&1 | tee loki-make
+make install 2>&1 | tee loki-install
+sed -e "s|<BASE>|$tgt|g" ../../setup/load-scalapack.sh > $tgt/load.sh
+cd ../..
+
+# fix permissions
+chmod -R g+rwX $tgt
+chmod -R o+rX $tgt

gpaw/build/examples/davide/scalapack/setup/SLmake.inc

+############################################################################
+#
+#  Program:         ScaLAPACK
+#
+#  Module:          SLmake.inc
+#
+#  Purpose:         Top-level Definitions
+#
+#  Creation date:   February 15, 2000
+#
+#  Modified:        October 13, 2011
+#
+#  Send bug reports, comments or suggestions to scalapack@cs.utk.edu
+#
+############################################################################
+#
+#  C preprocessor definitions:  set CDEFS to one of the following:
+#
+#     -DNoChange (fortran subprogram names are lower case without any suffix)
+#     -DUpCase   (fortran subprogram names are upper case without any suffix)
+#     -DAdd_     (fortran subprogram names are lower case with "_" appended)
+
+CDEFS         = -DAdd_
+
+#
+#  The fortran and C compilers, loaders, and their flags
+#
+
+FC            = mpif90
+CC            = mpicc 
+NOOPT         = -mcpu=power8 -O0
+FCFLAGS       = -mcpu=power8 -O3 -shared -fPIC
+CCFLAGS       = -mcpu=power8 -O3 -shared -fPIC
+FCLOADER      = $(FC)
+CCLOADER      = $(CC)
+FCLOADFLAGS   = $(FCFLAGS)
+CCLOADFLAGS   = $(CCFLAGS)
+
+#
+#  The archiver and the flag(s) to use when building archive (library)
+#  Also the ranlib routine.  If your system has no ranlib, set RANLIB = echo
+#
+
+ARCH          = ar
+ARCHFLAGS     = cr
+RANLIB        = ranlib
+
+#
+#  The name of the ScaLAPACK library to be created
+#
+
+SCALAPACKLIB  = libscalapack.so
+
+#
+#  BLAS, LAPACK (and possibly other) libraries needed for linking test programs
+#
+
+BLASLIB       = -lopenblas -L$(OPENBLAS_ROOT)/lib -I$(OPENBLAS_ROOT)/include
+LAPACKLIB     = 
+LIBS          = $(LAPACKLIB) $(BLASLIB)
+

gpaw/build/examples/davide/scalapack/setup/load-scalapack.sh

+#!/bin/bash
+export SCALAPACK_ROOT=<BASE>
+export LD_LIBRARY_PATH=$SCALAPACK_ROOT/lib:$LD_LIBRARY_PATH

gpaw/build/examples/juwels/gpaw/build.sh

+### GPAW installation script for JUWELS
+