Add build scripts for Piz Daint

gpaw/build/examples/piz-daint/gpaw/build.sh

+### GPAW installation script for Piz Daint
+
+# version numbers (modify if needed)
+gpaw_version=cuda
+
+# installation directory (modify!)
+tgt=$SCRATCH/lib/gpaw-${gpaw_version}
+
+# setup build environment
+module swap PrgEnv-cray PrgEnv-gnu
+module unload xalt
+module load daint-gpu
+module load craype-accel-nvidia60
+source $SCRATCH/lib/python-2018-12-cuda/load.sh 2016-06
+export GPAW_SETUP_PATH=$SCRATCH/lib/gpaw-setups-0.9.11271
+export CFLAGS=""
+
+# gpaw
+git clone https://gitlab.com/mlouhivu/gpaw.git gpaw-$gpaw_version
+cd gpaw-$gpaw_version
+git checkout $gpaw_version
+patch gpaw/eigensolvers/rmm_diis.py ../setup/patch-rmmdiis.diff
+ln -s ../setup/gcc.py
+cd c/cuda
+cp ../../../setup/make.inc .
+make 2>&1 | tee loki-make
+cd -
+python setup.py install --customize=../setup/customize-piz-daint.py --prefix=$tgt 2>&1 | tee loki-inst
+cd ..
+sed -e "s|<BASE>|$tgt|g" -e "s|<PYTHONHOME>|$PYTHONHOME|" setup/load-gpaw.sh > $tgt/load.sh
+
+# fix permissions
+chmod -R g+rwX $tgt
+chmod -R o+rX $tgt

gpaw/build/examples/piz-daint/gpaw/setup/customize-piz-daint.py

+# Setup customisation for gpaw/cuda
+import os
+
+# compiler and linker
+compiler = './gcc.py'
+mpicompiler = './gcc.py'
+mpilinker = 'cc'
+extra_compile_args = ['-std=c99', '-fopenmp-simd']
+
+# libraries
+libraries = ['z']
+
+# cuda
+library_dirs += [os.environ['CUDATOOLKIT_HOME'] + '/lib64', './c/cuda']
+include_dirs += [os.environ['CUDATOOLKIT_HOME'] + '/include']
+libraries += ['gpaw-cuda', 'cublas', 'cudart', 'stdc++']
+
+# libxc
+library_dirs += [os.environ['LIBXCDIR'] + '/lib']
+include_dirs += [os.environ['LIBXCDIR'] + '/include']
+libraries += ['xc']
+
+# GPAW defines
+define_macros += [('GPAW_NO_UNDERSCORE_CBLACS', '1')]
+define_macros += [('GPAW_NO_UNDERSCORE_CSCALAPACK', '1')]
+define_macros += [("GPAW_ASYNC",1)]
+define_macros += [("GPAW_MPI2",1)]
+define_macros += [('GPAW_CUDA', '1')]
+
+# ScaLAPACK
+scalapack = True
+
+# HDF5
+hdf5 = False
+

gpaw/build/examples/piz-daint/gpaw/setup/gcc.py

+#!/usr/bin/env python
+"""Wrapper for the GNU compiler that converts / removes incompatible
+   compiler options and allows for file-specific tailoring."""
+
+import sys
+from subprocess import call
+
+# Default compiler and options
+compiler = 'gcc'
+args2change = {}
+fragile_files = ['c/xc/tpss.c']
+# Default optimisation settings
+default_level = 3
+default_flags = ['-funroll-loops']
+fragile_level = 2
+fragile_flags = []
+
+# Sisu (Cray XC40)
+if True:
+    compiler = 'cc'
+    default_flags += ['-march=haswell -mtune=haswell -mavx2']
+    fragile_files += ['c/xc/revtpss.c']
+
+# Taito (HP cluster)
+if not True:
+    compiler = 'mpicc'
+    default_flags += ['-ffast-math -march=sandybridge -mtune=haswell']
+
+optimise = None  # optimisation level 0/1/2/3
+debug = False    # use -g or not
+fragile = False  # use special flags for current file?
+sandwich = True  # use optimisation flag twice (= no override possible)
+
+# process arguments
+args = []
+for arg in sys.argv[1:]:
+    arg = arg.strip()
+    if arg.startswith('-O'):
+        level = int(arg.replace('-O',''))
+        if not optimise or level > optimise:
+            optimise = level
+    elif arg == '-g':
+        debug = True
+    elif arg in args2change:
+        if args2change[arg]:
+            args.append(args2change[arg])
+    else:
+        if arg in fragile_files:
+            fragile = True
+        args.append(arg)
+
+# set default optimisation level and flags
+if fragile:
+    optimise = min(fragile_level, optimise)
+    flags = fragile_flags
+else:
+    optimise = max(default_level, optimise)
+    flags = default_flags
+
+# add optimisation level to flags
+if optimise is not None:
+    flags.insert(0, '-O{0}'.format(optimise))
+    if sandwich:
+        args.append('-O{0}'.format(optimise))
+# make sure -g is always the _first_ flag, so it doesn't mess e.g. with the
+# optimisation level
+if debug:
+    flags.insert(0, '-g')
+
+# construct and execute the compile command
+cmd = '{0} {1} {2}'.format(compiler, ' '.join(flags), ' '.join(args))
+print(cmd)
+call(cmd, shell=True)

gpaw/build/examples/piz-daint/gpaw/setup/load-gpaw.sh

+#!/bin/bash
+module swap PrgEnv-cray PrgEnv-gnu
+module unload xalt
+module load daint-gpu
+module load craype-accel-nvidia60
+source <PYTHONHOME>/load.sh
+export GPAW_SETUP_PATH=$SCRATCH/lib/gpaw-setups-0.9.11271
+export PATH=<BASE>/bin:$PATH
+export PYTHONPATH=<BASE>/lib/python2.7/site-packages:$PYTHONPATH

gpaw/build/examples/piz-daint/gpaw/setup/make.inc

+####################################################################
+#  make include file.                                              #
+####################################################################
+#
+SHELL = /bin/sh
+
+# ----------------------------------------------------------------------
+# - gpaw-cuda Directory Structure / gpaw-cuda library --------------------
+# ----------------------------------------------------------------------
+#
+TOPdir        = .
+INCdir        = $(TOPdir)
+PYTHONINCdir ?= $(PYTHONHOME)/include/python2.7/
+PYTHONLIBdir ?= $(PYTHONHOME)/lib/
+NUMPYINCdir  ?= `python -c "import numpy; print numpy.get_include()"`
+MPIINCdir    ?= $(MPICH_DIR)/include
+LIBdir        = $(TOPdir)
+CUGPAWLIB     = $(LIBdir)/libgpaw-cuda.a
+
+#
+# ----------------------------------------------------------------------
+# - NVIDIA CUDA includes / libraries / specifics -----------------------
+# ----------------------------------------------------------------------
+CUDAINCdir    = $(CUDATOOLKIT_HOME)/include
+CUDALIBdir    = $(CUDATOOLKIT_HOME)/lib64
+CUDA_OPTS     =
+
+#
+# ----------------------------------------------------------------------
+# - gpaw-cuda includes / libraries / specifics -------------------------------
+# ----------------------------------------------------------------------
+#
+
+CUGPAW_INCLUDES =  -I$(INCdir) -I$(CUDAINCdir) -I$(MPIINCdir) -I$(NUMPYINCdir)  -I$(PYTHONINCdir)
+CUGPAW_OPTS     = -DPARALLEL=1 -DGPAW_CUDA=1
+
+#
+# ----------------------------------------------------------------------
+#
+
+CUGPAW_DEFS     = $(CUGPAW_OPTS) $(CUDA_OPTS)  $(CUGPAW_INCLUDES)
+
+#
+# ----------------------------------------------------------------------
+# - Compilers / linkers - Optimization flags ---------------------------
+# ----------------------------------------------------------------------
+CC           = gcc
+CCNOOPT      = $(CUGPAW_DEFS)
+CCFLAGS      = $(CUGPAW_DEFS) -g -fPIC -std=c99 -m64 -O3
+
+NVCC           = nvcc
+NVCCFLAGS      = $(CUGPAW_DEFS) -O3 -g -gencode arch=compute_60,code=sm_60 -m64 --compiler-options '-O3 -g -std=c99 -fPIC'
+
+ARCH     = ar
+ARCHFLAGS= cr
+RANLIB   = ranlib
+

gpaw/build/examples/piz-daint/gpaw/setup/patch-rmmdiis.diff

+commit 761cba649d58e2d2f24c0a1e2fdad917b5929679
+Author: Martti Louhivuori <martti.louhivuori@csc.fi>
+Date:   Thu May 18 10:56:07 2017 +0300
+
+    Remove obsolete error calculation from DIIS step
+
+diff --git a/gpaw/eigensolvers/rmmdiis.py b/gpaw/eigensolvers/rmmdiis.py
+index 7d60553..d182713 100644
+--- a/gpaw/eigensolvers/rmmdiis.py
++++ b/gpaw/eigensolvers/rmmdiis.py
+@@ -299,23 +299,6 @@ class RMMDIIS(Eigensolver):
+                                              P_axi, kpt.eps_n[n_x], R_xG, n_x,
+                                              calculate_change=True)
+                     self.timer.stop('Calculate residuals')
+-                    self.timer.start('Calculate errors')
+-                    errors_new_x = np.zeros(B)
+-                    # errors_x[:] = 0.0
+-                    for n in range(n1, n2):
+-                        if kpt.f_n is None:
+-                            weight = kpt.weight
+-                        else:
+-                            weight = kpt.f_n[n]
+-                        if self.nbands_converge != 'occupied':
+-                            if wfs.bd.global_index(n) < self.nbands_converge:
+-                                weight = kpt.weight
+-                            else:
+-                                weight = 0.0
+-                        errors_new_x[n-n1] += weight * integrate(R_xG[n - n1],
+-                                                                 R_xG[n - n1])
+-                    comm.sum(errors_x)
+-                    self.timer.stop('Calculate errors')
+ 
+             self.timer.stop('DIIS step')
+             # Final trial step

gpaw/build/examples/piz-daint/python/build-extra.sh

+### Python (extra) modules installation script for Piz Daint
+###   uses PYTHONUSERBASE to bundle all modules into a separate location
+###   away from the base python installation
+
+# load Python
+source $PYTHONHOME/load.sh
+
+# bundle ID (e.g. time of release) (modify if needed)
+bundle=2016-06
+
+# version numbers (modify if needed)
+numpy_version=1.10.4
+scipy_version=0.17.1
+ase_version=3.11.0
+pycuda_version=2017.1.1
+libxc_version=2.1.3
+libsci_version=18.07.1
+
+# installation directory (modify!)
+tgt=$PYTHONHOME/bundle/$bundle
+
+# setup build environment
+export CFLAGS="-fPIC $CFLAGS"
+export FFLAGS="-fPIC $FFLAGS"
+
+# use --user to install modules
+export PYTHONUSERBASE=$tgt
+mkdir -p $PYTHONUSERBASE/lib/python2.7/site-packages
+
+# build in a separate directory
+mkdir bundle-$bundle
+cd bundle-$bundle
+
+# cython + mpi4py
+pip install --user cython
+pip install --user mpi4py
+
+# numpy
+git clone git://github.com/numpy/numpy.git numpy-$numpy_version
+cd numpy-$numpy_version
+git checkout v$numpy_version
+sed -e 's/<ARCH>/haswell/g' -e "s/<LIBSCI>/$libsci_version/g" ../../setup/piz-daint.cfg >| site.cfg
+python setup.py build -j 4 install --user 2>&1 | tee loki-inst
+cd ..
+
+# scipy
+git clone git://github.com/scipy/scipy.git scipy-$scipy_version
+cd scipy-$scipy_version
+git checkout v$scipy_version
+python setup.py build -j 4 install --user 2>&1 | tee loki-inst
+cd ..
+
+# ase
+git clone https://gitlab.com/ase/ase.git ase-$ase_version
+cd ase-$ase_version
+git checkout $ase_version
+python setup.py install --user 2>&1 | tee loki-inst
+cd ..
+
+# libxc
+tar xvfz ~/src/libxc-${libxc_version}.tar.gz
+cd libxc-$libxc_version
+./configure --prefix=$PYTHONUSERBASE --enable-shared | tee loki-conf
+make | tee loki-make
+make install | tee loki-inst
+export LD_LIBRARY_PATH=$PYTHONUSERBASE/lib:$LD_LIBRARY_PATH
+cd ..
+
+# pycuda
+pip install --user pycuda==$pycuda_version
+
+# go back to the main build directory
+cd ..
+
+# if this is the first bundle, use it as default
+if [ ! -e $PYTHONHOME/bundle/default ]
+then
+    cd $PYTHONHOME/bundle
+    ln -s $bundle default
+    cd -
+fi
+
+# fix permissions
+chmod -R g+rwX $tgt
+chmod -R o+rX $tgt

gpaw/build/examples/piz-daint/python/build.sh

+### Python installation script for Piz Daint
+###   uses --prefix to set a custom installation directory
+
+# version numbers (modify if needed)
+python_version=2.7.13
+
+# installation directory (modify!)
+tgt=$SCRATCH/lib/python-2018-12-cuda
+
+# setup build environment
+module swap PrgEnv-cray PrgEnv-gnu
+module unload xalt
+module load daint-gpu
+module load craype-accel-nvidia60
+export CC=cc
+export CFLAGS='-O3'
+export CXX=CC
+export CXXFLAGS='-O3'
+export FC=ftn
+export FFLAGS='-O3'
+export LINKFORSHARED='-Wl,-export-dynamic -dynamic'
+export MPI_LINKFORSHARED='-Wl,-export-dynamic -dynamic'
+
+# python
+git clone https://github.com/python/cpython.git python-$python_version
+cd python-$python_version
+git checkout v$python_version
+./configure --prefix=$tgt --enable-shared --disable-ipv6 --enable-unicode=ucs4 2>&1 | tee loki-conf
+make 2>&1 | tee loki-make
+make install 2>&1 | tee loki-inst
+cd ..
+sed -e "s|<BASE>|$tgt|g" setup/load-python.sh > $tgt/load.sh
+
+# install pip
+source $tgt/load.sh
+python -m ensurepip
+pip install --upgrade pip
+
+# fix permissions
+chmod -R g+rwX $tgt
+chmod -R o+rX $tgt

gpaw/build/examples/piz-daint/python/setup/load-python.sh

+#!/bin/bash
+export PYTHONHOME=<BASE>
+export PYTHONPATH=$PYTHONHOME/lib
+export PATH=$PYTHONHOME/bin:$PATH
+export MANPATH=$PYTHONHOME/share/man:$MANPATH
+export LD_LIBRARY_PATH=$PYTHONHOME/lib:$LD_LIBRARY_PATH
+
+if [[ $# -gt 0 ]]
+then
+    export PYTHONUSERBASE=$PYTHONHOME/bundle/$1
+    export PATH=$PYTHONUSERBASE/bin:$PATH
+    export LD_LIBRARY_PATH=$PYTHONUSERBASE/lib:$LD_LIBRARY_PATH
+elif [[ -e $PYTHONHOME/bundle/default ]]
+then
+    export PYTHONUSERBASE=$PYTHONHOME/bundle/default
+    export PATH=$PYTHONUSERBASE/bin:$PATH
+    export LD_LIBRARY_PATH=$PYTHONUSERBASE/lib:$LD_LIBRARY_PATH
+fi
+if [[ -e $PYTHONUSERBASE/include/xc.h ]]
+then
+    export LIBXCDIR=$PYTHONUSERBASE
+fi

gpaw/build/examples/piz-daint/python/setup/piz-daint.cfg

+[atlas]
+atlas_libs = sci_gnu
+library_dirs = /opt/cray/pe/libsci/<LIBSCI>/gnu/6.1/<ARCH>/lib
+include_dirs = /opt/cray/pe/libsci/<LIBSCI>/gnu/6.1/<ARCH>/include
+[lapack_atlas]
+lapack_libs = sci_gnu
+library_dirs = /opt/cray/pe/libsci/<LIBSCI>/gnu/6.1/<ARCH>/lib
+include_dirs = /opt/cray/pe/libsci/<LIBSCI>/gnu/6.1/<ARCH>/include
+[lapack]
+lapack_libs = sci_gnu
+library_dirs = /opt/cray/pe/libsci/<LIBSCI>/gnu/6.1/<ARCH>/lib
+include_dirs = /opt/cray/pe/libsci/<LIBSCI>/gnu/6.1/<ARCH>/include