gpaw/build/examples/davide/gpaw/setup/customize-cuda.py

-# Setup customisation for gpaw/cuda
-import os
-
-# compiler and linker
-compiler = 'gcc'
-mpicompiler = 'mpicc'
-mpilinker = 'mpicc'
-extra_compile_args = ['-std=c99', '-mcpu=power8']
-
-# libraries
-libraries = ['z']
-
-# cuda
-library_dirs += [os.environ['CUDA_LIB'], './c/cuda']
-include_dirs += [os.environ['CUDA_HOME'] + '/include']
-libraries += ['gpaw-cuda', 'cublas', 'cudart', 'stdc++']
-
-# openblas
-library_dirs += [os.environ['OPENBLAS_ROOT'] + '/lib']
-include_dirs += [os.environ['OPENBLAS_ROOT'] + '/include']
-libraries += ['openblas']
-
-# scalapack
-library_dirs += [os.environ['SCALAPACK_ROOT'] + '/lib']
-libraries += ['scalapack']
-
-# libxc
-library_dirs += [os.environ['LIBXCDIR'] + '/lib']
-include_dirs += [os.environ['LIBXCDIR'] + '/include']
-libraries += ['xc']
-
-# GPAW defines
-define_macros += [('GPAW_NO_UNDERSCORE_CBLACS', '1')]
-define_macros += [('GPAW_NO_UNDERSCORE_CSCALAPACK', '1')]
-define_macros += [("GPAW_ASYNC",1)]
-define_macros += [("GPAW_MPI2",1)]
-define_macros += [('GPAW_CUDA', '1')]
-
-# ScaLAPACK
-scalapack = True
-
-# HDF5
-hdf5 = False
-

gpaw/build/examples/davide/gpaw/setup/load-gpaw.sh

-#!/bin/bash
-module load cuda/9.2.88
-module load cudnn/7.1.4--cuda--9.2.88
-module load gnu/6.4.0
-module load openmpi/3.1.0--gnu--6.4.0
-source $CINECA_SCRATCH/lib/openblas-0.3.4-openmp/load.sh
-source <PYTHONHOME>/load.sh
-source $CINECA_SCRATCH/lib/scalapack-2.0.2/load.sh
-export GPAW_SETUP_PATH=$CINECA_SCRATCH/lib/gpaw-setups-0.9.11271
-export PATH=<BASE>/bin:$PATH
-export PYTHONPATH=<BASE>/lib/python2.7/site-packages:$PYTHONPATH

gpaw/build/examples/davide/gpaw/setup/make.inc

-####################################################################
-#  make include file.                                              #
-####################################################################
-#
-SHELL = /bin/sh
-
-# ----------------------------------------------------------------------
-# - gpaw-cuda Directory Structure / gpaw-cuda library --------------------
-# ----------------------------------------------------------------------
-#
-TOPdir        = .
-INCdir        = $(TOPdir)
-PYTHONINCdir ?= $(PYTHONHOME)/include/python2.7/
-PYTHONLIBdir ?= $(PYTHONHOME)/lib/
-NUMPYINCdir  ?= `python -c "import numpy; print numpy.get_include()"`
-MPIINCdir    ?= $(OPENMPI_HOME)/include
-LIBdir        = $(TOPdir)
-CUGPAWLIB     = $(LIBdir)/libgpaw-cuda.a
-
-#
-# ----------------------------------------------------------------------
-# - NVIDIA CUDA includes / libraries / specifics -----------------------
-# ----------------------------------------------------------------------
-CUDAINCdir    = $(CUDADIR)/include
-CUDALIBdir    = $(CUDADIR)/lib64
-CUDA_OPTS     =
-
-#
-# ----------------------------------------------------------------------
-# - gpaw-cuda includes / libraries / specifics -------------------------------
-# ----------------------------------------------------------------------
-#
-
-CUGPAW_INCLUDES =  -I$(INCdir) -I$(CUDAINCdir) -I$(MPIINCdir) -I$(NUMPYINCdir)  -I$(PYTHONINCdir)
-CUGPAW_OPTS     = -DPARALLEL=1 -DGPAW_CUDA=1
-
-#
-# ----------------------------------------------------------------------
-#
-
-CUGPAW_DEFS     = $(CUGPAW_OPTS) $(CUDA_OPTS)  $(CUGPAW_INCLUDES)
-
-#
-# ----------------------------------------------------------------------
-# - Compilers / linkers - Optimization flags ---------------------------
-# ----------------------------------------------------------------------
-CC           = gcc
-CCNOOPT      = $(CUGPAW_DEFS)
-CCFLAGS      = $(CUGPAW_DEFS) -g -fPIC -std=c99 -m64 -O3
-
-NVCC           = nvcc
-NVCCFLAGS      = $(CUGPAW_DEFS) -O3 -g -gencode arch=compute_60,code=sm_60 -m64 --compiler-options '-O3 -g -std=c99 -fPIC'
-
-ARCH     = ar
-ARCHFLAGS= cr
-RANLIB   = ranlib
-

gpaw/build/examples/davide/gpaw/setup/patch-rmmdiis.diff

-commit 761cba649d58e2d2f24c0a1e2fdad917b5929679
-Author: Martti Louhivuori <martti.louhivuori@csc.fi>
-Date:   Thu May 18 10:56:07 2017 +0300
-
-    Remove obsolete error calculation from DIIS step
-
-diff --git a/gpaw/eigensolvers/rmmdiis.py b/gpaw/eigensolvers/rmmdiis.py
-index 7d60553..d182713 100644
---- a/gpaw/eigensolvers/rmmdiis.py
-+++ b/gpaw/eigensolvers/rmmdiis.py
-@@ -299,23 +299,6 @@ class RMMDIIS(Eigensolver):
-                                              P_axi, kpt.eps_n[n_x], R_xG, n_x,
-                                              calculate_change=True)
-                     self.timer.stop('Calculate residuals')
--                    self.timer.start('Calculate errors')
--                    errors_new_x = np.zeros(B)
--                    # errors_x[:] = 0.0
--                    for n in range(n1, n2):
--                        if kpt.f_n is None:
--                            weight = kpt.weight
--                        else:
--                            weight = kpt.f_n[n]
--                        if self.nbands_converge != 'occupied':
--                            if wfs.bd.global_index(n) < self.nbands_converge:
--                                weight = kpt.weight
--                            else:
--                                weight = 0.0
--                        errors_new_x[n-n1] += weight * integrate(R_xG[n - n1],
--                                                                 R_xG[n - n1])
--                    comm.sum(errors_x)
--                    self.timer.stop('Calculate errors')
- 
-             self.timer.stop('DIIS step')
-             # Final trial step

gpaw/build/examples/davide/openblas/build.sh

-### OpenBLAS installation script for D.A.V.I.D.E
-
-# version numbers (modify if needed)
-openblas_version=0.3.4
-
-# installation directory (modify!)
-tgt=$CINECA_SCRATCH/lib/openblas-${openblas_version}-openmp
-
-# setup build environment
-module load cuda/9.2.88
-module load cudnn/7.1.4--cuda--9.2.88
-module load gnu/6.4.0
-module load openmpi/3.1.0--gnu--6.4.0
-export CC=gcc
-export CFLAGS='-mcpu=power8 -O3'
-export CXX=g++
-export CXXFLAGS='-mcpu=power8 -O3'
-export FC=gfortran
-export FFLAGS='-mcpu=power8 -O3'
-
-# openblas
-git clone https://github.com/xianyi/OpenBLAS
-cd OpenBLAS
-git checkout v$openblas_version
-make TARGET=POWER8 USE_OPENMP=1 2>&1 | tee loki-make
-make install PREFIX=$tgt 2>&1 | tee loki-install
-sed -e "s|<BASE>|$tgt|g" ../setup/load-openblas.sh > $tgt/load.sh
-cd ..
-
-# fix permissions
-chmod -R g+rwX $tgt
-chmod -R o+rX $tgt

gpaw/build/examples/davide/openblas/setup/load-openblas.sh

-#!/bin/bash
-export OPENBLAS_ROOT=<BASE>
-export LD_LIBRARY_PATH=$OPENBLAS_ROOT/lib:$LD_LIBRARY_PATH

gpaw/build/examples/davide/python/build-extra.sh

-### Python (extra) modules installation script for D.A.V.I.D.E
-###   uses PYTHONUSERBASE to bundle all modules into a separate location
-###   away from the base python installation
-
-# load Python
-source $PYTHONHOME/load.sh
-
-# bundle ID (e.g. time of release) (modify if needed)
-bundle=2016-06
-
-# version numbers (modify if needed)
-numpy_version=1.10.4
-scipy_version=0.17.1
-ase_version=3.11.0
-pycuda_version=2017.1.1
-libxc_version=2.1.3
-
-# installation directory (modify!)
-tgt=$PYTHONHOME/bundle/$bundle
-
-# setup build environment
-export CFLAGS="-fPIC $CFLAGS"
-export FFLAGS="-fPIC $FFLAGS"
-
-# use --user to install modules
-export PYTHONUSERBASE=$tgt
-mkdir -p $PYTHONUSERBASE/lib/python2.7/site-packages
-
-# build in a separate directory
-mkdir bundle-$bundle
-cd bundle-$bundle
-
-# cython + mpi4py
-pip install --user cython
-pip install --user mpi4py
-
-# numpy
-git clone git://github.com/numpy/numpy.git numpy-$numpy_version
-cd numpy-$numpy_version
-git checkout v$numpy_version
-sed -e "s|<OPENBLAS_ROOT>|$OPENBLAS_ROOT|g" ../../setup/davide-openblas.cfg > site.cfg
-python setup.py build -j 4 install --user 2>&1 | tee loki-inst
-cd ..
-
-# scipy
-git clone git://github.com/scipy/scipy.git scipy-$scipy_version
-cd scipy-$scipy_version
-git checkout v$scipy_version
-python setup.py build -j 4 install --user 2>&1 | tee loki-inst
-cd ..
-
-# ase
-git clone https://gitlab.com/ase/ase.git ase-$ase_version
-cd ase-$ase_version
-git checkout $ase_version
-python setup.py install --user 2>&1 | tee loki-inst
-cd ..
-
-# libxc
-tar xvfz ~/src/libxc-${libxc_version}.tar.gz
-cd libxc-$libxc_version
-./configure --prefix=$PYTHONUSERBASE --enable-shared | tee loki-conf
-make | tee loki-make
-make install | tee loki-inst
-export LD_LIBRARY_PATH=$PYTHONUSERBASE/lib:$LD_LIBRARY_PATH
-cd ..
-
-# pycuda
-pip install --user pycuda==$pycuda_version
-
-# go back to the main build directory
-cd ..
-
-# if this is the first bundle, use it as default
-if [ ! -e $PYTHONHOME/bundle/default ]
-then
-    cd $PYTHONHOME/bundle
-    ln -s $bundle default
-    cd -
-fi
-
-# fix permissions
-chmod -R g+rwX $tgt
-chmod -R o+rX $tgt

gpaw/build/examples/davide/python/setup/davide-openblas.cfg

-[openblas]
-libraries = openblas
-library_dirs = <OPENBLAS_ROOT>/lib
-include_dirs = <OPENBLAS_ROOT>/include

gpaw/build/examples/davide/scalapack/setup/load-scalapack.sh

-#!/bin/bash
-export SCALAPACK_ROOT=<BASE>
-export LD_LIBRARY_PATH=$SCALAPACK_ROOT/lib:$LD_LIBRARY_PATH

gpaw/build/examples/juwels/gpaw/setup/load-gpaw.sh

-#!/bin/bash
-module load CUDA/9.2.88
-module load Intel/2019.0.117-GCC-7.3.0
-module load IntelMPI/2018.4.274
-module load imkl/2019.0.117
-source $HOME/lib/python-2019-01/load.sh 2016-06
-export GPAW_SETUP_PATH=$HOME/lib/gpaw-setups-0.9.11271
-export PATH=<BASE>/bin:$PATH
-export PYTHONPATH=<BASE>/lib/python2.7/site-packages:$PYTHONPATH