Commit c7f11e63 authored by Martti Louhivuori's avatar Martti Louhivuori
Browse files

Example build scripts on JUWELS and DAVIDE

parent 6a1c02ae
### GPAW installation script for D.A.V.I.D.E
# version numbers (modify if needed)
gpaw_version=1.1.0
# installation directory (modify!)
tgt=$CINECA_SCRATCH/lib/gpaw-${gpaw_version}
# setup build environment
module load cuda/9.2.88
module load cudnn/7.1.4--cuda--9.2.88
module load gnu/6.4.0
module load openmpi/3.1.0--gnu--6.4.0
source $CINECA_SCRATCH/lib/openblas-0.3.4-openmp/load.sh
source $CINECA_SCRATCH/lib/python-2018-12-cuda/load.sh 2016-06
source $CINECA_SCRATCH/lib/scalapack-2.0.2/load.sh
export GPAW_SETUP_PATH=$CINECA_SCRATCH/lib/gpaw-setups-0.9.11271
export CFLAGS=""
# gpaw
git clone https://gitlab.com/mlouhivu/gpaw.git gpaw-$gpaw_version
cd gpaw-$gpaw_version
git checkout $gpaw_version
patch gpaw/eigensolvers/rmm_diis.py ../setup/patch-rmmdiis.diff
cp ../setup/customize-power8.py .
python setup.py install --customize=customize-power8.py --prefix=$tgt 2>&1 | tee loki-inst
cd ..
sed -e "s|<BASE>|$tgt|g" -e "s|<PYTHONHOME>|$PYTHONHOME|" setup/load-gpaw.sh > $tgt/load.sh
# fix permissions
chmod -R g+rwX $tgt
chmod -R o+rX $tgt
# Setup customisation for gpaw/cuda
import os
# compiler and linker
compiler = 'gcc'
mpicompiler = 'mpicc'
mpilinker = 'mpicc'
extra_compile_args = ['-std=c99', '-mcpu=power8']
# libraries
libraries = ['z']
# openblas
library_dirs += [os.environ['OPENBLAS_ROOT'] + '/lib']
include_dirs += [os.environ['OPENBLAS_ROOT'] + '/include']
libraries += ['openblas']
# scalapack
library_dirs += [os.environ['SCALAPACK_ROOT'] + '/lib']
libraries += ['scalapack']
# libxc
library_dirs += [os.environ['LIBXCDIR'] + '/lib']
include_dirs += [os.environ['LIBXCDIR'] + '/include']
libraries += ['xc']
# GPAW defines
define_macros += [('GPAW_NO_UNDERSCORE_CBLACS', '1')]
define_macros += [('GPAW_NO_UNDERSCORE_CSCALAPACK', '1')]
define_macros += [("GPAW_ASYNC",1)]
define_macros += [("GPAW_MPI2",1)]
define_macros += [('GPAW_CUDA', '1')]
# ScaLAPACK
scalapack = True
# HDF5
hdf5 = False
#!/bin/bash
module load cuda/9.2.88
module load cudnn/7.1.4--cuda--9.2.88
module load gnu/6.4.0
module load openmpi/3.1.0--gnu--6.4.0
source $CINECA_SCRATCH/lib/openblas-0.3.4-openmp/load.sh
source <PYTHONHOME>/load.sh
source $CINECA_SCRATCH/lib/scalapack-2.0.2/load.sh
export GPAW_SETUP_PATH=$CINECA_SCRATCH/lib/gpaw-setups-0.9.11271
export PATH=<BASE>/bin:$PATH
export PYTHONPATH=<BASE>/lib/python2.7/site-packages:$PYTHONPATH
commit 761cba649d58e2d2f24c0a1e2fdad917b5929679
Author: Martti Louhivuori <martti.louhivuori@csc.fi>
Date: Thu May 18 10:56:07 2017 +0300
Remove obsolete error calculation from DIIS step
diff --git a/gpaw/eigensolvers/rmmdiis.py b/gpaw/eigensolvers/rmmdiis.py
index 7d60553..d182713 100644
--- a/gpaw/eigensolvers/rmmdiis.py
+++ b/gpaw/eigensolvers/rmmdiis.py
@@ -299,23 +299,6 @@ class RMMDIIS(Eigensolver):
P_axi, kpt.eps_n[n_x], R_xG, n_x,
calculate_change=True)
self.timer.stop('Calculate residuals')
- self.timer.start('Calculate errors')
- errors_new_x = np.zeros(B)
- # errors_x[:] = 0.0
- for n in range(n1, n2):
- if kpt.f_n is None:
- weight = kpt.weight
- else:
- weight = kpt.f_n[n]
- if self.nbands_converge != 'occupied':
- if wfs.bd.global_index(n) < self.nbands_converge:
- weight = kpt.weight
- else:
- weight = 0.0
- errors_new_x[n-n1] += weight * integrate(R_xG[n - n1],
- R_xG[n - n1])
- comm.sum(errors_x)
- self.timer.stop('Calculate errors')
self.timer.stop('DIIS step')
# Final trial step
### GPAW installation script for D.A.V.I.D.E
# version numbers (modify if needed)
gpaw_version=cuda
# installation directory (modify!)
tgt=$CINECA_SCRATCH/lib/gpaw-${gpaw_version}
# setup build environment
module load cuda/9.2.88
module load cudnn/7.1.4--cuda--9.2.88
module load gnu/6.4.0
module load openmpi/3.1.0--gnu--6.4.0
source $CINECA_SCRATCH/lib/openblas-0.3.4-openmp/load.sh
source $CINECA_SCRATCH/lib/python-2018-12-cuda/load.sh 2016-06
source $CINECA_SCRATCH/lib/scalapack-2.0.2/load.sh
export GPAW_SETUP_PATH=$CINECA_SCRATCH/lib/gpaw-setups-0.9.11271
export CFLAGS=""
# gpaw
git clone https://gitlab.com/mlouhivu/gpaw.git gpaw-$gpaw_version
cd gpaw-$gpaw_version
git checkout $gpaw_version
patch gpaw/eigensolvers/rmm_diis.py ../setup/patch-rmmdiis.diff
cp ../setup/customize-cuda.py .
cd c/cuda
cp ../../../setup/make.inc .
make 2>&1 | tee loki-make
cd -
python setup.py install --customize=customize-cuda.py --prefix=$tgt 2>&1 | tee loki-inst
cd ..
sed -e "s|<BASE>|$tgt|g" -e "s|<PYTHONHOME>|$PYTHONHOME|" setup/load-gpaw.sh > $tgt/load.sh
# fix permissions
chmod -R g+rwX $tgt
chmod -R o+rX $tgt
# Setup customisation for gpaw/cuda
import os
# compiler and linker
compiler = 'gcc'
mpicompiler = 'mpicc'
mpilinker = 'mpicc'
extra_compile_args = ['-std=c99', '-mcpu=power8']
# libraries
libraries = ['z']
# cuda
library_dirs += [os.environ['CUDA_LIB'], './c/cuda']
include_dirs += [os.environ['CUDA_HOME'] + '/include']
libraries += ['gpaw-cuda', 'cublas', 'cudart', 'stdc++']
# openblas
library_dirs += [os.environ['OPENBLAS_ROOT'] + '/lib']
include_dirs += [os.environ['OPENBLAS_ROOT'] + '/include']
libraries += ['openblas']
# scalapack
library_dirs += [os.environ['SCALAPACK_ROOT'] + '/lib']
libraries += ['scalapack']
# libxc
library_dirs += [os.environ['LIBXCDIR'] + '/lib']
include_dirs += [os.environ['LIBXCDIR'] + '/include']
libraries += ['xc']
# GPAW defines
define_macros += [('GPAW_NO_UNDERSCORE_CBLACS', '1')]
define_macros += [('GPAW_NO_UNDERSCORE_CSCALAPACK', '1')]
define_macros += [("GPAW_ASYNC",1)]
define_macros += [("GPAW_MPI2",1)]
define_macros += [('GPAW_CUDA', '1')]
# ScaLAPACK
scalapack = True
# HDF5
hdf5 = False
#!/bin/bash
module load cuda/9.2.88
module load cudnn/7.1.4--cuda--9.2.88
module load gnu/6.4.0
module load openmpi/3.1.0--gnu--6.4.0
source $CINECA_SCRATCH/lib/openblas-0.3.4-openmp/load.sh
source <PYTHONHOME>/load.sh
source $CINECA_SCRATCH/lib/scalapack-2.0.2/load.sh
export GPAW_SETUP_PATH=$CINECA_SCRATCH/lib/gpaw-setups-0.9.11271
export PATH=<BASE>/bin:$PATH
export PYTHONPATH=<BASE>/lib/python2.7/site-packages:$PYTHONPATH
####################################################################
# make include file. #
####################################################################
#
SHELL = /bin/sh
# ----------------------------------------------------------------------
# - gpaw-cuda Directory Structure / gpaw-cuda library --------------------
# ----------------------------------------------------------------------
#
TOPdir = .
INCdir = $(TOPdir)
PYTHONINCdir ?= $(PYTHONHOME)/include/python2.7/
PYTHONLIBdir ?= $(PYTHONHOME)/lib/
NUMPYINCdir ?= `python -c "import numpy; print numpy.get_include()"`
MPIINCdir ?= $(OPENMPI_HOME)/include
LIBdir = $(TOPdir)
CUGPAWLIB = $(LIBdir)/libgpaw-cuda.a
#
# ----------------------------------------------------------------------
# - NVIDIA CUDA includes / libraries / specifics -----------------------
# ----------------------------------------------------------------------
CUDAINCdir = $(CUDADIR)/include
CUDALIBdir = $(CUDADIR)/lib64
CUDA_OPTS =
#
# ----------------------------------------------------------------------
# - gpaw-cuda includes / libraries / specifics -------------------------------
# ----------------------------------------------------------------------
#
CUGPAW_INCLUDES = -I$(INCdir) -I$(CUDAINCdir) -I$(MPIINCdir) -I$(NUMPYINCdir) -I$(PYTHONINCdir)
CUGPAW_OPTS = -DPARALLEL=1 -DGPAW_CUDA=1
#
# ----------------------------------------------------------------------
#
CUGPAW_DEFS = $(CUGPAW_OPTS) $(CUDA_OPTS) $(CUGPAW_INCLUDES)
#
# ----------------------------------------------------------------------
# - Compilers / linkers - Optimization flags ---------------------------
# ----------------------------------------------------------------------
CC = gcc
CCNOOPT = $(CUGPAW_DEFS)
CCFLAGS = $(CUGPAW_DEFS) -g -fPIC -std=c99 -m64 -O3
NVCC = nvcc
NVCCFLAGS = $(CUGPAW_DEFS) -O3 -g -gencode arch=compute_60,code=sm_60 -m64 --compiler-options '-O3 -g -std=c99 -fPIC'
ARCH = ar
ARCHFLAGS= cr
RANLIB = ranlib
commit 761cba649d58e2d2f24c0a1e2fdad917b5929679
Author: Martti Louhivuori <martti.louhivuori@csc.fi>
Date: Thu May 18 10:56:07 2017 +0300
Remove obsolete error calculation from DIIS step
diff --git a/gpaw/eigensolvers/rmmdiis.py b/gpaw/eigensolvers/rmmdiis.py
index 7d60553..d182713 100644
--- a/gpaw/eigensolvers/rmmdiis.py
+++ b/gpaw/eigensolvers/rmmdiis.py
@@ -299,23 +299,6 @@ class RMMDIIS(Eigensolver):
P_axi, kpt.eps_n[n_x], R_xG, n_x,
calculate_change=True)
self.timer.stop('Calculate residuals')
- self.timer.start('Calculate errors')
- errors_new_x = np.zeros(B)
- # errors_x[:] = 0.0
- for n in range(n1, n2):
- if kpt.f_n is None:
- weight = kpt.weight
- else:
- weight = kpt.f_n[n]
- if self.nbands_converge != 'occupied':
- if wfs.bd.global_index(n) < self.nbands_converge:
- weight = kpt.weight
- else:
- weight = 0.0
- errors_new_x[n-n1] += weight * integrate(R_xG[n - n1],
- R_xG[n - n1])
- comm.sum(errors_x)
- self.timer.stop('Calculate errors')
self.timer.stop('DIIS step')
# Final trial step
### OpenBLAS installation script for D.A.V.I.D.E
# version numbers (modify if needed)
openblas_version=0.3.4
# installation directory (modify!)
tgt=$CINECA_SCRATCH/lib/openblas-${openblas_version}-openmp
# setup build environment
module load cuda/9.2.88
module load cudnn/7.1.4--cuda--9.2.88
module load gnu/6.4.0
module load openmpi/3.1.0--gnu--6.4.0
export CC=gcc
export CFLAGS='-mcpu=power8 -O3'
export CXX=g++
export CXXFLAGS='-mcpu=power8 -O3'
export FC=gfortran
export FFLAGS='-mcpu=power8 -O3'
# openblas
git clone https://github.com/xianyi/OpenBLAS
cd OpenBLAS
git checkout v$openblas_version
make TARGET=POWER8 USE_OPENMP=1 2>&1 | tee loki-make
make install PREFIX=$tgt 2>&1 | tee loki-install
sed -e "s|<BASE>|$tgt|g" ../setup/load-openblas.sh > $tgt/load.sh
cd ..
# fix permissions
chmod -R g+rwX $tgt
chmod -R o+rX $tgt
#!/bin/bash
export OPENBLAS_ROOT=<BASE>
export LD_LIBRARY_PATH=$OPENBLAS_ROOT/lib:$LD_LIBRARY_PATH
### Python (extra) modules installation script for D.A.V.I.D.E
### uses PYTHONUSERBASE to bundle all modules into a separate location
### away from the base python installation
# load Python
source $PYTHONHOME/load.sh
# bundle ID (e.g. time of release) (modify if needed)
bundle=2016-06
# version numbers (modify if needed)
numpy_version=1.10.4
scipy_version=0.17.1
ase_version=3.11.0
pycuda_version=2017.1.1
libxc_version=2.1.3
# installation directory (modify!)
tgt=$PYTHONHOME/bundle/$bundle
# setup build environment
export CFLAGS="-fPIC $CFLAGS"
export FFLAGS="-fPIC $FFLAGS"
# use --user to install modules
export PYTHONUSERBASE=$tgt
mkdir -p $PYTHONUSERBASE/lib/python2.7/site-packages
# build in a separate directory
mkdir bundle-$bundle
cd bundle-$bundle
# cython + mpi4py
pip install --user cython
pip install --user mpi4py
# numpy
git clone git://github.com/numpy/numpy.git numpy-$numpy_version
cd numpy-$numpy_version
git checkout v$numpy_version
sed -e "s|<OPENBLAS_ROOT>|$OPENBLAS_ROOT|g" ../../setup/davide-openblas.cfg > site.cfg
python setup.py build -j 4 install --user 2>&1 | tee loki-inst
cd ..
# scipy
git clone git://github.com/scipy/scipy.git scipy-$scipy_version
cd scipy-$scipy_version
git checkout v$scipy_version
python setup.py build -j 4 install --user 2>&1 | tee loki-inst
cd ..
# ase
git clone https://gitlab.com/ase/ase.git ase-$ase_version
cd ase-$ase_version
git checkout $ase_version
python setup.py install --user 2>&1 | tee loki-inst
cd ..
# libxc
tar xvfz ~/src/libxc-${libxc_version}.tar.gz
cd libxc-$libxc_version
./configure --prefix=$PYTHONUSERBASE --enable-shared | tee loki-conf
make | tee loki-make
make install | tee loki-inst
export LD_LIBRARY_PATH=$PYTHONUSERBASE/lib:$LD_LIBRARY_PATH
cd ..
# pycuda
pip install --user pycuda==$pycuda_version
# go back to the main build directory
cd ..
# if this is the first bundle, use it as default
if [ ! -e $PYTHONHOME/bundle/default ]
then
cd $PYTHONHOME/bundle
ln -s $bundle default
cd -
fi
# fix permissions
chmod -R g+rwX $tgt
chmod -R o+rX $tgt
### Python installation script for D.A.V.I.D.E
### uses --prefix to set a custom installation directory
# version numbers (modify if needed)
python_version=2.7.13
# installation directory (modify!)
tgt=$CINECA_SCRATCH/lib/python-2018-12-cuda
# setup build environment
module load cuda/9.2.88
module load cudnn/7.1.4--cuda--9.2.88
module load gnu/6.4.0
module load openmpi/3.1.0--gnu--6.4.0
source $CINECA_SCRATCH/lib/openblas-0.3.4-openmp/load.sh
export CC=gcc
export CFLAGS='-mcpu=power8 -O3'
export CXX=g++
export CXXFLAGS='-mcpu=power8 -O3'
export F77=gfortran
export FFLAGS='-mcpu=power8 -O3'
# python
git clone https://github.com/python/cpython.git python-$python_version
cd python-$python_version
git checkout v$python_version
./configure --prefix=$tgt --enable-shared --disable-ipv6 --enable-unicode=ucs4 2>&1 | tee loki-conf
make 2>&1 | tee loki-make
make install 2>&1 | tee loki-inst
cd ..
sed -e "s|<BASE>|$tgt|g" setup/load-python.sh > $tgt/load.sh
# install pip
source $tgt/load.sh
python -m ensurepip
pip install --upgrade pip
# fix permissions
chmod -R g+rwX $tgt
chmod -R o+rX $tgt
[openblas]
libraries = openblas
library_dirs = <OPENBLAS_ROOT>/lib
include_dirs = <OPENBLAS_ROOT>/include
#!/bin/bash
export PYTHONHOME=<BASE>
export PYTHONPATH=$PYTHONHOME/lib
export PATH=$PYTHONHOME/bin:$PATH
export MANPATH=$PYTHONHOME/share/man:$MANPATH
export LD_LIBRARY_PATH=$PYTHONHOME/lib:$LD_LIBRARY_PATH
if [[ $# -gt 0 ]]
then
export PYTHONUSERBASE=$PYTHONHOME/bundle/$1
export PATH=$PYTHONUSERBASE/bin:$PATH
export LD_LIBRARY_PATH=$PYTHONUSERBASE/lib:$LD_LIBRARY_PATH
elif [[ -e $PYTHONHOME/bundle/default ]]
then
export PYTHONUSERBASE=$PYTHONHOME/bundle/default
export PATH=$PYTHONUSERBASE/bin:$PATH
export LD_LIBRARY_PATH=$PYTHONUSERBASE/lib:$LD_LIBRARY_PATH
fi
if [[ -e $PYTHONUSERBASE/include/xc.h ]]
then
export LIBXCDIR=$PYTHONUSERBASE
fi
### ScaLAPACK installation script for D.A.V.I.D.E
# version numbers (modify if needed)
scalapack_version=2.0.2
# installation directory (modify!)
tgt=$CINECA_SCRATCH/lib/scalapack-${scalapack_version}
# setup build environment
module load cuda/9.2.88
module load cudnn/7.1.4--cuda--9.2.88
module load gnu/6.4.0
module load openmpi/3.1.0--gnu--6.4.0
source $CINECA_SCRATCH/lib/openblas-0.3.4-openmp/load.sh
export CFLAGS="-mcpu=power8 -O3"
export FFLAGS="-mcpu=power8 -O3"
# scalapack
tar xvfz ~/scalapack-${scalapack_version}.tgz
cd scalapack-${scalapack_version}
cp ../setup/SLmake.inc .
mkdir build
cd build
cmake -DBLAS_LIBRARIES=$OPENBLAS_ROOT/lib/libopenblas.so -DLAPACK_LIBRARIES=$OPENBLAS_ROOT/lib/libopenblas.so -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=$tgt ..
make 2>&1 | tee loki-make
make install 2>&1 | tee loki-install
sed -e "s|<BASE>|$tgt|g" ../../setup/load-scalapack.sh > $tgt/load.sh
cd ../..
# fix permissions
chmod -R g+rwX $tgt
chmod -R o+rX $tgt
############################################################################
#
# Program: ScaLAPACK
#
# Module: SLmake.inc
#
# Purpose: Top-level Definitions
#
# Creation date: February 15, 2000
#
# Modified: October 13, 2011
#
# Send bug reports, comments or suggestions to scalapack@cs.utk.edu
#
############################################################################
#
# C preprocessor definitions: set CDEFS to one of the following:
#
# -DNoChange (fortran subprogram names are lower case without any suffix)
# -DUpCase (fortran subprogram names are upper case without any suffix)
# -DAdd_ (fortran subprogram names are lower case with "_" appended)
CDEFS = -DAdd_
#
# The fortran and C compilers, loaders, and their flags
#
FC = mpif90
CC = mpicc
NOOPT = -mcpu=power8 -O0
FCFLAGS = -mcpu=power8 -O3 -shared -fPIC
CCFLAGS = -mcpu=power8 -O3 -shared -fPIC
FCLOADER = $(FC)
CCLOADER = $(CC)
FCLOADFLAGS = $(FCFLAGS)
CCLOADFLAGS = $(CCFLAGS)
#
# The archiver and the flag(s) to use when building archive (library)
# Also the ranlib routine. If your system has no ranlib, set RANLIB = echo
#
ARCH = ar
ARCHFLAGS = cr
RANLIB = ranlib
#
# The name of the ScaLAPACK library to be created
#
SCALAPACKLIB = libscalapack.so
#
# BLAS, LAPACK (and possibly other) libraries needed for linking test programs
#
BLASLIB = -lopenblas -L$(OPENBLAS_ROOT)/lib -I$(OPENBLAS_ROOT)/include
LAPACKLIB =
LIBS = $(LAPACKLIB) $(BLASLIB)
#!/bin/bash
export SCALAPACK_ROOT=<BASE>
export LD_LIBRARY_PATH=$SCALAPACK_ROOT/lib:$LD_LIBRARY_PATH
### GPAW installation script for JUWELS