Commit 5a03180f authored by Martti Louhivuori's avatar Martti Louhivuori
Browse files

Add build scripts for Piz Daint

parent 4a25bc42
### GPAW installation script for Piz Daint
# version numbers (modify if needed)
gpaw_version=cuda
# installation directory (modify!)
tgt=$SCRATCH/lib/gpaw-${gpaw_version}
# setup build environment
module swap PrgEnv-cray PrgEnv-gnu
module unload xalt
module load daint-gpu
module load craype-accel-nvidia60
source $SCRATCH/lib/python-2018-12-cuda/load.sh 2016-06
export GPAW_SETUP_PATH=$SCRATCH/lib/gpaw-setups-0.9.11271
export CFLAGS=""
# gpaw
git clone https://gitlab.com/mlouhivu/gpaw.git gpaw-$gpaw_version
cd gpaw-$gpaw_version
git checkout $gpaw_version
patch gpaw/eigensolvers/rmm_diis.py ../setup/patch-rmmdiis.diff
ln -s ../setup/gcc.py
cd c/cuda
cp ../../../setup/make.inc .
make 2>&1 | tee loki-make
cd -
python setup.py install --customize=../setup/customize-piz-daint.py --prefix=$tgt 2>&1 | tee loki-inst
cd ..
sed -e "s|<BASE>|$tgt|g" -e "s|<PYTHONHOME>|$PYTHONHOME|" setup/load-gpaw.sh > $tgt/load.sh
# fix permissions
chmod -R g+rwX $tgt
chmod -R o+rX $tgt
# Setup customisation for gpaw/cuda
import os
# compiler and linker
compiler = './gcc.py'
mpicompiler = './gcc.py'
mpilinker = 'cc'
extra_compile_args = ['-std=c99', '-fopenmp-simd']
# libraries
libraries = ['z']
# cuda
library_dirs += [os.environ['CUDATOOLKIT_HOME'] + '/lib64', './c/cuda']
include_dirs += [os.environ['CUDATOOLKIT_HOME'] + '/include']
libraries += ['gpaw-cuda', 'cublas', 'cudart', 'stdc++']
# libxc
library_dirs += [os.environ['LIBXCDIR'] + '/lib']
include_dirs += [os.environ['LIBXCDIR'] + '/include']
libraries += ['xc']
# GPAW defines
define_macros += [('GPAW_NO_UNDERSCORE_CBLACS', '1')]
define_macros += [('GPAW_NO_UNDERSCORE_CSCALAPACK', '1')]
define_macros += [("GPAW_ASYNC",1)]
define_macros += [("GPAW_MPI2",1)]
define_macros += [('GPAW_CUDA', '1')]
# ScaLAPACK
scalapack = True
# HDF5
hdf5 = False
#!/usr/bin/env python
"""Wrapper for the GNU compiler that converts / removes incompatible
compiler options and allows for file-specific tailoring."""
import sys
from subprocess import call
# Default compiler and options
compiler = 'gcc'
args2change = {}
fragile_files = ['c/xc/tpss.c']
# Default optimisation settings
default_level = 3
default_flags = ['-funroll-loops']
fragile_level = 2
fragile_flags = []
# Sisu (Cray XC40)
if True:
compiler = 'cc'
default_flags += ['-march=haswell -mtune=haswell -mavx2']
fragile_files += ['c/xc/revtpss.c']
# Taito (HP cluster)
if not True:
compiler = 'mpicc'
default_flags += ['-ffast-math -march=sandybridge -mtune=haswell']
optimise = None # optimisation level 0/1/2/3
debug = False # use -g or not
fragile = False # use special flags for current file?
sandwich = True # use optimisation flag twice (= no override possible)
# process arguments
args = []
for arg in sys.argv[1:]:
arg = arg.strip()
if arg.startswith('-O'):
level = int(arg.replace('-O',''))
if not optimise or level > optimise:
optimise = level
elif arg == '-g':
debug = True
elif arg in args2change:
if args2change[arg]:
args.append(args2change[arg])
else:
if arg in fragile_files:
fragile = True
args.append(arg)
# set default optimisation level and flags
if fragile:
optimise = min(fragile_level, optimise)
flags = fragile_flags
else:
optimise = max(default_level, optimise)
flags = default_flags
# add optimisation level to flags
if optimise is not None:
flags.insert(0, '-O{0}'.format(optimise))
if sandwich:
args.append('-O{0}'.format(optimise))
# make sure -g is always the _first_ flag, so it doesn't mess e.g. with the
# optimisation level
if debug:
flags.insert(0, '-g')
# construct and execute the compile command
cmd = '{0} {1} {2}'.format(compiler, ' '.join(flags), ' '.join(args))
print(cmd)
call(cmd, shell=True)
#!/bin/bash
module swap PrgEnv-cray PrgEnv-gnu
module unload xalt
module load daint-gpu
module load craype-accel-nvidia60
source <PYTHONHOME>/load.sh
export GPAW_SETUP_PATH=$SCRATCH/lib/gpaw-setups-0.9.11271
export PATH=<BASE>/bin:$PATH
export PYTHONPATH=<BASE>/lib/python2.7/site-packages:$PYTHONPATH
####################################################################
# make include file. #
####################################################################
#
SHELL = /bin/sh
# ----------------------------------------------------------------------
# - gpaw-cuda Directory Structure / gpaw-cuda library --------------------
# ----------------------------------------------------------------------
#
TOPdir = .
INCdir = $(TOPdir)
PYTHONINCdir ?= $(PYTHONHOME)/include/python2.7/
PYTHONLIBdir ?= $(PYTHONHOME)/lib/
NUMPYINCdir ?= `python -c "import numpy; print numpy.get_include()"`
MPIINCdir ?= $(MPICH_DIR)/include
LIBdir = $(TOPdir)
CUGPAWLIB = $(LIBdir)/libgpaw-cuda.a
#
# ----------------------------------------------------------------------
# - NVIDIA CUDA includes / libraries / specifics -----------------------
# ----------------------------------------------------------------------
CUDAINCdir = $(CUDATOOLKIT_HOME)/include
CUDALIBdir = $(CUDATOOLKIT_HOME)/lib64
CUDA_OPTS =
#
# ----------------------------------------------------------------------
# - gpaw-cuda includes / libraries / specifics -------------------------------
# ----------------------------------------------------------------------
#
CUGPAW_INCLUDES = -I$(INCdir) -I$(CUDAINCdir) -I$(MPIINCdir) -I$(NUMPYINCdir) -I$(PYTHONINCdir)
CUGPAW_OPTS = -DPARALLEL=1 -DGPAW_CUDA=1
#
# ----------------------------------------------------------------------
#
CUGPAW_DEFS = $(CUGPAW_OPTS) $(CUDA_OPTS) $(CUGPAW_INCLUDES)
#
# ----------------------------------------------------------------------
# - Compilers / linkers - Optimization flags ---------------------------
# ----------------------------------------------------------------------
CC = gcc
CCNOOPT = $(CUGPAW_DEFS)
CCFLAGS = $(CUGPAW_DEFS) -g -fPIC -std=c99 -m64 -O3
NVCC = nvcc
NVCCFLAGS = $(CUGPAW_DEFS) -O3 -g -gencode arch=compute_60,code=sm_60 -m64 --compiler-options '-O3 -g -std=c99 -fPIC'
ARCH = ar
ARCHFLAGS= cr
RANLIB = ranlib
commit 761cba649d58e2d2f24c0a1e2fdad917b5929679
Author: Martti Louhivuori <martti.louhivuori@csc.fi>
Date: Thu May 18 10:56:07 2017 +0300
Remove obsolete error calculation from DIIS step
diff --git a/gpaw/eigensolvers/rmmdiis.py b/gpaw/eigensolvers/rmmdiis.py
index 7d60553..d182713 100644
--- a/gpaw/eigensolvers/rmmdiis.py
+++ b/gpaw/eigensolvers/rmmdiis.py
@@ -299,23 +299,6 @@ class RMMDIIS(Eigensolver):
P_axi, kpt.eps_n[n_x], R_xG, n_x,
calculate_change=True)
self.timer.stop('Calculate residuals')
- self.timer.start('Calculate errors')
- errors_new_x = np.zeros(B)
- # errors_x[:] = 0.0
- for n in range(n1, n2):
- if kpt.f_n is None:
- weight = kpt.weight
- else:
- weight = kpt.f_n[n]
- if self.nbands_converge != 'occupied':
- if wfs.bd.global_index(n) < self.nbands_converge:
- weight = kpt.weight
- else:
- weight = 0.0
- errors_new_x[n-n1] += weight * integrate(R_xG[n - n1],
- R_xG[n - n1])
- comm.sum(errors_x)
- self.timer.stop('Calculate errors')
self.timer.stop('DIIS step')
# Final trial step
### Python (extra) modules installation script for Piz Daint
### uses PYTHONUSERBASE to bundle all modules into a separate location
### away from the base python installation
# load Python
source $PYTHONHOME/load.sh
# bundle ID (e.g. time of release) (modify if needed)
bundle=2016-06
# version numbers (modify if needed)
numpy_version=1.10.4
scipy_version=0.17.1
ase_version=3.11.0
pycuda_version=2017.1.1
libxc_version=2.1.3
libsci_version=18.07.1
# installation directory (modify!)
tgt=$PYTHONHOME/bundle/$bundle
# setup build environment
export CFLAGS="-fPIC $CFLAGS"
export FFLAGS="-fPIC $FFLAGS"
# use --user to install modules
export PYTHONUSERBASE=$tgt
mkdir -p $PYTHONUSERBASE/lib/python2.7/site-packages
# build in a separate directory
mkdir bundle-$bundle
cd bundle-$bundle
# cython + mpi4py
pip install --user cython
pip install --user mpi4py
# numpy
git clone git://github.com/numpy/numpy.git numpy-$numpy_version
cd numpy-$numpy_version
git checkout v$numpy_version
sed -e 's/<ARCH>/haswell/g' -e "s/<LIBSCI>/$libsci_version/g" ../../setup/piz-daint.cfg >| site.cfg
python setup.py build -j 4 install --user 2>&1 | tee loki-inst
cd ..
# scipy
git clone git://github.com/scipy/scipy.git scipy-$scipy_version
cd scipy-$scipy_version
git checkout v$scipy_version
python setup.py build -j 4 install --user 2>&1 | tee loki-inst
cd ..
# ase
git clone https://gitlab.com/ase/ase.git ase-$ase_version
cd ase-$ase_version
git checkout $ase_version
python setup.py install --user 2>&1 | tee loki-inst
cd ..
# libxc
tar xvfz ~/src/libxc-${libxc_version}.tar.gz
cd libxc-$libxc_version
./configure --prefix=$PYTHONUSERBASE --enable-shared | tee loki-conf
make | tee loki-make
make install | tee loki-inst
export LD_LIBRARY_PATH=$PYTHONUSERBASE/lib:$LD_LIBRARY_PATH
cd ..
# pycuda
pip install --user pycuda==$pycuda_version
# go back to the main build directory
cd ..
# if this is the first bundle, use it as default
if [ ! -e $PYTHONHOME/bundle/default ]
then
cd $PYTHONHOME/bundle
ln -s $bundle default
cd -
fi
# fix permissions
chmod -R g+rwX $tgt
chmod -R o+rX $tgt
### Python installation script for Piz Daint
### uses --prefix to set a custom installation directory
# version numbers (modify if needed)
python_version=2.7.13
# installation directory (modify!)
tgt=$SCRATCH/lib/python-2018-12-cuda
# setup build environment
module swap PrgEnv-cray PrgEnv-gnu
module unload xalt
module load daint-gpu
module load craype-accel-nvidia60
export CC=cc
export CFLAGS='-O3'
export CXX=CC
export CXXFLAGS='-O3'
export FC=ftn
export FFLAGS='-O3'
export LINKFORSHARED='-Wl,-export-dynamic -dynamic'
export MPI_LINKFORSHARED='-Wl,-export-dynamic -dynamic'
# python
git clone https://github.com/python/cpython.git python-$python_version
cd python-$python_version
git checkout v$python_version
./configure --prefix=$tgt --enable-shared --disable-ipv6 --enable-unicode=ucs4 2>&1 | tee loki-conf
make 2>&1 | tee loki-make
make install 2>&1 | tee loki-inst
cd ..
sed -e "s|<BASE>|$tgt|g" setup/load-python.sh > $tgt/load.sh
# install pip
source $tgt/load.sh
python -m ensurepip
pip install --upgrade pip
# fix permissions
chmod -R g+rwX $tgt
chmod -R o+rX $tgt
#!/bin/bash
export PYTHONHOME=<BASE>
export PYTHONPATH=$PYTHONHOME/lib
export PATH=$PYTHONHOME/bin:$PATH
export MANPATH=$PYTHONHOME/share/man:$MANPATH
export LD_LIBRARY_PATH=$PYTHONHOME/lib:$LD_LIBRARY_PATH
if [[ $# -gt 0 ]]
then
export PYTHONUSERBASE=$PYTHONHOME/bundle/$1
export PATH=$PYTHONUSERBASE/bin:$PATH
export LD_LIBRARY_PATH=$PYTHONUSERBASE/lib:$LD_LIBRARY_PATH
elif [[ -e $PYTHONHOME/bundle/default ]]
then
export PYTHONUSERBASE=$PYTHONHOME/bundle/default
export PATH=$PYTHONUSERBASE/bin:$PATH
export LD_LIBRARY_PATH=$PYTHONUSERBASE/lib:$LD_LIBRARY_PATH
fi
if [[ -e $PYTHONUSERBASE/include/xc.h ]]
then
export LIBXCDIR=$PYTHONUSERBASE
fi
[atlas]
atlas_libs = sci_gnu
library_dirs = /opt/cray/pe/libsci/<LIBSCI>/gnu/6.1/<ARCH>/lib
include_dirs = /opt/cray/pe/libsci/<LIBSCI>/gnu/6.1/<ARCH>/include
[lapack_atlas]
lapack_libs = sci_gnu
library_dirs = /opt/cray/pe/libsci/<LIBSCI>/gnu/6.1/<ARCH>/lib
include_dirs = /opt/cray/pe/libsci/<LIBSCI>/gnu/6.1/<ARCH>/include
[lapack]
lapack_libs = sci_gnu
library_dirs = /opt/cray/pe/libsci/<LIBSCI>/gnu/6.1/<ARCH>/lib
include_dirs = /opt/cray/pe/libsci/<LIBSCI>/gnu/6.1/<ARCH>/include
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment