Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
UEABS
ueabs
Commits
5a03180f
Commit
5a03180f
authored
Apr 16, 2019
by
Martti Louhivuori
Browse files
Add build scripts for Piz Daint
parent
4a25bc42
Changes
10
Hide whitespace changes
Inline
Side-by-side
gpaw/build/examples/piz-daint/gpaw/build.sh
0 → 100644
View file @
5a03180f
### GPAW installation script for Piz Daint
# version numbers (modify if needed)
gpaw_version
=
cuda
# installation directory (modify!)
tgt
=
$SCRATCH
/lib/gpaw-
${
gpaw_version
}
# setup build environment
module swap PrgEnv-cray PrgEnv-gnu
module unload xalt
module load daint-gpu
module load craype-accel-nvidia60
source
$SCRATCH
/lib/python-2018-12-cuda/load.sh 2016-06
export
GPAW_SETUP_PATH
=
$SCRATCH
/lib/gpaw-setups-0.9.11271
export
CFLAGS
=
""
# gpaw
git clone https://gitlab.com/mlouhivu/gpaw.git gpaw-
$gpaw_version
cd
gpaw-
$gpaw_version
git checkout
$gpaw_version
patch gpaw/eigensolvers/rmm_diis.py ../setup/patch-rmmdiis.diff
ln
-s
../setup/gcc.py
cd
c/cuda
cp
../../../setup/make.inc
.
make 2>&1 |
tee
loki-make
cd
-
python setup.py
install
--customize
=
../setup/customize-piz-daint.py
--prefix
=
$tgt
2>&1 |
tee
loki-inst
cd
..
sed
-e
"s|<BASE>|
$tgt
|g"
-e
"s|<PYTHONHOME>|
$PYTHONHOME
|"
setup/load-gpaw.sh
>
$tgt
/load.sh
# fix permissions
chmod
-R
g+rwX
$tgt
chmod
-R
o+rX
$tgt
gpaw/build/examples/piz-daint/gpaw/setup/customize-piz-daint.py
0 → 100644
View file @
5a03180f
# Setup customisation for gpaw/cuda
import
os
# compiler and linker
compiler
=
'./gcc.py'
mpicompiler
=
'./gcc.py'
mpilinker
=
'cc'
extra_compile_args
=
[
'-std=c99'
,
'-fopenmp-simd'
]
# libraries
libraries
=
[
'z'
]
# cuda
library_dirs
+=
[
os
.
environ
[
'CUDATOOLKIT_HOME'
]
+
'/lib64'
,
'./c/cuda'
]
include_dirs
+=
[
os
.
environ
[
'CUDATOOLKIT_HOME'
]
+
'/include'
]
libraries
+=
[
'gpaw-cuda'
,
'cublas'
,
'cudart'
,
'stdc++'
]
# libxc
library_dirs
+=
[
os
.
environ
[
'LIBXCDIR'
]
+
'/lib'
]
include_dirs
+=
[
os
.
environ
[
'LIBXCDIR'
]
+
'/include'
]
libraries
+=
[
'xc'
]
# GPAW defines
define_macros
+=
[(
'GPAW_NO_UNDERSCORE_CBLACS'
,
'1'
)]
define_macros
+=
[(
'GPAW_NO_UNDERSCORE_CSCALAPACK'
,
'1'
)]
define_macros
+=
[(
"GPAW_ASYNC"
,
1
)]
define_macros
+=
[(
"GPAW_MPI2"
,
1
)]
define_macros
+=
[(
'GPAW_CUDA'
,
'1'
)]
# ScaLAPACK
scalapack
=
True
# HDF5
hdf5
=
False
gpaw/build/examples/piz-daint/gpaw/setup/gcc.py
0 → 100755
View file @
5a03180f
#!/usr/bin/env python
"""Wrapper for the GNU compiler that converts / removes incompatible
compiler options and allows for file-specific tailoring."""
import
sys
from
subprocess
import
call
# Default compiler and options
compiler
=
'gcc'
args2change
=
{}
fragile_files
=
[
'c/xc/tpss.c'
]
# Default optimisation settings
default_level
=
3
default_flags
=
[
'-funroll-loops'
]
fragile_level
=
2
fragile_flags
=
[]
# Sisu (Cray XC40)
if
True
:
compiler
=
'cc'
default_flags
+=
[
'-march=haswell -mtune=haswell -mavx2'
]
fragile_files
+=
[
'c/xc/revtpss.c'
]
# Taito (HP cluster)
if
not
True
:
compiler
=
'mpicc'
default_flags
+=
[
'-ffast-math -march=sandybridge -mtune=haswell'
]
optimise
=
None
# optimisation level 0/1/2/3
debug
=
False
# use -g or not
fragile
=
False
# use special flags for current file?
sandwich
=
True
# use optimisation flag twice (= no override possible)
# process arguments
args
=
[]
for
arg
in
sys
.
argv
[
1
:]:
arg
=
arg
.
strip
()
if
arg
.
startswith
(
'-O'
):
level
=
int
(
arg
.
replace
(
'-O'
,
''
))
if
not
optimise
or
level
>
optimise
:
optimise
=
level
elif
arg
==
'-g'
:
debug
=
True
elif
arg
in
args2change
:
if
args2change
[
arg
]:
args
.
append
(
args2change
[
arg
])
else
:
if
arg
in
fragile_files
:
fragile
=
True
args
.
append
(
arg
)
# set default optimisation level and flags
if
fragile
:
optimise
=
min
(
fragile_level
,
optimise
)
flags
=
fragile_flags
else
:
optimise
=
max
(
default_level
,
optimise
)
flags
=
default_flags
# add optimisation level to flags
if
optimise
is
not
None
:
flags
.
insert
(
0
,
'-O{0}'
.
format
(
optimise
))
if
sandwich
:
args
.
append
(
'-O{0}'
.
format
(
optimise
))
# make sure -g is always the _first_ flag, so it doesn't mess e.g. with the
# optimisation level
if
debug
:
flags
.
insert
(
0
,
'-g'
)
# construct and execute the compile command
cmd
=
'{0} {1} {2}'
.
format
(
compiler
,
' '
.
join
(
flags
),
' '
.
join
(
args
))
print
(
cmd
)
call
(
cmd
,
shell
=
True
)
gpaw/build/examples/piz-daint/gpaw/setup/load-gpaw.sh
0 → 100644
View file @
5a03180f
#!/bin/bash
module swap PrgEnv-cray PrgEnv-gnu
module unload xalt
module load daint-gpu
module load craype-accel-nvidia60
source
<PYTHONHOME>/load.sh
export
GPAW_SETUP_PATH
=
$SCRATCH
/lib/gpaw-setups-0.9.11271
export
PATH
=
<BASE>/bin:
$PATH
export
PYTHONPATH
=
<BASE>/lib/python2.7/site-packages:
$PYTHONPATH
gpaw/build/examples/piz-daint/gpaw/setup/make.inc
0 → 100644
View file @
5a03180f
####################################################################
# make include file. #
####################################################################
#
SHELL
=
/
bin
/
sh
# ----------------------------------------------------------------------
# - gpaw-cuda Directory Structure / gpaw-cuda library --------------------
# ----------------------------------------------------------------------
#
TOPdir
=
.
INCdir
=
$
(
TOPdir
)
PYTHONINCdir
?=
$
(
PYTHONHOME
)
/
include
/
python2
.7
/
PYTHONLIBdir
?=
$
(
PYTHONHOME
)
/
lib
/
NUMPYINCdir
?=
`python -c "import numpy; print numpy.get_include()"`
MPIINCdir
?=
$
(
MPICH_DIR
)
/
include
LIBdir
=
$
(
TOPdir
)
CUGPAWLIB
=
$
(
LIBdir
)
/
libgpaw
-
cuda
.
a
#
# ----------------------------------------------------------------------
# - NVIDIA CUDA includes / libraries / specifics -----------------------
# ----------------------------------------------------------------------
CUDAINCdir
=
$
(
CUDATOOLKIT_HOME
)
/
include
CUDALIBdir
=
$
(
CUDATOOLKIT_HOME
)
/
lib64
CUDA_OPTS
=
#
# ----------------------------------------------------------------------
# - gpaw-cuda includes / libraries / specifics -------------------------------
# ----------------------------------------------------------------------
#
CUGPAW_INCLUDES
=
-
I
$
(
INCdir
)
-
I
$
(
CUDAINCdir
)
-
I
$
(
MPIINCdir
)
-
I
$
(
NUMPYINCdir
)
-
I
$
(
PYTHONINCdir
)
CUGPAW_OPTS
=
-
DPARALLEL
=
1
-
DGPAW_CUDA
=
1
#
# ----------------------------------------------------------------------
#
CUGPAW_DEFS
=
$
(
CUGPAW_OPTS
)
$
(
CUDA_OPTS
)
$
(
CUGPAW_INCLUDES
)
#
# ----------------------------------------------------------------------
# - Compilers / linkers - Optimization flags ---------------------------
# ----------------------------------------------------------------------
CC
=
gcc
CCNOOPT
=
$
(
CUGPAW_DEFS
)
CCFLAGS
=
$
(
CUGPAW_DEFS
)
-
g
-
fPIC
-
std
=
c99
-
m64
-
O3
NVCC
=
nvcc
NVCCFLAGS
=
$
(
CUGPAW_DEFS
)
-
O3
-
g
-
gencode
arch
=
compute_60
,
code
=
sm_60
-
m64
--
compiler
-
options
'-O3 -g -std=c99 -fPIC'
ARCH
=
ar
ARCHFLAGS
=
cr
RANLIB
=
ranlib
gpaw/build/examples/piz-daint/gpaw/setup/patch-rmmdiis.diff
0 → 100644
View file @
5a03180f
commit 761cba649d58e2d2f24c0a1e2fdad917b5929679
Author: Martti Louhivuori <martti.louhivuori@csc.fi>
Date: Thu May 18 10:56:07 2017 +0300
Remove obsolete error calculation from DIIS step
diff --git a/gpaw/eigensolvers/rmmdiis.py b/gpaw/eigensolvers/rmmdiis.py
index 7d60553..d182713 100644
--- a/gpaw/eigensolvers/rmmdiis.py
+++ b/gpaw/eigensolvers/rmmdiis.py
@@ -299,23 +299,6 @@
class RMMDIIS(Eigensolver):
P_axi, kpt.eps_n[n_x], R_xG, n_x,
calculate_change=True)
self.timer.stop('Calculate residuals')
- self.timer.start('Calculate errors')
- errors_new_x = np.zeros(B)
- # errors_x[:] = 0.0
- for n in range(n1, n2):
- if kpt.f_n is None:
- weight = kpt.weight
- else:
- weight = kpt.f_n[n]
- if self.nbands_converge != 'occupied':
- if wfs.bd.global_index(n) < self.nbands_converge:
- weight = kpt.weight
- else:
- weight = 0.0
- errors_new_x[n-n1] += weight * integrate(R_xG[n - n1],
- R_xG[n - n1])
- comm.sum(errors_x)
- self.timer.stop('Calculate errors')
self.timer.stop('DIIS step')
# Final trial step
gpaw/build/examples/piz-daint/python/build-extra.sh
0 → 100644
View file @
5a03180f
### Python (extra) modules installation script for Piz Daint
### uses PYTHONUSERBASE to bundle all modules into a separate location
### away from the base python installation
# load Python
source
$PYTHONHOME
/load.sh
# bundle ID (e.g. time of release) (modify if needed)
bundle
=
2016-06
# version numbers (modify if needed)
numpy_version
=
1.10.4
scipy_version
=
0.17.1
ase_version
=
3.11.0
pycuda_version
=
2017.1.1
libxc_version
=
2.1.3
libsci_version
=
18.07.1
# installation directory (modify!)
tgt
=
$PYTHONHOME
/bundle/
$bundle
# setup build environment
export
CFLAGS
=
"-fPIC
$CFLAGS
"
export
FFLAGS
=
"-fPIC
$FFLAGS
"
# use --user to install modules
export
PYTHONUSERBASE
=
$tgt
mkdir
-p
$PYTHONUSERBASE
/lib/python2.7/site-packages
# build in a separate directory
mkdir
bundle-
$bundle
cd
bundle-
$bundle
# cython + mpi4py
pip
install
--user
cython
pip
install
--user
mpi4py
# numpy
git clone git://github.com/numpy/numpy.git numpy-
$numpy_version
cd
numpy-
$numpy_version
git checkout v
$numpy_version
sed
-e
's/<ARCH>/haswell/g'
-e
"s/<LIBSCI>/
$libsci_version
/g"
../../setup/piz-daint.cfg
>
| site.cfg
python setup.py build
-j
4
install
--user
2>&1 |
tee
loki-inst
cd
..
# scipy
git clone git://github.com/scipy/scipy.git scipy-
$scipy_version
cd
scipy-
$scipy_version
git checkout v
$scipy_version
python setup.py build
-j
4
install
--user
2>&1 |
tee
loki-inst
cd
..
# ase
git clone https://gitlab.com/ase/ase.git ase-
$ase_version
cd
ase-
$ase_version
git checkout
$ase_version
python setup.py
install
--user
2>&1 |
tee
loki-inst
cd
..
# libxc
tar
xvfz ~/src/libxc-
${
libxc_version
}
.tar.gz
cd
libxc-
$libxc_version
./configure
--prefix
=
$PYTHONUSERBASE
--enable-shared
|
tee
loki-conf
make |
tee
loki-make
make
install
|
tee
loki-inst
export
LD_LIBRARY_PATH
=
$PYTHONUSERBASE
/lib:
$LD_LIBRARY_PATH
cd
..
# pycuda
pip
install
--user
pycuda
==
$pycuda_version
# go back to the main build directory
cd
..
# if this is the first bundle, use it as default
if
[
!
-e
$PYTHONHOME
/bundle/default
]
then
cd
$PYTHONHOME
/bundle
ln
-s
$bundle
default
cd
-
fi
# fix permissions
chmod
-R
g+rwX
$tgt
chmod
-R
o+rX
$tgt
gpaw/build/examples/piz-daint/python/build.sh
0 → 100644
View file @
5a03180f
### Python installation script for Piz Daint
### uses --prefix to set a custom installation directory
# version numbers (modify if needed)
python_version
=
2.7.13
# installation directory (modify!)
tgt
=
$SCRATCH
/lib/python-2018-12-cuda
# setup build environment
module swap PrgEnv-cray PrgEnv-gnu
module unload xalt
module load daint-gpu
module load craype-accel-nvidia60
export
CC
=
cc
export
CFLAGS
=
'-O3'
export
CXX
=
CC
export
CXXFLAGS
=
'-O3'
export
FC
=
ftn
export
FFLAGS
=
'-O3'
export
LINKFORSHARED
=
'-Wl,-export-dynamic -dynamic'
export
MPI_LINKFORSHARED
=
'-Wl,-export-dynamic -dynamic'
# python
git clone https://github.com/python/cpython.git python-
$python_version
cd
python-
$python_version
git checkout v
$python_version
./configure
--prefix
=
$tgt
--enable-shared
--disable-ipv6
--enable-unicode
=
ucs4 2>&1 |
tee
loki-conf
make 2>&1 |
tee
loki-make
make
install
2>&1 |
tee
loki-inst
cd
..
sed
-e
"s|<BASE>|
$tgt
|g"
setup/load-python.sh
>
$tgt
/load.sh
# install pip
source
$tgt
/load.sh
python
-m
ensurepip
pip
install
--upgrade
pip
# fix permissions
chmod
-R
g+rwX
$tgt
chmod
-R
o+rX
$tgt
gpaw/build/examples/piz-daint/python/setup/load-python.sh
0 → 100644
View file @
5a03180f
#!/bin/bash
export
PYTHONHOME
=
<BASE>
export
PYTHONPATH
=
$PYTHONHOME
/lib
export
PATH
=
$PYTHONHOME
/bin:
$PATH
export
MANPATH
=
$PYTHONHOME
/share/man:
$MANPATH
export
LD_LIBRARY_PATH
=
$PYTHONHOME
/lib:
$LD_LIBRARY_PATH
if
[[
$#
-gt
0
]]
then
export
PYTHONUSERBASE
=
$PYTHONHOME
/bundle/
$1
export
PATH
=
$PYTHONUSERBASE
/bin:
$PATH
export
LD_LIBRARY_PATH
=
$PYTHONUSERBASE
/lib:
$LD_LIBRARY_PATH
elif
[[
-e
$PYTHONHOME
/bundle/default
]]
then
export
PYTHONUSERBASE
=
$PYTHONHOME
/bundle/default
export
PATH
=
$PYTHONUSERBASE
/bin:
$PATH
export
LD_LIBRARY_PATH
=
$PYTHONUSERBASE
/lib:
$LD_LIBRARY_PATH
fi
if
[[
-e
$PYTHONUSERBASE
/include/xc.h
]]
then
export
LIBXCDIR
=
$PYTHONUSERBASE
fi
gpaw/build/examples/piz-daint/python/setup/piz-daint.cfg
0 → 100644
View file @
5a03180f
[atlas]
atlas_libs = sci_gnu
library_dirs = /opt/cray/pe/libsci/<LIBSCI>/gnu/6.1/<ARCH>/lib
include_dirs = /opt/cray/pe/libsci/<LIBSCI>/gnu/6.1/<ARCH>/include
[lapack_atlas]
lapack_libs = sci_gnu
library_dirs = /opt/cray/pe/libsci/<LIBSCI>/gnu/6.1/<ARCH>/lib
include_dirs = /opt/cray/pe/libsci/<LIBSCI>/gnu/6.1/<ARCH>/include
[lapack]
lapack_libs = sci_gnu
library_dirs = /opt/cray/pe/libsci/<LIBSCI>/gnu/6.1/<ARCH>/lib
include_dirs = /opt/cray/pe/libsci/<LIBSCI>/gnu/6.1/<ARCH>/include
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment