Commit 42482007 authored by Andrew Emerson's avatar Andrew Emerson
Browse files

JUWELS and ARM files added

parent 6abed49d
# Quantum Espresso v6.3 on ARM v8
## Benchmark system
CARMEN (CINECA) 8-node ARM v8 cluster, 2x32 cores + 256G RAM/node.
For more details:
[Cineca documentation](
## Installation
Installed with ARM v19 compilers (flang and clang) and the ARM performance
library. This provides threaded BLAS, LAPACK and FFTW libraries. You will need to modify the file to make use of these.
Remember also to include the following flags:
-mcpu=native -armpl
Currently the OpenMP version (with ```-fopenmp```) flag does not compile. (Interanal compiler error).
- Because of fairly long execution times and limited number of nodes only AUSURF has been tested.
- Run time error causes QE to crash just after final iteration. Probably due to FoX XMl library (an issue has been raised). Walltimes therefore estimated from time of final iteration.
See job files for example execution.
## Profiling
Use the ARM MAP profiler. See example job scripts.
# system supplied modules
module load autoload armcompiler/19.0--binary
# the following is the ARM performance library including BLAS, LAPACK + FFTW
module load armpl-for-armcompiler/19.0.0--binary
# own-compiled OpenMPI 4.0.0 (for ARM compilers)
module use $HOME/modules
module load openmpi/openmpi-4.0.0_Arm
# QE configure
CC=mpicc FC=mpifort ./configure
# you need also to modify the since configure does not set it correctly
#SBATCH --tasks-per-node=64
#SBATCH -A cin_staff
#SBATCH -t 3:00:00
module load autoload armpl-for-armcompiler/19.0.0--binary
module load armcompiler/19.0--binary
module load arm-forge
module use $HOME/modules
module load openmpi/openmpi-4.0.0_Arm
start_time=$(date +"%s")
mpirun -np 64 $HOME/qe-6.3-fft/bin/pw.x -npool 2 -ndiag 16 -input
end_time=$(date +"%s")
echo "walltime $walltime"
#SBATCH --tasks-per-node=64
#SBATCH -A cin_staff
#SBATCH -t 3:00:00
module load autoload armpl-for-armcompiler/19.0.0--binary
module load armcompiler/19.0--binary
module load arm-forge
module use $HOME/modules
module load openmpi/openmpi-4.0.0_Arm
start_time=$(date +"%s")
#map --verbose --start-after 60 --stop-after 600 --profile mpirun -np 64 $HOME/qe-6.3-fft/bin/pw.x -npool 2 -input
map --verbose --profile mpirun -np 64 $HOME/qe-6.3-fft/bin/pw.x -npool 2 -ndiag 16 -input
end_time=$(date +"%s")
echo "walltime $walltime"
# Generated from by configure.
# compilation rules
.SUFFIXES : .o .c .f .f90
# most fortran compilers can directly preprocess c-like directives: use
# $(MPIF90) $(F90FLAGS) -c $<
# if explicit preprocessing by the C preprocessor is needed, use:
# $(CPP) $(CPPFLAGS) $< -o $*.F90
# $(MPIF90) $(F90FLAGS) -c $*.F90 -o $*.o
# remember the tabulator in the first column !!!
$(CPP) $(CPPFLAGS) $< -o $(*)_tmp.f90 ; \
$(MPIF90) $(F90FLAGS) -c $(*)_tmp.f90 -o $(*).o
# .f.o and .c.o: do not modify
$(F77) $(FFLAGS) -c $<
$(CC) $(CFLAGS) -c $<
# Top QE directory, useful for locating libraries, linking QE with plugins
# The following syntax should always point to TOPDIR:
TOPDIR = $(dir $(abspath $(filter,$(MAKEFILE_LIST))))
# if it doesn't work, uncomment the following line (edit if needed):
# TOPDIR = /home/userinternal/aemerson/qe-6.3
# DFLAGS = precompilation options (possible arguments to -D and -U)
# used by the C compiler and preprocessor
# To use libxc (v>=3.0.1), add -D__LIBXC to DFLAGS
# See include/defs.h.README for a list of options and their meaning
# With the exception of IBM xlf, FDFLAGS = $(DFLAGS)
# For IBM xlf, FDFLAGS is the same as DFLAGS with separating commas
# MANUAL_DFLAGS = additional precompilation option(s), if desired
# BEWARE: it does not work for IBM xlf! Manually edit FDFLAGS
# IFLAGS = how to locate directories with *.h or *.f90 file to be included
# typically -I$(TOPDIR)/include -I/some/other/directory/
# the latter contains .e.g. files needed by FFT libraries
# for libxc add -I/path/to/libxc/include/
IFLAGS = -I$(TOPDIR)/include -I$(TOPDIR)/FoX/finclude -I$(TOPDIR)/S3DE/iotk/include/
# MOD_FLAG = flag used by f90 compiler to locate modules
# BASEMOD_FLAGS points to directories containing basic modules,
# while BASEMODS points to the corresponding module libraries
# Each Makefile can add directories to MODFLAGS and libraries to QEMODS
$(MOD_FLAG)$(TOPDIR)/Modules \
$(MOD_FLAG)$(TOPDIR)/UtilXlib \
# Compilers: fortran-90, fortran-77, C
# If a parallel compilation is desired, MPIF90 should be a fortran-90
# compiler that produces executables for parallel execution using MPI
# (such as for instance mpif90, mpf90, mpxlf90,...);
# otherwise, an ordinary fortran-90 compiler (f90, g95, xlf90, ifort,...)
# If you have a parallel machine but no suitable candidate for MPIF90,
# try to specify the directory containing "mpif.h" in IFLAGS
# and to specify the location of MPI libraries in MPI_LIBS
MPIF90 = mpif90
F90 = mpifort
CC = mpicc
F77 = mpifort
# GPU architecture (Kepler: 35, Pascal: 60, Volta: 70 )
# CUDA runtime (Pascal: 8.0, Volta: 9.0)
# CUDA F90 Flags
# C preprocessor and preprocessing flags - for explicit preprocessing,
# if needed (see the compilation rules above)
# preprocessing flags must include DFLAGS and IFLAGS
CPP = cpp
CPPFLAGS = -P -traditional $(DFLAGS) $(IFLAGS)
# compiler flags: C, F90, F77
# C flags must include DFLAGS and IFLAGS
# F90 flags must include MODFLAGS, IFLAGS, and FDFLAGS with appropriate syntax
CFLAGS = -O3 -mcpu=native -armpl -fopenmp $(DFLAGS) $(IFLAGS)
FFLAGS = -O2 -g -mcpu=native -armpl -fopenmp
# compiler flags without optimization for fortran-77
# the latter is NEEDED to properly compile dlamch.f, used by lapack
# compiler flag needed by some compilers when the main program is not fortran
# Currently used for Yambo
# Linker, linker-specific flags (if any)
# Typically LD coincides with F90 or MPIF90, LD_LIBS is empty
# for libxc, set LD_LIBS=-L/path/to/libxc/lib/ -lxcf90 -lxc
LD = mpif90
LDFLAGS = -mcpu=native -armpl -fopenmp
# External Libraries (if any) : blas, lapack, fft, MPI
# If you have nothing better, use the local copy via "--with-netlib" :
# BLAS_LIBS = /your/path/to/espresso/LAPACK/blas.a
# BLAS_LIBS_SWITCH = internal
BLAS_LIBS = /cineca/prod/opt/tools/arm-compiler-for-hpc/19.0/none/opt/arm/armpl-19.0.0_ThunderX2CN99_RHEL-7_arm-hpc-compiler_19.0_aarch64-linux/lib/libarmpl_lp64.a
# If you have nothing better, use the local copy via "--with-netlib" :
# LAPACK_LIBS = /your/path/to/espresso/LAPACK/lapack.a
# For IBM machines with essl (-D__ESSL): load essl BEFORE lapack !
# remember that LAPACK_LIBS precedes BLAS_LIBS in loading order
LAPACK_LIBS = /cineca/prod/opt/tools/arm-compiler-for-hpc/19.0/none/opt/arm/armpl-19.0.0_ThunderX2CN99_RHEL-7_arm-hpc-compiler_19.0_aarch64-linux/lib/libarmpl_lp64.a
# nothing needed here if the the internal copy of FFTW is compiled
# (needs -D__FFTW in DFLAGS)
FFT_LIBS = /cineca/prod/opt/tools/arm-compiler-for-hpc/19.0/none/opt/arm/armpl-19.0.0_ThunderX2CN99_RHEL-7_arm-hpc-compiler_19.0_aarch64-linux/lib/libarmpl_lp64.a
# HDF5
FOX_LIB = -L$(TOPDIR)/FoX/lib -lFoX_dom -lFoX_sax -lFoX_wxml -lFoX_common\
-lFoX_utils -lFoX_fsys
# For parallel execution, the correct path to MPI libraries must
# be specified in MPI_LIBS (except for IBM if you use mpxlf)
# IBM-specific: MASS libraries, if available and if -D__MASS is defined in FDFLAGS
# CUDA libraries
# ar command and flags - for most architectures: AR = ar, ARFLAGS = ruv
AR = ar
# ranlib command. If ranlib is not needed (it isn't in most cases) use
# RANLIB = echo
RANLIB = ranlib
# all internal and external libraries - do not modify
LIBOBJS = $(TOPDIR)/clib/clib.a $(TOPDIR)/iotk/src/libiotk.a
# wget or curl - useful to download from network
WGET = wget -O
# Install directory - not currently used
PREFIX = /usr/local
......@@ -7,10 +7,12 @@
#SBATCH --error=ta205-err.%j
#SBATCH --mem=90GB
#SBATCH --time=01:30:00
#SBATCH --partition=batch
#SBATCH --account=prpb74
#module load Intel IntelMPI imkl
module load intel-para/2018b-mt
module load Intel IntelMPI imkl
#module load intel-para/2018b-mt
srun -n 650 $QE_HOME/bin/pw.x -npool 26 -input
# Example job script for AUSURF (2 k points)
#SBATCH --nodes=1
#SBATCH --ntasks=48
#SBATCH --ntasks-per-node=48
#SBATCH --output=mpi-out.%j
#SBATCH --error=mpi-err.%j
#SBATCH --time=01:00:00
#SBATCH --ntasks=16
#SBATCH --ntasks-per-node=16
#SBATCH --output=ausurf-out.%j
#SBATCH --error=ausurf-err.%j
#SBATCH --time=00:30:00
#SBATCH --partition=batch
#SBATCH --account=prpb74
module load Intel IntelMPI imkl
srun $QE_HOME/bin/pw.x -npool 2 -ndiag 16 -input
srun -n 16 $QE_HOME/bin/pw.x -npool 2 -ndiag 4 -input
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment