# Example PFARM (EXDIG) MPI/GPU job with test_case_1 (atomic) using SLURM.
# This configuration has 8 nodes with 1 MPI Task per node and 4 GPU devices per node

#!/bin/bash
#SBATCH --job-name=gpu_atom_8x4   # job name
#SBATCH --nodes=8               # 1 node
#SBATCH --ntasks-per-node=1        # set 1 MPI task per node
#SBATCH --gres=gpu:4                # no. gpus per node
#SBATCH --exclusive                  # exclusive access for benchmarking
#SBATCH --time=1:00:00               # time limits: 1 hour
#SBATCH --error=%x_%j.err         # standard error file
#SBATCH --output=%x_%j.out        # standard output file
#SBATCH --account=<set this>       # set account name
# #SBATCH --partition=<set this>     # set partition name
#SBATCH --reservation=<set this>     # set reservation name
# #SBATCH --qos=noQos                # set quality of service

echo $LD_LIBRARY_PATH

# Set the number of CPU threads (MAGMA uses both CPUs and GPUs)
export OMP_NUM_THREADS=1

# Environment settings for exdig_atom
export RMX_NSECT_FINE=16
export RMX_NSECT_COARSE=16
export RMX_NL_FINE=22
export RMX_NL_COARSE=10

# Set the number of GPUs per node here
export RMX_NGPU=4

# Remove output from previous runs
rm -f AMP*

# List the CPU/GPU node 
srun -l hostname
srun -l echo $CUDA_VISIBLE_DEVICES

# Run the PRMAT executable
date
srun ./exdig_mpi_gpu_atom
date
