##
##  RUN - Strong -scaling
##
##  Before starting this job-script replace "SUBMIT" with the submition-command of the local queing system.
##  Additional in the script submit_job the execution command has to be adjusted to the local machine.
##  
##
##  Script for a parallelization of 2 4 8 16 32 64 KNLs
##
#!/bin/bash

EXE=/gpfs/projects/pr1ehq00/bench/qphix/time_clov_noqdp
## Set scaling-mode: Strong or Weak
sca_mode="Strong"
#sca_mode="Weak"
## mode="Analysis"
mode="Run"

##	sbatch_on=1
exe_perm=1 ## use chmod to allow execution of submit_job_Nx_Gx.sh
g=8 ##MPItaskperNODE
openmp=6  ##OMP
cpuptask=6 ## Total number of CPUS / MpitaskperNODE (= openmp if Hyperthreaded Cores are used,  = 2* openmp if Hyperthreading is enabled but not used )


## lattice size (size strong 1)
#gx=32
#gt=96
## lattice size (size strong 2)
gx=64
gt=128
## lattice size (size weak 1)
#gx=48
#gt=24

## use smaller lattice size of weak scaling mode: like gx=24 gt=24
##

lt=$gt
lx=$gx
ly=$gx
lz=$gx

# for gpus_per_node in 1 2; do
    cards_per_node=1
    # n=1
#    for n in 1 2 4 8 16 32 64 128 256 512 1024 2048 4096 8192 16384; do
    for n in 8 16 32 64 128 256 512 1024 2048 4096 8192 16384; do
#    for n in 8; do
       for p in "s" "d" ; do
#         p="d"
            case $p in 
                "s" )
                    prec="f"
                    ;;
                "d" )
                    prec="d"
                    ;;
                "h" )
                    prec="h"
                    ;;
            esac
            px=1
            py=1
            pz=1
            pt=$n
            
            if [ $n -eq 16 ];then
	        pz=2
                pt=8
	    fi
	    if [ $n -eq 32 ];then
	        pz=4
                pt=8
	    fi
            if [ $n -eq 64 ];then
                py=2
                pz=4
                pt=8
            fi
            if [ $n -eq 128 ];then
                py=2
                pz=8
                pt=8
            fi
            if [ $n -eq 256 ];then
                py=4
                pz=8
                pt=8
            fi
            if [ $n -eq 512 ];then
                px=2
                py=4
                pz=8
                pt=8
            fi
            if [ $n -eq 1024 ];then
                px=4
	        py=4		
                pz=8
                pt=8
            fi
            if [ $n -eq 2048 ];then
                px=8
                py=4
                pz=8
                pt=8
            fi
            if [ $n -eq 4096 ];then
                px=8
                py=8
                pz=8
                pt=8
            fi
            if [ $n -eq 8192 ];then
                px=8
                py=8
                pz=8
                pt=16
            fi
            if [ $n -eq 16384 ];then
                px=8
                py=8
                pz=16
                pt=16
            fi

            if [ $sca_mode = "Strong" ];then
                lt1=$((gt/pt))
                lx1=$((gx/px))
                ly1=$((gx/py))
                lz1=$((gx/pz))
            else
                lt1=$lt
                lx1=$lx
                ly1=$ly
                lz1=$lz

                lt=$((gt*pt))
                lx=$((gx*px))
                ly=$((gx*py))
                lz=$((gx*pz))
            fi
            Node=$((n/g))
	    name=${sca_mode}_qphix_${pt}x${pz}x${py}x${px}_${lt}x${lz}x${ly}x${lx}_${p}
            if [ $mode != "Analysis" ];then
            	echo $name
		submitscript=submit_job_N${n}_${p}.sh
		./prepare_submit_job.sh '00:10:00' ${Node} ${n} ${g} ${openmp} ${cpuptask}  ${exe_perm} ${submitscript}
		sbatch ./$submitscript ${pt} ${pz} ${py} $px $EXE $name $lx $lz $ly $lt $prec
                sleep 1
            ## Scaning the output and save the data in dat_nameif
      	    else   
                case $p in
                "s" )
                        echo $name >> Sca_s.log
                        less $name | grep "Time" -A 1 >> Sca_s.log
                    ;;
                "d" )   
                        echo $name >> Sca_d.log
                        less $name | grep "Time" -A 1 >> Sca_d.log
                    ;;
                "h" )
                        echo $name >> Sca_h.log
                        less $name | grep "Time" -A 1 >> Sca_h.log
                    ;;
                 esac
            fi
    done
done


