##
##  RUN - Strong -scaling
##
##  Before starting this job-script replace "SUBMIT" with the submition-command of the local queing system.
##  Additional in the script submit_job the execution command has to be adjusted to the local machine.
##  
##
##  Script for a parallelization of 2 4 8 16 32 64 KNLs
##
#!/bin/bash

EXE=/ccc/cont005/home/unicy/finkenrj/run/qphix/time_clov_noqdp
## Set scaling-mode: Strong or Weak
sca_mode="Strong"
#sca_mode="OneNode"
#sca_mode="Weak"
## mode="Analysis"
mode="Run"

##	sbatch_on=1
exe_perm=1 ## use chmod to allow execution of submit_job_Nx_Gx.sh
g=8 ##MPItaskperNODE
openmp=6  ##OMP
cpuptask=6 ## Total number of CPUS / MpitaskperNODE (= openmp if Hyperthreaded Cores are used,  = 2* openmp if Hyperthreading is enabled but not used )


## lattice size (size strong 1)
gx=96
gy=96
gz=96
gt=192
## lattice size (size strong 2)
#gx=8
#gy=4
#gz=4
#gt=8
## lattice size (size weak 1)
#gx=48
#gt=24

## use smaller lattice size of weak scaling mode: like gx=24 gt=24
##

#gy=$gx
#gz=$gx

lt=$gt
lx=$gx
ly=$gy
lz=$gz

# for gpus_per_node in 1 2; do
    cards_per_node=1
    #n=1
#    for n in 1 2 4 8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 65536 131072 262144 524288 1048576; do
    for n in 8 16 32 64 128 256 512 1024 2048 4096 8192 16384; do
#    for n in 8 16 32 64 128 256 512 1024 2048 4096 8192 16384; do
#   for n in 2 4 8 16 32 64 128 256 512 1024 2048 4096 8192 16384; do
#    for n in 8; do
       for p in "s" "d" ; do
#         p="d"
            case $p in 
                "s" )
                    prec="f"
                    ;;
                "d" )
                    prec="d"
                    ;;
                "h" )
                    prec="h"
                    ;;
            esac
            px=1
            py=1
            pz=1
            pt=$n
            
            if [ $n -eq 16 ];then
	        pz=2
                pt=8
	    fi
	    if [ $n -eq 32 ];then
	        pz=4
                pt=8
	    fi
            if [ $n -eq 64 ];then
                py=2
                pz=4
                pt=8
            fi
            if [ $n -eq 128 ];then
                py=2
                pz=8
                pt=8
            fi
            if [ $n -eq 256 ];then
                py=4
                pz=8
                pt=8
            fi
            if [ $n -eq 512 ];then
                px=2
                py=4
                pz=8
                pt=8
            fi
            if [ $n -eq 1024 ];then
                px=4
	        py=4		
                pz=8
                pt=8
            fi
            if [ $n -eq 2048 ];then
                px=8
                py=4
                pz=8
                pt=8
            fi
            if [ $n -eq 4096 ];then
                px=8
                py=8
                pz=8
                pt=8
            fi
            if [ $n -eq 8192 ];then
                px=8
                py=8
                pz=8
                pt=16
            fi
            if [ $n -eq 16384 ];then
                px=8
                py=8
                pz=16
                pt=16
            fi
            if [ $n -eq 32768 ];then
                px=8
                py=16
                pz=16
                pt=16
            fi 
            if [ $n -eq 65536 ];then
                px=16
                py=16
                pz=16
                pt=16
            fi
            if [ $n -eq 131072 ];then
                px=16
                py=16
                pz=16
                pt=32
            fi
            if [ $n -eq 262144 ];then
                px=16
                py=16
                pz=32
                pt=32
            fi
            if [ $n -eq 524288 ];then
                px=16
                py=32
                pz=32
                pt=32
            fi
            if [ $n -eq 1048576 ];then
                px=32
                py=32
                pz=32
                pt=32
            fi
            nt=$n
            if [ $sca_mode = "Strong" ];then
                lt1=$((gt/pt))
                lx1=$((gx/px))
                ly1=$((gy/py))
                lz1=$((gz/pz))
            elif [ $sca_mode = "OneNode" ]; then
                lx1=$((gx*px))
                ly1=$((gy*py))
                lz1=$((gz*pz))
                lt1=$((gt*pt/g))
                nt=$g
                lx=$((gx*px))
                ly=$((gy*py))
                lz=$((gz*pz))
                lt=$((gt*pt))
                px=1
                py=1
                pz=1
                pt=$g
            else
                lt1=$lt
                lx1=$lx
                ly1=$ly
                lz1=$lz

                lt=$((gt*pt))
                lx=$((gx*px))
                ly=$((gy*py))
                lz=$((gz*pz))
            fi
            Node=$((nt/g))
	    name=${sca_mode}_qphix_${pt}x${pz}x${py}x${px}_${lt}x${lz}x${ly}x${lx}_${p}
            if [ $mode != "Analysis" ];then
            	echo $name
        	submitscript=submit_job_N${nt}_${p}.sh
		        ./prepare_submit_job.sh '00:10:00' ${Node} ${nt} ${g} ${openmp} ${cpuptask}  ${exe_perm} ${submitscript} ${pt} ${pz} ${py} $px $EXE $name $lx $lz $ly $lt $prec
                ccc_msub ./$submitscript
                sleep 1
            ## Scaning the output and save the data in dat_nameif
      	    else   
                case $p in
                "s" )
                        echo $name >> Sca_s.log
                        less $name | grep "Time" -A 1 >> Sca_s.log
                    ;;
                "d" )   
                        echo $name >> Sca_d.log
                        less $name | grep "Time" -A 1 >> Sca_d.log
                    ;;
                "h" )
                        echo $name >> Sca_h.log
                        less $name | grep "Time" -A 1 >> Sca_h.log
                    ;;
                 esac
            fi
    done
done