##
##  RUN - Strong -scaling
##
##  Before starting this job-script replace "SUBMIT" with the submition-command of the local queing system.
##  Additional in the script submit_job the execution command has to be adjusted to the local machine.
##  
##
##  Script for a parallelization of 2 4 8 16 32 64 KNLs
##
#!/bin/bash

EXE=$PATH2EXE
## Set scaling-mode: Strong or Weak
sca_mode="Strong"
## sca_mode="Weak"
mode="Analysis"
##mode="Run"

##	sbatch_on=1
exe_perm=1 ## use chmod to allow execution of submit_job_Nx_Gx.sh

## lattice size (size strong 1)
gx=32
gt=96
## lattice size (size strong 2)
# gx=32
# gt=96
## lattice size (size weak 1)
#gx=48
#gt=24
## use smaller lattice size of weak scaling mode: like gx=24 gt=24
##

lt=$gt
lx=$gx
ly=$gx
lz=$gx

# for gpus_per_node in 1 2; do
    cards_per_node=1
    # n=1
    for n in 1 2 4 8 16; do
#       for p in "s" "d" "h" ; do
         p="h"
            case $p in 
                "s" )
                    prec="f"
                    ;;
                "d" )
                    prec="d"
                    ;;
                "h" )
                    prec="h"
                    ;;
            esac
            g=$cards_per_node
            n0=$n
            m=1
            v=1
            if [ $n -eq 16 ];then
	        m=2
                n0=8
	    fi
	    if [ $n -eq 32 ];then
	        m=4
                n0=8
	    fi
            if [ $n -eq 64 ];then
                v=2
                m=4
                n0=8
            fi

            if [ $sca_mode = "Strong" ];then
                lt1=$((gt/n0))
                lx1=$((gx/v))
                ly1=$((gx/cards_per_node))
                lz1=$((gx/m))
            else
                lt1=$gt
                lx1=$gx
                ly1=$gx
                lz1=$gx

                lt=$((gt*n0))
                lx=$((gx*v))
                ly=$((gx*cards_per_node))
                lz=$((gx*m))
            fi
	    name=${sca_mode}_Xeon_${n0}x${m}x${g}x${v}_gv${lt1}x${lz1}x${ly1}x${lx1}_${p}
            if [ $mode != "Analysis" ];then
            	echo $name
		submitscript=submit_job_N${n}_G${g}_${p}.sh
		./prepare_submit_job.sh '00:10:00' ${n} ${g} ${exe_perm} ${submitscript}
		sbatch ./$submitscript ${n0} ${m} ${g} $v $EXE $name $lx $lz $ly $lt $prec
                sleep 60    
            ## Scaning the output and save the data in dat_nameif
      	    else    
                case $p in
                "s" )
                        less $name | grep "CG GFLOPS=" | cut -c11- >> Sca_s.log
                    ;;
                "d" )
			less $name | grep "CG GFLOPS=" | cut -c11- >> Sca_d.log
                    ;;
                "h" )
                        less $name | grep "CG GFLOPS=" | cut -c11- >> Sca_h.log
                    ;;
                 esac
	    fi
     #  done
    done
# done


