##
##  RUN - Strong -scaling
##
##  Before starting this job-script replace "SUBMIT" with the submition-command of the local queing system.
##  Additional in the script submit_job the execution command has to be adjusted to the local machine.
##  
##
##  Script for a parallelization of 2 4 8 16 32 64 GPU's
##
#!/bin/bash

export QUDA_RESOURCE_PATH=/users/jff/benchmark/tune
EXE=/users/jff/benchmark/runs_32c96/invert_test
## Set scaling-mode: Strong or Weak
##sca_mode="Strong"
sca_mode="Weak"

mode="Analysis"
##mode="Run"

sbatch_on=0
##	sbatch_on=1
QUDA_version=85   ## this script supports QUDA versions from 0.7 to 0.85, version 0.7 = 70, 0.8 = 80 and 0.85 =85

exe_perm=1 ## use chmod to allow execution of submit_job_Nx_Gx.sh

## lattice size 
gx=48
gt=24
## use smaller lattice size of weak scaling mode: like gx=24 gt=24
##

lt=$gt
lx=$gx
ly=$gx
lz=$gx
recon=12  ## reconstruction of the SU3-links

# for gpus_per_node in 1 2; do
    gpus_per_node=1
    # n=1
    for n in 1 2 4 8 16 32 64; do
#       for p in "ss" "dd" "hd" ; do
         p="dd"
            case $p in 
                "ss" )
                    prec="single"
                    prec_sloppy="single"
                    ;;
                "dd" )
                    prec="double"
                    prec_sloppy="double"
                    ;;
                "hd" )
                    prec="double"
                    prec_sloppy="half"
                    ;;
            esac
            g=$gpus_per_node
            n0=$n
            m=1
            v=1
            if [ $n -eq 16 ];then
	        m=2
                n0=8
	    fi
	    if [ $n -eq 32 ];then
	        m=4
                n0=8
	    fi
            if [ $n -eq 64 ];then
                v=2
                m=4
                n0=8
            fi

            if [ $sca_mode = "Strong" ];then
                lt=$((gt/n0))
                lx=$((gx/v))
                ly=$((gx/gpus_per_node))
                lz=$((gx/m))
            fi
	    name=${sca_mode}_gpu_${n0}x${m}x${g}x${v}_gv${lt}x${lz}x${ly}x${lx}_${p}
	    datname=${sca_mode}_GPUperNode${g}_${p}.log
            if [ $mode != "Analysis" ];then
            	echo $name
	           ##echo -t 00:20:00 -N ${n} --tasks-per-node ${g} -p gpu_short -o strong_sca_n${n}_g${g}_prec_${prec}_${prec_sloppy}.out -e strong_sca_n${n}_g${g}_prec_${prec}_${prec_sloppy}.err submit_job.sh ${n} ${m} ${g} $EXE $name $lx $lz $ly $lt $recon $prec $prec_sloppy $name
        	   if [ $sbatch_on -eq 1 ];then
				batch -t 00:20:00 -N ${n} --gres=gpu:${g} -p gpus -o ${sca_mode}_sca_${n}x${m}x${g}_prec${prec}_${prec_sloppy}.out -e ${sca_mode}_sca_${n}x${m}x${g}_prec${prec}_${prec_sloppy}.err submit_job_JURECA.sh ${n} ${m} ${g} $EXE $name $lx $lz $ly $lt $recon $prec $prec_sloppy
	           else
        	               submitscript=submit_job_N${n}_G${g}_${p}.sh
				./prepare_submit_job.sh '00:10:00' ${n} ${g} ${exe_perm} ${submitscript}
				sbatch ./$submitscript ${n0} ${m} ${g} $v $EXE $name $lx $lz $ly $lt $recon $prec $prec_sloppy $QUDA_version
                               sleep 60
		   fi
            
            ## Scaning the output and save the data in dat_nameif
      	    else    
                case $p in
                "ss" )
                        less $name.log | grep Gflops | awk '{printf("%f %f \n",$8,$13)}' >> Sca_ss.log
                    ;;
                "dd" )
                        less $name.log | grep Gflops | awk '{printf("%f %f \n",$8,$13)}' >> Sca_dd.log                    
                    ;;
                "hd" )
                        less $name.log | grep Gflops | awk '{printf("%f %f \n",$8,$13)}' >> Sca_hd.log
                    ;;
                 esac
              less $name.log | grep Gflops | awk '{printf("%f %f \n",$8,$13)}' >> Overview.log
	    fi
     #  done
    done
# done


