<execution>


<!-- ******************************************************************************** -->

<execute cname="Intel-SNB-supermuc">
    <input files="$pdir/Intel-SNB-supermuc/ibm_llsubmit.job.in" />

    <substitute infile="ibm_llsubmit.job.in" outfile="ibm_llsubmit.job">
        <sub from="#OUTDIR#"            to="$outdir" />
        <sub from="#STDOUTLOGFILE#"     to="$stdoutlogfile" />
        <sub from="#STDERRLOGFILE#"     to="$stderrlogfile" />
        <sub from="#JOB_CLASS#"         to="$job_class" />
        <sub from="#DATA_LIMIT#"        to="0.75GB" />
        <sub from="#STACK_LIMIT#"       to="0.25GB" />
        <sub from="#MEMORYPERTASK#"     to="1.0GB" />
        <sub from="#BENCHNAME#"         to="$benchname" />
        <sub from="#NODEUSAGE#"         to="shared" />
        <sub from="#TIME_LIMIT#"        to="00:15:00" />
        <sub from="#NODES#"             to="$nodes" />
        <sub from="#TASKS#"             to="$tasks" />
        <sub from="#TAFFINITY#"         to="$taffinity" />
        <sub from="#TASKSPERNODE#"      to="$taskspernode" />
        <sub from="#NOTIFICATION#"      to="never" />
        <sub from="#NOTIFY_EMAIL#"      to="XXX@sd.ds" />
        <sub from="#MY_ENERGYTAG#"      to="$my_energytag" />
        <sub from="#THREADSPERTASK#"    to="$threadspertask" />
        <sub from="#EXECUTABLE#"        to="$executable" />
        <sub from="#ENV#"               to="$env" />
        <sub from="#PREPROCESS#"        to="" />
        <sub from="#POSTPROCESS#"       to="echo 'JuBE: $COMMENT'" />
        <sub from="#STARTER#"           to="mpiexec -n $tasks" />
        <sub from="#ARGS_STARTER#"      to="" />
        <sub from="#MEASUREMENT#"       to="" />
        <sub from="#ARGS_EXECUTABLE#"   to="-nodes ${nodes}" />
    </substitute>

    <environment>
        <env var = "MP_LABELIO"           value="yes" />
        <env var = "MP_INFOLEVEL"         value="2" />
        <env var = "MP_SHARED_MEMORY"     value="yes" />
        <env var = "MP_TASK_AFFINITY"     value="MCM" />
        <env var = "MEMORY_AFFINITY"      value="MCM" />
        <env var = "HPM_STDOUT"           value="0" />
        <env var = "HPM_UNIQUE_FILE_NAME" value="1" />
        <env var = "HPM_EVENT_SET"        value="$IHPCT_HWC_GRP" />
    </environment>

    <command>llsubmit ibm_llsubmit.job</command>

</execute>

<!-- ******************************************************************************** -->

<execute cname="Cray-XT4-Louhi">
    <input files="$pdir/Cray-XT4-Louhi/cray_qsub.job.in" />

    <substitute infile="cray_qsub.job.in" outfile="cray_qsub.job">
        <sub from="#BENCHNAME#"         to="$benchname" />

        <sub from="#OUTDIR#"            to="$outdir" />
        <sub from="#STDOUTLOGFILE#"     to="$stdoutlogfile" />
        <sub from="#STDERRLOGFILE#"     to="$stderrlogfile" />

        <sub from="#TIME_LIMIT#"        to="00:10:00" />

        <sub from="#TASKS#"             to="$tasks" />
        <sub from="#TASKSPERNODE#"      to="$taskspernode" />
        <sub from="#THREADSPERTASK#"    to="$threadspertask" />

        <sub from="#NOTIFICATION#"      to="n" />
        <sub from="#NOTIFY_EMAIL#"      to="l.arnold@fz-juelich.de" />

        <sub from="#MEMORYPERTASK#"     to="`($datalimit+$stacklimit)*$threadspertask`M" />

        <sub from="#EXECUTABLE#"        to="${outdir}/qcd-submit.exe.tmp" />
        <sub from="#ENV#"               to="$env" />
        <sub from="#PREPROCESS#"        to="module load xt-craypat; cp $executable ${outdir}/qcd-submit.exe.tmp" />
        <sub from="#POSTPROCESS#"       to="" />
        <sub from="#STARTER#"           to="aprun -n $tasks -m ${datalimit}M -N ${taskspernode}" />
        <sub from="#ARGS_STARTER#"      to="" />
        <sub from="#MEASUREMENT#"       to="time" />
        <sub from="#ARGS_EXECUTABLE#"   to="" />
    </substitute>
  
    <environment>
        <env var="PAT_RT_HWPC"   value="$CRAYPAT_HWC" />    
    </environment>
  
    <command>qsub -q prace cray_qsub.job</command>
</execute>

<!-- ******************************************************************************** -->

<execute cname="Cray-XE6-HECToR">
    <input files="$pdir/Cray-XE6-HECToR/cray_PBSsubmit.job.in" />

    <substitute infile="cray_PBSsubmit.job.in" outfile="cray_PBSsubmit.job">
        <sub from="#BENCHNAME#"         to="$benchname" />

        <sub from="#OUTDIR#"            to="$outdir" />
        <sub from="#STDOUTLOGFILE#"     to="$stdoutlogfile" />
        <sub from="#STDERRLOGFILE#"     to="$stderrlogfile" />
        <sub from="#ACCOUNTING#"        to="pr1uqqcd" />

        <sub from="#TIME_LIMIT#"        to="00:10:00" />

        <sub from="#TASKS#"             to="$tasks" />
        <sub from="#TASKSPERNODE#"      to="$taskspernode" />
        <sub from="#THREADSPERTASK#"    to="$threadspertask" />

        <sub from="#EXECUTABLE#"        to="${outdir}/qcd-submit.exe.tmp" />
        <sub from="#ENV#"               to="$env" />
        <sub from="#PREPROCESS#"        to="cp $executable ${outdir}/qcd-submit.exe.tmp" />
        <sub from="#POSTPROCESS#"       to="" />
        <sub from="#STARTER#"           to="aprun" />
        <sub from="#ARGS_STARTER#"      to="-n $tasks -N ${taskspernode}" />
        <sub from="#MEASUREMENT#"       to="time" />
        <sub from="#ARGS_EXECUTABLE#"   to="" />
    </substitute>
  
    <environment>
    </environment>
  
    <command>qsub cray_PBSsubmit.job</command>
</execute>

<!-- ******************************************************************************** -->

<execute cname="Cray-XE6-HERMIT">
    <input files="$pdir/Cray-XE6-HERMIT/cray_PBSsubmit.job.in" />

    <substitute infile="cray_PBSsubmit.job.in" outfile="cray_PBSsubmit.job">
        <sub from="#BENCHNAME#"         to="$benchname" />

        <sub from="#OUTDIR#"            to="$outdir" />
        <sub from="#STDOUTLOGFILE#"     to="$stdoutlogfile" />
        <sub from="#STDERRLOGFILE#"     to="$stderrlogfile" />

        <sub from="#TIME_LIMIT#"        to="00:10:00" />

        <sub from="#TASKS#"             to="$tasks" />
        <sub from="#TASKSPERNODE#"      to="$taskspernode" />
        <sub from="#THREADSPERTASK#"    to="$threadspertask" />

        <sub from="#EXECUTABLE#"        to="${outdir}/qcd-submit.exe.tmp" />
        <sub from="#ENV#"               to="$env" />
        <sub from="#PREPROCESS#"        to="cp $executable ${outdir}/qcd-submit.exe.tmp" />
        <sub from="#POSTPROCESS#"       to="" />
        <sub from="#STARTER#"           to="aprun" />
        <sub from="#ARGS_STARTER#"      to="-n $tasks -N ${taskspernode}" />
        <sub from="#MEASUREMENT#"       to="time" />
        <sub from="#ARGS_EXECUTABLE#"   to="" />
    </substitute>
  
    <environment>
    </environment>
  
    <command>qsub intel_PBSsubmit.job</command>
</execute>

<!-- ******************************************************************************** -->

<execute cname="Intel-Nehalem-JUROPA">
    <input files="$pdir/Intel-Nehalem-JUROPA/intel_PBSsubmit.job.in" />

    <substitute infile="intel_PBSsubmit.job.in" outfile="intel_PBSsubmit.job">
        <sub from="#BENCHNAME#"         to="$benchname" />

        <sub from="#OUTDIR#"            to="$outdir" />
        <sub from="#STDOUTLOGFILE#"     to="$stdoutlogfile" />
        <sub from="#STDERRLOGFILE#"     to="$stderrlogfile" />

        <sub from="#TIME_LIMIT#"        to="00:10:00" />

	<sub from="#PPN"                to="8" />
        <sub from="#TASKS#"             to="$tasks" />
        <sub from="#TASKSPERNODE#"      to="$taskspernode" />
        <sub from="#NODES#"             to="$nodes" />
        <sub from="#THREADSPERTASK#"    to="$threadspertask" />

        <sub from="#NOTIFICATION#"      to="n" />
        <sub from="#NOTIFY_EMAIL#"      to="l.arnold@fz-juelich.de" />

        <sub from="#EXECUTABLE#"        to="${outdir}/qcd-submit.exe.tmp" />
        <sub from="#ENV#"               to="$env" />
        <sub from="#PREPROCESS#"        to="cp $executable ${outdir}/qcd-submit.exe.tmp" />
        <sub from="#POSTPROCESS#"       to="" />
        <sub from="#STARTER#"           to="mpiexec" />
        <sub from="#ARGS_STARTER#"      to="-np $ncpus" />
        <sub from="#MEASUREMENT#"       to="time" />
        <sub from="#ARGS_EXECUTABLE#"   to="" />
    </substitute>
  
    <environment>
    </environment>
  
    <command>msub intel_PBSsubmit.job</command>
</execute>

<!-- ******************************************************************************** -->

<execute cname="IBM-SP6-Huygens">
    <input files="$pdir/IBM-SP6-Huygens/ibm_llsubmit.job.in" />

    <substitute infile="ibm_llsubmit.job.in" outfile="ibm_llsubmit.job">
        <sub from="#OUTDIR#"            to="$outdir" />
        <sub from="#STDOUTLOGFILE#"     to="$stdoutlogfile" />
        <sub from="#STDERRLOGFILE#"     to="$stderrlogfile" />
        <sub from="#BENCHNAME#"         to="$benchname" />
        <sub from="#NODEUSAGE#"         to="not_shared" />
        <sub from="#TIME_LIMIT#"        to="00:30:00" />
	<sub from="#DATA_LIMIT#"        to="0.75GB" />
	<sub from="#STACK_LIMIT#"       to="0.25GB" />
        <sub from="#NODES#"             to="$nodes" />
        <sub from="#TASKS#"             to="$tasks" />
        <sub from="#TASKSPERNODE#"      to="$taskspernode" />
        <sub from="#NOTIFICATION#"      to="never" />
        <sub from="#NOTIFY_EMAIL#"      to="l.arnold@fz-juelich.de" />
        <sub from="#THREADSPERTASK#"    to="$threadspertask" />
        <sub from="#EXECUTABLE#"        to="$executable" />
        <sub from="#ENV#"               to="$env" />
        <sub from="#PREPROCESS#"        to="`index('$IHPCT_HWC','on')==0 || index('$IHPCT_MPITR','on')==0 ? 'module load hpct;' : ' '`" />
        <sub from="#POSTPROCESS#"       to="" />
        <sub from="#STARTER#"           to="poe" />
        <sub from="#ARGS_STARTER#"      to="" />
        <sub from="#MEASUREMENT#"       to="time" />
        <sub from="#ARGS_EXECUTABLE#"   to="" />
    </substitute>
  
    <environment>
        <env var = "MP_LABELIO"           value="yes" />
        <env var = "MP_INFOLEVEL"         value="2" />
        <env var = "MP_SHARED_MEMORY"     value="yes" />
        <env var = "MP_TASK_AFFINITY"     value="MCM" />
        <env var = "MEMORY_AFFINITY"      value="MCM" />
	<env var = "HPM_STDOUT"           value="0" />
	<env var = "HPM_UNIQUE_FILE_NAME" value="1" />
	<env var = "HPM_EVENT_SET"        value="$IHPCT_HWC_GRP" />
    </environment>
  
    <command>llsubmit ibm_llsubmit.job</command>
</execute>

<!-- ******************************************************************************** -->

<execute cname="IBM-BGP-Jugene">
    <input files="$pdir/IBM-BGP-Jugene/ibm_llsubmit.job.in" />

    <substitute infile="ibm_llsubmit.job.in" outfile="ibm_llsubmit.job">
        <sub from="#OUTDIR#"            to="$outdir" />
        <sub from="#STDOUTLOGFILE#"     to="$stdoutlogfile" />
        <sub from="#STDERRLOGFILE#"     to="$stderrlogfile" />
        <sub from="#BENCHNAME#"         to="$benchname" />
        <sub from="#TIME_LIMIT#"        to="00:30:00" />
        <sub from="#BGSIZE#"            to="$nodes" />
        <sub from="#BGCONNECTION#"      to="`int('$nodes')>256 ? 'TORUS' : 'MESH'`" />
        <sub from="#NOTIFICATION#"      to="never" />
        <sub from="#NOTIFY_EMAIL#"      to="l.arnold@fz-juelich.de" />
        <sub from="#EXECUTABLE#"        to="${outdir}/qcd-submit.exe.tmp" />
        <sub from="#ENV#"               to="$env" />
        <sub from="#PREPROCESS#"        to="cp $executable ${outdir}/qcd-submit.exe.tmp" />
        <sub from="#POSTPROCESS#"       to="" />
        <sub from="#STARTER#"           to="mpirun -env 'BG_MAPPING=TXYZ'" />
        <sub from="#ARGS_STARTER#"      to="`$taskspernode == 1 ? '-mode SMP' : ($taskspernode == 2 ? '-mode DUAL' : '-mode VN')` 
	                                    -np $ncpus -verbose 1" />
        <sub from="#MEASUREMENT#"       to="time" />
        <sub from="#ARGS_EXECUTABLE#"   to="" />
    </substitute>
  
    <environment>
        <env var = "MP_LABELIO"           value="yes" />
        <env var = "MP_INFOLEVEL"         value="2" />
        <env var = "MP_SHARED_MEMORY"     value="yes" />
        <env var = "MP_TASK_AFFINITY"     value="MCM" />
        <env var = "MEMORY_AFFINITY"      value="MCM" />
	<env var = "HPM_STDOUT"           value="0" />
	<env var = "HPM_UNIQUE_FILE_NAME" value="1" />
	<env var = "HPM_EVENT_SET"        value="$IHPCT_HWC_GRP" />
    </environment>
  
    <command>llsubmit ibm_llsubmit.job</command>
</execute>

<!-- ******************************************************************************** -->

<execute cname="IBM-BGQ-Juqueen">
    <input files="$pdir/IBM-BGQ-Juqueen/ibm_llsubmit.job.in" />

    <substitute infile="ibm_llsubmit.job.in" outfile="ibm_llsubmit.job">
        <sub from="#OUTDIR#"            to="$outdir" />
        <sub from="#STDOUTLOGFILE#"     to="$stdoutlogfile" />
        <sub from="#STDERRLOGFILE#"     to="$stderrlogfile" />
        <sub from="#BENCHNAME#"         to="$benchname" />
        <sub from="#TIME_LIMIT#"        to="00:30:00" />
        <sub from="#BGSIZE#"            to="$nodes" />
        <sub from="#BGCONNECTIVITY#"    to="`int('$nodes')>256 ? 'TORUS' : 'MESH'`" />
        <sub from="#NOTIFICATION#"      to="never" />
        <sub from="#NOTIFY_EMAIL#"      to="st.janetzko@fz-juelich.de" />
        <sub from="#EXECUTABLE#"        to="${outdir}/qcd-submit.exe.tmp" />
        <sub from="#ENV#"               to="$env" />
        <sub from="#PREPROCESS#"        to="cp $executable ${outdir}/qcd-submit.exe.tmp" />
        <sub from="#POSTPROCESS#"       to="" />
        <sub from="#STARTER#"           to="runjob" />
        <sub from="#ARGS_STARTER#"      to="--ranks-per-node $taskspernode --np $ncpus" />
        <sub from="#MEASUREMENT#"       to="time" />
        <sub from="#ARGS_EXECUTABLE#"   to="" />
    </substitute>
  
    <environment>
        <env var = "MP_LABELIO"           value="yes" />
        <env var = "MP_INFOLEVEL"         value="2" />
        <env var = "MP_SHARED_MEMORY"     value="yes" />
        <env var = "MP_TASK_AFFINITY"     value="MCM" />
        <env var = "MEMORY_AFFINITY"      value="MCM" />
	<env var = "HPM_STDOUT"           value="0" />
	<env var = "HPM_UNIQUE_FILE_NAME" value="1" />
	<env var = "HPM_EVENT_SET"        value="$IHPCT_HWC_GRP" />
    </environment>
  
    <command>llsubmit ibm_llsubmit.job</command>
</execute>

<!-- ******************************************************************************** -->

<execute cname="IBM-SP6-Jump">
    <input files="$pdir/IBM-SP6-Jump/ibm_llsubmit.job.in" />

    <substitute infile="ibm_llsubmit.job.in" outfile="ibm_llsubmit.job">
        <sub from="#OUTDIR#"            to="$outdir" />
        <sub from="#STDOUTLOGFILE#"     to="$stdoutlogfile" />
        <sub from="#STDERRLOGFILE#"     to="$stderrlogfile" />
        <sub from="#CLASS#"             to="$class" />
	<sub from="#DATA_LIMIT#"        to="0.75GB" />
	<sub from="#STACK_LIMIT#"       to="0.25GB" />
	<sub from="#MEMORYPERTASK#"     to="1.0GB" />
        <sub from="#BENCHNAME#"         to="$benchname" />
        <sub from="#NODEUSAGE#"         to="shared" />
        <sub from="#TIME_LIMIT#"        to="00:05:00" />
        <sub from="#NODES#"             to="$nodes" />
        <sub from="#TASKS#"             to="$tasks" />
        <sub from="#TAFFINITY#"         to="$taffinity" />
        <sub from="#TASKSPERNODE#"      to="$taskspernode" />
        <sub from="#NOTIFICATION#"      to="never" />
        <sub from="#NOTIFY_EMAIL#"      to="l.arnold@fz-juelich.de" />
        <sub from="#THREADSPERTASK#"    to="$threadspertask" />
        <sub from="#EXECUTABLE#"        to="$executable" />
        <sub from="#ENV#"               to="$env" />
        <sub from="#PREPROCESS#"        to="" />
        <sub from="#POSTPROCESS#"       to="echo 'JuBE: $COMMENT'" />
        <sub from="#STARTER#"           to="poe" />
        <sub from="#ARGS_STARTER#"      to="" />
        <sub from="#MEASUREMENT#"       to="time" />
        <sub from="#ARGS_EXECUTABLE#"   to="" />
    </substitute>
  
    <environment>
        <env var = "MP_LABELIO"           value="yes" />
        <env var = "MP_INFOLEVEL"         value="2" />
        <env var = "MP_SHARED_MEMORY"     value="yes" />
        <env var = "MP_TASK_AFFINITY"     value="MCM" />
        <env var = "MEMORY_AFFINITY"      value="MCM" />
	<env var = "HPM_STDOUT"           value="0" />
	<env var = "HPM_UNIQUE_FILE_NAME" value="1" />
	<env var = "HPM_EVENT_SET"        value="$IHPCT_HWC_GRP" />
    </environment>
  
    <command>llsubmit ibm_llsubmit.job</command>
</execute>

<!-- ******************************************************************************** -->

<execute cname="Intel-Haswell-Cartesius">
    <input files="$pdir/Intel-Haswell-Cartesius/intel_SLURMsubmit.job.in" />

    <substitute infile="intel_SLURMsubmit.job.in" outfile="intel_SLURMsubmit.job">
        <sub from="#BENCHNAME#"         to="$benchname" />

        <sub from="#OUTDIR#"            to="$outdir" />
        <sub from="#STDOUTLOGFILE#"     to="$stdoutlogfile" />
        <sub from="#STDERRLOGFILE#"     to="$stderrlogfile" />

        <sub from="#TIME_LIMIT#"        to="00:10:00" />

        <sub from="#PARTITION#"         to="short" />
        <sub from="#CONSTRAINT#"        to="haswell" />

        <sub from="#TASKS#"             to="$tasks" />
        <sub from="#TASKSPERNODE#"      to="$taskspernode" />
        <sub from="#NODES#"             to="$nodes" />
        <sub from="#THREADSPERTASK#"    to="$threadspertask" />

        <sub from="#EXECUTABLE#"        to="${outdir}/qcd-submit.exe.tmp" />
        <sub from="#ENV#"               to="$env" />
        <sub from="#PREPROCESS#"        to="cp $executable ${outdir}/qcd-submit.exe.tmp" />
        <sub from="#POSTPROCESS#"       to="" />
        <sub from="#STARTER#"           to="srun" />
        <sub from="#ARGS_STARTER#"      to="" />
        <sub from="#MEASUREMENT#"       to="time" />
        <sub from="#ARGS_EXECUTABLE#"   to="" />
    </substitute>
  
    <environment>
    </environment>
  
    <command>sbatch intel_SLURMsubmit.job</command>
</execute>

<!-- ******************************************************************************** -->

<execute cname="Intel-Broadwell-Marconi">
    <input files="$pdir/Intel-Broadwell-Marconi/intel_PBSsubmit.job.in" />

    <substitute infile="intel_PBSsubmit.job.in" outfile="intel_PBSsubmit.job">
        <sub from="#BENCHNAME#"         to="$benchname" />

        <sub from="#OUTDIR#"            to="$outdir" />
        <sub from="#STDOUTLOGFILE#"     to="$stdoutlogfile" />
        <sub from="#STDERRLOGFILE#"     to="$stderrlogfile" />

        <sub from="#TIME_LIMIT#"        to="00:10:00" />

        <sub from="#PARTITION#"         to="short" />
        <sub from="#CONSTRAINT#"        to="broadwell" />

        <sub from="#TASKS#"             to="$tasks" />
        <sub from="#TASKSPERNODE#"      to="$taskspernode" />
        <sub from="#NODES#"             to="$nodes" />
        <sub from="#THREADSPERTASK#"    to="$threadspertask" />

        <sub from="#EXECUTABLE#"        to="${outdir}/qcd-submit.exe.tmp" />
        <sub from="#ENV#"               to="$env" />
        <sub from="#PREPROCESS#"        to="cp $executable ${outdir}/qcd-submit.exe.tmp" />
        <sub from="#POSTPROCESS#"       to="" />
        <sub from="#STARTER#"           to="mpirun" />
        <sub from="#ARGS_STARTER#"      to="" />
        <sub from="#MEASUREMENT#"       to="time" />
        <sub from="#ARGS_EXECUTABLE#"   to="" />
    </substitute>
  
    <environment>
    </environment>
  
    <command>qsub intel_PBSsubmit.job</command>
</execute>

</execution>



