#!/usr/bin/perl -w
#
#####################################################################################
#                                                                                   #
#        JuBE: Juelich Benchmarking Environment                                     #
#                                                                                   #
#####################################################################################


#   Copyright (C) 2008, Forschungszentrum Juelich GmbH, Federal Republic of
#   Germany. All rights reserved.
#
#   Redistribution and use in source and binary forms, with or without
#   modification, are permitted provided that the following conditions are met:
#
#   Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
#
#     - Redistributions of source code must retain the above copyright notice,
#       this list of conditions and the following disclaimer.
#
#     - Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#
#     - Any publications that result from the use of this software shall
#       reasonably refer to the Research Centre's development.
#
#     - All advertising materials mentioning features or use of this software
#       must display the following acknowledgement:
#
#           This product includes software developed by Forschungszentrum
#           Juelich GmbH, Federal Republic of Germany.
#
#     - Forschungszentrum Juelich GmbH is not obligated to provide the user with
#       any support, consulting, training or assistance of any kind with regard
#       to the use, operation and performance of this software or to provide
#       the user with any updates, revisions or new versions.
#
#
#   THIS SOFTWARE IS PROVIED BY FORSCHUNGSZENTRUM JUELICH GMBH "AS IS" AND ANY
#   EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
#   WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
#   DISCLAIMED. IN NO EVENT SHALL FORSCHUNGSZENTRUM JUELICH GMBH BE LIABLE FOR
#   ANY SPECIAL, DIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
#   RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
#   CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
#   CONNECTION WITH THE ACCESS, USE OR PERFORMANCE OF THIS SOFTWARE.


use strict;
use Carp;

# get installation path of jube perl script
use FindBin;
my $instpath="$FindBin::RealBin";

my $patint="([\\+\\-\\d]+)";    # Pattern for Integer number
my $patfp ="([\\+\\-\\d.Ee]+)"; # Pattern for Floating Point number
my $patwrd="([\^\\s]+)";        # Pattern for Work (all noblank characters)
my $patnint="[\\+\\-\\d]+";     # Pattern for Integer number, no () 
my $patnfp ="[\\+\\-\\d.Ee]+";  # Pattern for Floating Point number, no () 
my $patnwrd="[\^\\s]+";         # Pattern for Work (all noblank characters), no () 
my $patbl ="\\s+";              # Pattern for blank space (variable length)

my @columns;

use Getopt::Long qw(:config no_ignore_case);

my $pwd=`pwd`;
chomp($pwd);

my $opt_verbose=0;
my $opt_dump=0;
my $opt_debug=undef;
my $opt_start=undef;
my $opt_update=undef;
my $opt_result=undef;
my $opt_force=undef;
my $opt_rmtmp=undef;
my $opt_tmpdir=undef;
my $opt_configdir="$pwd";
my $opt_platformsdir="$instpath/../platform";
my $opt_result_showall=0;
my $opt_cmpdir=undef;
my $opt_version=undef;
my $SUBSTITUTE_NOTFOUND="";
my $colw_default=10;

# if set, indicates that job stdout/err should be keep in 
# tmpdir (will be copied to logs dir when calling jube with -update)
my $defer_stdout_to_tmpdir=0;

usage($0) if( ! GetOptions( 
			    'verbose=i'        => \$opt_verbose,
			    'dump'             => \$opt_dump,
			    'debug'            => \$opt_debug,
			    'update'           => \$opt_update,
			    'result'           => \$opt_result,
			    'start|submit'     => \$opt_start,
			    'force'            => \$opt_force,
			    'cdir=s'           => \$opt_configdir,
			    'pdir=s'           => \$opt_platformsdir,
			    'tmpdir=s'         => \$opt_tmpdir,
			    'showall'          => \$opt_result_showall,
			    'rmtmp'            => \$opt_rmtmp,
			    'cmpdir=s'         => \$opt_cmpdir,
			    'Version|V'        => \$opt_version
			    ) );

 
&version() if ($opt_version);

use FindBin;
use lib "$FindBin::RealBin/lib";

use XML::Simple;
use Data::Dumper;
use File::Listing;
use Time::HiRes qw ( time );

my $idfile="./.bench_current_id.dat";


my $startdate=localtime(time());


my $logdir="$pwd/xmllogs";
if(! -d $logdir) {
    mkdir $logdir;
}

my $stdlogdir="$pwd/logs";
if(! -d $stdlogdir) {
    mkdir $stdlogdir;
}
my $benchlogdir="$pwd/benchlog";
if(! -d $benchlogdir) {
    mkdir $benchlogdir;
}
my $tmpdir="$pwd/tmp";
if($opt_tmpdir) {
    $tmpdir=$opt_tmpdir;
    $defer_stdout_to_tmpdir=1;
}
if(! -d $tmpdir) {
    mkdir $tmpdir;
}
my $tmplogdir="$tmpdir/logs";
if($defer_stdout_to_tmpdir) {
    if(! -d $tmplogdir) {
	mkdir $tmplogdir;
    }
}

my $rundir="$pwd/run";
if(! -d $rundir) {
    mkdir $rundir;
}
my $resdir="$pwd/results";
if(! -d $resdir) {
    mkdir $resdir;
}


my $logfilelevel=0;
my $loglinelength=140;
my $compilexmlfile="$opt_configdir/compile.xml";
my $preparexmlfile="$opt_configdir/prepare.xml";
my $executexmlfile="$opt_configdir/execute.xml";
my $verifyxmlfile="$opt_configdir/verify.xml";
my $analysexmlfile="$opt_configdir/analyse.xml";
my $resultxmlfile="$opt_configdir/result.xml";
my $platformxmlfile="$opt_platformsdir/platform.xml";

my ($benchxmlfile,$benchlogfile,$startspec,$enddate);
my(@attrsortlist); # sort order for result lists

my(%generatedexecutables);

my $tstart=time;
my $xs=XML::Simple->new();
my $tdiff=time-$tstart;

if((!$opt_update) && (!$opt_result)) {
    $opt_start=1;
}

if($opt_start) {
    # submit new benchmarks

    if(!$ARGV[0]) {
	if(-f "$opt_configdir/bench.xml") {
	    $benchxmlfile="$opt_configdir/bench.xml";
	} elsif(-f "$opt_configdir/bench1.xml") {
	    $benchxmlfile="$opt_configdir/bench1.xml";
	} else {
	    usage($0);
	}
    } else {
	$benchxmlfile=$ARGV[0];
    }
    $benchlogfile=sprintf("%s/benchlog_%06d.log",$benchlogdir,&get_identifier_ro());
    open(BENCHLOG,"> $benchlogfile");
} 
if($opt_update) {
    # update logfiles (verifying and analyse)
    $benchlogfile=sprintf("%s/benchupdatelog_%06d.log",$benchlogdir,&get_identifier_ro());
    open(BENCHLOG,"> $benchlogfile");
    $startspec="0+";
    $startspec = $ARGV[0] if($ARGV[0]);
    if($startspec=~/last(\-\d+)?/) {
	my $id=&get_identifier_ro();
	my $offset=-0;
	$offset=$1 if($1);
	if($id>0) { $startspec=sprintf("%d",$id-1+$offset);} 
	else      { $startspec="0+";	}
    }
    
    printlog(0," Looking for new log files with spec \#%s\n",$startspec);
}

if($opt_result) {
    # show results from logfiles
    $benchlogfile=sprintf("%s/benchresultlog_%06d.log",$benchlogdir,&get_identifier_ro());
    open(BENCHLOG,"> $benchlogfile");
    $startspec="0+";
    $startspec = $ARGV[0] if($ARGV[0]);
    if($startspec=~/last(\-\d+)?/) {
	my $id=&get_identifier_ro();
	my $offset=-0;
	$offset=$1 if($1);
	if($id>0) { $startspec=sprintf("%d",$id-1+$offset);} 
	else      { $startspec="0+";	}
    }
    printlog(0," Looking for log files with spec \#%s\n",$startspec);
}


printlog(0,"%s\n","-"x80); 
printlog(0," Benchmark-Suite: starting at %s\n",$startdate);
printlog(0,"%s\n","-"x80); 
printlog(0," %s\n", &getversion());
printlog(0,"%s\n","-"x80);
printlog(0," OPTIONS: %-25s = %d \n","start",  $opt_start) if ($opt_start);
printlog(0," OPTIONS: %-25s = %d \n","update", $opt_update) if ($opt_update);
printlog(0," OPTIONS: %-25s = %d \n","result", $opt_result) if ($opt_result);
printlog(0,"%s\n","-"x80); 
printlog(0," OPTIONS: %-25s = %d \n","force update", $opt_force) if ($opt_force);
printlog(0," OPTIONS: %-25s = %d \n","verbose",$opt_verbose) if (defined($opt_verbose));
printlog(0," OPTIONS: %-25s = %d \n","dump",   $opt_dump) if ($opt_dump);
printlog(0," OPTIONS: %-25s = %s \n","Benchmark XML-file",$benchxmlfile) if ($benchxmlfile);
printlog(0," OPTIONS: %-25s = %s \n","Compile   XML-file",$compilexmlfile);
printlog(0," OPTIONS: %-25s = %-25s\n","benchlogfile",$benchlogfile);
printlog(0," OPTIONS: %-25s = %-25s\n","configdir",$opt_configdir);
printlog(0," OPTIONS: %-25s = %-25s\n","platformdir",$opt_platformsdir);
printlog(0," OPTIONS: %-25s = %-25s\n","tmpdir",$opt_tmpdir) if ($opt_tmpdir);
printlog(0," OPTIONS: %-25s = %-25s\n","cmpdir",$opt_cmpdir) if ($opt_cmpdir);
printlog(0,"%s\n","-"x80); 


&benchmark($benchxmlfile) if ($opt_start);
&check_defered_logdir() if(($defer_stdout_to_tmpdir) && ($opt_update));
&update()    if ($opt_update);
&result()    if ($opt_result);

$enddate=localtime(time());
printlog(0,"%s\n","-"x80); 
printlog(0," Benchmark-Suite: ending at %s\n",$enddate);
printlog(0,"%s\n","-"x80); 

close(BENCHLOG);

sub benchmark {
    my($benchxmlfile)=@_;
    my($tstart,$tdiff,$benchref,$compileref,$prepareref,$executeref,$platformtopref,$platformref);
    my($first_id,$last_id,$benchname,$platform,$i,$bench,$dir,$executable,$cproto,$bc,$taskref);
    my($nodes,$taskspernode,$threadspertask,$param,$c,$t,$p,@param_ptr,$iter,$subid,$subdir);
    my($protofile,$rc,$done,$cmd,$key,$platformparamsref,$val);


    printlog(0, "--->\tprocessing %s ...\n", $benchxmlfile);
    $tstart=time;
    $benchref=$xs->XMLin($benchxmlfile, KeyAttr => { 'map' => "n",
						     'benchmark' => "+name"
						     },
			 ForceArray => 1);
    $tdiff=time-$tstart;
    printlog(3,"parsing $benchxmlfile in %6.4f sec\n",$tdiff);
    
    

    printlog(0, "--->\tprocessing %s ...\n", $compilexmlfile);
    $tstart=time;
    $compileref=$xs->XMLin($compilexmlfile, KeyAttr => { compile => "+cname", 
							 substitute => "+infile",
							 'sub' => "+from"
							 },
			   ForceArray => 1);
    $tdiff=time-$tstart;
    printlog(3,"parsing $compilexmlfile in %6.4f sec\n",$tdiff);
    
    printlog(0, "--->\tprocessing %s ...\n", $preparexmlfile);
    $tstart=time;
    $prepareref=$xs->XMLin($preparexmlfile, KeyAttr => { prepare => "+cname", 
						     substitute => "+infile",
							 'sub' => "+from",
							 'mkdir' => "+directory",
						     },
			   ForceArray => 1);
    $tdiff=time-$tstart;
    printlog(3,"parsing $preparexmlfile in %6.4f sec\n",$tdiff);
    
    printlog(0, "--->\tprocessing %s ...\n", $executexmlfile);
    $tstart=time;
    $executeref=$xs->XMLin($executexmlfile, KeyAttr => { execute => "+cname", 
							 substitute => "+infile",
							 'sub' => "+from",
							 'env' => '+var',
							 'mkdir' => "+directory",
						     },
			   ForceArray => 1);
    $tdiff=time-$tstart;
    printlog(3,"parsing $executexmlfile in %6.4f sec\n",$tdiff);
    
    $first_id=-1;
    $last_id=-1;
    
    if($benchref) {
 
	if($opt_dump) {
	    printlog(-1,"%s",Dumper($benchref));
	    exit(1);
	}
    
	$benchname= $benchref->{'name'};
	$platform = $benchref->{'platform'};


	# scan platform.xml for coressponding entry
	printlog(0, "--->\tprocessing %s ...\n", $platformxmlfile);
	$tstart=time;
	$platformtopref=$xs->XMLin($platformxmlfile, KeyAttr => { platform => "+name"
							     },
			       ForceArray => 1);
	$tdiff=time-$tstart;
	printlog(3,"parsing $platformxmlfile in %6.4f sec\n",$tdiff);

	
	printlog(4,"platform=>%s<\n",$platform,);
	if($platformtopref->{'platform'}->{$platform}) {
	    $platformref=$platformtopref->{'platform'}->{$platform};
#	    printlog(-1,"%s",Dumper($platformref));
	} else {
	    printlog(0,"No platform description found for %s in %s\n",$platform,$platformxmlfile);
	    $platformref=undef;
	}

        # Evaluate expressions in platform.xml

        $platformparamsref=$platformref->{'params'}->[0];
	foreach $key (keys(%{$platformparamsref})) {
            $val=$platformparamsref->{$key};
	    $rc=&substitute(\$val,$platformparamsref);
            $platformparamsref->{$key}=$val;
        }

	printlog(0,"scanning benchmarks for $benchname on $platform: \n","");
	$i=0;
	foreach $bench (keys(%{$benchref->{'benchmark'}})) {
	    my $active=$benchref->{'benchmark'}->{$bench}->{'active'};
	    next if(!$active);
	    my $cname =    $benchref->{'benchmark'}->{$bench}->{'compile'}->[0]->{'cname'};
	    my $cname_expand=$cname; 
	    my $cversion = $benchref->{'benchmark'}->{$bench}->{'compile'}->[0]->{'version'};
	    my $params=    $benchref->{'benchmark'}->{$bench}->{'params'}->[0];
	    my $prep=      $benchref->{'benchmark'}->{$bench}->{'prepare'}->[0];
	    my $execu=     $benchref->{'benchmark'}->{$bench}->{'execution'}->[0];
	    my $analyse=   $benchref->{'benchmark'}->{$bench}->{'analyse'}->[0];
	    my $verify=    $benchref->{'benchmark'}->{$bench}->{'verify'}->[0];
	    my $iteration= $benchref->{'benchmark'}->{$bench}->{'execution'}->[0]->{'iteration'};
	    my $addopt;
	    my $lastcommand=""; # will be executed after last benchmark, e.g. for job chains

	    if(exists($benchref->{'benchmark'}->{$bench}->{'execution'}->[0]->{'addopt'})) {
		$addopt =    $benchref->{'benchmark'}->{$bench}->{'execution'}->[0]->{'addopt'};
	    } else {
		$addopt = "";
	    }

	    my $id=&get_identifier(); $last_id=$id;	$first_id=$id if($first_id==-1);
	    $cversion="new" if(!$cversion);
	    
	    my $identifier=sprintf("%s_%s_%s_i%06d",$benchname,$platform,$bench,$id);
	    printlog(0,"\t %02d: %-20s cname=%-10s (%s) -> Identifier=%s\n",++$i,$bench,$cname,$platform,$identifier);
	    
	    $dir="$tmpdir/$identifier";
	    printlog(0,"\t\t\t\t  -> generating temporary directory %s\n",$dir);
	    mkdir $dir;
	    if(! -d $dir) {
		printlog(-1,"... failed to create directory $dir\n");
		exit(-1);
	    }

# if -cmpdir is set the compile step has to be performed in the applied compile directory
# this causes a redefinition of $dir

	    my $cmpdir;
	    if($opt_cmpdir){
		$cmpdir="$opt_cmpdir/$identifier";
		printlog(0,"\t\t\t\t  -> generating temporary directory %s\n",$cmpdir);
		system "mkdir -p $cmpdir";
		if(! -d $cmpdir) {
		    printlog(-1,"... failed to create directory $cmpdir\n");
		    exit(-1);
		}
	    } else {
		$cmpdir=$dir;
	    }




	    printlog(0,"\t\t\t\t  -> generating run step %s\n",$cname);
	    $bc=0;
	    
	    my $aref;
	    if(ref($benchref->{'benchmark'}->{$bench}->{'tasks'}) eq "ARRAY") {
		$aref=$benchref->{'benchmark'}->{$bench}->{'tasks'};
	    } else {
		$aref=[$benchref->{'benchmark'}->{$bench}->{'tasks'}];
	    }
	    foreach $taskref (@{$aref}) {
		my($threadspertaskspec,$taskspernodespec,$nodesspec)=(1,1,1);
		$threadspertaskspec=$taskref->{threadspertask} if ($taskref->{threadspertask});
		$taskspernodespec=  $taskref->{taskspernode}   if ($taskref->{taskspernode});
		$nodesspec=         $taskref->{nodes}          if ($taskref->{nodes});
				
		my $bproto="  <benchmark";
		$bproto.=" name=\"$bench\"\n";
		$bproto.="     compile_cname=\"$cname\"\n";
		$bproto.="     compile_version=\"$cversion\"\n";
		$bproto.="     prepare_cname=\"".$prep->{'cname'}."\"\n";
		$bproto.="     execute_cname=\"".$execu->{'cname'}."\"\n";
		$bproto.="     verify_cname=\"".$verify->{'cname'}."\"\n";
		$bproto.="     analyse_cname=\"".$analyse->{'cname'}."\"\n";
		foreach $nodes (&getsequence($nodesspec)) {
		    foreach $taskspernode (&getsequence($taskspernodespec)) {
			foreach $threadspertask (&getsequence($threadspertaskspec)) {
			    my (%paramlist,@param_name,@param_cnt,@param_pt,$numparam,$numtests,%phash);
			    printlog(0,"\t\t\t\t\t%-2d: %d nodes %d tasks %d threads\n",++$bc,
				     $nodes,$taskspernode,$threadspertask);

			    $phash{nodes}=$nodes;		    $phash{taskspernode}=$taskspernode;
			    $phash{threadspertask}=$threadspertask; $phash{tasks}=$taskspernode*$nodes;
			    $phash{ncpus}=$threadspertask*$taskspernode*$nodes;

			    # run twice over parms for substituting scalar parms
			    foreach my $step (1,2) {
				foreach $param (keys(%$params)) {
				    my$val=$params->{$param};
				    &substitute(\$val,\%phash);
				    $paramlist{$param}=[&getsequence($val,$benchref->{'benchmark'}->{$bench})];
				    if(scalar @{$paramlist{$param}} == 1) {
					$phash{$param}=$paramlist{$param}->[0];
				    }
				}
			    }

			    $c=0;
			    $numtests=1;
			    foreach $param ( sort { $#{$paramlist{$a}} <=> $#{$paramlist{$b}} } (keys(%paramlist))) {
				$param_name[$c]=$param;
				$param_cnt[$c]=$#{$paramlist{$param}}+1;
				$numtests*=$param_cnt[$c];
				$param_ptr[$c]=0;
#				print "debug: $c-> $param_name[$c] $param_cnt[$c]\n";
				$c++;			  
			    }
			    $numparam=$c;
			    
			    for($t=1;$t<=$numtests;$t++) { 
				my $str="";
				my %parmhash;
				my $tproto=$bproto;
				
				for($p=0;$p<$numparam;$p++) { 
				    $str.=sprintf("[%s->%s]",$param_name[$p],$paramlist{$param_name[$p]}->[$param_ptr[$p]]);
				    if($param_name[$p]=~/\-/) {
					#field of parms
					my @plist=split(/\-/,$param_name[$p]);
					my @vlist=split(/\:/,$paramlist{$param_name[$p]}->[$param_ptr[$p]]);
					for(my $j=0;$j<=$#plist;$j++) {
					    $parmhash{$plist[$j]}=$vlist[$j];
					}
				    } else {
					# scalar
					$parmhash{$param_name[$p]}=$paramlist{$param_name[$p]}->[$param_ptr[$p]];
				    }
				    $tproto.="     ".$param_name[$p]."=\"".$paramlist{$param_name[$p]}->[$param_ptr[$p]]."\"\n";
				}
				printlog(0,"\t\t\t\t\t   %2d: %s\n",$t,$str);
				$parmhash{platform}=$platform;
				$parmhash{pdir}=$opt_platformsdir;
				$parmhash{benchname}=$benchname;
				$parmhash{benchhome}=$pwd;
				$parmhash{name}=$bench;
				$parmhash{nodes}=$nodes;
				$parmhash{taskspernode}=$taskspernode;
				$parmhash{threadspertask}=$threadspertask;
				$parmhash{addopt}=$addopt;
				foreach $param (keys(%phash)) {
				    $parmhash{$param}=$phash{$param};
				}
				$tproto.="     nodes=\"".$nodes."\"\n";
				$tproto.="     taskspernode=\"".$taskspernode."\"\n";
				$tproto.="     threadspertask=\"".$threadspertask."\"\n";
				
				substitute(\$cname_expand,\%parmhash);

				$iteration=1 if(!$iteration);
				for($iter=1;$iter<=$iteration;$iter++) {
				    my $proto=$tproto;
				    
				    $subid=sprintf("n%dp%dt%d_t%03d_i%02d",$nodes,$taskspernode,$threadspertask,$t,$iter);


				    # $cmpsubdir holds the direction for the compile step if -cmpdir is set
				    my $cmpsubdir=$cmpdir."/".$subid;

				    $subdir=$dir."/".$subid;
				    $parmhash{id}=$identifier;
				    $parmhash{subid}=$subid;
				    $parmhash{rundir}=$subdir;
				    $parmhash{subdir}=$subdir;
				    printlog(1,"\t\t\t\t\t\t  -> generating temporary directory %s\n",$subdir);
				    mkdir $subdir;
				    if(! -d $subdir) {
					printlog(-1,"... failed to create directory $subdir\n");
					exit(-1);
				    }
				    if($subdir ne $cmpsubdir) {
					mkdir $cmpsubdir;
					if(! -d $cmpsubdir) {
					    printlog(-1,"... failed to create directory $cmpsubdir\n");
					    exit(-1);
					}
				    }
				    $proto.="     iteration=\"".$iter."\"\n";
				    $proto.="     subdir=\"".$subdir."\"\n";
				    $proto.="     identifier=\"".$identifier."\"\n";
				    $proto.="     subid=\"".$subid."\"\n";
				    
				    $protofile=sprintf("%s/benchlog_%s_%s.log",$logdir,$identifier,$subid);

				    printlog(0,"\t\t\t\t\t\t  -> compile step %s (%s)\n",$cname,$cname_expand);
				    
				    $executable=&compile($compileref,$subdir,$cmpsubdir,$identifier,$cname,$cversion,\$cproto,
							 $benchref->{'benchmark'}->{$bench}->{'compile'}->[0],
							 $platformref,\%parmhash);
#FJ
				    if(! $executable) {
					printlog(-1,"... failed to create executable %s\n",$cname);
					exit(-1);
				    }
			    
				    $rc=&pproto_open($protofile,$benchref,$bench,$startdate);

                                    # Write platform information to xmllog
                                    my $platform_proto="  <platform>\n";
                                    $platform_proto.="    <params\n"; 
                                    foreach my $platform_key (keys(%{$platformparamsref})) {
                                      $platform_proto.="        $platform_key=\"$platformparamsref->{$platform_key}\"\n";
                                    }
                                    $platform_proto.="    />\n"; 
                                    $platform_proto.="  </platform>\n";
                                    $rc=&pproto($protofile,$platform_proto);

                                    # Write compile information to xmllog
				    $rc=&pproto($protofile,$cproto);

                                    # Write prepare information to xmllog
				    $proto.="     />\n";
				    $rc=&pproto($protofile,$proto);

				    printlog(0,"\t\t\t\t\t\t  -> prepare step %s (%s)\n",$prep->{'cname'},$platform);
				    $rc=&prepare($prepareref,$subdir,$identifier,$prep->{'cname'},\%parmhash);
				    printlog(0,"\t\t\t\t\t\t  -> execute step %s (%s)\n",$execu->{'cname'},$platform);
				  
				    $rc=&execute($executeref,$subdir,$identifier,$subid,$executable,$execu,\%parmhash,\$lastcommand);
				    
				    $rc=&pproto_close($protofile,$benchref->{'benchmark'}->{$bench});
				    
				    
				}
				
				# search next parameter set
				if($numtests>1) {
				    $done=0;
				    $p=$numparam-1;
				    while(!$done) {
					$param_ptr[$p]++;
					if($param_ptr[$p]<$param_cnt[$p]) {
					    $done=1;
					} else {
					    $param_ptr[$p]=0;
					    $p--;
					    $done=1 if($p<0); # only for fun
					}
				    }
				}
			    }
			    
			}
		    }
		}
		printlog(0,"\t\t\t\t\t\n","");
	    }

	    # execute lastcommand
	    if($lastcommand ne "") {
		printlog(0,"\t\t\t\t\t-> last command: %s %s\n",($opt_debug)?"[debug]":"",$lastcommand);
		system($lastcommand) if (!$opt_debug);
		if($?) { printlog(-1,"... failed to execute %s\n",$lastcommand);  return (-1);}
	    }
	    
	    if($opt_rmtmp) {
		$cmd="rm -r $dir";
		printlog(0,"\t\t\t\t\texecuting: %s\n",$cmd);
		system($cmd);
		if($?) { printlog(-1,"... failed to execute \n",$?);  exit(-1);}
	    }
	    
	}
    } else {
	printlog(-1,"Error while processing XML file, exiting ...\n","");
    }

    printlog(0,"%s\n","-"x80); 
    printlog(0," JUBE: used id:                                %s\n",$last_id) if($first_id == $last_id); 
    printlog(0," JUBE: used id range:                      from %s_id to %s\n",$first_id,$last_id) if($first_id != $last_id); 
 
}


sub check_defered_logdir {
    my($rc,$fn,$id,$name,$destname,$cmd);
    $rc=opendir(DIR,$tmplogdir);
    while($fn=readdir(DIR)) {
	next if($fn!~/\.log$/);
	if($fn=~/.*_i$patint.*_i$patint\_/) {
	    $name=$tmplogdir."/".$fn;
	    $destname=$stdlogdir."/".$fn;
	    my($id)=($1);
	    if(&testspec($id,$startspec)) {
		printlog(0,"%s\n","-"x80); 
		printlog(0," JUBE: check log file in tmpdir: %s<=>%s \n",$name,$destname); 
		if(! -f $destname) {
		    printlog(0," JUBE: new file: cp -p %s %s \n",$name,$destname); 
		    $cmd="cp -p ${name} ${destname}";
		    printlog(3,"\texecuting: %s\n",$cmd);
		    system($cmd);
		    if($?) { printlog(-1,"... failed to execute %s\n",$?);  exit(-1);}

		} else {
		    my($dev1,$ino1,$mode1,$nlink1,$uid1,$gid1,$rdev1,$size1,
		       $atime1,$mtime1,$ctime1,$blksize1,$blocks1) = stat($name);
		    my($dev2,$ino2,$mode2,$nlink2,$uid2,$gid2,$rdev2,$size2,
		       $atime2,$mtime2,$ctime2,$blksize2,$blocks2) = stat($destname);

		    if( ($size1 != $size2) || ($mtime1 != $mtime2)) {
			printlog(0," JUBE: newer file: cp -p %s %s \n",$name,$destname); 
			$cmd="cp -p ${name} ${destname}";
			printlog(3,"\texecuting: %s\n",$cmd);
			system($cmd);
		    }
		}
	    }
	}
    }
}

sub update {
    my($entry,$logref);
    my($analyseref,$bench,$nextstep,$jobstartfile,$bstdout,$bstderr,$inpsep,$stdoutdata,$stderrdata);
    my($jobendfile,$verifyref,$verifyfile,$analysefile,$cname,$cmd,$rc,$fn,$name,$endmarkfound,$line);
    my($includeref,$includepattern,$file,$parm);
    my($precommand);

    printlog(0, "--->\tprocessing %s ...\n", $analysexmlfile);
    $tstart=time;
    $analyseref=$xs->XMLin($analysexmlfile, KeyAttr => { analyse => "+cname",
							 parm => "+name", 
							 includepattern => "+file" 
							 },
			   ForceArray => 1);
    $tdiff=time-$tstart;
    printlog(3,"parsing $analysexmlfile in %6.4f sec\n",$tdiff);

    printlog(0, "--->\tprocessing %s ...\n", $verifyxmlfile);
    $tstart=time;
    $verifyref=$xs->XMLin($verifyxmlfile, KeyAttr => {  verify => "+cname"},
			  ForceArray => 1);
    $tdiff=time-$tstart;
    printlog(3,"parsing $verifyxmlfile in %6.4f sec\n",$tdiff);

    foreach $cname (sort(keys(%{$analyseref->{analyse}}))) {
	foreach $includepattern (sort(keys(%{$analyseref->{analyse}->{$cname}->{includepattern}}))) {
	    $file=$includepattern;
	    printlog(3,"analysis: include patterns from %s\n",$file);
	    $tstart=time;
	    $includeref=$xs->XMLin($file, KeyAttr => { parm => "+name"},
				   ForceArray => 1);
	    $tdiff=time-$tstart;
	    printlog(3,"parsing includefile $file in %6.4f sec\n",$tdiff);
#	    printlog(-1,"%s",Dumper($includeref));
	    foreach $parm (keys(%{$includeref->{parm}})) {
		$analyseref->{analyse}->{$cname}->{parm}->{$parm}=$includeref->{parm}->{$parm};
	    }
	}
    }

# test if jube_report.xsl and style.css are put in xmllogs/
{
    my $xsl_file = "jube_report.xsl";
    my $css = "style.css";
    my $cmd1 = "cp $instpath/$xsl_file $logdir";
    my $cmd2 = "cp $instpath/$css $logdir";

    system($cmd1) unless -e "$logdir/$xsl_file";
    system($cmd2) unless -e "$logdir/$css";
}


#    foreach $entry (File::Listing::parse_dir(`ls -lrt $logdir/*.log`)) {
#	my($name,$type,$size,$mtime,$mode)=@$entry;

    $rc=opendir(DIR,$logdir);
    while($fn=readdir(DIR)) {
	next if($fn!~/\.log$/);
	if($fn=~/benchlog.*i$patint\_/) {
	    $name=$logdir."/".$fn;
	    my($id)=($1);
	    if(&testspec($id,$startspec)) {
		printlog(0,"%s\n","-"x80); 
		printlog(0," JUBE: update, parsing: %s \n",$name); 

                # check if xml log is complete
		$endmarkfound=0;
		open(IN,$name);
		while($line=<IN>) {
		    $endmarkfound=1 if($line=~/<\/benchrun>/);
		}
		close(IN);
		next if(!$endmarkfound);

                # read xml log file
		$tstart=time;
		$logref=$xs->XMLin($name, KeyAttr => { benchmark => "+name" },
				   ForceArray => 1);
#		printlog(-1,"%s",Dumper($logref));
		# bug fix
		if(exists($logref->{benchmark}->{""})) {
		    delete($logref->{benchmark}->{""});
		}
		$bench=(keys(%{$logref->{benchmark}}))[0];
		$tdiff=time-$tstart;
		printlog(2,"\t\tparsing $name in %6.4f sec bench=%s\n",$tdiff,$bench);
#		printlog(-1,"%s",Dumper($logref));

                # some fixes
		# ncpus is not stored in result XML file
		$logref->{benchmark}->{$bench}->{ncpus}=
		    $logref->{benchmark}->{$bench}->{threadspertask}
   		  * $logref->{benchmark}->{$bench}->{taskspernode}
		  * $logref->{benchmark}->{$bench}->{nodes};
#                printlog(-1,"%s",Dumper($logref));
                if(!exists($logref->{benchmark}->{$bench}->{platform})) {
                    $logref->{benchmark}->{$bench}->{platform}=$logref->{platform}->[0];
                }

		# testing job start
		$nextstep=1;
		$jobstartfile=$logref->{benchmark}->{$bench}->{subdir}."/start_info.xml";
#		print "debug: $bench $jobstartfile\n";
		if ((-f $jobstartfile) && ($nextstep)) {
		    my $startref=$xs->XMLin($jobstartfile, ForceArray => 1);
		    my $starttime=$startref->{at};
		    printlog(1,"\t\t job:     started at %s\n",$starttime);
		    $logref->{jobstartdate}=[$starttime];
		} else {$nextstep=0;}

                # test if start_info.xml exist
		if(!-f $jobstartfile) {
		    printlog(0,"%s doesn't exist\n",$jobstartfile);
		}

		# testing job end
		$jobendfile=$logref->{benchmark}->{$bench}->{subdir}."/end_info.xml";
		if ((-f $jobendfile) && ($nextstep)) {
		    my $endref=$xs->XMLin($jobendfile, ForceArray => 1);
		    my $endtime=$endref->{at};
		    printlog(1,"\t\t job:     ended  at %s\n",$endtime);
		    $logref->{jobenddate}=[$endtime];
		} else {$nextstep=0;}

                # test if start_info.xml exist
		if(!-f $jobendfile) {
		     printlog(0,"%s doesn't exist\n",$jobendfile);
		}

		# read stdout and stderr
		if($nextstep) {
		    $bstdout=sprintf("%s/%s.%s_stdout.log",$stdlogdir,
				     $logref->{benchmark}->{$bench}->{identifier},
				     $logref->{benchmark}->{$bench}->{subid});
		    $bstderr=sprintf("%s/%s.%s_stderr.log",$stdlogdir,
				     $logref->{benchmark}->{$bench}->{identifier},
				     $logref->{benchmark}->{$bench}->{subid});
		    
		    $inpsep=$/;$/=undef;
		    if(! open(IN,"$bstdout") ) { 
			$stdoutdata="";
			printlog(-1,"... failed to open stdout file %s\n",$bstdout);  
		    } else {
			$stdoutdata=<IN>;
			close(IN);
		    }
		    if(! open(IN,"$bstderr") ) { 
			$stderrdata="";
			printlog(-1," JUBE: no stderr file found or not possible to open. \n",$bstderr);  
		    } else {
			$stderrdata=<IN>;
			close(IN);
		    }
		    $/=$inpsep;

		    $logref->{stdoutfile}->[0]->{name}=$bstdout;
		    $logref->{stderrfile}->[0]->{name}=$bstderr;
		    printlog(1,"\t\t job:     stdout %d bytes\n",length($stdoutdata));
		    printlog(1,"\t\t job:     stderr %d bytes\n",length($stderrdata));

		}

		# call verify step, generates also verify.xml in subdir
		if($nextstep) {
		    $verifyfile=$logref->{benchmark}->{$bench}->{subdir}."/verify.xml";
		    if ( (! -f $verifyfile) || ($opt_force) ) {
			# call verify
			$cname=$logref->{benchmark}->{$bench}->{verify_cname};
			$cname=$logref->{benchmark}->{$bench}->{postp_cname} if(!$cname);
			&verify($logref->{benchmark}->{$bench}->{subdir},
				$bstdout,$bstderr,$cname,
				$verifyfile,$verifyref,$logref->{benchmark}->{$bench});
			printlog(1,"\t\t job:     verify done %s\n",$cname);
		    }
		    
		    # read verify results
		    if (-f $verifyfile) {
			# slurp verify data in
			my $vref=$xs->XMLin($verifyfile, KeyAttr => {parm => "+name"}, ForceArray => 1);
#			printlog(-1,"%s",Dumper($vref));

			$logref->{verify}=$vref;
			if($vref->{parm}->{vcheck}->{'value'}) { 
			    printlog(1,"\t\t job:     verify, bench check=%s\n",$vref->{parm}->{vcheck}->{'value'});
			} else {
			    printlog(1,"\t\t job:     verify, bench no results\n","");
			}
		    } else {$nextstep=0;}
		}
		

                # execute precommand
#                printlog(-1,"$cname %s",Dumper($analyseref));
                if(exists($analyseref->{analyse})) {
                    $cname=$logref->{benchmark}->{$bench}->{analyse_cname};
		    &substitute(\$cname,$logref->{benchmark}->{$bench});
                    $precommand  = $analyseref->{analyse}->{$cname}->{precommand}->[0];
                }
                if($precommand) {
                    my $command=$precommand;
                    my $subdir=$logref->{benchmark}->{$bench}->{subdir};
                    $rc=&substitute(\$command,$logref->{benchmark}->{$bench});
                    $rc=&substitute(\$command,$logref->{compile}->[0]->{params}->[0]);
                    printlog(3,"\t\t precommand     substitute param %s not found for cmd=>%s<\n",$SUBSTITUTE_NOTFOUND,$command,$rc) if($rc==-1);
                    printlog(3,"\t\t precommand     %s cmd=>%s< rc=%d\n",$cname,$command,$rc);
                    $cmd="(cd $subdir; $command 1>$subdir/precommand_out.log 2>$subdir/precommand_err.log)";
                    printlog(0,"\t\t\t\t\texecuting: %s\n",$cmd);
                    system($cmd);
#                    if($?) { printlog(-1,"... failed to execute precommand $?\n","");  return (undef);}
                }

		# analyse
		if($nextstep) {
		    $analysefile=$logref->{benchmark}->{$bench}->{subdir}."/analyse.xml";
		    if ( (! -f $analysefile) || ($opt_force) ) {
			# call analyse
			$cname=$logref->{benchmark}->{$bench}->{analyse_cname};
                        printlog(3,"\t\t analyse     cname=>%s<\n",$cname);
			&analyse(\$stdoutdata,\$stderrdata,$cname,$analysefile,$analyseref,$logref->{benchmark}->{$bench});
			printlog(1,"\t\t job:     analyse done\n","");
		    }
		    if (-f $analysefile) {
			# slurp analyse data in
			my $aref=$xs->XMLin($analysefile, ForceArray => 1);
			$logref->{analyse}=$aref;
			if($aref->{parm}->{walltime}->{'value'}) { 
			    printlog(1,"\t\t job:     analyse, bench runtime=%10.4f s\n",$aref->{parm}->{walltime}->{'value'});
			} else {
			    printlog(1,"\t\t job:     analyse, bench no results\n","");
			}
		    } else {$nextstep=0;}
		}
		

#		return();

		if(! open(OUT,"> ${name}.new") ) { 
		    printlog(-1,"... failed to open log file ${name}.new\n");  return (-1);
		}
		print OUT $xs->XMLout($logref, AttrIndent => 1, RootName => "benchrun" );
		close(OUT);
		if(0) {
		    $cmd="mv ${name} ${name}.old";
		    printlog(3,"\texecuting: %s\n",$cmd);
		    system($cmd);
		    if($?) { printlog(-1,"... failed to execute %s\n",$?);  exit(-1);}
		}
		$cmd="mv ${name}.new ${name}";
                printlog(3,"--->\tprocessing %s\n",$resultxmlfile);
		printlog(3,"\texecuting: %s\n",$cmd);
		system($cmd);
		if($?) { printlog(-1,"... failed to execute %s\n",$?);  exit(-1);}

		
		$logref->{stdoutfile}->[0]->{content}=$stdoutdata if($stdoutdata);
		$logref->{stderrfile}->[0]->{content}=$stderrdata if($stderrdata);

		my $longname=$name;
		if($longname=~s/\.log$/\.longlog/s) {
		    if(! open(OUT,"> ${longname}") ) { 
			printlog(-1,"... failed to open log file %s\n",$longname);  return (-1);
		    }
		 
		    my $header = '<?xml-stylesheet href="jube_report.xsl" type="text/xsl"?>';
		    print OUT $xs->XMLout($logref, AttrIndent => 1, RootName => "benchrun" , xmldecl => "$header", noescape => 1);
		    close(OUT);
		}


#		printlog(-1,"%s",Dumper($logref));
		
	    }

	    
	}
    }
}


sub result {
    my ($entry,$logref,$benchmref,$key,$bench);
    my($identifier,$subid,$result,%keylist,%keytype,$aref);
    my($resultref,$attrlist,@allattrsort,@allattr,@attr,$sortlist,$rc,$fn,$name,$endmarkfound,$line);
    my($compileparmref,$href,$resfile,$vref,$lcnt,$tabcnt,$tabcntsort,$tab,$showref,$sortref);

    my @colw;
    my $subw=18;
    my $digits=2;
    my @tabtitle;
    my @is_transposed;

    $tstart=time;
    $resultref=$xs->XMLin($resultxmlfile, KeyAttr => { },
			  ForceArray => 1,
			  ForceContent => 1);
    $tdiff=time-$tstart;
#    printlog(-1,"%s",Dumper($resultref));
    printlog(3,"parsing $resultxmlfile in %6.4f sec\n",$tdiff);
    $tabcnt=0;
    foreach $showref (@{$resultref->{'show'}}) {
	$attrlist=$showref->{'content'};
	$attrlist=~s/\s*//gs;
	if(exists($showref->{'active'})) {
	    next if($showref->{'active'} ne "1");
	}

	if(exists($showref->{'colw'})) {
	    $colw[$tabcnt] = $showref->{'colw'};
	    $subw = $colw[$tabcnt]+8;
	}

	if(exists($showref->{'digits'})) {
	    $digits = $showref->{'digits'};
	    $subw = $colw[$tabcnt]+8;
	}

	if(!exists($showref->{'colw'})) {
	   printlog(-1,"\nWARNING in result.xml: column width has to be defined ... !\n\n","");
	   printlog(-1, "Example: <show active=\"1\" colw=\"10\" > ... </show>\n\n","");
	   printlog(-1, "Using colw=\"$colw_default\" as default\n\n","");
	   $colw[$tabcnt] = $colw_default;
	   $subw = $colw[$tabcnt]+8;
	}

	if (exists $showref->{'title'}) {
	    $tabtitle[$tabcnt] = $showref->{'title'};
	}
	
	if (exists $showref->{'transpose'} && $showref->{'transpose'} eq "yes") {
	    $is_transposed[$tabcnt] = 1;
	}

	@{$attr[$tabcnt]}=(split(/,/,$attrlist)); 
        push(@allattr,split(/,/,$attrlist));
	$tabcnt++;
    }

    $tabcntsort=0;
    foreach $sortref (@{$resultref->{'sort'}}) {
	$sortlist=$sortref->{'content'};
	if(exists($sortref->{'active'})) {
	    next if($sortref->{'active'} ne "1");
	}
	$sortlist=~s/\s*//gs;
	@{$attrsortlist[$tabcntsort]}=(split(/,/,$sortlist));
        push(@allattrsort,split(/,/,$sortlist));
	$tabcntsort++;
    }
    
    if($tabcnt==0) {
	printlog(-1,"\nERROR in result: no show tab found, exiting ... !\n\n","");  return (-1);
    }

    if($tabcntsort==0) {
	printlog(-1,"\nERROR in result: no sort tab found, exiting ... !\n\n","");  return (-1);
    }

    if($tabcntsort < $tabcnt) {
	for($tab=$tabcntsort;$tab<$tabcnt;$tab++) {
	    $attrsortlist[$tab]=$attrsortlist[$tabcntsort-1];
	}
	$tabcntsort=$tabcnt;
    }

    if($tabcntsort != $tabcnt) {
	printlog(-1,"\nERROR in result: number of sort tabs and show tabs are different (%d != %d), exiting ... !\n\n",
		 $tabcnt,$tabcntsort);  return (-1);
    }

    $rc=opendir(DIR,$logdir);
    while($fn=readdir(DIR)) {
	next if($fn!~/\.log$/);
	$name=$logdir."/".$fn;
#    foreach $entry (File::Listing::parse_dir(`ls -l $logdir/*.log`)) {
#	my($name,$type,$size,$mtime,$mode)=@$entry;
	if($name=~/benchlog.*i$patint\_/) {
	    my($id)=($1);
	    if(&testspec($id,$startspec)) {
		printlog(0," JUBE: result,  parsing %s \n",$name); 
		open(IN,$name);
		$endmarkfound=0;
		while($line=<IN>) {
		    $endmarkfound=1 if($line=~/<\/benchrun>/);
		}
		close(IN);
		next if(!$endmarkfound);
		$tstart=time;
		$logref=$xs->XMLin($name, KeyAttr => { benchmark => "+name", values => "name" },
				   ForceArray => 1);
		$tdiff=time-$tstart;
		printlog(2,"\t\t parsing $name in %6.4f sec\n",$tdiff);
#		printlog(-1,"%s",Dumper($logref));
		# bug fix
		if(exists($logref->{benchmark}->{""})) {
		    delete($logref->{benchmark}->{""});
		}
		$bench=(keys(%{$logref->{benchmark}}))[0];
		$benchmref=$logref->{benchmark}->{$bench};
		$compileparmref=$logref->{compile}->[0]->{params}->[0];
		$identifier=$benchmref->{identifier};
		$subid=$benchmref->{subid};
		foreach $href ($benchmref,$compileparmref) {
		    foreach $key (keys(%$href)) {
			$result->{$identifier}->{$subid}->{$key}=$href->{$key};
			$keylist{$key}++;
			if(!exists($keytype{$key})) {
			    $keytype{$key}="string";
			    $keytype{$key}="float" if($result->{$identifier}->{$subid}->{$key}=~/^$patnfp$/);
			    $keytype{$key}="int"   if($result->{$identifier}->{$subid}->{$key}=~/^$patnint$/);
			}
		    }
		}

		foreach $key ("jobenddate") {
		    $result->{$identifier}->{$subid}->{$key}=$logref->{$key}->[0];
		    $keylist{$key}++;$keytype{$key}="string";
		}

	
		$aref=$logref->{analyse}->[0];
		foreach $key (keys(%$aref)) {
		    if(ref($aref->{$key})) {
#			printlog(-1,"%s",Dumper($aref->{$key}));
			if(exists($aref->{$key}->[0]->{value})) {
			    $result->{$identifier}->{$subid}->{$key}=$aref->{$key}->[0]->{value};
#			    print "debug: $key: $result->{$identifier}->{$subid}->{$key}\n";
			    $keylist{$key}++;
			    $keytype{$key}=$aref->{$key}->[0]->{type};
			} elsif(exists($aref->{$key}->[0]->{values})) {
			    $result->{$identifier}->{$subid}->{$key}=$aref->{$key}->[0]->{values};
			    $keylist{$key}++;
			    $keytype{$key}="index";
			}
		    }
		}

		$vref=$logref->{verify}->[0];
		foreach $key (keys(%$vref)) {
		    if(ref($vref->{$key})) {
#			printlog(-1,"%s",Dumper($aref->{$key}));
			if(exists($vref->{$key}->[0]->{value})) {
			    $result->{$identifier}->{$subid}->{$key}=$vref->{$key}->[0]->{value};
#			    print "debug: $key: $result->{$identifier}->{$subid}->{$key}\n";
			    $keylist{$key}++;
			    $keytype{$key}=$vref->{$key}->[0]->{type};
			} elsif(exists($vref->{$key}->[0]->{values})) {
			    $result->{$identifier}->{$subid}->{$key}=$vref->{$key}->[0]->{values};
			    $keylist{$key}++;
			    $keytype{$key}="index";
			}
		    }
		}

	    }
	}
    }

    foreach $key (@allattr,@allattrsort) {
	if(!defined($keytype{$key})) {
#	    print "debug: not found $key -> $result->{$identifier}->{$subid}->{$key}\n";
	    $keytype{$key}="int";
	}
    }

  
    foreach $identifier (sort {$a cmp $b} keys(%$result)) {
	$lcnt=0;
	printf("\n%-40s\n",$identifier);
	printf("%-40s\n","="x40);
	$resfile="$resdir"."/".$identifier.".dat";
	if(! open(DAT,"> $resfile") ) { 
	    printlog(-1,"... failed to open log file $resfile\n");  return (-1);
	}

	
	for($tab=0;$tab<$tabcnt;$tab++) {
	    
	    # header

	
            # Print title if it is given.
            if ($tabtitle[$tab]) {
                printf(" %s\n", $tabtitle[$tab]);
		printf(DAT " %s\n", $tabtitle[$tab]);
	    }
	    
	    if (not $is_transposed[$tab]) {
                #
                # Non-transposed output
		#

		printf("  %-${subw}s", "Subid");
		printf(DAT "#  %-${subw}s", "Subid");
		foreach $key (@{$attr[$tab]}) {
		    my $wkey=$key;
		    $wkey=~s/000000\b/M/s;
		    $wkey=~s/000\b/K/s;
		    
		    if(length($wkey)>$colw[$tab]) {
			printf(" %s%s",substr($wkey,0,$colw[$tab]-3),substr($wkey,-3));
			printf(DAT " %s%s",substr($wkey,0,$colw[$tab]-3),substr($wkey,-3));
		    } else {
			printf(" %$colw[$tab]s",$wkey);
			printf(DAT " %$colw[$tab]s",$wkey);
		    }
		}
		print "\n";
		print DAT "\n";
		printf("  %${subw}s", "-"x${subw});
		printf(DAT "#  %${subw}s", "-"x${subw});
		foreach $key (@{$attr[$tab]}) {
		printf(" %$colw[$tab]s","-"x$colw[$tab]);
		printf(DAT " %$colw[$tab]s","-"x$colw[$tab]);
		}
		print "\n";
		print DAT "\n";

		foreach $subid (sort {&attrsort($result->{$identifier},\%keytype,$attrsortlist[$tab],$a,$b) } 
				keys(%{$result->{$identifier}})) {
		    next if(!($result->{$identifier}));
		    next if(!exists($result->{$identifier}->{$subid}));
		    next if((!exists($result->{$identifier}->{$subid}->{'walltime'})) && (!$opt_result_showall));
		    if(($opt_result_showall) || ($result->{$identifier}->{$subid}->{'walltime'}>=0)) {
			$lcnt++;
			printf("  %${subw}s",$subid);
			printf(DAT "  %${subw}s",$subid);
			foreach $key (@{$attr[$tab]}) {
			    if($key=~/$patwrd\($patwrd\)/) {
				# index
				my($kkey,$kind)=($1,$2);
#		     print "debug: $key: ($kkey,$kind)\n";
				printf(" %$colw[$tab].${digits}f",$result->{$identifier}->{$subid}->{$kkey}->{$kind}->{"value"});
				printf(DAT " %$colw[$tab].${digits}f",$result->{$identifier}->{$subid}->{$kkey}->{$kind}->{"value"});
			    } else {
				if(exists($result->{$identifier}->{$subid}->{$key})) {
				    printf(" %$colw[$tab]d",$result->{$identifier}->{$subid}->{$key}) if($keytype{$key} eq "bool");
				    printf(" %$colw[$tab]s",$result->{$identifier}->{$subid}->{$key}) if($keytype{$key} eq "string");
				    printf(" %$colw[$tab].${digits}f",$result->{$identifier}->{$subid}->{$key}) if($keytype{$key} eq "float");
				    printf(" %$colw[$tab]d",$result->{$identifier}->{$subid}->{$key}) if($keytype{$key} eq "int");
				    printf(DAT " %$colw[$tab]d",$result->{$identifier}->{$subid}->{$key}) if($keytype{$key} eq "bool");
				    printf(DAT " %$colw[$tab]s",$result->{$identifier}->{$subid}->{$key}) if($keytype{$key} eq "string");
				    printf(DAT " %$colw[$tab].${digits}f",$result->{$identifier}->{$subid}->{$key}) if($keytype{$key} eq "float");
				    printf(DAT " %$colw[$tab]d",$result->{$identifier}->{$subid}->{$key}) if($keytype{$key} eq "int");
				} else {
				    printf(" %${colw[$tab]}s","      ---");
				    printf(DAT " %${colw[$tab]}s","      ---");
				}
			    }
			}
			print "\n";
			print DAT "\n";
		    }
		}
		print "\n";
		print DAT "\n";

	    } else {
                #
                # Transposed output
                #

		my $keyw;   # key width
		my $tcolw;  # data column width
                my @sorted_subids = sort {&attrsort($result->{$identifier}, \%keytype, $attrsortlist[$tab], $a, $b)}
		keys (%{$result->{$identifier}});
		
                # Get maximum key width.
                $keyw = 10;  # Has to fit "Subid:".
                foreach $key (@{$attr[$tab]}) {
		    $keyw = max(length($key) + 1, $keyw);
		}
		
                # Get maximum data column width. Take attribute 'colw' as a hint.
                $tcolw = $colw[$tab];
                for (@sorted_subids) {
		    $tcolw = max(length($_) + 1, $tcolw);
		}

                # Print the first row.
                printf("  %-${keyw}s|", "Subid:");
                printf(DAT "#  %-${keyw}s|", "Subid:");
                for (@sorted_subids) {
		    printf(" %${tcolw}s",$_);
		    printf(DAT " %${tcolw}s",$_);
		}
                # Print the second row (separator line).
                printf("\n  %s+", "-"x$keyw);
                printf(DAT "\n  %s+", "-"x$keyw);
                for (@sorted_subids) {
                    printf("-%s", "-"x$tcolw);
                    printf(DAT "-%s", "-"x$tcolw);
                }
                print "\n";
                print DAT "\n";
                # For each key print the key and its associated values.
                foreach $key (@{$attr[$tab]}) {
		    my $wkey = $key;
		    $wkey =~ s/000000\b/M/s;
		    $wkey =~ s/000\b/K/s;
                    printf("  %-${keyw}s|", $wkey);
                    printf(DAT "  %-${keyw}s|", $wkey);
                    undef $wkey;

                    foreach $subid (@sorted_subids) {
			next if (!($result->{$identifier}));
			next if (!exists($result->{$identifier}->{$subid}));
			next if ((!exists($result->{$identifier}->{$subid}->{'walltime'})) && (!$opt_result_showall));
			if (($opt_result_showall) || ($result->{$identifier}->{$subid}->{'walltime'} >= 0)) {
			    $lcnt++;
                            if ($key =~ /$patwrd\($patwrd\)/) {
				my ($kkey,$kind) = ($1,$2);
				printf(" %${tcolw}.2f",$result->{$identifier}->{$subid}->{$kkey}->{$kind}->{"value"});
				printf(DAT " %${tcolw}.2f",$result->{$identifier}->{$subid}->{$kkey}->{$kind}->{"value"});
                            } else {
				if (exists($result->{$identifier}->{$subid}->{$key})) {
				    printf(" %${tcolw}d",$result->{$identifier}->{$subid}->{$key})   if ($keytype{$key} eq "bool");
                                    printf(" %${tcolw}s",$result->{$identifier}->{$subid}->{$key})   if ($keytype{$key} eq "string");
                                    printf(" %${tcolw}.2f",$result->{$identifier}->{$subid}->{$key}) if ($keytype{$key} eq "float");
                                    printf(" %${tcolw}d",$result->{$identifier}->{$subid}->{$key})   if ($keytype{$key} eq "int");
				    printf(DAT " %${tcolw}d",$result->{$identifier}->{$subid}->{$key})   if ($keytype{$key} eq "bool");
                                    printf(DAT " %${tcolw}s",$result->{$identifier}->{$subid}->{$key})   if ($keytype{$key} eq "string");
                                    printf(DAT " %${tcolw}.2f",$result->{$identifier}->{$subid}->{$key}) if ($keytype{$key} eq "float");
                                    printf(DAT " %${tcolw}d",$result->{$identifier}->{$subid}->{$key})   if ($keytype{$key} eq "int");
                                } else {
                                    printf(" %${tcolw}s","      ---");
                                    printf(DAT " %${tcolw}s","      ---");
				}
                            }
                        }
                    }
                    print "\n";
                    print DAT "\n";
                }
		print "\n";
                print DAT "\n";
	    }
	}
	
	if($opt_verbose>3) {
	    print "\nKeylist: ";
	    foreach $key (sort {$a cmp $b} keys(%keylist)) {
		    print "$key,";
		}
	    print "\n";
	}
	close(DAT);
	if ($lcnt==0) {
	    # remove empty file
	    unlink($resfile);
	}
    }
}


sub attrsort {
    my($hashref,$keytyperef,$attrsortlistref,$aa,$bb)=@_;
    my($key,$val,$aaa,$bbb,$kkey,$dir);
    foreach $key (@{$attrsortlistref}) {
	$key=~/^([^\+\-]*)([\+\-])?$/;
	($kkey,$dir)=($1,$2);
	if((!$dir) || ($dir eq "+")) {$aaa=$aa;$bbb=$bb;}
	else                         {$aaa=$bb;$bbb=$aa;}

	return(-1) if(!exists($hashref->{$aaa}->{$kkey}));
	return(1) if(!exists($hashref->{$bbb}->{$kkey}));

	$val=$hashref->{$aaa}->{$kkey} cmp $hashref->{$bbb}->{$kkey} if($keytyperef->{$kkey} eq "string");
	$val=$hashref->{$aaa}->{$kkey} <=> $hashref->{$bbb}->{$kkey} if($keytyperef->{$kkey} eq "float");
	$val=$hashref->{$aaa}->{$kkey} <=> $hashref->{$bbb}->{$kkey} if($keytyperef->{$kkey} eq "int");
#	print "<$kkey,$hashref->{$aaa}->{$kkey},$hashref->{$bbb}->{$kkey},$val,$keytyperef->{$kkey}>";
	last if($val!=0);
    }
#    print "$val\n";

    $val=($aa cmp $bb) if($val==0);
    return($val);
}


sub verify {
    my($subdir,$stdoutfile,$stderrfile,$cname,$verifyfile,$verifyref,$parmhash)=@_;
    my($mode,$dtype,$val,$evalstr,$rc);
    my($vref,$command,$cmd);

    $parmhash->{stdoutfile}=$stdoutfile;
    $parmhash->{stderrfile}=$stderrfile;

    &substitute(\$cname,$parmhash);
    $vref=$verifyref->{verify}->{$cname};
    if (!$vref) { printlog(-1,"... no verify step '%s' found in verify.xml\n",$cname);  return (-2);}
    $command         = $vref->{command}->[0];

    $rc=&substitute(\$command,$parmhash);
    printlog(3,"\t\t verify     substitute param %s not found for cmd=>%s<\n",$SUBSTITUTE_NOTFOUND,$command,$rc) if($rc==-1);

    if($command) {
	printlog(3,"\t\t verify     %s cmd=>%s< rc=%d\n",$cname,$command,$rc);
	$cmd="($command 1>$subdir/verify_out.log 2>$subdir/verify_err.log)";
	printlog(0,"\t\t\t\t\texecuting: %s\n",$cmd);
	system($cmd);
	if($?) { printlog(-1,"... failed to verify $?\n","");  return (undef);}
    }

}

sub analyse {
    my($stdoutdataref,$stderrdataref,$cname,$analysefile,$analyseref,$gparmhashref)=@_;
    my($mode,$dtype,$val,$evalstr,$rc);
#    printlog(-1,"%s",Dumper($analyseref));

    &substitute(\$cname,$gparmhashref);
    printlog(3,"\t\t analyse2     cname=>%s<\n",$cname);
    my $aref=$analyseref->{analyse}->{$cname};
    if (!$aref) { printlog(-1,"... no analyse step '%s' found in analyse.xml\n",$cname);  return (-2);}

    my ($data,$inpsep,$parm,$regexp,$unit,$naref,$parmhashref);
    my $reghash= { 'patfp' => $patfp, 'patint' => $patint, 'patwrd' => $patwrd, 'patbl' => $patbl,
		   'patnfp' => $patnfp, 'patnint' => $patnint, 'patnwrd' => $patnwrd};

    $data=$$stdoutdataref.$$stderrdataref;

    return(-1) if(!$data);

#    printlog(-1,"%s",Dumper($aref));
    # test parameters

    if(exists($aref->{'input'})) {
#	printlog(-1,"%s",Dumper($gparmhashref));
	my $addfiles=$aref->{'input'}[0]->{'addfiles'};
	my($inputfile,$line,$lnr);
	foreach $inputfile (split(/\s,?\s*/,$addfiles)) {
	    &substitute(\$inputfile,$gparmhashref);
	    $inputfile=$gparmhashref->{'subdir'}."/".$inputfile if($inputfile!~/^\//);
	    if(-f $inputfile) {
		$lnr=0;
		open(IN,"$inputfile");
		while($line=<IN>) {
		    $lnr++;
		    $data.=$line;
		}
		close(IN);
		printlog(2,"\t\t\t  include additional input file: %-16s (%d lines)\n",$inputfile,$lnr);
	    } else {
		printlog(2,"\t\t\t  include additional input file: %-16s NOT FOUND\n",$inputfile);
	    }
	} 
    }
    
    foreach $parm (sort(keys(%{$aref->{parm}}))) {
	$regexp=$aref->{parm}->{$parm}->{content};
	$regexp=~s/^\s+//gs;
	$regexp=~s/\s+$//gs;
	$unit=$aref->{parm}->{$parm}->{unit};
	$mode=$aref->{parm}->{$parm}->{mode};
	$dtype=$aref->{parm}->{$parm}->{type};
	printlog(3,"\t\t search for     %-16s  --> %-8s (%s)\n",$parm,$unit,$regexp);
	&substitute(\$regexp,$reghash);
	if ($mode eq "line") {
	    if($data=~/$regexp/m) {
		$val=$1;
		printlog(3,"\t\t found     %-16s  --> %15s %-8s (%s)\n",$parm,$val,$unit,$regexp);
	    } else {
		$val="-1";
		printlog(3,"\t\t not found %-16s  --> %15s %-8s (%s)\n",$parm,"?",$unit,$regexp);
	    }
	    $naref->{parm}->{$parm}->{'unit'}=$unit;
	    $naref->{parm}->{$parm}->{'value'}=$val;
	    $naref->{parm}->{$parm}->{'type'}=$dtype;
	    $parmhashref->{$parm}=$val;
	} elsif($mode eq "line,last")  {
	    my $found=0;
	    while($data=~/$regexp/mg) {
		$val=$1;
		$found=1;
	    }
	    if($found) {
		printlog(3,"\t\t found     %-16s  --> %15s %-8s (%s)\n",$parm,$val,$unit,$regexp);
	    } else {
		$val="-1";
		printlog(3,"\t\t not found %-16s  --> %15s %-8s (%s)\n",$parm,"?",$unit,$regexp);
	    }
	    $parmhashref->{$parm}=$val;
	    $naref->{parm}->{$parm}->{'value'}=$val;
	    $naref->{parm}->{$parm}->{'unit'}=$unit;
	    $naref->{parm}->{$parm}->{'type'}=$dtype;
	} elsif($mode eq "line,add")  {
	    $naref->{parm}->{$parm}->{'count'}=0;
	    $naref->{parm}->{$parm}->{'value'}=0;
	    $naref->{parm}->{$parm}->{'unit'}=$unit;
	    $naref->{parm}->{$parm}->{'type'}=$dtype;
	    $naref->{parm}->{$parm."_cnt"}->{'value'}=0;
	    $naref->{parm}->{$parm."_cnt"}->{'unit'}="#";
	    $naref->{parm}->{$parm."_cnt"}->{'type'}="int";

	    while($data=~/$regexp/mg) {
		$val=$1;
		$naref->{parm}->{$parm}->{'value'}+=$val;
		$naref->{parm}->{$parm}->{'count'}++;
	    }
	    if($naref->{parm}->{$parm}->{'count'}>0) {
		printlog(3,"\t\t found     %-16s  --> %15s %-8s (%s) sum of %d matches (%s)\n",
			 $parm,$naref->{parm}->{$parm}->{'value'},$unit,$regexp,$naref->{parm}->{$parm}->{'count'},
			 $parm."_cnt");
		$naref->{parm}->{$parm."_cnt"}->{'value'}=$naref->{parm}->{$parm}->{'count'};
	    } else {
		$val="?";
		printlog(3,"\t\t not found %-16s  --> %15s %-8s (%s)\n",$parm,"?",$unit,$regexp);
	    }
#	    printlog(3,"\t\t set %s to '%s'\n",$parm,$naref->{parm}->{$parm}->{'value'});

	    $parmhashref->{$parm}=$naref->{parm}->{$parm}->{'value'};
	    $parmhashref->{$parm."_cnt"}=$naref->{parm}->{$parm."_cnt"}->{'value'};	   
	} elsif($mode eq "line,min")  {
	    $naref->{parm}->{$parm}->{'count'}=0;
	    $naref->{parm}->{$parm}->{'value'}=0;
	    $naref->{parm}->{$parm}->{'unit'}=$unit;
	    $naref->{parm}->{$parm}->{'type'}=$dtype;
	    $naref->{parm}->{$parm."_cnt"}->{'value'}=0;
	    $naref->{parm}->{$parm."_cnt"}->{'unit'}="#";
	    $naref->{parm}->{$parm."_cnt"}->{'type'}="int";

	    while($data=~/$regexp/mg) {
		$val=$1;
		if($naref->{parm}->{$parm}->{'count'} == 0) {
		    $naref->{parm}->{$parm}->{'value'} = $val;
		} 
		$naref->{parm}->{$parm}->{'value'}=min($val,$naref->{parm}->{$parm}->{'value'});
		$naref->{parm}->{$parm}->{'count'}++;
	    }
	    if($naref->{parm}->{$parm}->{'count'}>0) {
		printlog(3,"\t\t found     %-16s  --> %15s %-8s (%s) min of %d matches (%s)\n",
			 $parm,$naref->{parm}->{$parm}->{'value'},$unit,$regexp,$naref->{parm}->{$parm}->{'count'},
			 $parm."_cnt");
		$naref->{parm}->{$parm."_cnt"}->{'value'}=$naref->{parm}->{$parm}->{'count'};
	    } else {
		$val="?";
		printlog(3,"\t\t not found %-16s  --> %15s %-8s (%s)\n",$parm,"?",$unit,$regexp);
	    }
#	    printlog(3,"\t\t set %s to '%s'\n",$parm,$naref->{parm}->{$parm}->{'value'});

	    $parmhashref->{$parm}=$naref->{parm}->{$parm}->{'value'};
	    $parmhashref->{$parm."_cnt"}=$naref->{parm}->{$parm."_cnt"}->{'value'};
	} elsif($mode eq "line,max")  {
	    $naref->{parm}->{$parm}->{'count'}=0;
	    $naref->{parm}->{$parm}->{'value'}=0;
	    $naref->{parm}->{$parm}->{'unit'}=$unit;
	    $naref->{parm}->{$parm}->{'type'}=$dtype;
	    $naref->{parm}->{$parm."_cnt"}->{'value'}=0;
	    $naref->{parm}->{$parm."_cnt"}->{'unit'}="#";
	    $naref->{parm}->{$parm."_cnt"}->{'type'}="int";

	    while($data=~/$regexp/mg) {
		$val=$1;
		if($naref->{parm}->{$parm}->{'count'} == 0) {
		    $naref->{parm}->{$parm}->{'value'} = $val;
		} 
		$naref->{parm}->{$parm}->{'value'}=max($val,$naref->{parm}->{$parm}->{'value'});
		$naref->{parm}->{$parm}->{'count'}++;
	    }
	    if($naref->{parm}->{$parm}->{'count'}>0) {
		printlog(3,"\t\t found     %-16s  --> %15s %-8s (%s) max of %d matches (%s)\n",
			 $parm,$naref->{parm}->{$parm}->{'value'},$unit,$regexp,$naref->{parm}->{$parm}->{'count'},
			 $parm."_cnt");
		$naref->{parm}->{$parm."_cnt"}->{'value'}=$naref->{parm}->{$parm}->{'count'};
	    } else {
		$val="?";
		printlog(3,"\t\t not found %-16s  --> %15s %-8s (%s)\n",$parm,"?",$unit,$regexp);
	    }
#	    printlog(3,"\t\t set %s to '%s'\n",$parm,$naref->{parm}->{$parm}->{'value'});

	    $parmhashref->{$parm}=$naref->{parm}->{$parm}->{'value'};
	    $parmhashref->{$parm."_cnt"}=$naref->{parm}->{$parm."_cnt"}->{'value'};
	} elsif($mode eq "line,statistics")  {
	    $naref->{parm}->{$parm}->{'count'}=0;
	    $naref->{parm}->{$parm}->{'value'}=0;
	    $naref->{parm}->{$parm}->{'unit'}=$unit;
	    $naref->{parm}->{$parm}->{'type'}=$dtype;

	    $naref->{parm}->{$parm."_cnt"}->{'value'}=0;
	    $naref->{parm}->{$parm."_cnt"}->{'unit'}="#";
	    $naref->{parm}->{$parm."_cnt"}->{'type'}="int";

	    $naref->{parm}->{$parm."_min"}->{'value'}=0;
	    $naref->{parm}->{$parm."_min"}->{'unit'}=$unit;
	    $naref->{parm}->{$parm."_min"}->{'type'}="float";

	    $naref->{parm}->{$parm."_max"}->{'value'}=0;
	    $naref->{parm}->{$parm."_max"}->{'unit'}=$unit;
	    $naref->{parm}->{$parm."_max"}->{'type'}="float";

	    $naref->{parm}->{$parm."_avg"}->{'value'}=0;
	    $naref->{parm}->{$parm."_avg"}->{'unit'}=$unit;
	    $naref->{parm}->{$parm."_avg"}->{'type'}="float";

	    $naref->{parm}->{$parm."_std"}->{'value'}=0;
	    $naref->{parm}->{$parm."_std"}->{'unit'}=$unit;
	    $naref->{parm}->{$parm."_std"}->{'type'}="float";


	    while($data=~/$regexp/mg) {
		$val=$1;
		if($naref->{parm}->{$parm}->{'count'} == 0) {
		    $naref->{parm}->{$parm."_max"}->{'value'} = $val;
		    $naref->{parm}->{$parm."_min"}->{'value'} = $val;
		    $naref->{parm}->{$parm."_std"}->{'value'} = 0;
		    $naref->{parm}->{$parm."_avg"}->{'value'} = 0;
		    $naref->{parm}->{$parm."_sum"}->{'value'} = 0;
		} 
		$naref->{parm}->{$parm."_max"}->{'value'}=max($val,$naref->{parm}->{$parm."_max"}->{'value'});
		$naref->{parm}->{$parm."_min"}->{'value'}=min($val,$naref->{parm}->{$parm."_min"}->{'value'});
		$naref->{parm}->{$parm."_avg"}->{'value'}+=$val;
		$naref->{parm}->{$parm."_sum"}->{'value'}+=$val;
		$naref->{parm}->{$parm."_std"}->{'value'}+=$val*$val;
		$naref->{parm}->{$parm}->{'count'}++;
	    }

	    if($naref->{parm}->{$parm}->{'count'}>1 && $naref->{parm}->{$parm."_avg"}->{'value'}>0) {
		my $help=($naref->{parm}->{$parm}->{'count'}*$naref->{parm}->{$parm."_std"}->{'value'} - 
			  $naref->{parm}->{$parm."_avg"}->{'value'} * $naref->{parm}->{$parm."_avg"}->{'value'}) /
			  ($naref->{parm}->{$parm}->{'count'} * ($naref->{parm}->{$parm}->{'count'}-1));
		if($help<0) {
		    	   printlog(-1,"\nWARNING value $help <0 $parm ... ! setting to 0\n","");
			   $help=0;
		} 
		$naref->{parm}->{$parm."_std"}->{'value'} = sqrt($help);
		$naref->{parm}->{$parm."_avg"}->{'value'}/=$naref->{parm}->{$parm}->{'count'};
#Bug in statistics min/max should not be divided trough avg
#		$naref->{parm}->{$parm."_min"}->{'value'}/=$naref->{parm}->{$parm."_avg"}->{'value'};
#		$naref->{parm}->{$parm."_max"}->{'value'}/=$naref->{parm}->{$parm."_avg"}->{'value'};
		$naref->{parm}->{$parm."_std"}->{'value'}/=$naref->{parm}->{$parm."_avg"}->{'value'};
	    }
		    

	    if($naref->{parm}->{$parm}->{'count'}>0) {
		printlog(3,"\t\t found     %-16s  --> %15s %-8s (%s) statistics of %d matches (%s)\n",
			 $parm,$naref->{parm}->{$parm."_avg"}->{'value'},$unit,$regexp,$naref->{parm}->{$parm}->{'count'},
			 $parm."_cnt");
		$naref->{parm}->{$parm."_cnt"}->{'value'}=$naref->{parm}->{$parm}->{'count'};
	    } else {
		$val="?";
		printlog(3,"\t\t not found %-16s  --> %15s %-8s (%s)\n",$parm,"?",$unit,$regexp);
	    }
#	    printlog(3,"\t\t set %s to '%s'\n",$parm,$naref->{parm}->{$parm}->{'value'});

	    $parmhashref->{$parm}=$naref->{parm}->{$parm}->{'value'};
	    $parmhashref->{$parm."_cnt"}=$naref->{parm}->{$parm."_cnt"}->{'value'};
	    $parmhashref->{$parm."_min"}=$naref->{parm}->{$parm."_min"}->{'value'};
	    $parmhashref->{$parm."_max"}=$naref->{parm}->{$parm."_max"}->{'value'};
	    $parmhashref->{$parm."_avg"}=$naref->{parm}->{$parm."_avg"}->{'value'};
	    $parmhashref->{$parm."_std"}=$naref->{parm}->{$parm."_std"}->{'value'};
	} elsif($mode eq "span") {
	    # mode: span line
# index takes a table form input and writes out columns via indeces choosen by the user
	} elsif ($mode=~/line\,index\(([\d,]+){1,}\)/){
# the indeces are stored in @avalues
	    my @avalues = split /,/,$1;
# first index (key-value)
	    my $indexvalue = shift @avalues;
	
	    $naref->{parm}->{$parm}->{'count'}=0;
	    $naref->{parm}->{$parm}->{'unit'}=$unit;
	    $naref->{parm}->{$parm}->{'type'}=$dtype;
	    
	    while($data=~/$regexp/mg) {
	    my $correspondingValues = "";
	    my @val=($1,$2,$3,$4,$5,$6,$7,$8,$9);
	    foreach (@avalues) {
		$correspondingValues = $correspondingValues . " " . $val[$_-1];
	    }
#	    print "\$corrrespondingValues: $correspondingValues\n";
	    
		push(@{$naref->{parm}->{$parm}->{'values'}},{"key"   => $val[$indexvalue-1],
							     "value" => $correspondingValues});
		$naref->{parm}->{$parm}->{'count'}++;
	    }
	    if($naref->{parm}->{$parm}->{'count'}>0) {
		printlog(3,"\t\t found     %-16s  --> %15d idx ent. (%s) \n",
			 $parm,$naref->{parm}->{$parm}->{'count'},$regexp);
		$naref->{parm}->{$parm."_cnt"}->{'value'}=$naref->{parm}->{$parm}->{'count'};
		$naref->{parm}->{$parm."_cnt"}->{'unit'}="#";
		$naref->{parm}->{$parm."_cnt"}->{'type'}="int";
	    } else {
		$val="?";
		printlog(3,"\t\t not found %-16s  --> %15s %-8s (%s)\n",$parm,"?",$unit,$regexp);
	    }
	    $parmhashref->{$parm}=$naref->{parm}->{$parm}->{'value'};
	    $parmhashref->{$parm."_cnt"}=$naref->{parm}->{$parm."_cnt"}->{'value'};
	} else {
	    # e.g. derived
	}

    }

    foreach $parm (keys(%{$gparmhashref})) {
	$parmhashref->{$parm}=$gparmhashref->{$parm};
    }

    #derived parms
    my %derivedparms=();
    foreach $parm (keys(%{$aref->{parm}})) {
	$mode=$aref->{parm}->{$parm}->{mode};
	next if($mode ne "derived"); 
	$derivedparms{$parm}=1;
    }
    
    while((scalar keys(%derivedparms))>0) {
	foreach $parm (keys(%derivedparms)) {
	    $evalstr=$aref->{parm}->{$parm}->{content};
	    $evalstr=~s/^\s*//gs;
	    $evalstr=~s/\s*$//gs;
	    $unit=$aref->{parm}->{$parm}->{unit};
	    $dtype=$aref->{parm}->{$parm}->{type};
	    $val="\`".$evalstr."\`";
	    $rc=&substitute(\$val,$parmhashref);
	    if($rc>=0) {
		printlog(3,"\t\t derived[%1d]   %-16s  --> %15s %-12s (%s)\n",$derivedparms{$parm},$parm,$val,$unit,$evalstr);
		$naref->{parm}->{$parm}->{'unit'}=$unit;
		$naref->{parm}->{$parm}->{'value'}=$val;
		$naref->{parm}->{$parm}->{'type'}=$dtype;
		$parmhashref->{$parm}=$naref->{parm}->{$parm}->{'value'};
		delete($derivedparms{$parm});
	    } else {
		$derivedparms{$parm}++;
		if($derivedparms{$parm}>4) {
		    printlog(3,"\t\t derived[%1d]   %-16s  --> %15s %-8s could not be resolved\n",$derivedparms{$parm},$parm,$val,$unit);
		    delete($derivedparms{$parm});
		}
	    }
	}
    }

    if(! open(OUT,"> $analysefile") ) { 
	printlog(-1,"... failed to open log file $analysefile\n");  return (-1);
    }
#    printlog(-1,"%s",Dumper($naref));
    print OUT $xs->XMLout($naref, AttrIndent => 0, RootName => "analyse" );
    close(OUT);


}

sub compile {
    my($compileref,$dir,$cmpdir,$identifier,$cname,$cversion,$cproto,$cstepref,$platformref,$parmhash)=@_;
    my($cmd,$from,$to,$lparam,$execname,$execnamepath);
    my($file,$rc,$key,$spec,$var);
    

    my $tmpdir_for_copy = $dir;
    if ($opt_cmpdir)
    {
	$dir = $cmpdir;
    }
    


    &substitute(\$cname,$parmhash);
    my $cref=$compileref->{compile}->{$cname};
    if (!$cref) { printlog(-1,"... no compile step '%s' found in compile.xml\n",$cname);  return (-2);}

    my $execfound;
    return(undef) if (!$cstepref);

#    printlog(-1,"%s",Dumper($cstepref));

    my $srcdir          = $cref->{src}->[0]->{directory};
    my $srcfiles        = $cref->{src}->[0]->{files};
    my $param;
    if (defined ($cref->{param})) {
	$param           = $cref->{param}->[0];
    } else {
	$param           = $cref->{params}->[0];
    }
    my $command         = $cref->{command}->[0];
    my $executable      = $cref->{executable}->[0];
    my $platformparam   = $platformref->{params}->[0];
    my $platform        = $parmhash->{platform};

    $$cproto="  <compile>\n";

    # scan params and build executable name
    $execname=$parmhash->{benchname};
    $execname.="_".$parmhash->{platform};
    

    # parameters of cstep in top level xml file
    foreach $key (sort(keys(%$cstepref))) {
	next if($key eq "version");
	$var=$cstepref->{$key};
	$rc=&substitute(\$var,$parmhash);
	$execname.="_".$key."_".$var;
	$param->{$key}=$var;
    }
    $execname.=".exe";
    $execnamepath="$dir/$execname";

    # HK modification, 8.5.08
    foreach (keys(%$parmhash)){
	if($parmhash->{$_} ne "") {
	    $param->{$_} = $parmhash->{$_};
	}
    }

    if($command) {


	$param->{outdir}=$dir;
	$param->{rundir}="$pwd/run";
	$param->{id}=$identifier;
	$param->{execname}=$execnamepath;
	
	$execfound=0;

	# reuse old version
# FJ: Introduction of option "always new"
	if( ($cversion ne "always new") && (($cversion eq "reuse") 
	    || $generatedexecutables{$execname} )) {
	    if(-f "$pwd/run/$execname") {
		printlog(1,"\t\t\t\t\t\t\treusing executable: %s\n",$execname);
		$cmd="cp -p $pwd/run/$execname $execnamepath";
		printlog(1,"\t\t\t\t\t\t\t executing: %s\n",$cmd);
		system($cmd);
		$execfound=1;
	    }
	}

	# include platform specific parameter in param
	foreach $key (keys(%{$param})) {
	    my $var=$param->{$key};
	    $rc=&substitute(\$var,$platformparam);
	    $param->{$key}=$var;
	    printlog(4,"\t\t\t\t\t key= >%s< >%s< rc=%d\n",$key,$var,$rc);
		if($rc==-1) {
		    printlog(-1,"... parameter(s) not found in platform.xml for key %s (%s)\n",$key,$var);
		    exit(-1);
		}
	}
	# merge platform parameter in bench hash
	foreach $key (keys(%{$platformparam})) {
	    $param->{$key}=$platformparam->{$key} if(!exists($param->{$key}));
	}
	# adjust command
	&substitute(\$command,$param);

	# copy src and execute cmd (e.g. make, configure)
	if(!$execfound) {
	    return(undef) if(! ((-d $srcdir) && ($srcfiles) ));
		    
	    # copy files
	    printlog(1,"\t\t\t\t\t\t\tcopy files/dirs: %s\n",$srcfiles);
	    system "mkdir -p $dir/src";
	    if(! -d "$dir/src") { printlog(-1,"... failed to create directory %s\n",$dir);  return (undef);}
	    foreach $file (split('[, ]',$srcfiles)) {
                if ($file =~ m/.*\.(tar|tar\.gz|tgz)/) {
		    $cmd="(cd $dir/src/; gunzip -c $opt_configdir/$srcdir/$file | tar xf -)";
		    printlog(2,"\t\t\t\t\texecuting: %s\n",$cmd);
		    system($cmd);
		    if($?) { printlog(-1,"... failed to extract $file to %s\n","$dir/src");  return (undef);}
                }
		else
		{
		    $cmd="cp -rp $srcdir/$file $dir/src/";
		    printlog(2,"\t\t\t\t\t\t\texecuting: %s\n",$cmd);
		    system($cmd);
		    if($?) { printlog(-1,"... failed to copy file $file to %s\n","$dir/src");  return (undef);}
	        }
            }
	    
	    # substitute parameters
	    $rc=&substitute_files($cref->{substitute},$param,"$dir/src");
	    
	    # execute compile command
	    &substitute(\$command,$param);
	    $cmd="(cd $dir/src; $command 1>$dir/compile_out.log 2>$dir/compile_err.log)";
	    printlog(0,"\t\t\t\t\t\t\texecuting compile command: %s\n",$cmd);
	    system($cmd);
	    if($?) { printlog(-1,"... failed to compile\n","");  return (undef);}
  
	    # save version of executable
	    $cmd="cp -p $execnamepath $pwd/run/$execname";
	    printlog(1,"\t\t\t\t\t\t\texecuting: %s\n",$cmd);
	    system($cmd);
	    if($?) { printlog(-1,"... failed to copy file %s to %s\n","$execnamepath","$pwd/run/$execname");  return (undef);}
	    $generatedexecutables{$execname}=1;
	    
	    # generate XML compile description
	    # if(! open(CPROTO,"> $pwd/run/${execname}.xml") ) { 
	    #      printlog(-1,"... failed to open protocol file %s\n","$pwd/run/${execname}.xml");  return (-1);
	    # }
	    

	}

	# generate log file information
	$$cproto.="    <command>$command</command>\n";
	$$cproto.="    <params \n";
	foreach $key (keys(%$param)) {
	    $$cproto.="        $key=\"".$param->{$key}."\"\n";
	}
	$$cproto.="    />\n";
	

    } elsif ($executable) {
	# precompiled
	my $ex=  $executable->{name};
	my $dest=$executable->{destname};
	my $desc=$cref->{description}->[0];
	&substitute(\$dest,{"id" => "$identifier"});
	$cmd="cp -p $ex $dir/$dest";
	printlog(0,"\t\t\t\t\texecuting: %s\n",$cmd);
	system($cmd);
	if($?) { printlog(-1,"... failed to copy file $ex to $dir/$dest\n");  return (undef);}
	$execnamepath="$dir/$dest";
	$$cproto.="    <executable origname=\"$ex\" name=\"$execname\" />\n";
	if($desc) {
	    $$cproto.="    <description>$desc</description>\n";
	}
    } else {
	printlog(-1," problems to compile executable\n","");  return (undef);
	$execnamepath=undef;
    }
    $$cproto.="  </compile>\n";
   
# copy executable to tmpdir if tmpdir is specified in the option section

if($tmpdir_for_copy ne $cmpdir)
{
    my $archive = $identifier . ".tar";
    my $archive_zip = $archive . ".gz";
   
    printlog(0," JUBE: tar source code: tar cf $archive %s \n",$archive, "./src"); 
    my $cmd = "(cd $dir; tar cf $archive ./src)";
    printlog(3,"\texecuting: %s\n",$cmd);
    system($cmd);
    if($?) { printlog(-1,"... failed to execute %s\n",$?);  return (undef);}

    printlog(0," JUBE: zip tar file: gzip %s \n", $archive);
    $cmd = "(cd $dir; gzip $archive)";
    printlog(3,"\executing: %s\n", $cmd);
    system($cmd);
    if($?) { printlog(-1,"... failed to execute %s\n", $cmd); return (undef);}

    printlog(0," JUBE: new directory: mkdir -p %s \n", "$tmpdir/run");
    $cmd = "(mkdir -p $tmpdir/run)";
    printlog(3,"\executing: %s\n", $cmd);
    system($cmd);
    if($?) { printlog(-1,"... failed to execute %s\n", $cmd); return (undef);}
	     
    printlog(0," JUBE: new File: cp -p %s %s\n", $execnamepath, "$tmpdir/run");
    $cmd = "(cp -p $execnamepath $tmpdir/run)";
    printlog(3,"\executing: %s\n", $cmd);
    system($cmd);
    if($?) { printlog(-1,"... failed to execute %s\n", $cmd); return (undef);}

    printlog(0," JUBE: new File: cp -p %s %s\n", $archive_zip, "$tmpdir_for_copy");
    $cmd = "(cd $dir ; cp -p $archive_zip $tmpdir_for_copy)";
    printlog(3,"\executing: %s\n", $cmd);
    system($cmd);
    if($?) { printlog(-1,"... failed to execute %s\n", $cmd); return (undef);}
}

return($execnamepath);
}

sub prepare {
    my($prepareref,$dir,$identifier,$cname,$parmhash)=@_;
    my($cmd,$from,$to,$lparam,$file,$rc);

    &substitute(\$cname,$parmhash);
    my $pref=$prepareref->{prepare}->{$cname};
    if (!$pref) { printlog(-1,"... no prepare step '%s' found in prepare.xml\n",$cname);  return (-2);}

    my $inpfiles=$pref->{input}->[0]->{files};

#    printlog(-1,"%s",Dumper($pref));


    if($pref->{mkdir}) {
	# execute command
	my $subdir;
	printlog(1,"\t\t\t\t\t\t\tmkdirs: %s\n",join(" ",keys(%{$pref->{mkdir}})));
	foreach $subdir  (keys(%{$pref->{mkdir}})) {
	    my $cmd="(mkdir $dir/$subdir 1>$dir/prepare_mkdir_out.log 2>$dir/prepare_mkdir_err.log)";
	    $parmhash->{outdir}=$dir;
	    $parmhash->{identifier}=$identifier;
	    &substitute(\$cmd,$parmhash);
	    printlog(2,"\t\t\t\t\t\t\t\t  executing: %s\n",$cmd);
	    system($cmd);
	    if($?) { printlog(-1,"... failed to execute %s rc=%s\n",$cmd,$?);  return (undef);}
	}
    }

    if(($pref->{precommand}) && ($pref->{precommand}->[0]) && !ref($pref->{precommand}->[0]) && (($pref->{precommand})!~/^\s*$/)) {
	# execute command
	my $command=$pref->{precommand}->[0];
	$command=~s/\n/ /gs;
	my $cmd="($command 1>$dir/prepare_precmd_out.log 2>$dir/prepare_precmd_err.log)";
	$parmhash->{outdir}=$dir;
	$parmhash->{identifier}=$identifier;
#	printlog(-1,"%s",Dumper($parmhash));
	$rc=&substitute(\$cmd,$parmhash);
	printlog(1,"\t\t\t\t\t\t\texec. prep precommand: %s\n",$command);
	printlog(3,"\t\t prepare     substitute param %s not found for cmd=>%s<\n",$SUBSTITUTE_NOTFOUND,$cmd,$rc) if($rc==-1);
	printlog(2,"\t\t\t\t\t\t  executing: %s\n",$cmd);
	system($cmd);
	if($?) { printlog(-1,"... failed to execute %s rc=%s\n",$cmd,$?);  return (undef);}
    }


    if($inpfiles) {
	printlog(1,"\t\t\t\t\t\t\tprep input files: %s\n",$inpfiles);
	foreach $file (split('[, ]',$inpfiles)) {
	    &substitute(\$file,$parmhash);
	    $cmd="cp -rp $file $dir/";
	    printlog(2,"\t\t\t\t\t\t\t  executing: %s\n",$cmd);
	    system($cmd);
	    if($?) { printlog(-1,"... failed to copy file %s to %s\n",$file,$dir);  return (-1);}
	}
    }
    if($pref->{substitute}) {
	# substitute parameters
	$rc=&substitute_files($pref->{substitute},$parmhash,$dir);
    }
	
    if(($pref->{command}) && ($pref->{command}->[0]) && !ref($pref->{command}->[0]) && (($pref->{command})!~/^\s*$/)) {
	# execute command
	my $command=$pref->{command}->[0];
	$command=~s/\n/ /gs;
	my $cmd="($command 1>$dir/prepare_cmd_out.log 2>$dir/prepare_cmd_err.log)";
	$parmhash->{outdir}=$dir;
	$parmhash->{identifier}=$identifier;
#	printlog(-1,"%s",Dumper($parmhash));
	$rc=&substitute(\$cmd,$parmhash);
	printlog(1,"\t\t\t\t\t\t\texec. prep command: %s\n",$command);
	printlog(3,"\t\t prepare     substitute param %s not found for cmd=>%s<\n",$SUBSTITUTE_NOTFOUND,$cmd,$rc) if($rc==-1);
	printlog(2,"\t\t\t\t\t\t  executing: %s\n",$cmd);
	system($cmd);
	if($?) { printlog(-1,"... failed to execute %s rc=%s\n",$cmd,$?);  return (undef);}
    }
}


sub execute {
    my($executeref,$dir,$identifier,$subid,$executable,$estepref,$parmhash,$lastcommandref)=@_;
    my($cmd,$from,$to,$lparam,$envstr,$envvar,$file,$rc,$var,$cname,$key);

    $cname=$estepref->{'cname'};
    
    # parameters of cstep in top level xml file
    foreach $key (sort(keys(%$estepref))) {
	next if($key eq "cname");
	$var=$estepref->{$key};
	$parmhash->{$key}=$var;
    }

#    printlog(-1,"in execute: %s",Dumper($parmhash));


    &substitute(\$cname,$parmhash);
    my $eref=$executeref->{execute}->{$cname};
    if (!$eref) { printlog(-1,"... no execution step '%s' found in execute.xml\n",$cname);  return (-2);}

    my $inpfiles=$eref->{input}->[0]->{files};
    my $envref=$eref->{environment}->[0]->{env};
    my $command =$eref->{command}->[0];
    my $lastcommand =$eref->{lastcommand}->[0];
    if($defer_stdout_to_tmpdir) {
	$parmhash->{logdir}=$tmplogdir;
	$parmhash->{stdoutlogfile}=sprintf("%s/%s.%s_stdout.log",$tmplogdir,$identifier,$subid);
	$parmhash->{stderrlogfile}=sprintf("%s/%s.%s_stderr.log",$tmplogdir,$identifier,$subid);
    } else {
	$parmhash->{logdir}=$stdlogdir;
	$parmhash->{stdoutlogfile}=sprintf("%s/%s.%s_stdout.log",$stdlogdir,$identifier,$subid);
	$parmhash->{stderrlogfile}=sprintf("%s/%s.%s_stderr.log",$stdlogdir,$identifier,$subid);
    }

    $parmhash->{outdir}=$dir; 
    $parmhash->{id}=$identifier;
    $parmhash->{subid}=$subid;
    $parmhash->{executable}=$executable;

#    printlog(-1,Dumper($eref));

    printlog(1,"\t\t\t\t\t\t\tcopy files: %s\n",$inpfiles);
    foreach $file (split('[, ]',$inpfiles)) {
	&substitute(\$file,$parmhash);
	$cmd="cp -rp $file $dir/";
	printlog(2,"\t\t\t\t\t\t\t  executing: %s\n",$cmd);
	system($cmd);
	if($?) { printlog(-1,"... failed to copy file $file to $dir\n");  return (-1);}
    }

    $envstr="";
    foreach $envvar (keys(%{$envref})) {
	my $envval=$envref->{$envvar}->{'value'};
	&substitute(\$envval,$parmhash);
	$envstr.="export $envvar=\"$envval\"\n";
    }
    $parmhash->{env}=$envstr;
    $rc=&substitute_files($eref->{substitute},$parmhash,$dir);
    $parmhash->{env}="";
    
    # execute command
    if($command) {
  	&substitute(\$command,$parmhash);
	$cmd="(cd $dir; $command 1>$dir/execute_out.log 2>$dir/execute_err.log)";
	printlog(0,"\t\t\t\t\t\t\t-> submit job command: %s %s\n",($opt_debug)?"[debug]":"",$command);
	system($cmd) if (!$opt_debug);
	if($?) { printlog(-1,"... failed to execute %s\n",$command);  return (-1);}
    }

    # execute command
    if($lastcommand) {
  	&substitute(\$lastcommand,$parmhash);
	$$lastcommandref="(cd $dir; $lastcommand 1>$dir/execute_out.log 2>$dir/execute_err.log)";
    }
    
}

sub pproto_open {
    my($protofile,$benchref)=@_;
    my($startdate,$help1,$help2);
    if(! open(PROTO,"> $protofile") ) { 
	printlog(-1,"... failed to open protocol file %s\n",$protofile);  return (-1);
    }
    
    $help1= $benchref->{'name'};
    $help2 = $benchref->{'platform'};

    print PROTO "<benchrun name=\"$help1\" platform=\"$help2\">\n";
    $startdate=localtime(time());
    print PROTO "  <bstartdate>$startdate</bstartdate>\n";
}

sub pproto {
    my($protofile,$str)=@_;
    if(! open(PROTO,">> $protofile") ) { 
	printlog(-1,"... failed to open protocol file %s\n",$protofile);  return (-1);
    }
    
    print PROTO $str;
}


sub pproto_close {
    my($protofile,$benchref)=@_;

    if(! open(PROTO,">> $protofile") ) { 
	printlog(-1,"... failed to open protocol file %s\n",$protofile);  return (-1);
    }
    $enddate=localtime(time());
    print PROTO "  <benddate>$enddate</benddate>\n";
    print PROTO "</benchrun>\n";
}

sub getsequence {
    my($str,$benchref)=@_;
    my($spec,$i);
    my (@sequence,@result);
    # only if evaluated
    return if ($str=~/\$/);
    if($str=~/^$patwrd\($patwrd\)$/) {
	my ($func,$parms)=($1,$2);
#	print "getsequence: $func $parms\n";
#	printlog(-1,"%s",Dumper($benchref));
	@result=&readperm(split(/\s*,\s*/,$parms),$benchref) if ($func eq "readperm");
	@result=&factorperm(split(/\s*,\s*/,$parms),$benchref) if ($func eq "factorperm");
	@result=&factorpermbound(split(/\s*,\s*/,$parms),$benchref) if ($func eq "factorpermbound");
	@result=&iterator(split(/\s*,\s*/,$parms),$benchref) if ($func eq "iterator");	
	@result=&predefinedparams(split(/\s*,\s*/,$parms),$benchref) if ($func eq "predefinedparams");
    } else {
	if($str!~/\,/) {
	    push(@sequence,$str);
	} else {
	    @sequence=split(/,/,$str);
	}
	foreach $spec (@sequence) {
	    if($spec=~/(\d+)\.\.(\d+)/) {
		my($a,$e)=($1,$2);
		for($i=$a;$i<=$e;$i++) {
		    push(@result,$i);
		}
	    } else {
		push(@result,$spec);
	    }
	}
    }
#    print "debug: getsequence >$str< -> $result[0]:$result[1]:$result[2]\n";
    return(@result);
}

sub readperm {
    my($maptagname,$x,$y,$z,$t,$n,$benchref)=@_;
    my(@result);
    if($benchref->{$maptagname} && $benchref->{$maptagname}->[0]->{"map"}) {
	my $ref=$benchref->{$maptagname}->[0]->{"map"};
	if($ref->{$n}) {
	    my $spec; 
	    foreach $spec (split(/\s+/,$ref->{$n}->{"spec"})) {
		my ($px,$py,$pz,$pt)=split(/:/,$spec);
		if( ($px*$py*$pz*$pt == $n) 
		    && ($x % $px == 0)
		    && ($y % $py == 0)
		    && ($z % $pz == 0)
		    && ($t % $pt == 0)
		    )
		 {
		    push(@result,$spec);
		}
	    }
	} else {
	    printlog(-1,"... no mapping entry found for $maptagname and tasknumber=%s\n",$n);
	}
    } else {
	printlog(-1,"... no mapping entry found for %s\n",$maptagname);
    }
    return(@result);
}

sub iterator {
    my($start,$end,$op,$benchref)=@_;
    my(@result,$val,$opr);
    if($op=~/[+]$patfp/) {
	$opr=$1;
	$val=$start;
	while($val<=$end) {
	    push(@result,$val);
	    $val+=$opr;
	}
    } elsif($op=~/[*]$patfp/) {
	$opr=$1;
	$val=$start;
	while($val<$end) {
	    push(@result,$val);
	    $val*=$opr;
	}
    }elsif($op=~/[-]$patfp/) {
	$opr=$1;
	$val=$start;
	while($val>=$end) {
	    push(@result,$val);
	    $val-=$opr;
	}
    } elsif($op=~/[\/]$patfp/) {
	$opr=$1;
	$val=$start;
	while($val>=$end) {
	    push(@result,$val);
	    $val/=$opr;
	}
    }else {
	@result=();
    }
    return(@result);
}

# subrutine to read in predefined parameter sets, as a function of tasks and tagname
# args:  - xml target tag name
#        - number of tasks
#        - request, i.e. which parameter shall be read in
#        - reference to bench
sub predefinedparams {
    my($paramtagname, $tasks, $request, $benchref)=@_;
    my $result;

    printlog(5,"predefined parameter call: $paramtagname, $tasks, $request\n", 0);

    if($benchref->{$paramtagname} && $benchref->{$paramtagname}->[0]->{"predefparam"}) {
#	print Dumper($benchref->{$paramtagname}->[0]->{"predefparam"});
	my $paranmb = @{$benchref->{$paramtagname}->[0]->{"predefparam"}};
	my $parafound = 0;
	for(my $paracnt=0; $paracnt < $paranmb; $paracnt++)
	    {
		my $ref=$benchref->{$paramtagname}->[0]->{"predefparam"}->[$paracnt];
		if( $ref->{$request} && $ref->{"tasks"}==$tasks)
		{
		    $result=$ref->{$request};
		    $parafound = 1;
		}
	    }
	if($parafound == 0)
	{
	    printlog(-1, "... could not find request $request in predefined params ($paramtagname) for $tasks tasks\n",0);
	}
	else
	{
	    printlog(5, "request: $request; value: $result\n", 0);
	}

    }
    else {
	printlog(-1,"... no mapping entry found for %s\n",$paramtagname);
    }
    return $result;
}


sub factorperm {
    my($x,$y,$z,$t,$n,$benchref)=@_;
    my(@result);
    my (@factors, $i, $limit);
    # factors of $n
    for (my $i = 2; $i <= $n; $i++) {
        last if $i > ($n / 2);
        if ($n % $i == 0)  {  
            push @factors, $i;
        }
    }
    # matches
    my (@matches, @previous_bases, $skip);
    my @base1 = my @base2 = my @base3 = my @base4 = @factors;

    for my $base1 (@base1) { 
	for my $base2 (@base2) { 
	    for my $base3 (@base3) { 
		for my $base4 (@base4) { 
		    if ($base1 * $base2 * $base3 * $base4 == $n) {
			$skip=0;
			$skip=1 if (( ($x % $base1) != 0) && ($x != $base1));
			$skip=1 if (( ($y % $base2) != 0) && ($y != $base2));
			$skip=1 if (( ($z % $base3) != 0) && ($z != $base3));
			$skip=1 if (( ($t % $base4) != 0) && ($t != $base4));
			push(@result, "$base1:$base2:$base3:$base4") if(!$skip);
		    }
		}
	    }
	}
    }
    return(@result);
}

sub factorpermbound {
    my($x,$y,$z,$t,$n,$benchref)=@_;
    my(@result);
    my (@factors, $i, $limit);
    # factors of $n
    for (my $i = 2; $i <= $n; $i++) {
        last if $i > ($n / 2);
        if ($n % $i == 0)  {  
            push @factors, $i;
        }
    }
    push(@factors,1);
    push(@factors,$n);

    # matches
    my (@matches, @previous_bases, $skip);
    my @base1 = my @base2 = my @base3 = my @base4 = @factors;

    for my $base1 (@base1) { 
	for my $base2 (@base2) { 
	    for my $base3 (@base3) { 
		for my $base4 (@base4) { 
		    if ($base1 * $base2 * $base3 * $base4 == $n) {
			$skip=0;
			$skip=1 if (( ($x % $base1) != 0) && ($x != $base1));
			$skip=1 if (( ($y % $base2) != 0) && ($y != $base2));
			$skip=1 if (( ($z % $base3) != 0) && ($z != $base3));
			$skip=1 if (( ($t % $base4) != 0) && ($t != $base4));
			push(@result, "$base1:$base2:$base3:$base4") if(!$skip);
		    }
		}
	    }
	}
    }
    return(@result);
}

sub substitute_files {
    my($substhashref,$parmhash,$dir)=@_;
    my($subst,$data,$from,$to,$nc);

    foreach $subst (keys(%{$substhashref})) {
	my $infile=$subst;
	my $outfile=$substhashref->{$subst}->{outfile};
	&substitute(\$infile,$parmhash);
	&substitute(\$outfile,$parmhash);
	
	printlog(2,"\t\t\t\t\t\t\t sub: %s -> %s\n",$infile,$outfile);
	
	# read infile
	$data="";
	if(! open(IN,"$dir/$infile") ) { 
	    printlog(-1,"... failed to open input file to %s/%s\n",$dir,$infile);  return (-1);
	}
	while (<IN>) {
	    $data.=$_;
	}
	close(IN);
	
	# substitute params
	my $subhash=$substhashref->{$subst}->{'sub'};
	foreach $from (keys(%$subhash)) {
	    $to=$subhash->{$from}->{to};
	    &substitute(\$to,$parmhash);
	    $nc= $data=~s/$from/$to/gs;
	    my $tto=substr($to,0,80);
	    $tto.="..." if(length($to)>80);
	    $tto=~s/\n/ /gs;
	    printlog(2,"\t\t\t\t\t\t(1)  #%02d %-20s -> %s\n",$nc,$from,$tto);
	}
	# second substitute step for recursive definitions
	foreach $from (keys(%$subhash)) {
	    $to=$subhash->{$from}->{to};
	    &substitute(\$to,$parmhash);
	    $nc= $data=~s/$from/$to/gs;
	    my $tto=substr($to,0,80);
	    $tto.="..." if(length($to)>80);
	    $tto=~s/\n/ /gs;
	    printlog(2,"\t\t\t\t\t\t(2)  #%02d %-20s -> %s\n",$nc,$from,$tto);
	}
	
	#write outfile
	if(! open(OUT,"> $dir/$outfile") ) { 
	    printlog(-1,"... failed to open output file to %s/%s\n",$dir,$outfile);  return (-1);
	}
	print OUT $data;
	close(OUT);
    }
}

sub substitute {
    my($strref,$hashref)=@_;
    my($found,$c,@varlist1,@varlist2,$var);
    $c=0;
    $found=0;

#    return(0) if($$strref eq "");
    $$strref=" " if($$strref eq "");

    # search normal variables
    @varlist1=($$strref=~/\$([^\{\[\$\\\s\.\,\*\/\+\-\\\`\(\)\'\?\:\;\}]+)/g);
    foreach $var (sort {length($b) <=> length($a)} (@varlist1)) {
	if(exists($hashref->{$var})) {
	    my $val=$hashref->{$var};
	    $$strref=~s/\$$var/$val/egs;
	    printlog(5,"                      substitute var1: %s = %s\n",$var,$val);
	    $found=1;
	}
    }

    # search variables in following form: ${name}
    @varlist2=($$strref=~/\$\{([^\{\[\$\\\s\.\,\*\/\+\-\\\`\(\)\'\?\:\;\}]+)\}/g);
    foreach $var (sort {length($b) <=> length($a)} (@varlist2)) {
	if(exists($hashref->{$var})) {
	    my $val=$hashref->{$var};
	    $$strref=~s/\$\{$var\}/$val/egs;
	    printlog(5,"                      substitute var2: %s = %s\n",$var,$val);
	    $found=1;
	} 
    }

    # search eval strings (`...`)
    while($$strref=~/^(.*)(\`(.*?)\`)(.*)$/) {
	my ($before,$evalall,$evalstr,$after)=($1,$2,$3,$4);
	my($val,$executeval);
        $val=undef;

        if($evalstr=~/^\s*getstdout\((.*)\)\s*$/) {
            $executeval=$1;
	    eval("{\$val=`$executeval`}");
            $val=~s/\n/ /gs;
        } 
	if(!defined($val)) {
	    eval("{\$val=$evalstr;}");
	}
	if(!defined($val)) {
	    $val=eval("{$evalstr;}");
	}
	$val="" if(!defined($val));
	if($val ne "") {
	    $$strref=$before.$val.$after;
	} else {
	    last;
	}
	printlog(5,"                      eval %s -> %s >%s<\n",$val,$$strref,$evalall);
    }

    # search for variables which could not be substitute
    @varlist1=($$strref=~/\$([^\{\[\$\\\s\.\,\*\/\+\-\\\`\(\)\'\?\:\;\}]+)/g);
    @varlist2=($$strref=~/\$\{([^\{\[\$\\\s\.\,\*\/\+\-\\\`\(\)\'\?\:\;\}]+)\}/g);
    if ( (@varlist1) || (@varlist2) ) {
	$SUBSTITUTE_NOTFOUND=join(',',@varlist1,@varlist2);
	$found=-1;
	printlog(5,"                      unknown vars in %s: %s\n",$$strref,$SUBSTITUTE_NOTFOUND);
    }
    return($found);
}

sub substitute_ori {
    my($strref,$hashref)=@_;
    my($found,$c);
    $found=1;
    $c=0;
#    print "debug: $val -> '$$strref', >$lparam<\n";
    while($found>0) {
	$c++; last if($c>10);
	# variable replacement
	if($$strref=~/^.*\$([^\{\[\$\\\s\.\,\*\/\+\-\/\\\`\(\)\'\?\:\;]+).*$/) {
	    my $lparam=$1;
#	    print "debug: -> '$$strref', >$lparam<\n";
	    if(exists($hashref->{$lparam})) {
		my $val=$hashref->{$lparam};
		$$strref=~s/\$$lparam/$val/es;
#		print "debug: val = >$val<\n";
		$found=1;
	    } else {
		# eval string could also handle variables
		if(($$strref!~/^(.*)(\`(.*)\`)(.*)$/)) {
		    $SUBSTITUTE_NOTFOUND=$lparam;$found=-1;
		}
	    }
	    # ${..}
	} elsif ($$strref=~/^.*\$\{([^\}]+)\}.*$/) {
	    my $lparam=$1;
#	    print "debug: {}-> '$$strref', >$lparam<\n";
	    if(exists($hashref->{$lparam})) {
		my $val=$hashref->{$lparam};
		$$strref=~s/\$\{$lparam\}/$val/es;
#		print "debug: {} val = >$val<\n";
		$found=1;
	    } else {$SUBSTITUTE_NOTFOUND=$lparam;$found=-1;}

	} else {$found=0;}
    }
    if($found==0) {
	if($$strref=~/^(.*)(\`(.*)\`)(.*)$/) {
	    my ($before,$evalall,$evalstr,$after)=($1,$2,$3,$4);
	    my($val);
	    eval("{\$val=$evalstr;}");
	    $val="" if(!defined($val));
	    $$strref=$before.$val.$after;
#	    print "debug: $val -> '$$strref', >$evalall<\n";
	}
    }
    return($found);
}

sub printlog {
    my($level,$format,@parms)=@_;
    my($i);
    my $PWD='$PWD';
    $format=~s/$pwd/$PWD/es;

#    print "printlog: :$level:$format ",caller(),"\n";
    for($i=0;$i<=$#parms;$i++) {
        next if($parms[$i] eq "");
	$parms[$i]=~s/$pwd/$PWD/es;
    }

    if($level==-1) {
	# error message
	croak "error in printlog" if(!$format);
	croak "error in printlog" if(!@parms);
	printf (STDERR $format,@parms);
	printf(BENCHLOG "ERROR: $format",@parms);
    } else {
	if(defined($opt_verbose)) {
	    if($opt_verbose>=$level) {
		my $str=sprintf($format,@parms);
		
	if(length($str)>=$loglinelength) {
		    print STDOUT substr($str,0,$loglinelength)," ...\n";
		} else {
		    print STDOUT $str;
		}
	    }
	    croak "error in printlog" if(!@parms);
	    croak "error in printlog" if(!$format);
#	    print "debug: >$format<>@parms<>$opt_verbose>=$logfilelevel<\n";
	    printf(BENCHLOG $format,@parms) if($opt_verbose>=$logfilelevel);
	}
    }
}

sub get_identifier {
    my($id);
    if(-f $idfile) {
	open(IN,"$idfile");
	$id=<IN>;
	chomp($id);
    } else {
	$id=0;
    }
    $id++;
    open(OUT,">$idfile");
    print OUT $id,"\n";
    close(OUT);
    
    return($id);
}

sub get_identifier_ro {
    my($id);
    if(-f $idfile) {
	open(IN,"$idfile");
	$id=<IN>;
	chomp($id);
    } else {
	$id=0;
    }
    $id++;
    
    return($id);
}

sub testspec {
    my($id,$spec)=@_;
    my $rc=0;

    if($spec=~/^(\d+)$/) {
	$rc=($id==$1);
    }
    if($spec=~/^(\d+)\+$/) {
	$rc=($id>=$1);
    }
    if($spec=~/^(\d+)\-$/) {
	$rc=($id<=$1);
    }
    if($spec=~/^(\d+)\.\.(\d+)$/) {
	$rc=(($id>=$1) && ($id<=$2));
    }

    return($rc);
}

sub min {
    my($a,$b)=@_;
    if ($a<$b) {
	return($a); 
    } else {
	return($b); 
    }
}

sub max {
    my($a,$b)=@_;
    if ($a>$b) {
	return($a); 
    } else {
	return($b); 
    }
}


sub version {
    print &getversion(); print "\n";
    exit(0);
}


sub getversion {
    my $text = "jube version 1.1p20";
    return $text;
}

sub usage {
    die "Usage: $_[0] <options> <xml-file> <id-range>

                -start, -submit *  : submit new set of benchmark runs (defined in xml-file) 
                -update         +  : scans for results of finished jobs
                -result         +  : shows results of benchmark runs (tables) 
                -force          +  : force a rescan of benchmark output files for new results
                -cdir <dir>        : directory containing the xml files 
                                     (default: ./)
                -pdir <dir>        : directory containing platforms definition XML files 
                                     (default: ../platforms)
                -tmpdir <dir>      : directory which is used for running the job in,
                                     please use only an absolute path
                                     (default: tmp in benchmark directory)
                -verbose level     : verbose
                -dump              : dump XML-file structure
                -showall           : shows all results, incl. failed and queued runs
                -debug             : don't submit jobs
                -rmtmp             : remove temp directory directly
                -cmpdir <dir>      : directory which is used for running the compile step in,
                                     please use only an absolute path
                -Version           : prints out the current version
      * : needs XML top level file  <xml-file>
      + : a range of benchmark run ids can be specified <id-range>

";
}

