#include <stdio.h>
#include <assert.h>
#include <sys/resource.h>
#include "mpi.h"

#ifdef UNDERSCORE_CALLS

#define kernel_a kernel_a_
#define jube_kernel_init_f jube_kernel_init_f_
#define jube_kernel_run_f jube_kernel_run_f_
#define jube_kernel_finalize_f jube_kernel_finalize_f_
#define jube_kernel_end_f jube_kernel_end_f_
#endif


#ifdef IHPCT_HWC
#include "libhpm.h"
#endif

#ifdef PAPI
#include <papi.h>

#define JUBE_PAPI_NMB_EVENTS 2
char jube_papi_str[PAPI_MAX_STR_LEN];
int jube_papi_events[JUBE_PAPI_NMB_EVENTS] = {PAPI_TOT_CYC, PAPI_FP_OPS};
long_long jube_papi_values[JUBE_PAPI_NMB_EVENTS];
#endif


#define NMB_KERNEL 5

#define HWPC_ALL      1
#define HWPC_INIT     2
#define HWPC_KERNEL   3
#define HWPC_FINALIZE 4

int jube_mympirank; 

int jube_current_kernel;

int jube_kernel_active[NMB_KERNEL];

double jube_time_kernel_init[NMB_KERNEL], 
  jube_time_kernel_run[NMB_KERNEL], 
  jube_time_kernel_finalize[NMB_KERNEL], 
  jube_time_kernel_end[NMB_KERNEL];

char *jube_kernel_names[] = {"kernel_A", "kernel_B", "kernel_C", "kernel_D", "kernel_E"};

unsigned jube_mem_usage();

void jube_init()
{
  
  jube_current_kernel = -1;

  printf("JuBE: init QCD benchmark\n");

  MPI_Comm_rank(MPI_COMM_WORLD, &jube_mympirank);

#ifdef IHPCT_HWC
  printf("JuBE: starting HPM\n");
  hpmInit(0, "QCD");
#ifdef IHPCT_ALL
  hpmStart(1, "QCD");
#endif
#endif

#ifdef PAPI
  assert( PAPI_start_counters(jube_papi_events, JUBE_PAPI_NMB_EVENTS) == PAPI_OK );
#endif

}

void jube_end()
{
  int cnt, num_proc;
  double time_total, time_kernel;

  double timings[3*NMB_KERNEL], timings_sq[NMB_KERNEL];
  double buf_timings[3*NMB_KERNEL], buf_timings_sq[NMB_KERNEL];

#ifdef IHPCT_HWC
#ifdef IHPCT_ALL
  hpmStop(1);
#endif
  hpmTerminate(0);
  printf("JuBE: terminated HPM\n");
#endif

#ifdef PAPI
  assert( PAPI_stop_counters(jube_papi_values, JUBE_PAPI_NMB_EVENTS) == PAPI_OK );
#endif


  printf("JuBE: end QCD benchmark\n");

  MPI_Barrier(MPI_COMM_WORLD);

  printf("JuBE: \tkernelname \tinit \t\trun \t\tfinalize\n");
  
  for(cnt=0; cnt < NMB_KERNEL; cnt++)
    {
      timings[cnt +            0] = jube_kernel_active[cnt] ? jube_time_kernel_run[cnt]      - jube_time_kernel_init[cnt]     : 0;
      timings[cnt +   NMB_KERNEL] = jube_kernel_active[cnt] ? jube_time_kernel_finalize[cnt] - jube_time_kernel_run[cnt]      : 0;
      timings[cnt + 2*NMB_KERNEL] = jube_kernel_active[cnt] ? jube_time_kernel_end[cnt]      - jube_time_kernel_finalize[cnt] : 0;

      timings_sq[cnt] = timings[cnt +0] + timings[cnt + NMB_KERNEL] + timings[cnt + 2*NMB_KERNEL];
      timings_sq[cnt] = timings_sq[cnt] * timings_sq[cnt];
    }

  for(cnt=0; cnt < NMB_KERNEL; cnt++)
    {
      if(jube_kernel_active[cnt])
	printf("JuBE: \t%s \t%.3e \t%.3e \t%.3e\n", jube_kernel_names[cnt], 
	       timings[cnt], timings[cnt + NMB_KERNEL], timings[cnt + 2*NMB_KERNEL]);
    }

  time_total = 0;
  for(cnt=0; cnt < NMB_KERNEL; cnt++)
    {
      if(jube_kernel_active[cnt])
	time_total += timings[cnt] + timings[cnt + NMB_KERNEL] + timings[cnt + 2*NMB_KERNEL];
    }

  printf("JuBE: total time on process %d: %.3e\n", jube_mympirank, time_total);

  MPI_Reduce(timings,    buf_timings,    3*NMB_KERNEL, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
  MPI_Reduce(timings_sq, buf_timings_sq,   NMB_KERNEL, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
  
  if(jube_mympirank == 0)
    {
      MPI_Comm_size(MPI_COMM_WORLD, &num_proc);

      printf("JuBE global mean timing statistics: \tkernelname \tinit \t\trun \t\tfinalize \ttotal \t\ttotal(stddev)\n");

      time_total = 0;
      for(cnt=0; cnt < NMB_KERNEL; cnt++)
	{
	  time_kernel = buf_timings[cnt] + buf_timings[cnt + NMB_KERNEL] + buf_timings[cnt + 2*NMB_KERNEL];
	  printf("JuBE global mean timing statistics: \t%s \t%.3e \t%.3e \t%.3e \t%.3e \t%.3e\n", jube_kernel_names[cnt],
		 buf_timings[cnt]/num_proc, buf_timings[cnt + NMB_KERNEL]/num_proc, buf_timings[cnt + 2*NMB_KERNEL]/num_proc,
		 (time_kernel)/num_proc,
		 sqrt((num_proc*buf_timings_sq[cnt] - time_kernel*time_kernel)/num_proc/(num_proc-1)));
	  time_total += time_kernel/num_proc;
	}
      printf("JuBE: total mean run time: %e\n", time_total);
	     
    }

  printf("JuBE: total max mem: %d\n", jube_mem_usage());
}

double jube_time_get()
{
  return MPI_Wtime();
}

unsigned jube_mem_avail()
{
  return 0;
}

unsigned jube_mem_usage()
{
#ifdef HUYGENS
  char buf[30];
  unsigned size; //       total program size
  unsigned resident;//   resident set size
  unsigned share;//      shared pages
  unsigned text;//       text (code)
  unsigned lib;//        library
  unsigned data;//       data/stack
  unsigned dt;//         dirty pages (unused in Linux 2.6)
  
  snprintf(buf, 30, "/proc/%u/statm", (unsigned)getpid());
  FILE* pf = fopen(buf, "r");
  if (pf) {
    fscanf(pf, "%u" /* %u %u %u %u %u"*/, &size/*, &resident, &share, &text, &lib, &data*/);
  }
  else
    {
      size = 0;
    }
  fclose(pf);
  
  return size;
#else
  struct rusage rus;
  int ret_rus = getrusage(RUSAGE_SELF, &rus);
  assert(0 == ret_rus);
  
  return rus.ru_maxrss;
#endif

}

void jube_kernel_init(int* kernelnumber);
void jube_kernel_init_(int* kernelnumber)
{
  jube_kernel_init(kernelnumber);
}
void jube_kernel_init(int* kernelnumber)
{
  jube_current_kernel = *kernelnumber;

  printf("JuBE: init kernel %s\n", jube_kernel_names[jube_current_kernel]);

  jube_time_kernel_init[jube_current_kernel] = jube_time_get();

#ifdef IHPCT_HWC
#ifndef IHPCT_ALL
  hpmStart(jube_current_kernel, jube_kernel_names[jube_current_kernel]);
#endif
#endif

#ifdef PAPI
  assert( PAPI_read_counters(jube_papi_values, JUBE_PAPI_NMB_EVENTS) == PAPI_OK );
#endif

}

void jube_kernel_run();
void jube_kernel_run_()
{
  jube_kernel_run();
}
void jube_kernel_run()
{

  printf("JuBE: run kernel %s\n", jube_kernel_names[jube_current_kernel]);

  jube_time_kernel_run[jube_current_kernel] = jube_time_get();
}


void jube_kernel_finalize();
void jube_kernel_finalize_()
{
  jube_kernel_finalize();
}
void jube_kernel_finalize()
{

  printf("JuBE: finalize kernel %s\n", jube_kernel_names[jube_current_kernel]);

  jube_time_kernel_finalize[jube_current_kernel] = jube_time_get();
}

void jube_kernel_end();
void jube_kernel_end_()
{
  jube_kernel_end();
}
void jube_kernel_end()
{
  int cnt;

#ifdef IHPCT_HWC
#ifndef IHPCT_ALL
  hpmStop(jube_current_kernel);
#endif
#endif

#ifdef PAPI
  assert( PAPI_read_counters(jube_papi_values, JUBE_PAPI_NMB_EVENTS) == PAPI_OK );
  for(cnt=0; cnt<JUBE_PAPI_NMB_EVENTS; cnt++)
    {
      PAPI_event_code_to_name(jube_papi_events[cnt], jube_papi_str);
      printf("JuBE: PAPI counter for %s: %s: %lld\n", jube_kernel_names[jube_current_kernel], jube_papi_str, jube_papi_values[cnt]);
    }
#endif

  
  printf("JuBE: end kernel %s\n", jube_kernel_names[jube_current_kernel]);

  jube_time_kernel_end[jube_current_kernel] = jube_time_get();

  printf("JuBE: max mem for %s: %d\n", jube_kernel_names[jube_current_kernel], jube_mem_usage());

}
