main.c

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <mpi.h>

#include "configuration.h"
#include "mpitypes.h"
#include "io.h"
#include "world.h"
#include "simulation.h"

void broadcast_configuration(conf_t *c);

void read_input(const conf_t *c, world_t *world);

void iterate(const conf_t *c, world_t *world);

void print_avg_timings(const conf_t *c, double total_time, double sim_time, double io_time);

int main(int argc, char* argv[])
{
   int rank, nprocs;
   conf_t config;
   world_t world;

   MPI_Init(&argc, &argv);

   MPI_Comm_rank(MPI_COMM_WORLD, &rank);
   MPI_Comm_size(MPI_COMM_WORLD, &nprocs);

   // We want the program to stop on I/O errors
   // -> Change the default I/O error hander from MPI_ERRORS_RETURN to MPI_ERRORS_ARE_FATAL
   // Notes:
   //  * An individual I/O error handler can be associated to each file handle.
   //  * The default I/O error handler is associated to the null file handle, i.e. MPI_FILE_NULL.
   MPI_File_set_errhandler(MPI_FILE_NULL, MPI_ERRORS_ARE_FATAL);

   // Parse command line arguments and broadcast configuration
   if(rank == 0) {
      conf_init_from_args(&config, argc, argv);
   }
   broadcast_configuration(&config);

   if(info_enabled(&config)) conf_print(&config, stdout);

   // Read input file
   read_input(&config, &world);

   // Run simulation
   iterate(&config, &world);

   if(info_enabled(&config)) printf("Done.\n");

   MPI_Finalize();
   return EXIT_SUCCESS;
}

void broadcast_configuration(conf_t *c) {
   MPI_Datatype conf_type;

   mpitype_conf_init(&conf_type);

   MPI_Bcast(c, 1, conf_type, 0, MPI_COMM_WORLD);

   mpitype_conf_free(&conf_type);
}

void read_input(const conf_t *c, world_t *world) {
   int rank;
   MPI_Comm cart_comm;

   char input_filename[FILE_NAME_SZ];
   MPI_File file;

   size_t global_sizes[2];
   size_t header_length;

   MPI_Comm_rank(MPI_COMM_WORLD, &rank);

   // Determine input file name
   snprintf(input_filename, sizeof(input_filename), "%s%s", c->file_basename, FILE_EXT);

   if(info_enabled(c)) printf("Reading '%s'...\n\n", input_filename);
   MPI_File_open(MPI_COMM_WORLD, input_filename, MPI_MODE_RDONLY, MPI_INFO_NULL, &file);

   // Read header
   file_read_header(file, global_sizes, &header_length);

   if(debug_enabled(c)) printf(
      "Read header (%ld characters).\n"
      "Global size: %ld x %ld\n\n",
      header_length, global_sizes[0], global_sizes[1]
   );
   // Initialize cells (determine local tile, allocate memory)
   world_init(world, global_sizes, c);

   // Collectively read cell data
   file_read_world(file, world, header_length);
   MPI_File_close(&file);
}

void iterate(const conf_t *c, world_t *world)
{
   const int  n_it  = c->n_iterations;
   const long n_gen = c->n_generations_per_iteration;

   size_t i, g;
   double total_time, sim_time = 0, io_time = 0;

   char output_filename[FILE_NAME_SZ+10];
   MPI_File file;
   size_t header_length;

   if(info_enabled(c)) printf(
      "Running %d iteration%s with %ld generation%s per iteration.\n\n",
      n_it,  n_it  == 1 ? "" : "s",
      n_gen, n_gen == 1 ? "" : "s"
   );

   total_time = -MPI_Wtime();
   g = 0;
   for(i = 1; i <= n_it; i++) {

      // Run n_gen generations
      sim_time -= MPI_Wtime();
      do_simulation(world, n_gen, c); g += n_gen;
      sim_time += MPI_Wtime();

      // Determine output filename
      snprintf(
         output_filename, sizeof(output_filename),
         "%s+%09ld%s",
         c->file_basename, g, FILE_EXT
      );

      // Write output file
      io_time -= MPI_Wtime();
      MPI_File_open(
         MPI_COMM_WORLD, output_filename,
         MPI_MODE_CREATE | MPI_MODE_WRONLY,
         MPI_INFO_NULL, &file
      );
      file_write_header(file, world, &header_length);
      file_write_world(file, world, header_length);
      MPI_File_close(&file);
      io_time += MPI_Wtime();

      if(info_enabled(c)) printf("Generation %-9ld - written '%s'.\n", g, output_filename);
   }
   total_time += MPI_Wtime();

   print_avg_timings(c, total_time, sim_time, io_time);
}

void print_avg_timings(const conf_t *c, double total_time, double sim_time, double io_time)
{
   const int root = 0;
   const size_t total_generations = c->n_iterations * c->n_generations_per_iteration;
   int rank, nprocs;
   double times[] = {total_time, sim_time, io_time};
   void *sendbuf, *recvbuf;

   MPI_Comm_rank(MPI_COMM_WORLD, &rank);
   MPI_Comm_size(MPI_COMM_WORLD, &nprocs);

   sendbuf = rank == root ? MPI_IN_PLACE : times;
   recvbuf = rank == root ? times        : NULL;

   MPI_Reduce(
      sendbuf, recvbuf, 3, MPI_DOUBLE,
      MPI_SUM, root, MPI_COMM_WORLD
   );

   if(info_enabled(c)) {
      total_time = times[0] / nprocs;
      sim_time =   times[1] / nprocs;
      io_time =    times[2] / nprocs;

      printf(
         "\nStatistics:\n"
         " * Generations per second:  %.0f\n"
         " * Net simulation time (s): %f\n"
         " * Net I/O time (s):        %f\n"
         " * Total time (s):          %f\n\n",
         total_generations / sim_time, sim_time, io_time, total_time
      );
   }
}