Commit 7336bd74 authored by Thomas Ponweiser's avatar Thomas Ponweiser
Browse files

Halo exchange sample (unstructured grids):

 * added switch for halo width
 * added workaround for OpenMPI bug #1304 (compile with -DOMPI_BUG_1304 to enable workaround)
parent 79c821dc
......@@ -98,6 +98,7 @@ either on the command line or in your batch script.
* `-n [ncells-per-proc]`: Approximate average number of mesh cells per processor; Default: 16k (= 16 * 1024).
* `-N [ncells-total]`: Approximate total number of mesh cells (a nearby cubic number will be chosen)
* `-e [edge-length]`: Edge length of cube (mesh domain).
* `-w [halo-width]`: Halo width (in number of cells); Default: 1.
* `-i [iterations]`: Number of iterations for halo-exchange; Default: 100.
* Selecting halo-exchange mode:
* `--graph` (Default): Use MPI Graph topology and neighborhood collectives. If supported, this allows MPI to reorder the processes in order to choose a good embedding of the virtual topology to the physical machine.
......
......@@ -33,6 +33,7 @@ void conf_init(conf_t *c)
#pragma message("MPI_VERSION < 3 => Default transfer mode for halo exchange will be collective.")
c->transfer_mode = COLLECTIVE;
#endif
c->halo_width = 1;
c->n_iterations = 100;
// Default values:
......@@ -48,6 +49,7 @@ const char* conf_set_from_args(conf_t *c, int argc, char* argv[])
{"ncells-per-proc", required_argument, NULL, 'n'},
{"ncells", required_argument, NULL, 'N'},
{"edge-length", required_argument, NULL, 'e'},
{"halo-width", required_argument, NULL, 'w'},
{"graph", no_argument, &c->transfer_mode, SPARSE_COLLECTIVE},
{"collective", no_argument, &c->transfer_mode, COLLECTIVE},
......@@ -60,13 +62,14 @@ const char* conf_set_from_args(conf_t *c, int argc, char* argv[])
};
int option;
while((option = getopt_long(argc, argv, "v:g:n:N:e:i:", long_options, NULL)) >= 0) {
while((option = getopt_long(argc, argv, "v:g:n:N:e:w:i:", long_options, NULL)) >= 0) {
switch(option) {
case 'v': c->verbosity_level = atoi(optarg); break;
case 'g': c->debug_rank = atoi(optarg); break;
case 'n': set_ncells_per_proc(c, parse_long(optarg)); break;
case 'N': set_ncells_total(c, parse_long(optarg)); break;
case 'e': set_global_domain(c, atoi(optarg)); break;
case 'w': c->halo_width = atoi(optarg); break;
case 'i': c->n_iterations = parse_long(optarg); break;
}
}
......@@ -89,6 +92,7 @@ void conf_print(const conf_t *c, FILE *f)
fprintf(f, "Configuration:\n");
fprintf(f, " * Verbosity level: %s (%d)\n", verbosity_levels[i], c->verbosity_level);
fprintf(f, " * Mesh domain (cube): %s\n", box_to_string(&c->global_domain, buf, 128));
fprintf(f, " * Halo width: %d cell%s\n", c->halo_width, c->halo_width == 1 ? "" : "s");
fprintf(f, " * Halo transfer mode: %s\n", transfer_mode_strs[c->transfer_mode]);
fprintf(f, " * Number of iterations: %d\n", c->n_iterations);
fprintf(f, "\n");
......
......@@ -26,15 +26,16 @@ typedef struct
{
int verbosity_level;
int debug_rank;
int transfer_mode;
int halo_width;
int transfer_mode;
int n_iterations;
box_t global_domain;
} conf_t;
// Meta-information for MPI-Datatype creation (see mpitypes.c)
#define CONF_T_N_INT_MEMBERS 4
#define CONF_T_N_INT_MEMBERS 5
#define CONF_T_FIRST_INT_MEMBER verbosity_level
#define CONF_T_N_BOX_MEMBERS 1
#define CONF_T_FIRST_BOX_MEMBER global_domain
......
......@@ -219,7 +219,7 @@ void print_adjacency_matrix(const mesh_t *mesh, const conf_t *configuration, FIL
if(n > 0 && min > n) min = recvbuf[i];
if(max < n) max = n;
}
fprintf(f, "Number of adjacent cells per neighbor (min-max): %d - %d\n\n", min, max);
fprintf(f, "Number of halo cells per neighbor (min-max): %d - %d\n\n", min, max);
free(recvbuf);
}
free(sendbuf);
......
......@@ -84,7 +84,7 @@ void mesh_find_neighbors(mesh_t *mesh, const conf_t *configuration, const box_t
// Determine local domain (without and with halo cells)
local_domain = domains[mpi_rank];
tmp = local_domain;
box_grow(&tmp, 1);
box_grow(&tmp, configuration->halo_width);
box_intersect(&tmp, &configuration->global_domain, &extended_local_domain);
// Reserve space for neighbors
......@@ -107,7 +107,7 @@ void mesh_find_neighbors(mesh_t *mesh, const conf_t *configuration, const box_t
// Determine cells for outgoing halo data
extended_remote_domain = domains[i];
box_grow(&extended_remote_domain, 1);
box_grow(&extended_remote_domain, configuration->halo_width);
box_intersect(&local_domain, &extended_remote_domain, &halo_outgoing);
// Initialize neighbor info
......
......@@ -33,7 +33,9 @@ void mpitype_box_init(MPI_Datatype *new_type)
void mpitype_box_free(MPI_Datatype *type)
{
#ifndef OMPI_BUG_1304 // see https://github.com/open-mpi/ompi/issues/1304
MPI_Type_free(type);
#endif
}
void mpitype_conf_init(MPI_Datatype *new_type)
......@@ -64,7 +66,9 @@ void mpitype_conf_init(MPI_Datatype *new_type)
void mpitype_conf_free(MPI_Datatype *type)
{
#ifndef OMPI_BUG_1304 // see https://github.com/open-mpi/ompi/issues/1304
MPI_Type_free(type);
#endif
}
// --------------------------------------------------------- Helper functions
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment