Commit 7336bd74 authored by Thomas Ponweiser's avatar Thomas Ponweiser
Browse files

Halo exchange sample (unstructured grids):

 * added switch for halo width
 * added workaround for OpenMPI bug #1304 (compile with -DOMPI_BUG_1304 to enable workaround)
parent 79c821dc
...@@ -98,6 +98,7 @@ either on the command line or in your batch script. ...@@ -98,6 +98,7 @@ either on the command line or in your batch script.
* `-n [ncells-per-proc]`: Approximate average number of mesh cells per processor; Default: 16k (= 16 * 1024). * `-n [ncells-per-proc]`: Approximate average number of mesh cells per processor; Default: 16k (= 16 * 1024).
* `-N [ncells-total]`: Approximate total number of mesh cells (a nearby cubic number will be chosen) * `-N [ncells-total]`: Approximate total number of mesh cells (a nearby cubic number will be chosen)
* `-e [edge-length]`: Edge length of cube (mesh domain). * `-e [edge-length]`: Edge length of cube (mesh domain).
* `-w [halo-width]`: Halo width (in number of cells); Default: 1.
* `-i [iterations]`: Number of iterations for halo-exchange; Default: 100. * `-i [iterations]`: Number of iterations for halo-exchange; Default: 100.
* Selecting halo-exchange mode: * Selecting halo-exchange mode:
* `--graph` (Default): Use MPI Graph topology and neighborhood collectives. If supported, this allows MPI to reorder the processes in order to choose a good embedding of the virtual topology to the physical machine. * `--graph` (Default): Use MPI Graph topology and neighborhood collectives. If supported, this allows MPI to reorder the processes in order to choose a good embedding of the virtual topology to the physical machine.
......
...@@ -33,6 +33,7 @@ void conf_init(conf_t *c) ...@@ -33,6 +33,7 @@ void conf_init(conf_t *c)
#pragma message("MPI_VERSION < 3 => Default transfer mode for halo exchange will be collective.") #pragma message("MPI_VERSION < 3 => Default transfer mode for halo exchange will be collective.")
c->transfer_mode = COLLECTIVE; c->transfer_mode = COLLECTIVE;
#endif #endif
c->halo_width = 1;
c->n_iterations = 100; c->n_iterations = 100;
// Default values: // Default values:
...@@ -48,6 +49,7 @@ const char* conf_set_from_args(conf_t *c, int argc, char* argv[]) ...@@ -48,6 +49,7 @@ const char* conf_set_from_args(conf_t *c, int argc, char* argv[])
{"ncells-per-proc", required_argument, NULL, 'n'}, {"ncells-per-proc", required_argument, NULL, 'n'},
{"ncells", required_argument, NULL, 'N'}, {"ncells", required_argument, NULL, 'N'},
{"edge-length", required_argument, NULL, 'e'}, {"edge-length", required_argument, NULL, 'e'},
{"halo-width", required_argument, NULL, 'w'},
{"graph", no_argument, &c->transfer_mode, SPARSE_COLLECTIVE}, {"graph", no_argument, &c->transfer_mode, SPARSE_COLLECTIVE},
{"collective", no_argument, &c->transfer_mode, COLLECTIVE}, {"collective", no_argument, &c->transfer_mode, COLLECTIVE},
...@@ -60,13 +62,14 @@ const char* conf_set_from_args(conf_t *c, int argc, char* argv[]) ...@@ -60,13 +62,14 @@ const char* conf_set_from_args(conf_t *c, int argc, char* argv[])
}; };
int option; int option;
while((option = getopt_long(argc, argv, "v:g:n:N:e:i:", long_options, NULL)) >= 0) { while((option = getopt_long(argc, argv, "v:g:n:N:e:w:i:", long_options, NULL)) >= 0) {
switch(option) { switch(option) {
case 'v': c->verbosity_level = atoi(optarg); break; case 'v': c->verbosity_level = atoi(optarg); break;
case 'g': c->debug_rank = atoi(optarg); break; case 'g': c->debug_rank = atoi(optarg); break;
case 'n': set_ncells_per_proc(c, parse_long(optarg)); break; case 'n': set_ncells_per_proc(c, parse_long(optarg)); break;
case 'N': set_ncells_total(c, parse_long(optarg)); break; case 'N': set_ncells_total(c, parse_long(optarg)); break;
case 'e': set_global_domain(c, atoi(optarg)); break; case 'e': set_global_domain(c, atoi(optarg)); break;
case 'w': c->halo_width = atoi(optarg); break;
case 'i': c->n_iterations = parse_long(optarg); break; case 'i': c->n_iterations = parse_long(optarg); break;
} }
} }
...@@ -89,6 +92,7 @@ void conf_print(const conf_t *c, FILE *f) ...@@ -89,6 +92,7 @@ void conf_print(const conf_t *c, FILE *f)
fprintf(f, "Configuration:\n"); fprintf(f, "Configuration:\n");
fprintf(f, " * Verbosity level: %s (%d)\n", verbosity_levels[i], c->verbosity_level); fprintf(f, " * Verbosity level: %s (%d)\n", verbosity_levels[i], c->verbosity_level);
fprintf(f, " * Mesh domain (cube): %s\n", box_to_string(&c->global_domain, buf, 128)); fprintf(f, " * Mesh domain (cube): %s\n", box_to_string(&c->global_domain, buf, 128));
fprintf(f, " * Halo width: %d cell%s\n", c->halo_width, c->halo_width == 1 ? "" : "s");
fprintf(f, " * Halo transfer mode: %s\n", transfer_mode_strs[c->transfer_mode]); fprintf(f, " * Halo transfer mode: %s\n", transfer_mode_strs[c->transfer_mode]);
fprintf(f, " * Number of iterations: %d\n", c->n_iterations); fprintf(f, " * Number of iterations: %d\n", c->n_iterations);
fprintf(f, "\n"); fprintf(f, "\n");
......
...@@ -26,15 +26,16 @@ typedef struct ...@@ -26,15 +26,16 @@ typedef struct
{ {
int verbosity_level; int verbosity_level;
int debug_rank; int debug_rank;
int transfer_mode;
int halo_width;
int transfer_mode;
int n_iterations; int n_iterations;
box_t global_domain; box_t global_domain;
} conf_t; } conf_t;
// Meta-information for MPI-Datatype creation (see mpitypes.c) // Meta-information for MPI-Datatype creation (see mpitypes.c)
#define CONF_T_N_INT_MEMBERS 4 #define CONF_T_N_INT_MEMBERS 5
#define CONF_T_FIRST_INT_MEMBER verbosity_level #define CONF_T_FIRST_INT_MEMBER verbosity_level
#define CONF_T_N_BOX_MEMBERS 1 #define CONF_T_N_BOX_MEMBERS 1
#define CONF_T_FIRST_BOX_MEMBER global_domain #define CONF_T_FIRST_BOX_MEMBER global_domain
......
...@@ -219,7 +219,7 @@ void print_adjacency_matrix(const mesh_t *mesh, const conf_t *configuration, FIL ...@@ -219,7 +219,7 @@ void print_adjacency_matrix(const mesh_t *mesh, const conf_t *configuration, FIL
if(n > 0 && min > n) min = recvbuf[i]; if(n > 0 && min > n) min = recvbuf[i];
if(max < n) max = n; if(max < n) max = n;
} }
fprintf(f, "Number of adjacent cells per neighbor (min-max): %d - %d\n\n", min, max); fprintf(f, "Number of halo cells per neighbor (min-max): %d - %d\n\n", min, max);
free(recvbuf); free(recvbuf);
} }
free(sendbuf); free(sendbuf);
......
...@@ -84,7 +84,7 @@ void mesh_find_neighbors(mesh_t *mesh, const conf_t *configuration, const box_t ...@@ -84,7 +84,7 @@ void mesh_find_neighbors(mesh_t *mesh, const conf_t *configuration, const box_t
// Determine local domain (without and with halo cells) // Determine local domain (without and with halo cells)
local_domain = domains[mpi_rank]; local_domain = domains[mpi_rank];
tmp = local_domain; tmp = local_domain;
box_grow(&tmp, 1); box_grow(&tmp, configuration->halo_width);
box_intersect(&tmp, &configuration->global_domain, &extended_local_domain); box_intersect(&tmp, &configuration->global_domain, &extended_local_domain);
// Reserve space for neighbors // Reserve space for neighbors
...@@ -107,7 +107,7 @@ void mesh_find_neighbors(mesh_t *mesh, const conf_t *configuration, const box_t ...@@ -107,7 +107,7 @@ void mesh_find_neighbors(mesh_t *mesh, const conf_t *configuration, const box_t
// Determine cells for outgoing halo data // Determine cells for outgoing halo data
extended_remote_domain = domains[i]; extended_remote_domain = domains[i];
box_grow(&extended_remote_domain, 1); box_grow(&extended_remote_domain, configuration->halo_width);
box_intersect(&local_domain, &extended_remote_domain, &halo_outgoing); box_intersect(&local_domain, &extended_remote_domain, &halo_outgoing);
// Initialize neighbor info // Initialize neighbor info
......
...@@ -33,7 +33,9 @@ void mpitype_box_init(MPI_Datatype *new_type) ...@@ -33,7 +33,9 @@ void mpitype_box_init(MPI_Datatype *new_type)
void mpitype_box_free(MPI_Datatype *type) void mpitype_box_free(MPI_Datatype *type)
{ {
#ifndef OMPI_BUG_1304 // see https://github.com/open-mpi/ompi/issues/1304
MPI_Type_free(type); MPI_Type_free(type);
#endif
} }
void mpitype_conf_init(MPI_Datatype *new_type) void mpitype_conf_init(MPI_Datatype *new_type)
...@@ -64,7 +66,9 @@ void mpitype_conf_init(MPI_Datatype *new_type) ...@@ -64,7 +66,9 @@ void mpitype_conf_init(MPI_Datatype *new_type)
void mpitype_conf_free(MPI_Datatype *type) void mpitype_conf_free(MPI_Datatype *type)
{ {
#ifndef OMPI_BUG_1304 // see https://github.com/open-mpi/ompi/issues/1304
MPI_Type_free(type); MPI_Type_free(type);
#endif
} }
// --------------------------------------------------------- Helper functions // --------------------------------------------------------- Helper functions
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment