#pragma once #include #include #include "gsl/multi_span" #include #include "Configuration.hpp" #include "MpiEnvironment.hpp" #include "Util.hpp" #include "state.hpp" class Communicator { constexpr static std::size_t NoNeighbors{8}; std::vector _neighbors; std::vector _sizes; std::vector _sendTypes; const std::vector& _recvTypes{_sendTypes}; std::vector _sendDisplacements; std::vector _recvDisplacements; MPI_Comm _commDistGraph{MPI_COMM_NULL}; MPI_Datatype _haloRowType; MPI_Datatype _haloColumnType; MPI_Datatype _haloCornerType{MPI_CHAR}; public: Communicator(){}; Communicator(const MpiEnvironment& env, const Size& gridSize, const Size& tileSize) { // Begin definition of basic types MPI_Type_contiguous(tileSize.Cols, MPI_CHAR, &_haloRowType); MPI_Type_commit(&_haloRowType); MPI_Type_vector(tileSize.Rows, 1, tileSize.Cols + 2, MPI_CHAR, &_haloColumnType); MPI_Type_commit(&_haloColumnType); // End definition of basic types // Begin definition of datastructures for a general cell const std::array generalSendTypes{ _haloCornerType, _haloRowType, _haloCornerType, // _haloColumnType, _haloColumnType, // _haloCornerType, _haloRowType, _haloCornerType // }; const auto tCols = tileSize.Cols; const auto tRows = tileSize.Rows; const auto dp = [&](std::size_t x, std::size_t y) { // character coordinates to displacement return static_cast(y * (tCols + 2) + x); }; const std::array generalSendDisplacements{ dp(1, 1), dp(1, 1), dp(tCols, 1), // dp(1, 1), dp(tCols, 1), // dp(1, tRows), dp(1, tRows), dp(tCols, tRows) // }; const std::array generalRecvDisplacements{ dp(0, 0), dp(1, 0), dp(tCols + 1, 0), // dp(0, 1), dp(tCols + 1, 1), // dp(0, tRows + 1), dp(1, tRows + 1), dp(tCols + 1, tRows + 1) // }; const std::array generalSizes{ 1, 1, 1, // 1, 1, // 1, 1, 1 // }; // End definition of datastructures for a general cell // Begin definition of datastructures for this particular cell const auto rank2coord = [&](std::size_t rank) { return Coord{ rank % gridSize.Cols, // rank / gridSize.Cols // }; }; const auto coord2rank = [&](Coord c) { return gridSize.Cols * c.Y + c.X; }; const auto isInsideGrid = [&](Coord c) { return c.X < gridSize.Cols && c.Y < gridSize.Rows; }; const auto myCoord = rank2coord(env.worldRank()); const std::array virtualNeighborCoords{{ {myCoord.X - 1, myCoord.Y - 1}, // intentional signed underflow {myCoord.X + 0, myCoord.Y - 1}, // {myCoord.X + 1, myCoord.Y - 1}, // {myCoord.X - 1, myCoord.Y + 0}, // {myCoord.X + 1, myCoord.Y + 0}, // {myCoord.X - 1, myCoord.Y + 1}, // {myCoord.X + 0, myCoord.Y + 1}, // {myCoord.X + 1, myCoord.Y + 1} // }}; for (std::size_t i{0}; i < NoNeighbors; ++i) { const auto nbrCoord = virtualNeighborCoords[i]; if (isInsideGrid(nbrCoord)) { _neighbors.push_back(coord2rank(nbrCoord)); _sendTypes.push_back(generalSendTypes[i]); _sendDisplacements.push_back(generalSendDisplacements[i]); _recvDisplacements.push_back(generalRecvDisplacements[i]); _sizes.push_back(generalSizes[i]); } } // if (env.worldRank() == 0) { // std::cout << "neighbors:\n"; // for (const auto i : neighbors) std::cout << i << ",\n"; // std::cout << '\n'; //} MPI_Dist_graph_create_adjacent( MPI_COMM_WORLD, // comm_old _neighbors.size(), // indegree _neighbors.data(), // sources reinterpret_cast(MPI_UNWEIGHTED), // sourceweights _neighbors.size(), // outdegree _neighbors.data(), // destinations reinterpret_cast(MPI_UNWEIGHTED), // destweights MPI_INFO_NULL, // info 0, // reorder &_commDistGraph // comm_dist_graph ); // End definition of datastructures for this particular cell } ~Communicator() { if (_commDistGraph != MPI_COMM_NULL) { MPI_Comm_free(&_commDistGraph); MPI_Type_free(&_haloColumnType); MPI_Type_free(&_haloRowType); } } friend void swap(Communicator& first, Communicator& second) { using std::swap; swap(first._neighbors, second._neighbors); swap(first._sizes, second._sizes); swap(first._sendTypes, second._sendTypes); swap(first._sendDisplacements, second._sendDisplacements); swap(first._recvDisplacements, second._recvDisplacements); swap(first._commDistGraph, second._commDistGraph); swap(first._haloRowType, second._haloRowType); swap(first._haloColumnType, second._haloColumnType); swap(first._haloCornerType, second._haloCornerType); } Communicator(Communicator&) = delete; Communicator& operator=(Communicator&) = delete; Communicator(Communicator&& other) noexcept { swap(*this, other); } Communicator& operator=(Communicator&& other) noexcept { swap(*this, other); return *this; } void Communicate(gsl::multi_span& model) { if (_commDistGraph == MPI_COMM_NULL) throw std::logic_error("Communicator not initialized"); MPI_Neighbor_alltoallw(model.data(), // sendbuf _sizes.data(), // sendcounts _sendDisplacements.data(), // sdispl _sendTypes.data(), // sendtypes model.data(), // recvbuf _sizes.data(), // recvcounts _recvDisplacements.data(), // rdispls _recvTypes.data(), // recvtypes _commDistGraph // comm ); } class MpiRequest { MPI_Request _req{MPI_REQUEST_NULL}; bool finished{}; public: MpiRequest(MPI_Request req) : _req(req) {} void Wait() { MPI_Wait(&_req, MPI_STATUS_IGNORE); finished = true; } ~MpiRequest() { if (!finished) { std::cerr << "Forgot to Wait for MPI_Request\n"; MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); } } }; auto AsyncCommunicate(gsl::multi_span& model) { if (_commDistGraph == MPI_COMM_NULL) throw std::logic_error("Communicator not initialized"); MPI_Request req; MPI_Ineighbor_alltoallw(model.data(), // sendbuf _sizes.data(), // sendcounts _sendDisplacements.data(), // sdispl _sendTypes.data(), // sendtypes model.data(), // recvbuf _sizes.data(), // recvcounts _recvDisplacements.data(), // rdispls _recvTypes.data(), // recvtypes _commDistGraph, // comm &req // request ); return MpiRequest{req}; } };