Communicator.hpp

#pragma once

#include <boost/container/static_vector.hpp>
#include <mpi.h>

#include "Configuration.hpp"
#include "MpiEnvironment.hpp"
#include "State.hpp"
#include "Util.hpp"

// creates the graph topology and does the communication
class Communicator {
	constexpr static std::size_t NoNeighbors{8};

	// for very small container with known max size, this is a performance
	// improvement
	template <typename T>
	using Vector = boost::container::static_vector<T, NoNeighbors>;

  public:
	// Life cycle handling for MPI_Requests
	class MpiRequest { // TODO: unnest
	  public:
		template <typename T>
		using DoubleVector =
		    boost::container::static_vector<T, NoNeighbors * 2>;

	  private:
		DoubleVector<MPI_Request> _reqs;
		bool finished{};

	  public:
		MpiRequest(DoubleVector<MPI_Request> reqs);
		MpiRequest(const MpiRequest&) = default;
		MpiRequest(MpiRequest&&) = default;
		MpiRequest& operator=(const MpiRequest&) = default;
		MpiRequest& operator=(MpiRequest&&) = default;
		void Wait();
		~MpiRequest();
	};

  private:
	CommunicationMode _commMode;

	// data members for graph topology
	Vector<int> _neighbors;
	Vector<int> _sizes;
	Vector<MPI_Datatype> _sendTypes;
	const Vector<MPI_Datatype>& _recvTypes{_sendTypes};
	Vector<MPI_Aint> _sendDisplacements;
	Vector<MPI_Aint> _recvDisplacements;
	MPI_Comm _commDistGraph{MPI_COMM_NULL};

	// data types
	MPI_Datatype _haloRowType{};
	MPI_Datatype _haloColumnType{};
	MPI_Datatype _haloCornerType{MPI_CHAR};

  public:
	Communicator() = default;
	Communicator(const MpiEnvironment& env, CommunicationMode commMode,
	             const Size& gridSize, const Size& tileSize);
	~Communicator();
	void swap(Communicator& first, Communicator& second);
	Communicator(Communicator&) = delete;
	Communicator& operator=(Communicator&) = delete;
	Communicator(Communicator&& other) noexcept;
	Communicator& operator=(Communicator&& other) noexcept;

	void Communicate(State* model);
	MpiRequest AsyncCommunicate(State* model);
};