#include <iostream>
#include <cmath>
#include <cfloat>
#include <cstring>
#include <Body.hpp>
#include <Box.hpp>
#include <Tree.hpp>
#include <Node.hpp>
#include <iostream>
#include <BarnesHutTree.hpp>
#include "MpiSimulation.hpp"
namespace nbody {
using namespace std;
MpiSimulation::MpiSimulation(int& argc, char**& argv) {
this->bodyType = MPI_DATATYPE_NULL;
this->boxType = MPI_DATATYPE_NULL;
int bodyBlocklengths[6] = {1, 3, 3, 3, 1, 1};
MPI_Datatype bodyDatatypes[6] = {MPI_UNSIGNED_LONG, MPI_DOUBLE, MPI_DOUBLE, MPI_DOUBLE, MPI_DOUBLE, MPI_INT};
MPI_Aint bodyOffsets[6];
bodyOffsets[0] = offsetof(Body, id);
bodyOffsets[1] = offsetof(Body, position);
bodyOffsets[2] = offsetof(Body, velocity);
bodyOffsets[3] = offsetof(Body, acceleration);
bodyOffsets[4] = offsetof(Body, mass);
bodyOffsets[5] = offsetof(Body, refinement);
MPI_Type_create_struct(6, bodyBlocklengths, bodyOffsets, bodyDatatypes, &this->bodyType);
MPI_Type_commit(&this->bodyType);
int boxBlocklengths[2] = {3, 3};
MPI_Datatype boxDatatypes[2] = {MPI_DOUBLE, MPI_DOUBLE};
MPI_Aint boxOffsets[2];
boxOffsets[0] = offsetof(Box, min);
boxOffsets[1] = offsetof(Box, max);
MPI_Type_create_struct(2, boxBlocklengths, boxOffsets, boxDatatypes, &this->boxType);
MPI_Type_commit(&this->boxType);
MPI_Comm_size(MPI_COMM_WORLD, &this->parallelSize);
MPI_Comm_rank(MPI_COMM_WORLD, &this->parallelRank);
this->domains = new Box[this->parallelSize];
this->correctState = true;
this->tree = new BarnesHutTree(this->parallelRank);
if (argc == 2) {
this->correctState = this->readInputData(string(argv[1]));
} else {
this->correctState = false;
}
MPI_Bcast(&this->correctState, 1, MPI_INT, 0, MPI_COMM_WORLD);
if (!this->correctState) {
cerr << "Error occurred: terminating ..." << endl;
MPI_Type_free(&this->bodyType);
MPI_Type_free(&this->boxType);
MPI_Finalize();
exit(-1);
}
}
void MpiSimulation::cleanup() {
MPI_Type_free(&this->bodyType);
MPI_Type_free(&this->boxType);
delete this->tree;
this->tree = NULL;
}
bool MpiSimulation::stateCorrect() {
return this->correctState != 0;
}
MpiSimulation::~MpiSimulation() {
delete[] this->domains;
delete this->tree;
while (!this->sendStores.empty()) {
delete[] this->sendStores.back().bodies;
this->sendStores.pop_back();
}
}
int MpiSimulation::getNumberOfProcesses() {
return this->parallelSize;
}
int MpiSimulation::getProcessId() {
return this->parallelRank;
}
MPI_Datatype* MpiSimulation::getDatatype() {
return &this->bodyType;
}
void MpiSimulation::send(vector<Body> bodies, int target) {
int bodySize = bodies.size();
SendStore* store = this->availableSendStore(bodySize);
memcpy(store->bodies, &(bodies[0]), bodySize * sizeof(Body));
MPI_Isend(store->bodies, bodySize, this->bodyType, target, 0, MPI_COMM_WORLD, &store->request);
//MPI_Send(&(bodies[0]), bodySize, this->bodyType, target, 0, MPI_COMM_WORLD);
}
void MpiSimulation::recv(vector<Body>& bodies) {
MPI_Status status;
int count;
Body* lb;
MPI_Probe(0, 0, MPI_COMM_WORLD, &status);
MPI_Get_count(&status, this->bodyType, &count);
lb = new Body[count];
MPI_Recv(lb, count, this->bodyType, status.MPI_SOURCE, 0, MPI_COMM_WORLD, &status);
bodies = vector<Body>(lb, lb + count);
delete[] lb;
}
void MpiSimulation::distributeBodies() {
if (this->parallelRank == 0) {
vector<Node> nodes;
Box bb;
init(bb);
nodes.push_back(Node(NULL));
nodes.front().setBodies(this->bodies);
extendForBodies(bb, this->bodies);
nodes.front().setBB(bb);
for (unsigned int i = 0; i < nodes.size(); i++) {
printBB(i, nodes.front().getBB());
}
while (nodes.size() < (unsigned int) this->parallelSize) {
int mostBodiesIndex = 0;
for (unsigned int i = 1; i < nodes.size(); i++) {
if (nodes[i].getBodies().size() > nodes[mostBodiesIndex].getBodies().size()) {
mostBodiesIndex = i;
}
}
vector<Box> subdomains = splitLongestSide(nodes[mostBodiesIndex].getBB());
vector<Body> buf = nodes[mostBodiesIndex].getBodies();
Node n(NULL);
n.setBodies(extractBodies(subdomains[0], buf));
n.setBB(subdomains[0]);
nodes.insert(nodes.begin() + mostBodiesIndex, n);
n = Node(NULL);
n.setBodies(extractBodies(subdomains[1], buf));
n.setBB(subdomains[1]);
nodes.insert(nodes.begin() + mostBodiesIndex, n);
nodes.erase(nodes.begin() + mostBodiesIndex + 2);
}
this->bodies = nodes[0].getBodies();
for (unsigned int i = 1; i < nodes.size(); i++) {
this->send(nodes[i].getBodies(), i);
}
} else {
this->recv(this->bodies);
}
}
void MpiSimulation::distributeDomains(vector<Body> localBodies) {
Box localDomain;
init(localDomain);
extendForBodies(localDomain, localBodies);
this->distributeDomains(localDomain);
}
void MpiSimulation::distributeDomains() {
this->distributeDomains(this->bodies);
}
void MpiSimulation::distributeDomains(Box localDomain) {
this->domains[this->parallelRank] = localDomain;
MPI_Allgather(&this->domains[this->parallelRank], 1, this->boxType, &this->domains[0], 1, this->boxType, MPI_COMM_WORLD);
this->overallDomain = localDomain;
for (int i = 0; i < this->parallelSize; i++) {
extend(this->overallDomain, this->domains[i]);
}
}
SendStore* MpiSimulation::availableSendStore(int numElems) {
vector<SendStore>::iterator it = this->sendStores.begin();
while (it != this->sendStores.end()) {
int completed;
MPI_Test(&it->request, &completed, MPI_STATUS_IGNORE);
if (it->size >= numElems && completed) {
return &(*it);
} else if (completed) {
delete[] it->bodies;
it = this->sendStores.erase(it);
} else {
it++;
}
}
SendStore store;
store.bodies = new Body[numElems];
store.size = numElems;
this->sendStores.push_back(store);
return &(this->sendStores.back());
}
void MpiSimulation::distributeLETs() {
//send out locally essential trees (local bodies needed by remote simulations)
for (int i = 0; i < this->parallelSize; i++) {
if (i != this->parallelRank) {
//printBB(this->parallelRank, this->domains[i]);
vector<Body> refinements = this->tree->copyRefinements(this->domains[i]);
//cout << this->parallelRank << " -> " << i << ": " << refinements.size() << endl;
this->send(refinements, i);
}
}
//receive bodies and integrate them into local tree for simulation
int received = 0;
while (received < this->parallelSize - 1) {
vector<Body> refinements;
this->recv(refinements);
//this->tree->mergeLET(refinements);
/*
//integrate bodies in order of arrival to do communication/computation overlapping
this->comms[0].recvBlocking(MPI_ANY_SOURCE, refinements);
//cout << "recv: " << refinements.size() << endl;
this->tree->mergeLET(refinements);
//this->tree.getRootBB().print();
*/
received++;
}
if (!this->tree->isCorrect()) {
cout << "WRONG" << endl;
}
}
void MpiSimulation::buildTree() {
if (this->bodies.size() > 0 && isValid(this->overallDomain)) {
this->tree->build(this->bodies, this->overallDomain);
} else if (this->bodies.size() > 0) {
this->tree->build(this->bodies);
}
if (!this->tree->isCorrect()) {
cout << "wrong tree" << endl;
} else {
cout << this->parallelRank << " tree: " << this->tree->numberOfNodes() << endl;
}
}
void MpiSimulation::rebuildTree() {
this->tree->rebuild(this->overallDomain);
}
void MpiSimulation::runStep() {
this->distributeLETs();
/*
this->tree->computeForces();
this->distributeDomains(this->tree->advance());
this->rebuildTree();
*/
if (!this->tree->isCorrect()) {
cout << "WRONG" << endl;
}
}
}