DomainDecompMPIBase.h

/*
 * DomainDecompBaseMPI.h
 *
 *  Created on: Nov 15, 2015
 *      Author: tchipevn
 */
 
#ifndef DOMAINDECOMPMPIBASE_H_
#define DOMAINDECOMPMPIBASE_H_
 
#include <mpi.h>
#include <vector>
#include <memory>
 
#include "utils/Logger.h"
#include "DomainDecompBase.h"
#include "CollectiveCommunicationInterface.h"
#include "CommunicationPartner.h"
#include "ParticleDataForwardDeclaration.h"
 
#define LOWER  0
#define HIGHER 1
 
#define DIMgeom 3
 
class NeighbourCommunicationScheme;
 
struct HaloRegion;
 
class DomainDecompMPIBase: public DomainDecompBase {
public:
    DomainDecompMPIBase();
    virtual ~~DomainDecompMPIBase();
 
    // documentation see father class (DomainDecompBase.h)
    void barrier() const override {
        MPI_CHECK(MPI_Barrier(_comm));
    }
 
    unsigned Ndistribution(unsigned localN, float* minrnd, float* maxrnd);
 
    void assertIntIdentity(int IX);
    void assertDisjunctivity(ParticleContainer* moleculeContainer) const override;
 
    //##################################################################
    // The following methods with prefix "collComm" are all used
    // in the context of collective communication. Each of the methods
    // basically has to call the corresponding method from the class
    // CollectiveCommunication (or CollectiveCommDummy in the sequential
    // case). To get information about how to use this methods, read
    // the documentation of the class CollectiveCommunication and of the
    // father class of this class (DomainDecompBase.h)
    //##################################################################
    void collCommInit(int numValues, int key=0) override {
        _collCommunication->init(_comm, numValues, key);
    }
 
    void collCommFinalize() override {
        _collCommunication->finalize();
    }
 
    void collCommAppendInt(int intValue) override {
        _collCommunication->appendInt(intValue);
    }
 
    void collCommAppendUnsLong(unsigned long unsLongValue) override {
        _collCommunication->appendUnsLong(unsLongValue);
    }
 
    void collCommAppendFloat(float floatValue) override {
        _collCommunication->appendFloat(floatValue);
    }
 
    void collCommAppendDouble(double doubleValue) override {
        _collCommunication->appendDouble(doubleValue);
    }
 
    void collCommAppendLongDouble(long double longDoubleValue) override {
        _collCommunication->appendLongDouble(longDoubleValue);
    }
 
    int collCommGetInt() override {
        return _collCommunication->getInt();
    }
 
    unsigned long collCommGetUnsLong() override {
        return _collCommunication->getUnsLong();
    }
 
    float collCommGetFloat() override {
        return _collCommunication->getFloat();
    }
 
    double collCommGetDouble() override {
        return _collCommunication->getDouble();
    }
 
    long double collCommGetLongDouble() override {
        return _collCommunication->getLongDouble();
    }
 
    void collCommAllreduceSum() override {
        _collCommunication->allreduceSum();
    }
 
    void collCommAllreduceSumAllowPrevious() override;
 
    void collCommAllreduceCustom(ReduceType type) override {
        _collCommunication->allreduceCustom(type);
    }
 
    void collCommScanSum() override {
        _collCommunication->scanSum();
    }
 
    void collCommBroadcast(int root = 0) override {
        _collCommunication->broadcast(root);
    }
 
    virtual void balanceAndExchangeInitNonBlocking(bool forceRebalancing, ParticleContainer* moleculeContainer,
            Domain* domain);
 
    virtual void prepareNonBlockingStage(bool forceRebalancing, ParticleContainer* moleculeContainer, Domain* domain,
            unsigned int stageNumber) = 0;
 
    virtual void finishNonBlockingStage(bool forceRebalancing, ParticleContainer* moleculeContainer, Domain* domain,
            unsigned int stageNumber) = 0;
 
    void exchangeMoleculesMPI(ParticleContainer* moleculeContainer, Domain* domain, MessageType msgType,
                              bool doHaloPositionCheck = true, bool removeRecvDuplicates = false);
 
    void exchangeForces(ParticleContainer* moleculeContainer, Domain* domain) override;
 
    std::vector<int> getNeighbourRanks() override = 0;
    std::vector<int> getNeighbourRanksFullShell() override = 0;
 
    virtual std::vector<CommunicationPartner> getNeighboursFromHaloRegion(Domain* domain, const HaloRegion& haloRegion, double cutoff) = 0;
 
 
#if defined(ENABLE_MPI)
    MPI_Datatype getMPIParticleType() {
        return _mpiParticleType;
    }
    MPI_Datatype getMPIParticleForceType() {
        return _mpiParticleForceType;
    }
    MPI_Comm getCommunicator() override {
        return _comm;
    }
#endif
 
    virtual void readXML(XMLfileUnits& xmlconfig);
 
    virtual void setCommunicationScheme(const std::string& scheme, const std::string& comScheme);
 
    // documentation in base class
    virtual int getNonBlockingStageCount() override;
 
    virtual size_t getTotalSize() override;
 
    virtual void printSubInfo(int offset) override;
 
    virtual void printDecomp(const std::string &filename, Domain *domain, ParticleContainer *particleContainer) override;
 
    virtual std::string getName() override {
        return "DomainDecompMPIBase";
    }
 
    void printCommunicationPartners(std::string filename) const override;
protected:
 
    virtual void prepareNonBlockingStageImpl(ParticleContainer* moleculeContainer, Domain* domain,
            unsigned int stageNumber, MessageType msgType, bool removeRecvDuplicates = false);
 
    virtual void finishNonBlockingStageImpl(ParticleContainer* moleculeContainer, Domain* domain,
            unsigned int stageNumber, MessageType msgType, bool removeRecvDuplicates = false);
 
    MPI_Datatype _mpiParticleType;
    MPI_Datatype _mpiParticleForceType;
 
    MPI_Comm _comm;
 
    std::unique_ptr<NeighbourCommunicationScheme> _neighbourCommunicationScheme;
 
    bool _forceDirectPP{false};
private:
    std::unique_ptr<CollectiveCommunicationInterface> _collCommunication;
    unsigned long _overlappingStartAtStep {5ul};
};
 
#endif /* DOMAINDECOMPMPIBASE_H_ */