DomainDecompBase.h

#ifndef DOMAINDECOMPBASE_H_
#define DOMAINDECOMPBASE_H_
 
#include "parallel/CollectiveCommBase.h"
#include <string>
 
#ifdef ENABLE_MPI
#include <mpi.h>
#endif
#include <particleContainer/RegionParticleIterator.h>
#include <iostream>
#include <variant>
 
#include "HaloRegion.h"
#include "io/MemoryProfiler.h"
#include "molecules/MoleculeForwardDeclaration.h"
#include "utils/Logger.h"  // is this used?
 
class Component;
class Domain;
class ParticleContainer;
class XMLfileUnits;
 
class DomainDecompBase: public MemoryProfilable {
    friend class NeighbourCommunicationScheme;
    friend class IndirectNeighbourCommunicationScheme;
    friend class DirectNeighbourCommunicationScheme;
public:
    DomainDecompBase();
 
    ~DomainDecompBase() override;
 
    virtual void readXML(XMLfileUnits& xmlconfig);
 
    void exchangeMolecules(ParticleContainer* moleculeContainer, Domain* domain);
 
    virtual void exchangeForces(ParticleContainer* moleculeContainer, Domain* domain);
 
    virtual int getNonBlockingStageCount();
 
    virtual bool queryBalanceAndExchangeNonBlocking(bool forceRebalancing, ParticleContainer* moleculeContainer, Domain* domain, double etime);
 
    virtual void balanceAndExchange(double lastTraversalTime, bool forceRebalancing, ParticleContainer* moleculeContainer, Domain* domain);
 
    virtual bool procOwnsPos(double x, double y, double z, Domain* domain) final;
 
    void getBoundingBoxMinMax(Domain* domain, double* min, double* max);
 
    virtual double getBoundingBoxMin(int dimension, Domain* domain);
 
    virtual double getBoundingBoxMax(int dimension, Domain* domain);
 
    virtual void printDecomp(const std::string &filename, Domain *domain, ParticleContainer *particleContainer);
 
 
    virtual int getRank() const;
 
    virtual int getNumProcs() const;
 
    virtual void barrier() const;
 
    virtual double getTime() const;
 
    virtual unsigned Ndistribution(unsigned localN, float* minrnd, float* maxrnd);
 
    virtual void assertIntIdentity(int IX);
    virtual void assertDisjunctivity(ParticleContainer* moleculeContainer) const;
 
    double getIOCutoffRadius(int dim, Domain* domain, ParticleContainer* moleculeContainer);
 
 
#ifdef ENABLE_MPI
    void writeMoleculesToMPIFileBinary(const std::string& filename, ParticleContainer* moleculeContainer) const;
#endif // ENABLE_MPI
 
    void writeMoleculesToFile(const std::string& filename, ParticleContainer* moleculeContainer, bool binary = false) const;
 
 
    void updateSendLeavingWithCopies(bool sendTogether){
        using Log::global_log;
        // Count all processes that need to send separately
        collCommInit(1);
        collCommAppendInt(!sendTogether);
        collCommAllreduceSum();
        _sendLeavingAndCopiesSeparately = collCommGetInt();
        collCommFinalize();
 
 
        global_log->info() << "Sending leaving particles and halo copies "
                << (sendLeavingWithCopies() ? "together" : "separately") << std::endl;
    }
 
    bool sendLeavingWithCopies() const{
        // No process needs to send separately => send together
        return _sendLeavingAndCopiesSeparately == 0;
    }
 
 
    //##################################################################
    // The following methods with prefix "collComm" are all used
    // in the context of collective communication. Each of the methods
    // basically has to call the corresponding method from the class
    // CollectiveCommunication (or CollectiveCommDummy in the sequential
    // case). To get information about how to use this methods, read
    // the documentation of the class CollectiveCommunication.
    //##################################################################
    virtual void collCommInit(int numValues, int key=0);
    virtual void collCommFinalize();
    virtual void collCommAppendInt(int intValue);
    virtual void collCommAppendUnsLong(unsigned long unsLongValue);
    virtual void collCommAppendFloat(float floatValue);
    virtual void collCommAppendDouble(double doubleValue);
    virtual void collCommAppendLongDouble(long double longDoubleValue);
    virtual int collCommGetInt();
    virtual unsigned long collCommGetUnsLong();
    virtual float collCommGetFloat();
    virtual double collCommGetDouble();
    virtual long double collCommGetLongDouble();
    virtual void collCommAllreduceSum();
    virtual void collCommAllreduceSumAllowPrevious();
    virtual void collCommAllreduceCustom(ReduceType type);
    virtual void collCommScanSum();
    virtual void collCommBroadcast(int root = 0);
    //returns the ranks of the neighbours
    virtual std::vector<int> getNeighbourRanks(){
        return std::vector<int>(0);
    }
    //returns the ranks of the neighbours
    virtual std::vector<int> getNeighbourRanksFullShell(){
        std::cout << "Not yet implemented";
        return std::vector<int>(0);
    }
 
    //returns the ranks of all ranks
    virtual std::vector<std::vector<std::vector<int>>> getAllRanks(){
        std::cout << "Not yet implemented";
        return std::vector<std::vector<std::vector<int>>>(0);
    }
#if defined(ENABLE_MPI)
    virtual MPI_Comm getCommunicator(){
        return MPI_COMM_WORLD;
    }
#endif
 
    virtual size_t getTotalSize() override {
        return _collCommBase.getTotalSize();
    }
    virtual void printSubInfo(int offset) override {
        return;
    }
    virtual std::string getName() override {
        return "DomainDecompBase";
    }
 
    virtual void printCommunicationPartners(std::string filename) const {};
 
protected:
    void addLeavingMolecules(std::vector<Molecule>&& invalidMolecules, ParticleContainer* moleculeContainer);
 
    void handleDomainLeavingParticles(unsigned dim, ParticleContainer* moleculeContainer) const;
 
    void handleDomainLeavingParticlesDirect(const HaloRegion& haloRegion, ParticleContainer* moleculeContainer,
                                            std::vector<Molecule>& invalidParticles) const;
 
    void handleForceExchange(unsigned dim, ParticleContainer* moleculeContainer) const;
 
    virtual void handleForceExchangeDirect(const HaloRegion& haloRegion, ParticleContainer* moleculeContainer) const;
 
    void populateHaloLayerWithCopies(unsigned dim, ParticleContainer* moleculeContainer) const;
 
    void populateHaloLayerWithCopiesDirect(const HaloRegion& haloRegion, ParticleContainer* moleculeContainer,
                                           bool positionCheck = true) const;
 
    int _rank;
 
    int _numProcs;
 
private:
    CollectiveCommBase _collCommBase;
    int _sendLeavingAndCopiesSeparately = 0;
};
 
#endif /* DOMAINDECOMPBASE_H_ */