MoFEM::CommInterface Struct Reference

Managing BitRefLevels. More...

#include <src/interfaces/CommInterface.hpp>

Inheritance diagram for MoFEM::CommInterface:

Collaboration diagram for MoFEM::CommInterface:

Public Member Functions
MoFEMErrorCode	query_interface (boost::typeindex::type_index type_index, UnknownInterface **iface) const
	CommInterface (const MoFEM::Core &core)
	~CommInterface ()=default
	Destructor. More...
Public Member Functions inherited from MoFEM::UnknownInterface
virtual MoFEMErrorCode	query_interface (boost::typeindex::type_index type_index, UnknownInterface **iface) const =0
template<class IFACE >
MoFEMErrorCode	registerInterface (bool error_if_registration_failed=true)
	Register interface. More...
template<class IFACE >
MoFEMErrorCode	getInterface (IFACE *&iface) const
	Get interface refernce to pointer of interface. More...
template<class IFACE >
MoFEMErrorCode	getInterface (IFACE **const iface) const
	Get interface pointer to pointer of interface. More...
template<class IFACE , typename boost::enable_if< boost::is_pointer< IFACE >, int >::type = 0>
IFACE	getInterface () const
	Get interface pointer to pointer of interface. More...
template<class IFACE , typename boost::enable_if< boost::is_reference< IFACE >, int >::type = 0>
IFACE	getInterface () const
	Get reference to interface. More...
template<class IFACE >
IFACE *	getInterface () const
	Function returning pointer to interface. More...
virtual	~UnknownInterface ()=default

Public Attributes
MoFEM::Core &	cOre
bool	dEbug

Make elemnts multishared
MoFEMErrorCode	resolveSharedFiniteElements (const Problem *problem_ptr, const std::string &fe_name, int verb=DEFAULT_VERBOSITY)
	resolve shared entities for finite elements in the problem More...
MoFEMErrorCode	resolveSharedFiniteElements (const std::string name, const std::string &fe_name, int verb=DEFAULT_VERBOSITY)
	resolve shared entities for finite elements in the problem More...

Make entities multishared
MoFEMErrorCode	makeEntitiesMultishared (const EntityHandle *entities, const int num_entities, const int owner_proc=0, int verb=DEFAULT_VERBOSITY)
	make entities from proc 0 shared on all proc More...
MoFEMErrorCode	makeEntitiesMultishared (Range &entities, const int owner_proc=0, int verb=DEFAULT_VERBOSITY)
	make entities from proc 0 shared on all proc More...
MoFEMErrorCode	makeFieldEntitiesMultishared (const std::string field_name, const int owner_proc=0, int verb=DEFAULT_VERBOSITY)
	make field entities multi shared More...
MoFEMErrorCode	exchangeFieldData (const std::string field_name, int verb=DEFAULT_VERBOSITY)
	Exchange field data. More...

Synchronize entities (Following functions in future will be
deprecated)
MoFEMErrorCode	synchroniseEntities (Range &ent, std::map< int, Range > *received_ents, int verb=DEFAULT_VERBOSITY)
	synchronize entity range on processors (collective) More...
MoFEMErrorCode	synchroniseEntities (Range &ent, int verb=DEFAULT_VERBOSITY)
	synchronize entity range on processors (collective) More...
MoFEMErrorCode	synchroniseFieldEntities (const std::string name, int verb=DEFAULT_VERBOSITY)
MoFEMErrorCode	resolveParentEntities (const Range &ent, int verb=DEFAULT_VERBOSITY)
	Synchronise parent entities. More...

Read load and boradcoast
@
MoFEMErrorCode	partitionMesh (const Range &ents, const int dim, const int adj_dim, const int n_parts, Tag *th_vertex_weights=nullptr, Tag *th_edge_weights=nullptr, Tag *th_part_weights=nullptr, int verb=VERBOSE, const bool debug=false)
	Set partition tag to each finite element in the problem. More...

Additional Inherited Members
Static Public Member Functions inherited from MoFEM::UnknownInterface
static MoFEMErrorCode	getLibVersion (Version &version)
	Get library version. More...
static MoFEMErrorCode	getFileVersion (moab::Interface &moab, Version &version)
	Get database major version. More...
static MoFEMErrorCode	setFileVersion (moab::Interface &moab, Version version=Version(MoFEM_VERSION_MAJOR, MoFEM_VERSION_MINOR, MoFEM_VERSION_BUILD))
	Get database major version. More...
static MoFEMErrorCode	getInterfaceVersion (Version &version)
	Get database major version. More...

Detailed Description

Managing BitRefLevels.

Examples

dm_partitioned_no_field.cpp, elasticity.cpp, free_surface.cpp, level_set.cpp, mesh_smoothing.cpp, mixed_poisson.cpp, navier_stokes.cpp, partition_mesh.cpp, plastic.cpp, and thermo_elastic.cpp.

Definition at line 21 of file CommInterface.hpp.

Constructor & Destructor Documentation

CommInterface()

MoFEM::CommInterface::CommInterface

(

const MoFEM::Core &

core

)

Definition at line 23 of file CommInterface.cpp.

    : cOre(const_cast<MoFEM::Core &>(core)), dEbug(false) {}

~CommInterface()

MoFEM::CommInterface::~CommInterface ( )

default

Destructor.

Member Function Documentation

exchangeFieldData()

MoFEMErrorCode MoFEM::CommInterface::exchangeFieldData	(	const std::string	field_name,
		int	verb = DEFAULT_VERBOSITY
	)

Exchange field data.

Exchange field for all shared and ghosted entities. This function should be called collectively over the communicator for this ParallelComm. If the entities vector is empty, all shared entities participate in the exchange. If a proc has no owned entities this function must still be called since it is collective.

Note

collective - need tu be run on all processors in communicator

Todo:

It is not working if field has entities diffrent than vertices.

Parameters

verb
field_name

Returns

MoFEMErrorCode

Definition at line 818 of file CommInterface.cpp.

                                                          {
  MoFEM::Interface &m_field = cOre;
  MoFEMFunctionBegin;
  if (m_field.get_comm_size() > 1) {
 
    Range exchange_ents_data_verts, exchange_ents_data;
 
    auto *field_ents = m_field.get_field_ents();
    auto field_bit_number = m_field.get_field_bit_number(field_name);
    auto lo = field_ents->get<Unique_mi_tag>().lower_bound(
        FieldEntity::getLoBitNumberUId(field_bit_number));
    auto hi = field_ents->get<Unique_mi_tag>().lower_bound(
        FieldEntity::getHiBitNumberUId(field_bit_number));
 
    for (auto it = lo; it != hi; ++it)
      if (
 
          ((*it)->getPStatus()) &&
 
          (*it)->getNbDofsOnEnt()
 
      ) {
        if ((*it)->getEntType() == MBVERTEX)
          exchange_ents_data_verts.insert((*it)->getEnt());
        else
          exchange_ents_data.insert((*it)->getEnt());
      }
 
    auto field_ptr = m_field.get_field_structure(field_name);
    ParallelComm *pcomm = ParallelComm::get_pcomm(
        &m_field.get_moab(), m_field.get_basic_entity_data_ptr()->pcommID);
 
    auto exchange = [&](const Range &ents, Tag th) {
      MoFEMFunctionBegin;
      if (!ents.empty()) {
        std::vector<Tag> tags;
        tags.push_back(th);
        CHKERR pcomm->exchange_tags(tags, tags, ents);
      }
      MoFEMFunctionReturn(0);
    };
 
    CHKERR exchange(exchange_ents_data_verts, field_ptr->th_FieldDataVerts);
    CHKERR exchange(exchange_ents_data, field_ptr->th_FieldData);
  }
  MoFEMFunctionReturn(0);
}

makeEntitiesMultishared() [1/2]

MoFEMErrorCode MoFEM::CommInterface::makeEntitiesMultishared	(	const EntityHandle *	entities,
		const int	num_entities,
		const int	owner_proc = 0,
		int	verb = DEFAULT_VERBOSITY
	)

make entities from proc 0 shared on all proc

Note

collective - need tu be run on all processors in communicator

Parameters

entities
num_entities
my_proc	default proc id to share from
verb

Returns

MoFEMErrorCode

Definition at line 654 of file CommInterface.cpp.

                                                                       {
  MoFEM::Interface &m_field = cOre;
  MoFEMFunctionBegin;
 
  if (m_field.get_comm_size() > 1) {
 
    ParallelComm *pcomm = ParallelComm::get_pcomm(
        &m_field.get_moab(), m_field.get_basic_entity_data_ptr()->pcommID);
 
    Range all_ents_range;
    all_ents_range.insert_list(entities, entities + num_entities);
 
    auto get_tag = [&]() { return m_field.get_moab().globalId_tag(); };
 
    auto delete_tag = [&](auto &&th_gid) {
      MoFEMFunctionBegin;
      CHKERR m_field.get_moab().tag_delete(th_gid);
      MoFEMFunctionReturn(0);
    };
 
    auto resolve_shared_ents = [&](auto &&th_gid, auto &all_ents_range) {
      auto set_gid = [&](auto &th_gid) {
        std::vector<int> gids(num_entities);
        for (size_t g = 0; g != all_ents_range.size(); ++g)
          gids[g] = g + 1;
        CHKERR m_field.get_moab().tag_set_data(th_gid, all_ents_range,
                                               &*gids.begin());
 
        return &th_gid;
      };
 
      auto get_skin_ents = [&](auto &all_ents_range) {
        std::array<Range, 4> proc_ents_skin;
        proc_ents_skin[3] = all_ents_range.subset_by_dimension(3);
        proc_ents_skin[2] = all_ents_range.subset_by_dimension(2);
        proc_ents_skin[1] = all_ents_range.subset_by_dimension(1);
        proc_ents_skin[0] = all_ents_range.subset_by_dimension(0);
        return proc_ents_skin;
      };
 
      auto resolve_dim = [&](auto &all_ents_range) {
        for (int resolve_dim = 3; resolve_dim >= 0; --resolve_dim) {
          if (all_ents_range.num_of_dimension(resolve_dim))
            return resolve_dim;
        }
        return -1;
      };
 
      auto get_proc_ent = [&](auto &all_ents_range) {
        Range proc_ent;
        if (m_field.get_comm_rank() == owner_proc)
          proc_ent = all_ents_range;
        return proc_ent;
      };
 
      auto resolve_shared_ents = [&](auto &&proc_ents, auto &&skin_ents) {
        return pcomm->resolve_shared_ents(
            0, proc_ents, resolve_dim(all_ents_range),
            resolve_dim(all_ents_range), skin_ents.data(), set_gid(th_gid));
      };
 
      CHKERR resolve_shared_ents(get_proc_ent(all_ents_range),
                                 get_skin_ents(all_ents_range));
 
      return th_gid;
    };
 
    CHKERR delete_tag(resolve_shared_ents(get_tag(), all_ents_range));
 
    if (verb >= NOISY) {
 
      auto print_owner = [&](const EntityHandle e) {
        MoFEMFunctionBegin;
        int moab_owner_proc;
        EntityHandle moab_owner_handle;
        CHKERR pcomm->get_owner_handle(e, moab_owner_proc, moab_owner_handle);
 
        unsigned char pstatus = 0;
 
        CHKERR m_field.get_moab().tag_get_data(pcomm->pstatus_tag(), &e, 1,
                                               &pstatus);
 
        std::vector<int> shprocs(MAX_SHARING_PROCS, 0);
        std::vector<EntityHandle> shhandles(MAX_SHARING_PROCS, 0);
 
        CHKERR m_field.get_moab().tag_get_data(pcomm->sharedp_tag(), &e, 1,
                                               &shprocs[0]);
        CHKERR m_field.get_moab().tag_get_data(pcomm->sharedh_tag(), &e, 1,
                                               &shhandles[0]);
        if (pstatus & PSTATUS_MULTISHARED) {
          CHKERR m_field.get_moab().tag_get_data(pcomm->sharedps_tag(), &e, 1,
                                                 &shprocs[0]);
          CHKERR m_field.get_moab().tag_get_data(pcomm->sharedhs_tag(), &e, 1,
                                                 &shhandles[0]);
        }
 
        std::ostringstream ss;
 
        ss << "Rank " << m_field.get_comm_rank() << " ";
        if (!(pstatus & PSTATUS_NOT_OWNED))
          ss << "OWNER ";
        if (pstatus & PSTATUS_SHARED)
          ss << "PSTATUS_SHARED ";
        if (pstatus & PSTATUS_MULTISHARED)
          ss << "PSTATUS_MULTISHARED ";
 
        ss << "owner " << moab_owner_proc << " (" << owner_proc << ") ";
 
        ss << "shprocs: ";
        for (size_t r = 0; r != m_field.get_comm_size() + 1; ++r)
          ss << shprocs[r] << " ";
 
        ss << "shhandles: ";
        for (size_t r = 0; r != m_field.get_comm_size() + 1; ++r)
          ss << shhandles[r] << " ";
 
        ss << std::endl;
        MOFEM_LOG("SYNC", Sev::noisy) << ss.str();
        MOFEM_LOG_SYNCHRONISE(m_field.get_comm());
 
        MoFEMFunctionReturn(0);
      };
 
      for (auto e : all_ents_range)
        CHKERR print_owner(e);
    }
  }
 
  MoFEMFunctionReturn(0);
}

makeEntitiesMultishared() [2/2]

MoFEMErrorCode MoFEM::CommInterface::makeEntitiesMultishared	(	Range &	entities,
		const int	owner_proc = 0,
		int	verb = DEFAULT_VERBOSITY
	)

make entities from proc 0 shared on all proc

Note

collective - need tu be run on all processors in communicator

Parameters

entities
my_proc	default proc id to share from
verb

Returns

MoFEMErrorCode

Definition at line 787 of file CommInterface.cpp.

                                                                {
  MoFEM::Interface &m_field = cOre;
  MoFEMFunctionBegin;
  if (m_field.get_comm_size() > 1) {
    const int num_ents = entities.size();
    std::vector<EntityHandle> vec_ents(num_ents);
    std::copy(entities.begin(), entities.end(), vec_ents.begin());
    CHKERR makeEntitiesMultishared(&*vec_ents.begin(), num_ents, owner_proc,
                                   verb);
  }
  MoFEMFunctionReturn(0);
}

makeFieldEntitiesMultishared()

MoFEMErrorCode MoFEM::CommInterface::makeFieldEntitiesMultishared	(	const std::string	field_name,
		const int	owner_proc = 0,
		int	verb = DEFAULT_VERBOSITY
	)

make field entities multi shared

Note

collective - need tu be run on all processors in communicator

Parameters

field_name
owner_proc
verb

Returns

MoFEMErrorCode

Definition at line 803 of file CommInterface.cpp.

                                                                            {
  MoFEM::Interface &m_field = cOre;
  MoFEMFunctionBegin;
  if (m_field.get_comm_size() > 1) {
    EntityHandle field_meshset = m_field.get_field_meshset(field_name);
    std::vector<EntityHandle> field_ents;
    CHKERR m_field.get_moab().get_entities_by_handle(field_meshset, field_ents,
                                                     true);
    CHKERR makeEntitiesMultishared(&*field_ents.begin(), field_ents.size(),
                                   owner_proc, verb);
  }
  MoFEMFunctionReturn(0);
}

query_interface()

MoFEMErrorCode MoFEM::CommInterface::query_interface ( boost::typeindex::type_index  type_index, UnknownInterface **  iface  ) const

virtual

Implements MoFEM::UnknownInterface.

Definition at line 16 of file CommInterface.cpp.

                                                               {
  MoFEMFunctionBeginHot;
  *iface = const_cast<CommInterface *>(this);
  MoFEMFunctionReturnHot(0);
}

resolveParentEntities()

MoFEMErrorCode MoFEM::CommInterface::resolveParentEntities	(	const Range &	ent,
		int	verb = DEFAULT_VERBOSITY
	)

Synchronise parent entities.

Note

collective - need tu be run on all processors in communicator

Exchange parent entity tag and bitref of entity. Note thar parent handle can be different on each processor.

Parameters

ent
verb

Returns

MoFEMErrorCode

Definition at line 256 of file CommInterface.cpp.

                                                              {
  MoFEM::Interface &m_field = cOre;
  ParallelComm *pcomm = ParallelComm::get_pcomm(
      &m_field.get_moab(), m_field.get_basic_entity_data_ptr()->pcommID);
 
  MoFEMFunctionBegin;
 
  Range shared = ents;
  CHKERR pcomm->filter_pstatus(shared, PSTATUS_NOT_OWNED, PSTATUS_NOT, -1,
                               nullptr);
  CHKERR pcomm->filter_pstatus(shared, PSTATUS_SHARED | PSTATUS_MULTISHARED,
                               PSTATUS_OR, -1, nullptr);
 
  auto th_RefParentHandle = cOre.get_th_RefParentHandle();
  auto th_RefBitLevel = cOre.get_th_RefBitLevel();
 
  auto get_pstatus = [&](const auto ent) {
    unsigned char pstatus;
    CHK_MOAB_THROW(m_field.get_moab().tag_get_data(pcomm->pstatus_tag(), &ent,
                                                   1, &pstatus),
                   "can not get pstatus");
    return pstatus;
  };
 
  auto get_sharing_procs = [&](const auto ent, const auto pstatus) {
    std::vector<int> sharing_procs(MAX_SHARING_PROCS, -1);
    if (pstatus & PSTATUS_MULTISHARED) {
      // entity is multi shared
      CHK_MOAB_THROW(m_field.get_moab().tag_get_data(
                         pcomm->sharedps_tag(), &ent, 1, &sharing_procs[0]),
                     "can not ger sharing_procs_ptr");
    } else if (pstatus & PSTATUS_SHARED) {
      // shared
      CHK_MOAB_THROW(m_field.get_moab().tag_get_data(pcomm->sharedp_tag(), &ent,
                                                     1, &sharing_procs[0]),
                     "can not get sharing proc");
    }
    return sharing_procs;
  };
 
  auto get_sharing_handles = [&](const auto ent, const auto pstatus) {
    std::vector<EntityHandle> sharing_handles(MAX_SHARING_PROCS, 0);
    if (pstatus & PSTATUS_MULTISHARED) {
      // entity is multi shared
      CHK_MOAB_THROW(m_field.get_moab().tag_get_data(
                         pcomm->sharedhs_tag(), &ent, 1, &sharing_handles[0]),
                     "get shared handles");
    } else if (pstatus & PSTATUS_SHARED) {
      // shared
      CHK_MOAB_THROW(m_field.get_moab().tag_get_data(pcomm->sharedh_tag(), &ent,
                                                     1, &sharing_handles[0]),
                     "get sharing handle");
    }
    return sharing_handles;
  };
 
  auto get_parent_and_bit = [&](const auto ent) {
    EntityHandle parent;
    CHK_MOAB_THROW(
        m_field.get_moab().tag_get_data(th_RefParentHandle, &ent, 1, &parent),
        "get parent");
    BitRefLevel bit;
    CHK_MOAB_THROW(
        m_field.get_moab().tag_get_data(th_RefBitLevel, &ent, 1, &bit),
        "get parent");
    return std::make_pair(parent, bit);
  };
 
  auto set_parent = [&](const auto ent, const auto parent) {
    return m_field.get_moab().tag_set_data(th_RefParentHandle, &ent, 1,
                                           &parent);
  };
 
  auto set_bit = [&](const auto ent, const auto bit) {
    return m_field.get_moab().tag_set_data(th_RefBitLevel, &ent, 1, &bit);
  };
 
  // make a buffer
  std::vector<std::vector<unsigned long long>> sbuffer(m_field.get_comm_size());
 
  for (auto ent : shared) {
 
    auto pstatus = get_pstatus(ent);
    auto sharing_procs = get_sharing_procs(ent, pstatus);
    auto sharing_handles = get_sharing_handles(ent, pstatus);
    auto [parent, bit] = get_parent_and_bit(ent);
 
    if (verb >= NOISY)
      MOFEM_LOG("SYNC", Sev::noisy) << "pstatus " << std::bitset<8>(pstatus);
 
    if (parent) {
      auto pstatus_parent = get_pstatus(parent);
      auto sharing_procs_parent = get_sharing_procs(parent, pstatus_parent);
      auto sharing_handles_parent = get_sharing_handles(parent, pstatus_parent);
 
      for (int proc = 0; proc < MAX_SHARING_PROCS && -1 != sharing_procs[proc];
           proc++) {
        if (sharing_procs[proc] == -1)
          SETERRQ(PETSC_COMM_SELF, MOFEM_IMPOSSIBLE_CASE,
                  "sharing processor not set");
 
        if (sharing_procs[proc] != m_field.get_comm_rank()) {
 
          auto it = std::find(sharing_procs_parent.begin(),
                              sharing_procs_parent.end(), sharing_procs[proc]);
          if (it == sharing_procs_parent.end()) {
            SETERRQ1(
                PETSC_COMM_SELF, MOFEM_DATA_INCONSISTENCY,
                "Sharing proc for parent entity can not be found proc = %u",
                sharing_procs[proc]);
          }
 
          auto handle_on_sharing_proc = sharing_handles[proc];
          auto parent_handle_on_sharing_proc =
              sharing_handles_parent[std::distance(sharing_procs_parent.begin(),
                                                   it)];
          sbuffer[sharing_procs[proc]].push_back(handle_on_sharing_proc);
          sbuffer[sharing_procs[proc]].push_back(parent_handle_on_sharing_proc);
          try {
            sbuffer[sharing_procs[proc]].push_back(bit.to_ullong());
          } catch (std::exception &ex) {
            MOFEM_LOG("SELF", Sev::warning) << ex.what();
            MOFEM_LOG("SELF", Sev::warning)
                << "On " << ent << " "
                << moab::CN::EntityTypeName(type_from_handle(ent));
            MOFEM_LOG("SELF", Sev::warning) << "For bit ref " << bit;
          }          
          if (verb >= NOISY)
            MOFEM_LOG_C("SYNC", Sev::noisy, "send %lu (%lu) to %d at %d\n", ent,
                        handle_on_sharing_proc, sharing_procs[proc],
                        m_field.get_comm_rank());
 
          if (!(pstatus & PSTATUS_MULTISHARED))
            break;
        }
      }
    } else {
      for (int proc = 0; proc < MAX_SHARING_PROCS && -1 != sharing_procs[proc];
           proc++) {
        if (sharing_procs[proc] == -1)
          SETERRQ(PETSC_COMM_SELF, MOFEM_IMPOSSIBLE_CASE,
                  "sharing processor not set");
 
        if (sharing_procs[proc] != m_field.get_comm_rank()) {
          auto handle_on_sharing_proc = sharing_handles[proc];
          sbuffer[sharing_procs[proc]].push_back(handle_on_sharing_proc);
          sbuffer[sharing_procs[proc]].push_back(parent);
 
          try {
            sbuffer[sharing_procs[proc]].push_back(bit.to_ullong());
          } catch (std::exception &ex) {
            MOFEM_LOG("SELF", Sev::warning) << ex.what();
            MOFEM_LOG("SELF", Sev::warning)
                << "On " << ent << " "
                << moab::CN::EntityTypeName(type_from_handle(ent));
            MOFEM_LOG("SELF", Sev::warning) << "For bit ref " << bit;
          }
 
          if (verb >= NOISY)
            MOFEM_LOG_C("SYNC", Sev::noisy, "send %lu (%lu) to %d at %d\n", ent,
                        handle_on_sharing_proc, sharing_procs[proc],
                        m_field.get_comm_rank());
 
          if (!(pstatus & PSTATUS_MULTISHARED))
            break;
        }
      }
    }
  }
 
  int nsends = 0; // number of messages to send
  std::vector<int> sbuffer_lengths(
      m_field.get_comm_size()); // length of the message to proc
 
  const size_t block_size = sizeof(unsigned long long) / sizeof(int);
  for (int proc = 0; proc < m_field.get_comm_size(); proc++) {
 
    if (!sbuffer[proc].empty()) {
 
      sbuffer_lengths[proc] = sbuffer[proc].size() * block_size;
      nsends++;
 
    } else {
 
      sbuffer_lengths[proc] = 0;
    }
  }
 
  // Make sure it is a PETSc m_field.get_comm()
  MPI_Comm comm;
  CHKERR PetscCommDuplicate(m_field.get_comm(), &comm, NULL);
 
  std::vector<MPI_Status> status(m_field.get_comm_size());
 
  // Computes the number of messages a node expects to receive
  int nrecvs; // number of messages received
  CHKERR PetscGatherNumberOfMessages(comm, NULL, &sbuffer_lengths[0], &nrecvs);
 
  // Computes info about messages that a MPI-node will receive, including
  // (from-id,length) pairs for each message.
  int *onodes;   // list of node-ids from which messages are expected
  int *olengths; // corresponding message lengths
  CHKERR PetscGatherMessageLengths(comm, nsends, nrecvs, &sbuffer_lengths[0],
                                   &onodes, &olengths);
 
  // Gets a unique new tag from a PETSc communicator. All processors that share
  // the communicator MUST call this routine EXACTLY the same number of times.
  // This tag should only be used with the current objects communicator; do NOT
  // use it with any other MPI communicator.
  int tag;
  CHKERR PetscCommGetNewTag(comm, &tag);
 
  // Allocate a buffer sufficient to hold messages of size specified in
  // olengths. And post Irecvs on these buffers using node info from onodes
  int **rbuf;           // must bee freed by user
  MPI_Request *r_waits; // must bee freed by user
  // rbuf has a pointers to messages. It has size of of nrecvs (number of
  // messages) +1. In the first index a block is allocated,
  // such that rbuf[i] = rbuf[i-1]+olengths[i-1].
  CHKERR PetscPostIrecvInt(comm, tag, nrecvs, onodes, olengths, &rbuf,
                           &r_waits);
 
  MPI_Request *s_waits; // status of sens messages
  CHKERR PetscMalloc1(nsends, &s_waits);
 
  // Send messages
  for (int proc = 0, kk = 0; proc < m_field.get_comm_size(); proc++) {
    if (!sbuffer_lengths[proc])
      continue;                             // no message to send to this proc
    CHKERR MPI_Isend(&(sbuffer[proc])[0],   // buffer to send
                     sbuffer_lengths[proc], // message length
                     MPIU_INT, proc,        // to proc
                     tag, comm, s_waits + kk);
    kk++;
  }
 
  // Wait for received
  if (nrecvs)
    CHKERR MPI_Waitall(nrecvs, r_waits, &status[0]);
 
  // Wait for send messages
  if (nsends)
    CHKERR MPI_Waitall(nsends, s_waits, &status[0]);
 
  if (verb >= VERBOSE) {
    MOFEM_LOG_C("SYNC", Sev::verbose,
                "Rank %d nb. shared to synchronise parents ents %u\n",
                m_field.get_comm_rank(), shared.size());
  }
 
  // synchronise range
  for (int kk = 0; kk < nrecvs; kk++) {
 
    int len = olengths[kk];
    int *data_from_proc = rbuf[kk];
 
    for (int ee = 0; ee < len;) {
      EntityHandle ent;
      EntityHandle parent;
      unsigned long long uulong_bit;
      bcopy(&data_from_proc[ee], &ent, sizeof(EntityHandle));
      ee += block_size;
      bcopy(&data_from_proc[ee], &parent, sizeof(EntityHandle));
      ee += block_size;
      bcopy(&data_from_proc[ee], &uulong_bit, sizeof(unsigned long long));
      ee += block_size;
 
      CHKERR set_parent(ent, parent);
      CHKERR set_bit(ent, BitRefLevel(uulong_bit));
 
      if (verb >= VERBOSE) {
        MOFEM_LOG_C("SYNC", Sev::noisy, "received %lu (%lu) from %d at %d\n",
                    ent, parent, onodes[kk], m_field.get_comm_rank());
        MOFEM_LOG("SYNC", Sev::noisy) << "Bit " << BitRefLevel(uulong_bit);
      }
    }
  }
 
  // Cleaning
  CHKERR PetscFree(s_waits);
  CHKERR PetscFree(rbuf[0]);
  CHKERR PetscFree(rbuf);
  CHKERR PetscFree(r_waits);
  CHKERR PetscFree(onodes);
  CHKERR PetscFree(olengths);
  CHKERR PetscCommDestroy(&comm);
 
  if (verb >= VERBOSE)
    MOFEM_LOG_SYNCHRONISE(m_field.get_comm());
 
  MoFEMFunctionReturn(0);
}

synchroniseEntities() [1/2]

MoFEMErrorCode MoFEM::CommInterface::synchroniseEntities	(	Range &	ent,
		int	verb = DEFAULT_VERBOSITY
	)

synchronize entity range on processors (collective)

Note

collective - need tu be run on all processors in communicator

Parameters

ent	ents to send and received
verb

Returns

MoFEMErrorCode

Definition at line 240 of file CommInterface.cpp.

                                                                       {
  return synchroniseEntities(ents, nullptr, verb);
}

synchroniseEntities() [2/2]

MoFEMErrorCode MoFEM::CommInterface::synchroniseEntities	(	Range &	ent,
		std::map< int, Range > *	received_ents,
		int	verb = DEFAULT_VERBOSITY
	)

synchronize entity range on processors (collective)

synchronize entity range on processors (collective)

Note

collective - need tu be run on all processors in communicator

collective - need tu be run on all processors in communicator

Parameters

ent	ents to send and received
received_ents	pointer to map with received entities
verb

Returns

MoFEMErrorCode

Definition at line 26 of file CommInterface.cpp.

                                                              {
  MoFEM::Interface &m_field = cOre;
  ParallelComm *pcomm = ParallelComm::get_pcomm(
      &m_field.get_moab(), m_field.get_basic_entity_data_ptr()->pcommID);
  MoFEMFunctionBegin;
 
  auto get_pstatus = [&](const auto ent) {
    unsigned char pstatus;
    CHK_MOAB_THROW(m_field.get_moab().tag_get_data(pcomm->pstatus_tag(), &ent,
                                                   1, &pstatus),
                   "can not get pstatus");
    return pstatus;
  };
 
  auto get_sharing_procs = [&](const auto ent, const auto pstatus) {
    std::vector<int> sharing_procs(MAX_SHARING_PROCS, -1);
    if (pstatus & PSTATUS_MULTISHARED) {
      // entity is multi shared
      CHK_MOAB_THROW(m_field.get_moab().tag_get_data(
                         pcomm->sharedps_tag(), &ent, 1, &sharing_procs[0]),
                     "can not ger sharing_procs_ptr");
    } else if (pstatus & PSTATUS_SHARED) {
      // shared
      CHK_MOAB_THROW(m_field.get_moab().tag_get_data(pcomm->sharedp_tag(), &ent,
                                                     1, &sharing_procs[0]),
                     "can not get sharing proc");
    }
    return sharing_procs;
  };
 
  auto get_sharing_handles = [&](const auto ent, const auto pstatus) {
    std::vector<EntityHandle> sharing_handles(MAX_SHARING_PROCS, 0);
    if (pstatus & PSTATUS_MULTISHARED) {
      // entity is multi shared
      CHK_MOAB_THROW(m_field.get_moab().tag_get_data(
                         pcomm->sharedhs_tag(), &ent, 1, &sharing_handles[0]),
                     "get shared handles");
    } else if (pstatus & PSTATUS_SHARED) {
      // shared
      CHK_MOAB_THROW(m_field.get_moab().tag_get_data(pcomm->sharedh_tag(), &ent,
                                                     1, &sharing_handles[0]),
                     "get sharing handle");
    }
    return sharing_handles;
  };
 
  // make a buffer entities to send
  std::vector<std::vector<EntityHandle>> sbuffer(m_field.get_comm_size());
 
  for (auto ent : ents) {
 
    auto pstatus = get_pstatus(ent);
    if (pstatus) {
      auto sharing_procs = get_sharing_procs(ent, pstatus);
      auto sharing_handles = get_sharing_handles(ent, pstatus);
 
      if (verb >= NOISY) {
        MOFEM_LOG("SYNC", Sev::noisy) << "pstatus " << std::bitset<8>(pstatus);
      }
 
      for (int proc = 0; proc < MAX_SHARING_PROCS && -1 != sharing_procs[proc];
           proc++) {
        if (sharing_procs[proc] == -1)
          SETERRQ(PETSC_COMM_SELF, MOFEM_IMPOSSIBLE_CASE,
                  "sharing processor not set");
 
        if (sharing_procs[proc] == m_field.get_comm_rank())
          continue;
 
        const auto handle_on_sharing_proc = sharing_handles[proc];
        // add entity to send, handle on the other side
        sbuffer[sharing_procs[proc]].push_back(handle_on_sharing_proc);
        if (verb >= NOISY) {
          MOFEM_LOG_C("SYNC", Sev::noisy, "send %lu (%lu) to %d at %d\n", ent,
                      handle_on_sharing_proc, sharing_procs[proc],
                      m_field.get_comm_rank());
        }
 
        if (!(pstatus & PSTATUS_MULTISHARED))
          break;
      }
    }
  }
  if (verb >= NOISY) {
    MOFEM_LOG_SEVERITY_SYNC(m_field.get_comm(), Sev::noisy);
  }
 
  int nsends = 0; // number of messages to send
  std::vector<int> sbuffer_lengths(
      m_field.get_comm_size()); // length of the message to proc
  const size_t block_size =
      sizeof(EntityHandle) /
      sizeof(int); // FIXME check if that works on given architecture!
  for (int proc = 0; proc < m_field.get_comm_size(); proc++) {
 
    if (!sbuffer[proc].empty()) {
 
      sbuffer_lengths[proc] = sbuffer[proc].size() * block_size;
      nsends++;
 
    } else {
 
      sbuffer_lengths[proc] = 0;
    }
  }
 
  // Make sure it is a PETSc m_field.get_comm()
  MPI_Comm comm;
  CHKERR PetscCommDuplicate(m_field.get_comm(), &comm, NULL);
 
  std::vector<MPI_Status> status(m_field.get_comm_size());
 
  // Computes the number of messages a node expects to receive
  int nrecvs; // number of messages received
  CHKERR PetscGatherNumberOfMessages(comm, NULL, &sbuffer_lengths[0], &nrecvs);
 
  // Computes info about messages that a MPI-node will receive, including
  // (from-id,length) pairs for each message.
  int *onodes;   // list of node-ids from which messages are expected
  int *olengths; // corresponding message lengths
  CHKERR PetscGatherMessageLengths(comm, nsends, nrecvs, &sbuffer_lengths[0],
                                   &onodes, &olengths);
 
  // Gets a unique new tag from a PETSc communicator. All processors that share
  // the communicator MUST call this routine EXACTLY the same number of times.
  // This tag should only be used with the current objects communicator; do NOT
  // use it with any other MPI communicator.
  int tag;
  CHKERR PetscCommGetNewTag(comm, &tag);
 
  // Allocate a buffer sufficient to hold messages of size specified in
  // olengths. And post Irecvs on these buffers using node info from onodes
  int **rbuf;           // must bee freed by user
  MPI_Request *r_waits; // must bee freed by user
  // rbuf has a pointers to messages. It has size of of nrecvs (number of
  // messages) +1. In the first index a block is allocated,
  // such that rbuf[i] = rbuf[i-1]+olengths[i-1].
  CHKERR PetscPostIrecvInt(comm, tag, nrecvs, onodes, olengths, &rbuf,
                           &r_waits);
 
  MPI_Request *s_waits; // status of sens messages
  CHKERR PetscMalloc1(nsends, &s_waits);
 
  // Send messages
  for (int proc = 0, kk = 0; proc < m_field.get_comm_size(); proc++) {
    if (!sbuffer_lengths[proc])
      continue;                             // no message to send to this proc
    CHKERR MPI_Isend(&(sbuffer[proc])[0],   // buffer to send
                     sbuffer_lengths[proc], // message length
                     MPIU_INT, proc,        // to proc
                     tag, comm, s_waits + kk);
    kk++;
  }
 
  // Wait for received
  if (nrecvs)
    CHKERR MPI_Waitall(nrecvs, r_waits, &status[0]);
 
  // Wait for send messages
  if (nsends)
    CHKERR MPI_Waitall(nsends, s_waits, &status[0]);
 
  if (verb >= VERBOSE) {
    MOFEM_LOG_C("SYNC", Sev::verbose, "Rank %d nb. before ents %u\n",
                m_field.get_comm_rank(), ents.size());
    MOFEM_LOG_SEVERITY_SYNC(m_field.get_comm(), Sev::verbose);
  }
 
  // synchronise range
  for (int kk = 0; kk < nrecvs; kk++) {
 
    int len = olengths[kk];
    int *data_from_proc = rbuf[kk];
 
    for (int ee = 0; ee < len;) {
      EntityHandle ent;
      bcopy(&data_from_proc[ee], &ent, sizeof(EntityHandle));
      ents.insert(ent);
 
      if (received_ents) {
        (*received_ents)[onodes[kk]].insert(ent);
      }
 
      ee += block_size;
 
      if (verb >= VERBOSE) {
        MOFEM_LOG_C("SYNC", Sev::noisy, "received %lu from %d at %d\n", ent,
                    onodes[kk], m_field.get_comm_rank());
      }
    }
  }
  if (verb >= VERBOSE) {
    MOFEM_LOG_SEVERITY_SYNC(m_field.get_comm(), Sev::verbose);
  }
 
  if (verb >= VERBOSE) {
    MOFEM_LOG_C("SYNC", Sev::verbose, "Rank %d nb. after ents %u",
                m_field.get_comm_rank(), ents.size());
    MOFEM_LOG_SEVERITY_SYNC(m_field.get_comm(), Sev::verbose);
  }
 
  // Cleaning
  CHKERR PetscFree(s_waits);
  CHKERR PetscFree(rbuf[0]);
  CHKERR PetscFree(rbuf);
  CHKERR PetscFree(r_waits);
  CHKERR PetscFree(onodes);
  CHKERR PetscFree(olengths);
  CHKERR PetscCommDestroy(&comm);
 
  MoFEMFunctionReturn(0);
}

Member Data Documentation

cOre

MoFEM::Core& MoFEM::CommInterface::cOre

Definition at line 26 of file CommInterface.hpp.

dEbug

bool MoFEM::CommInterface::dEbug

Definition at line 27 of file CommInterface.hpp.

---

The documentation for this struct was generated from the following files:

• src/interfaces/CommInterface.hpp
• src/interfaces/impl/CommInterface.cpp