Synchronise parent entities.
Exchange parent entity tag and bitref of entity. Note thar parent handle can be different on each processor.
259 ParallelComm *pcomm = ParallelComm::get_pcomm(
265 CHKERR pcomm->filter_pstatus(shared, PSTATUS_NOT_OWNED, PSTATUS_NOT, -1,
267 CHKERR pcomm->filter_pstatus(shared, PSTATUS_SHARED | PSTATUS_MULTISHARED,
268 PSTATUS_OR, -1,
nullptr);
273 auto get_pstatus = [&](
const auto ent) {
274 unsigned char pstatus;
277 "can not get pstatus");
281 auto get_sharing_procs = [&](
const auto ent,
const auto pstatus) {
282 std::vector<int> sharing_procs(MAX_SHARING_PROCS, -1);
283 if (pstatus & PSTATUS_MULTISHARED) {
286 pcomm->sharedps_tag(), &ent, 1, &sharing_procs[0]),
287 "can not ger sharing_procs_ptr");
288 }
else if (pstatus & PSTATUS_SHARED) {
291 1, &sharing_procs[0]),
292 "can not get sharing proc");
294 return sharing_procs;
297 auto get_sharing_handles = [&](
const auto ent,
const auto pstatus) {
298 std::vector<EntityHandle> sharing_handles(MAX_SHARING_PROCS, 0);
299 if (pstatus & PSTATUS_MULTISHARED) {
302 pcomm->sharedhs_tag(), &ent, 1, &sharing_handles[0]),
303 "get shared handles");
304 }
else if (pstatus & PSTATUS_SHARED) {
307 1, &sharing_handles[0]),
308 "get sharing handle");
310 return sharing_handles;
313 auto get_parent_and_bit = [&](
const auto ent) {
316 m_field.
get_moab().tag_get_data(th_RefParentHandle, &ent, 1, &parent),
320 m_field.
get_moab().tag_get_data(th_RefBitLevel, &ent, 1, &
bit),
322 return std::make_pair(parent,
bit);
325 auto set_parent = [&](
const auto ent,
const auto parent) {
326 return m_field.
get_moab().tag_set_data(th_RefParentHandle, &ent, 1,
330 auto set_bit = [&](
const auto ent,
const auto bit) {
331 return m_field.
get_moab().tag_set_data(th_RefBitLevel, &ent, 1, &
bit);
335 std::vector<std::vector<unsigned long long>> sbuffer(m_field.
get_comm_size());
337 for (
auto ent : shared) {
339 auto pstatus = get_pstatus(ent);
340 auto sharing_procs = get_sharing_procs(ent, pstatus);
341 auto sharing_handles = get_sharing_handles(ent, pstatus);
342 auto [parent,
bit] = get_parent_and_bit(ent);
345 MOFEM_LOG(
"SYNC", Sev::noisy) <<
"pstatus " << std::bitset<8>(pstatus);
348 auto pstatus_parent = get_pstatus(parent);
349 auto sharing_procs_parent = get_sharing_procs(parent, pstatus_parent);
350 auto sharing_handles_parent = get_sharing_handles(parent, pstatus_parent);
352 for (
int proc = 0; proc < MAX_SHARING_PROCS && -1 != sharing_procs[proc];
354 if (sharing_procs[proc] == -1)
356 "sharing processor not set");
360 auto it = std::find(sharing_procs_parent.begin(),
361 sharing_procs_parent.end(), sharing_procs[proc]);
362 if (it == sharing_procs_parent.end()) {
365 "Sharing proc for parent entity can not be found proc = %u",
366 sharing_procs[proc]);
369 auto handle_on_sharing_proc = sharing_handles[proc];
370 auto parent_handle_on_sharing_proc =
371 sharing_handles_parent[std::distance(sharing_procs_parent.begin(),
373 sbuffer[sharing_procs[proc]].push_back(handle_on_sharing_proc);
374 sbuffer[sharing_procs[proc]].push_back(parent_handle_on_sharing_proc);
376 sbuffer[sharing_procs[proc]].push_back(
bit.to_ullong());
377 }
catch (std::exception &ex) {
378 MOFEM_LOG(
"SELF", Sev::warning) << ex.what();
380 <<
"On " << ent <<
" "
382 MOFEM_LOG(
"SELF", Sev::warning) <<
"For bit ref " <<
bit;
385 MOFEM_LOG_C(
"SYNC", Sev::noisy,
"send %lu (%lu) to %d at %d\n", ent,
386 handle_on_sharing_proc, sharing_procs[proc],
389 if (!(pstatus & PSTATUS_MULTISHARED))
394 for (
int proc = 0; proc < MAX_SHARING_PROCS && -1 != sharing_procs[proc];
396 if (sharing_procs[proc] == -1)
398 "sharing processor not set");
401 auto handle_on_sharing_proc = sharing_handles[proc];
402 sbuffer[sharing_procs[proc]].push_back(handle_on_sharing_proc);
403 sbuffer[sharing_procs[proc]].push_back(parent);
406 sbuffer[sharing_procs[proc]].push_back(
bit.to_ullong());
407 }
catch (std::exception &ex) {
408 MOFEM_LOG(
"SELF", Sev::warning) << ex.what();
410 <<
"On " << ent <<
" "
412 MOFEM_LOG(
"SELF", Sev::warning) <<
"For bit ref " <<
bit;
416 MOFEM_LOG_C(
"SYNC", Sev::noisy,
"send %lu (%lu) to %d at %d\n", ent,
417 handle_on_sharing_proc, sharing_procs[proc],
420 if (!(pstatus & PSTATUS_MULTISHARED))
428 std::vector<int> sbuffer_lengths(
431 const size_t block_size =
sizeof(
unsigned long long) /
sizeof(
int);
434 if (!sbuffer[proc].empty()) {
436 sbuffer_lengths[proc] = sbuffer[proc].size() * block_size;
441 sbuffer_lengths[proc] = 0;
453 CHKERR PetscGatherNumberOfMessages(comm, NULL, &sbuffer_lengths[0], &nrecvs);
459 CHKERR PetscGatherMessageLengths(comm, nsends, nrecvs, &sbuffer_lengths[0],
467 CHKERR PetscCommGetNewTag(comm, &tag);
472 MPI_Request *r_waits;
476 CHKERR PetscPostIrecvInt(comm, tag, nrecvs, onodes, olengths, &rbuf,
479 MPI_Request *s_waits;
480 CHKERR PetscMalloc1(nsends, &s_waits);
483 for (
int proc = 0, kk = 0; proc < m_field.
get_comm_size(); proc++) {
484 if (!sbuffer_lengths[proc])
486 CHKERR MPI_Isend(&(sbuffer[proc])[0],
487 sbuffer_lengths[proc],
489 tag, comm, s_waits + kk);
495 CHKERR MPI_Waitall(nrecvs, r_waits, &status[0]);
499 CHKERR MPI_Waitall(nsends, s_waits, &status[0]);
503 "Rank %d nb. shared to synchronise parents ents %u\n",
508 for (
int kk = 0; kk < nrecvs; kk++) {
510 int len = olengths[kk];
511 int *data_from_proc = rbuf[kk];
513 for (
int ee = 0; ee < len;) {
516 unsigned long long uulong_bit;
519 bcopy(&data_from_proc[ee], &parent,
sizeof(
EntityHandle));
521 bcopy(&data_from_proc[ee], &uulong_bit,
sizeof(
unsigned long long));
524 CHKERR set_parent(ent, parent);
528 MOFEM_LOG_C(
"SYNC", Sev::noisy,
"received %lu (%lu) from %d at %d\n",
536 CHKERR PetscFree(s_waits);
537 CHKERR PetscFree(rbuf[0]);
539 CHKERR PetscFree(r_waits);
541 CHKERR PetscFree(olengths);
542 CHKERR PetscCommDestroy(&comm);
Public Member Functions inherited from MoFEM::UnknownInterface