Synchronise parent entities.
Exchange parent entity tag and bitref of entity. Note thar parent handle can be different on each processor.
250 {
252 ParallelComm *pcomm = ParallelComm::get_pcomm(
254
256
258 CHKERR pcomm->filter_pstatus(shared, PSTATUS_NOT_OWNED, PSTATUS_NOT, -1,
259 nullptr);
260 CHKERR pcomm->filter_pstatus(shared, PSTATUS_SHARED | PSTATUS_MULTISHARED,
261 PSTATUS_OR, -1, nullptr);
262
265
266 auto get_pstatus = [&](const auto ent) {
267 unsigned char pstatus;
269 1, &pstatus),
270 "can not get pstatus");
271 return pstatus;
272 };
273
274 auto get_sharing_procs = [&](const auto ent, const auto pstatus) {
275 std::vector<int> sharing_procs(MAX_SHARING_PROCS, -1);
276 if (pstatus & PSTATUS_MULTISHARED) {
277
279 pcomm->sharedps_tag(), &ent, 1, &sharing_procs[0]),
280 "can not ger sharing_procs_ptr");
281 } else if (pstatus & PSTATUS_SHARED) {
282
284 1, &sharing_procs[0]),
285 "can not get sharing proc");
286 }
287 return sharing_procs;
288 };
289
290 auto get_sharing_handles = [&](const auto ent, const auto pstatus) {
291 std::vector<EntityHandle> sharing_handles(MAX_SHARING_PROCS, 0);
292 if (pstatus & PSTATUS_MULTISHARED) {
293
295 pcomm->sharedhs_tag(), &ent, 1, &sharing_handles[0]),
296 "get shared handles");
297 } else if (pstatus & PSTATUS_SHARED) {
298
300 1, &sharing_handles[0]),
301 "get sharing handle");
302 }
303 return sharing_handles;
304 };
305
306 auto get_parent_and_bit = [&](const auto ent) {
309 m_field.
get_moab().tag_get_data(th_RefParentHandle, &ent, 1, &parent),
310 "get parent");
313 m_field.
get_moab().tag_get_data(th_RefBitLevel, &ent, 1, &
bit),
314 "get parent");
315 return std::make_pair(parent,
bit);
316 };
317
318 auto set_parent = [&](const auto ent, const auto parent) {
319 return m_field.
get_moab().tag_set_data(th_RefParentHandle, &ent, 1,
320 &parent);
321 };
322
323 auto set_bit = [&](
const auto ent,
const auto bit) {
324 return m_field.
get_moab().tag_set_data(th_RefBitLevel, &ent, 1, &
bit);
325 };
326
327
328 std::vector<std::vector<unsigned long long>> sbuffer(m_field.
get_comm_size());
329
330 for (auto ent : shared) {
331
332 auto pstatus = get_pstatus(ent);
333 auto sharing_procs = get_sharing_procs(ent, pstatus);
334 auto sharing_handles = get_sharing_handles(ent, pstatus);
335 auto [parent,
bit] = get_parent_and_bit(ent);
336
338 MOFEM_LOG(
"SYNC", Sev::noisy) <<
"pstatus " << std::bitset<8>(pstatus);
339
340 if (parent) {
341 auto pstatus_parent = get_pstatus(parent);
342 auto sharing_procs_parent = get_sharing_procs(parent, pstatus_parent);
343 auto sharing_handles_parent = get_sharing_handles(parent, pstatus_parent);
344
345 for (int proc = 0; proc < MAX_SHARING_PROCS && -1 != sharing_procs[proc];
346 proc++) {
347 if (sharing_procs[proc] == -1)
349 "sharing processor not set");
350
352
353 auto it = std::find(sharing_procs_parent.begin(),
354 sharing_procs_parent.end(), sharing_procs[proc]);
355 if (it == sharing_procs_parent.end()) {
356 SETERRQ(
358 "Sharing proc for parent entity can not be found proc = %u",
359 sharing_procs[proc]);
360 }
361
362 auto handle_on_sharing_proc = sharing_handles[proc];
363 auto parent_handle_on_sharing_proc =
364 sharing_handles_parent[std::distance(sharing_procs_parent.begin(),
365 it)];
366 sbuffer[sharing_procs[proc]].push_back(handle_on_sharing_proc);
367 sbuffer[sharing_procs[proc]].push_back(parent_handle_on_sharing_proc);
368 try {
369 sbuffer[sharing_procs[proc]].push_back(
bit.to_ullong());
370 } catch (std::exception &ex) {
371 MOFEM_LOG(
"SELF", Sev::warning) << ex.what();
373 << "On " << ent << " "
375 MOFEM_LOG(
"SELF", Sev::warning) <<
"For bit ref " <<
bit;
376 }
378 MOFEM_LOG_C(
"SYNC", Sev::noisy,
"send %lu (%lu) to %d at %d\n", ent,
379 handle_on_sharing_proc, sharing_procs[proc],
381
382 if (!(pstatus & PSTATUS_MULTISHARED))
383 break;
384 }
385 }
386 } else {
387 for (int proc = 0; proc < MAX_SHARING_PROCS && -1 != sharing_procs[proc];
388 proc++) {
389 if (sharing_procs[proc] == -1)
391 "sharing processor not set");
392
394 auto handle_on_sharing_proc = sharing_handles[proc];
395 sbuffer[sharing_procs[proc]].push_back(handle_on_sharing_proc);
396 sbuffer[sharing_procs[proc]].push_back(parent);
397
398 try {
399 sbuffer[sharing_procs[proc]].push_back(
bit.to_ullong());
400 } catch (std::exception &ex) {
401 MOFEM_LOG(
"SELF", Sev::warning) << ex.what();
403 << "On " << ent << " "
405 MOFEM_LOG(
"SELF", Sev::warning) <<
"For bit ref " <<
bit;
406 }
407
409 MOFEM_LOG_C(
"SYNC", Sev::noisy,
"send %lu (%lu) to %d at %d\n", ent,
410 handle_on_sharing_proc, sharing_procs[proc],
412
413 if (!(pstatus & PSTATUS_MULTISHARED))
414 break;
415 }
416 }
417 }
418 }
419
420 int nsends = 0;
421 std::vector<int> sbuffer_lengths(
423
424 const size_t block_size = sizeof(unsigned long long) / sizeof(int);
426
427 if (!sbuffer[proc].empty()) {
428
429 sbuffer_lengths[proc] = sbuffer[proc].size() * block_size;
430 nsends++;
431
432 } else {
433
434 sbuffer_lengths[proc] = 0;
435 }
436 }
437
438
439 MPI_Comm comm;
441
443
444
445 int nrecvs;
446 CHKERR PetscGatherNumberOfMessages(comm, NULL, &sbuffer_lengths[0], &nrecvs);
447
448
449
450 int *onodes;
451 int *olengths;
452 CHKERR PetscGatherMessageLengths(comm, nsends, nrecvs, &sbuffer_lengths[0],
453 &onodes, &olengths);
454
455
456
457
458
459 int tag;
460 CHKERR PetscCommGetNewTag(comm, &tag);
461
462
463
464 int **rbuf;
465 MPI_Request *r_waits;
466
467
468
469 CHKERR PetscPostIrecvInt(comm, tag, nrecvs, onodes, olengths, &rbuf,
470 &r_waits);
471
472 MPI_Request *s_waits;
473 CHKERR PetscMalloc1(nsends, &s_waits);
474
475
476 for (
int proc = 0, kk = 0; proc < m_field.
get_comm_size(); proc++) {
477 if (!sbuffer_lengths[proc])
478 continue;
479 CHKERR MPI_Isend(&(sbuffer[proc])[0],
480 sbuffer_lengths[proc],
481 MPIU_INT, proc,
482 tag, comm, s_waits + kk);
483 kk++;
484 }
485
486
487 if (nrecvs)
488 CHKERR MPI_Waitall(nrecvs, r_waits, &status[0]);
489
490
491 if (nsends)
492 CHKERR MPI_Waitall(nsends, s_waits, &status[0]);
493
496 "Rank %d nb. shared to synchronise parents ents %u\n",
498 }
499
500
501 for (int kk = 0; kk < nrecvs; kk++) {
502
503 int len = olengths[kk];
504 int *data_from_proc = rbuf[kk];
505
506 for (int ee = 0; ee < len;) {
509 unsigned long long uulong_bit;
511 ee += block_size;
512 bcopy(&data_from_proc[ee], &parent,
sizeof(
EntityHandle));
513 ee += block_size;
514 bcopy(&data_from_proc[ee], &uulong_bit, sizeof(unsigned long long));
515 ee += block_size;
516
517 CHKERR set_parent(ent, parent);
519
521 MOFEM_LOG_C(
"SYNC", Sev::noisy,
"received %lu (%lu) from %d at %d\n",
524 }
525 }
526 }
527
528
529 CHKERR PetscFree(s_waits);
530 CHKERR PetscFree(rbuf[0]);
532 CHKERR PetscFree(r_waits);
534 CHKERR PetscFree(olengths);
535 CHKERR PetscCommDestroy(&comm);
536
539
541}
#define MOFEM_LOG_C(channel, severity, format,...)
std::bitset< BITREFLEVEL_SIZE > BitRefLevel
Bit structure attached to each entity identifying to what mesh entity is attached.
auto type_from_handle(const EntityHandle h)
get type from entity handle
Tag get_th_RefBitLevel() const
Tag get_th_RefParentHandle() const
Public Member Functions inherited from MoFEM::UnknownInterface