Synchronise parent entities.
Exchange parent entity tag and bitref of entity. Note thar parent handle can be different on each processor.
250 {
252 ParallelComm *pcomm = ParallelComm::get_pcomm(
254
256
258 CHKERR pcomm->filter_pstatus(shared, PSTATUS_NOT_OWNED, PSTATUS_NOT, -1,
259 nullptr);
260 CHKERR pcomm->filter_pstatus(shared, PSTATUS_SHARED | PSTATUS_MULTISHARED,
261 PSTATUS_OR, -1, nullptr);
262
265
266 auto get_pstatus = [&](const auto ent) {
267 unsigned char pstatus;
269 1, &pstatus),
270 "can not get pstatus");
271 return pstatus;
272 };
273
274 auto get_sharing_procs = [&](const auto ent, const auto pstatus) {
275 std::vector<int> sharing_procs(MAX_SHARING_PROCS, -1);
276 if (pstatus & PSTATUS_MULTISHARED) {
277
279 pcomm->sharedps_tag(), &ent, 1, &sharing_procs[0]),
280 "can not ger sharing_procs_ptr");
281 } else if (pstatus & PSTATUS_SHARED) {
282
284 1, &sharing_procs[0]),
285 "can not get sharing proc");
286 }
287 return sharing_procs;
288 };
289
290 auto get_sharing_handles = [&](const auto ent, const auto pstatus) {
291 std::vector<EntityHandle> sharing_handles(MAX_SHARING_PROCS, 0);
292 if (pstatus & PSTATUS_MULTISHARED) {
293
295 pcomm->sharedhs_tag(), &ent, 1, &sharing_handles[0]),
296 "get shared handles");
297 } else if (pstatus & PSTATUS_SHARED) {
298
300 1, &sharing_handles[0]),
301 "get sharing handle");
302 }
303 return sharing_handles;
304 };
305
306 auto get_parent_and_bit = [&](const auto ent) {
309 m_field.
get_moab().tag_get_data(th_RefParentHandle, &ent, 1, &parent),
310 "get parent");
313 m_field.
get_moab().tag_get_data(th_RefBitLevel, &ent, 1, &
bit),
314 "get parent");
315 return std::make_pair(parent,
bit);
316 };
317
318 auto set_parent = [&](const auto ent, const auto parent) {
319 return m_field.
get_moab().tag_set_data(th_RefParentHandle, &ent, 1,
320 &parent);
321 };
322
323 auto set_bit = [&](
const auto ent,
const auto bit) {
324 return m_field.
get_moab().tag_set_data(th_RefBitLevel, &ent, 1, &
bit);
325 };
326
327
328 std::vector<std::vector<unsigned long long>> sbuffer(m_field.
get_comm_size());
329
330 for (auto ent : shared) {
331
332 auto pstatus = get_pstatus(ent);
333 auto sharing_procs = get_sharing_procs(ent, pstatus);
334 auto sharing_handles = get_sharing_handles(ent, pstatus);
335 auto [parent,
bit] = get_parent_and_bit(ent);
336
338 MOFEM_LOG(
"SYNC", Sev::noisy) <<
"pstatus " << std::bitset<8>(pstatus);
339
340 if (parent) {
341 auto pstatus_parent = get_pstatus(parent);
342 auto sharing_procs_parent = get_sharing_procs(parent, pstatus_parent);
343 auto sharing_handles_parent = get_sharing_handles(parent, pstatus_parent);
344
345 for (int proc = 0; proc < MAX_SHARING_PROCS && -1 != sharing_procs[proc];
346 proc++) {
347 if (sharing_procs[proc] == -1)
349 "sharing processor not set");
350
352
353 auto it = std::find(sharing_procs_parent.begin(),
354 sharing_procs_parent.end(), sharing_procs[proc]);
355 if (it == sharing_procs_parent.end()) {
357 "Sharing proc for parent entity can not be found proc = %u",
358 sharing_procs[proc]);
359 }
360
361 auto handle_on_sharing_proc = sharing_handles[proc];
362 auto parent_handle_on_sharing_proc =
363 sharing_handles_parent[std::distance(sharing_procs_parent.begin(),
364 it)];
365 sbuffer[sharing_procs[proc]].push_back(handle_on_sharing_proc);
366 sbuffer[sharing_procs[proc]].push_back(parent_handle_on_sharing_proc);
367 try {
368 sbuffer[sharing_procs[proc]].push_back(
bit.to_ullong());
369 } catch (std::exception &ex) {
370 MOFEM_LOG(
"SELF", Sev::warning) << ex.what();
372 << "On " << ent << " "
374 MOFEM_LOG(
"SELF", Sev::warning) <<
"For bit ref " <<
bit;
375 }
377 MOFEM_LOG_C(
"SYNC", Sev::noisy,
"send %lu (%lu) to %d at %d\n", ent,
378 handle_on_sharing_proc, sharing_procs[proc],
380
381 if (!(pstatus & PSTATUS_MULTISHARED))
382 break;
383 }
384 }
385 } else {
386 for (int proc = 0; proc < MAX_SHARING_PROCS && -1 != sharing_procs[proc];
387 proc++) {
388 if (sharing_procs[proc] == -1)
390 "sharing processor not set");
391
393 auto handle_on_sharing_proc = sharing_handles[proc];
394 sbuffer[sharing_procs[proc]].push_back(handle_on_sharing_proc);
395 sbuffer[sharing_procs[proc]].push_back(parent);
396
397 try {
398 sbuffer[sharing_procs[proc]].push_back(
bit.to_ullong());
399 } catch (std::exception &ex) {
400 MOFEM_LOG(
"SELF", Sev::warning) << ex.what();
402 << "On " << ent << " "
404 MOFEM_LOG(
"SELF", Sev::warning) <<
"For bit ref " <<
bit;
405 }
406
408 MOFEM_LOG_C(
"SYNC", Sev::noisy,
"send %lu (%lu) to %d at %d\n", ent,
409 handle_on_sharing_proc, sharing_procs[proc],
411
412 if (!(pstatus & PSTATUS_MULTISHARED))
413 break;
414 }
415 }
416 }
417 }
418
419 int nsends = 0;
420 std::vector<int> sbuffer_lengths(
422
423 const size_t block_size = sizeof(unsigned long long) / sizeof(int);
425
426 if (!sbuffer[proc].empty()) {
427
428 sbuffer_lengths[proc] = sbuffer[proc].size() * block_size;
429 nsends++;
430
431 } else {
432
433 sbuffer_lengths[proc] = 0;
434 }
435 }
436
437
438 MPI_Comm comm;
440
442
443
444 int nrecvs;
445 CHKERR PetscGatherNumberOfMessages(comm, NULL, &sbuffer_lengths[0], &nrecvs);
446
447
448
449 int *onodes;
450 int *olengths;
451 CHKERR PetscGatherMessageLengths(comm, nsends, nrecvs, &sbuffer_lengths[0],
452 &onodes, &olengths);
453
454
455
456
457
458 int tag;
459 CHKERR PetscCommGetNewTag(comm, &tag);
460
461
462
463 int **rbuf;
464 MPI_Request *r_waits;
465
466
467
468 CHKERR PetscPostIrecvInt(comm, tag, nrecvs, onodes, olengths, &rbuf,
469 &r_waits);
470
471 MPI_Request *s_waits;
472 CHKERR PetscMalloc1(nsends, &s_waits);
473
474
475 for (
int proc = 0, kk = 0; proc < m_field.
get_comm_size(); proc++) {
476 if (!sbuffer_lengths[proc])
477 continue;
478 CHKERR MPI_Isend(&(sbuffer[proc])[0],
479 sbuffer_lengths[proc],
480 MPIU_INT, proc,
481 tag, comm, s_waits + kk);
482 kk++;
483 }
484
485
486 if (nrecvs)
487 CHKERR MPI_Waitall(nrecvs, r_waits, &status[0]);
488
489
490 if (nsends)
491 CHKERR MPI_Waitall(nsends, s_waits, &status[0]);
492
495 "Rank %d nb. shared to synchronise parents ents %zu\n",
497 }
498
499
500 for (int kk = 0; kk < nrecvs; kk++) {
501
502 int len = olengths[kk];
503 int *data_from_proc = rbuf[kk];
504
505 for (int ee = 0; ee < len;) {
508 unsigned long long uulong_bit;
510 ee += block_size;
511 bcopy(&data_from_proc[ee], &parent,
sizeof(
EntityHandle));
512 ee += block_size;
513 bcopy(&data_from_proc[ee], &uulong_bit, sizeof(unsigned long long));
514 ee += block_size;
515
516 CHKERR set_parent(ent, parent);
518
520 MOFEM_LOG_C(
"SYNC", Sev::noisy,
"received %lu (%lu) from %d at %d\n",
523 }
524 }
525 }
526
527
528 CHKERR PetscFree(s_waits);
529 CHKERR PetscFree(rbuf[0]);
531 CHKERR PetscFree(r_waits);
533 CHKERR PetscFree(olengths);
534 CHKERR PetscCommDestroy(&comm);
535
538
540}
#define MOFEM_LOG_C(channel, severity, format,...)
std::bitset< BITREFLEVEL_SIZE > BitRefLevel
Bit structure attached to each entity identifying to what mesh entity is attached.
auto type_from_handle(const EntityHandle h)
get type from entity handle
Tag get_th_RefBitLevel() const
Tag get_th_RefParentHandle() const
Public Member Functions inherited from MoFEM::UnknownInterface