20 Range &ents, std::map<int, Range> *received_ents,
int verb) {
22 ParallelComm *pcomm = ParallelComm::get_pcomm(
26 auto get_pstatus = [&](
const auto ent) {
27 unsigned char pstatus;
30 "can not get pstatus");
34 auto get_sharing_procs = [&](
const auto ent,
const auto pstatus) {
35 std::vector<int> sharing_procs(MAX_SHARING_PROCS, -1);
36 if (pstatus & PSTATUS_MULTISHARED) {
39 pcomm->sharedps_tag(), &ent, 1, &sharing_procs[0]),
40 "can not ger sharing_procs_ptr");
41 }
else if (pstatus & PSTATUS_SHARED) {
44 1, &sharing_procs[0]),
45 "can not get sharing proc");
50 auto get_sharing_handles = [&](
const auto ent,
const auto pstatus) {
51 std::vector<EntityHandle> sharing_handles(MAX_SHARING_PROCS, 0);
52 if (pstatus & PSTATUS_MULTISHARED) {
55 pcomm->sharedhs_tag(), &ent, 1, &sharing_handles[0]),
56 "get shared handles");
57 }
else if (pstatus & PSTATUS_SHARED) {
60 1, &sharing_handles[0]),
61 "get sharing handle");
63 return sharing_handles;
67 std::vector<std::vector<EntityHandle>> sbuffer(m_field.
get_comm_size());
69 for (
auto ent : ents) {
71 auto pstatus = get_pstatus(ent);
73 auto sharing_procs = get_sharing_procs(ent, pstatus);
74 auto sharing_handles = get_sharing_handles(ent, pstatus);
77 MOFEM_LOG(
"SYNC", Sev::noisy) <<
"pstatus " << std::bitset<8>(pstatus);
80 for (
int proc = 0; proc < MAX_SHARING_PROCS && -1 != sharing_procs[proc];
82 if (sharing_procs[proc] == -1)
84 "sharing processor not set");
89 const auto handle_on_sharing_proc = sharing_handles[proc];
91 sbuffer[sharing_procs[proc]].push_back(handle_on_sharing_proc);
93 MOFEM_LOG_C(
"SYNC", Sev::noisy,
"send %lu (%lu) to %d at %d\n", ent,
94 handle_on_sharing_proc, sharing_procs[proc],
98 if (!(pstatus & PSTATUS_MULTISHARED))
108 std::vector<int> sbuffer_lengths(
110 const size_t block_size =
115 if (!sbuffer[proc].empty()) {
117 sbuffer_lengths[proc] = sbuffer[proc].size() * block_size;
122 sbuffer_lengths[proc] = 0;
134 CHKERR PetscGatherNumberOfMessages(comm, NULL, &sbuffer_lengths[0], &nrecvs);
140 CHKERR PetscGatherMessageLengths(comm, nsends, nrecvs, &sbuffer_lengths[0],
148 CHKERR PetscCommGetNewTag(comm, &tag);
153 MPI_Request *r_waits;
157 CHKERR PetscPostIrecvInt(comm, tag, nrecvs, onodes, olengths, &rbuf,
160 MPI_Request *s_waits;
161 CHKERR PetscMalloc1(nsends, &s_waits);
164 for (
int proc = 0, kk = 0; proc < m_field.
get_comm_size(); proc++) {
165 if (!sbuffer_lengths[proc])
167 CHKERR MPI_Isend(&(sbuffer[proc])[0],
168 sbuffer_lengths[proc],
170 tag, comm, s_waits + kk);
176 CHKERR MPI_Waitall(nrecvs, r_waits, &status[0]);
180 CHKERR MPI_Waitall(nsends, s_waits, &status[0]);
183 MOFEM_LOG_C(
"SYNC", Sev::verbose,
"Rank %d nb. before ents %u\n",
189 for (
int kk = 0; kk < nrecvs; kk++) {
191 int len = olengths[kk];
192 int *data_from_proc = rbuf[kk];
194 for (
int ee = 0; ee < len;) {
200 (*received_ents)[onodes[kk]].insert(ent);
206 MOFEM_LOG_C(
"SYNC", Sev::noisy,
"received %lu from %d at %d\n", ent,
216 MOFEM_LOG_C(
"SYNC", Sev::verbose,
"Rank %d nb. after ents %u",
222 CHKERR PetscFree(s_waits);
223 CHKERR PetscFree(rbuf[0]);
225 CHKERR PetscFree(r_waits);
227 CHKERR PetscFree(olengths);
228 CHKERR PetscCommDestroy(&comm);
252 ParallelComm *pcomm = ParallelComm::get_pcomm(
258 CHKERR pcomm->filter_pstatus(shared, PSTATUS_NOT_OWNED, PSTATUS_NOT, -1,
260 CHKERR pcomm->filter_pstatus(shared, PSTATUS_SHARED | PSTATUS_MULTISHARED,
261 PSTATUS_OR, -1,
nullptr);
266 auto get_pstatus = [&](
const auto ent) {
267 unsigned char pstatus;
270 "can not get pstatus");
274 auto get_sharing_procs = [&](
const auto ent,
const auto pstatus) {
275 std::vector<int> sharing_procs(MAX_SHARING_PROCS, -1);
276 if (pstatus & PSTATUS_MULTISHARED) {
279 pcomm->sharedps_tag(), &ent, 1, &sharing_procs[0]),
280 "can not ger sharing_procs_ptr");
281 }
else if (pstatus & PSTATUS_SHARED) {
284 1, &sharing_procs[0]),
285 "can not get sharing proc");
287 return sharing_procs;
290 auto get_sharing_handles = [&](
const auto ent,
const auto pstatus) {
291 std::vector<EntityHandle> sharing_handles(MAX_SHARING_PROCS, 0);
292 if (pstatus & PSTATUS_MULTISHARED) {
295 pcomm->sharedhs_tag(), &ent, 1, &sharing_handles[0]),
296 "get shared handles");
297 }
else if (pstatus & PSTATUS_SHARED) {
300 1, &sharing_handles[0]),
301 "get sharing handle");
303 return sharing_handles;
306 auto get_parent_and_bit = [&](
const auto ent) {
309 m_field.
get_moab().tag_get_data(th_RefParentHandle, &ent, 1, &parent),
313 m_field.
get_moab().tag_get_data(th_RefBitLevel, &ent, 1, &
bit),
315 return std::make_pair(parent,
bit);
318 auto set_parent = [&](
const auto ent,
const auto parent) {
319 return m_field.
get_moab().tag_set_data(th_RefParentHandle, &ent, 1,
323 auto set_bit = [&](
const auto ent,
const auto bit) {
324 return m_field.
get_moab().tag_set_data(th_RefBitLevel, &ent, 1, &
bit);
328 std::vector<std::vector<unsigned long long>> sbuffer(m_field.
get_comm_size());
330 for (
auto ent : shared) {
332 auto pstatus = get_pstatus(ent);
333 auto sharing_procs = get_sharing_procs(ent, pstatus);
334 auto sharing_handles = get_sharing_handles(ent, pstatus);
335 auto [parent,
bit] = get_parent_and_bit(ent);
338 MOFEM_LOG(
"SYNC", Sev::noisy) <<
"pstatus " << std::bitset<8>(pstatus);
341 auto pstatus_parent = get_pstatus(parent);
342 auto sharing_procs_parent = get_sharing_procs(parent, pstatus_parent);
343 auto sharing_handles_parent = get_sharing_handles(parent, pstatus_parent);
345 for (
int proc = 0; proc < MAX_SHARING_PROCS && -1 != sharing_procs[proc];
347 if (sharing_procs[proc] == -1)
349 "sharing processor not set");
353 auto it = std::find(sharing_procs_parent.begin(),
354 sharing_procs_parent.end(), sharing_procs[proc]);
355 if (it == sharing_procs_parent.end()) {
358 "Sharing proc for parent entity can not be found proc = %u",
359 sharing_procs[proc]);
362 auto handle_on_sharing_proc = sharing_handles[proc];
363 auto parent_handle_on_sharing_proc =
364 sharing_handles_parent[std::distance(sharing_procs_parent.begin(),
366 sbuffer[sharing_procs[proc]].push_back(handle_on_sharing_proc);
367 sbuffer[sharing_procs[proc]].push_back(parent_handle_on_sharing_proc);
369 sbuffer[sharing_procs[proc]].push_back(
bit.to_ullong());
370 }
catch (std::exception &ex) {
371 MOFEM_LOG(
"SELF", Sev::warning) << ex.what();
373 <<
"On " << ent <<
" "
375 MOFEM_LOG(
"SELF", Sev::warning) <<
"For bit ref " <<
bit;
378 MOFEM_LOG_C(
"SYNC", Sev::noisy,
"send %lu (%lu) to %d at %d\n", ent,
379 handle_on_sharing_proc, sharing_procs[proc],
382 if (!(pstatus & PSTATUS_MULTISHARED))
387 for (
int proc = 0; proc < MAX_SHARING_PROCS && -1 != sharing_procs[proc];
389 if (sharing_procs[proc] == -1)
391 "sharing processor not set");
394 auto handle_on_sharing_proc = sharing_handles[proc];
395 sbuffer[sharing_procs[proc]].push_back(handle_on_sharing_proc);
396 sbuffer[sharing_procs[proc]].push_back(parent);
399 sbuffer[sharing_procs[proc]].push_back(
bit.to_ullong());
400 }
catch (std::exception &ex) {
401 MOFEM_LOG(
"SELF", Sev::warning) << ex.what();
403 <<
"On " << ent <<
" "
405 MOFEM_LOG(
"SELF", Sev::warning) <<
"For bit ref " <<
bit;
409 MOFEM_LOG_C(
"SYNC", Sev::noisy,
"send %lu (%lu) to %d at %d\n", ent,
410 handle_on_sharing_proc, sharing_procs[proc],
413 if (!(pstatus & PSTATUS_MULTISHARED))
421 std::vector<int> sbuffer_lengths(
424 const size_t block_size =
sizeof(
unsigned long long) /
sizeof(
int);
427 if (!sbuffer[proc].empty()) {
429 sbuffer_lengths[proc] = sbuffer[proc].size() * block_size;
434 sbuffer_lengths[proc] = 0;
446 CHKERR PetscGatherNumberOfMessages(comm, NULL, &sbuffer_lengths[0], &nrecvs);
452 CHKERR PetscGatherMessageLengths(comm, nsends, nrecvs, &sbuffer_lengths[0],
460 CHKERR PetscCommGetNewTag(comm, &tag);
465 MPI_Request *r_waits;
469 CHKERR PetscPostIrecvInt(comm, tag, nrecvs, onodes, olengths, &rbuf,
472 MPI_Request *s_waits;
473 CHKERR PetscMalloc1(nsends, &s_waits);
476 for (
int proc = 0, kk = 0; proc < m_field.
get_comm_size(); proc++) {
477 if (!sbuffer_lengths[proc])
479 CHKERR MPI_Isend(&(sbuffer[proc])[0],
480 sbuffer_lengths[proc],
482 tag, comm, s_waits + kk);
488 CHKERR MPI_Waitall(nrecvs, r_waits, &status[0]);
492 CHKERR MPI_Waitall(nsends, s_waits, &status[0]);
496 "Rank %d nb. shared to synchronise parents ents %u\n",
501 for (
int kk = 0; kk < nrecvs; kk++) {
503 int len = olengths[kk];
504 int *data_from_proc = rbuf[kk];
506 for (
int ee = 0; ee < len;) {
509 unsigned long long uulong_bit;
512 bcopy(&data_from_proc[ee], &parent,
sizeof(
EntityHandle));
514 bcopy(&data_from_proc[ee], &uulong_bit,
sizeof(
unsigned long long));
517 CHKERR set_parent(ent, parent);
521 MOFEM_LOG_C(
"SYNC", Sev::noisy,
"received %lu (%lu) from %d at %d\n",
529 CHKERR PetscFree(s_waits);
530 CHKERR PetscFree(rbuf[0]);
532 CHKERR PetscFree(r_waits);
534 CHKERR PetscFree(olengths);
535 CHKERR PetscCommDestroy(&comm);
544 const Problem *problem_ptr,
const std::string &fe_name,
int verb) {
547 ParallelComm *pcomm = ParallelComm::get_pcomm(
549 std::vector<int> shprocs(MAX_SHARING_PROCS, 0);
550 std::vector<EntityHandle> shhandles(MAX_SHARING_PROCS, 0);
557 CHKERR PetscLayoutGetRange(layout, &gid, &last_gid);
558 CHKERR PetscLayoutDestroy(&layout);
568 0, (*fe_miit)->getFEUId()));
573 for (; fit != hi_fe_it; ++fit) {
575 auto ent = (*fit)->getEnt();
576 auto part = (*fit)->getPart();
579 CHKERR m_field.
get_moab().tag_set_data(pcomm->part_tag(), &ent, 1, &part);
580 if (part == pcomm->rank()) {
587 if (pcomm->size() > 1) {
589 unsigned char pstatus = 0;
590 if (pcomm->rank() != part) {
591 pstatus = PSTATUS_NOT_OWNED;
592 pstatus |= PSTATUS_GHOST;
595 if (pcomm->size() > 2) {
596 pstatus |= PSTATUS_SHARED;
597 pstatus |= PSTATUS_MULTISHARED;
599 pstatus |= PSTATUS_SHARED;
603 for (
size_t rr = 0; rr < pcomm->size(); ++rr) {
604 if (rr != pcomm->rank()) {
605 shhandles[rrr] = ent;
610 for (; rrr != pcomm->size(); ++rrr)
613 if (pstatus & PSTATUS_SHARED) {
614 CHKERR m_field.
get_moab().tag_set_data(pcomm->sharedp_tag(), &ent, 1,
616 CHKERR m_field.
get_moab().tag_set_data(pcomm->sharedh_tag(), &ent, 1,
620 if (pstatus & PSTATUS_MULTISHARED) {
621 CHKERR m_field.
get_moab().tag_set_data(pcomm->sharedps_tag(), &ent, 1,
623 CHKERR m_field.
get_moab().tag_set_data(pcomm->sharedhs_tag(), &ent, 1,
626 CHKERR m_field.
get_moab().tag_set_data(pcomm->pstatus_tag(), &ent, 1,
632 CHKERR pcomm->exchange_tags(th_gid, ents);
648 const int num_entities,
649 const int owner_proc,
int verb) {
655 ParallelComm *pcomm = ParallelComm::get_pcomm(
658 Range all_ents_range;
659 all_ents_range.insert_list(entities, entities + num_entities);
661 auto get_tag = [&]() {
662 const int negone = -1;
664 m_field.
get_moab().tag_get_handle(
"TMP_GLOBAL_ID_TAG_NAME", 1,
666 MB_TAG_CREAT | MB_TAG_SPARSE, &negone);
670 auto delete_tag = [&](
auto &&th_gid) {
676 auto resolve_shared_ents = [&](
auto &&th_gid,
auto &all_ents_range) {
677 auto set_gid = [&](
auto &th_gid) {
678 std::vector<int> gids(num_entities);
679 for (
size_t g = 0;
g != all_ents_range.size(); ++
g)
687 auto get_skin_ents = [&](
auto &all_ents_range) {
688 std::array<Range, 4> proc_ents_skin;
689 proc_ents_skin[3] = all_ents_range.subset_by_dimension(3);
690 proc_ents_skin[2] = all_ents_range.subset_by_dimension(2);
691 proc_ents_skin[1] = all_ents_range.subset_by_dimension(1);
692 proc_ents_skin[0] = all_ents_range.subset_by_dimension(0);
693 return proc_ents_skin;
696 auto resolve_dim = [&](
auto &all_ents_range) {
697 for (
int resolve_dim = 3; resolve_dim >= 0; --resolve_dim) {
698 if (all_ents_range.num_of_dimension(resolve_dim))
704 auto get_proc_ent = [&](
auto &all_ents_range) {
707 proc_ent = all_ents_range;
711 auto resolve_shared_ents = [&](
auto &&proc_ents,
auto &&skin_ents) {
712 return pcomm->resolve_shared_ents(
713 0, proc_ents, resolve_dim(all_ents_range),
714 resolve_dim(all_ents_range), skin_ents.data(), set_gid(th_gid));
717 CHKERR resolve_shared_ents(get_proc_ent(all_ents_range),
718 get_skin_ents(all_ents_range));
723 CHKERR delete_tag(resolve_shared_ents(get_tag(), all_ents_range));
731 CHKERR pcomm->get_owner_handle(e, moab_owner_proc, moab_owner_handle);
733 unsigned char pstatus = 0;
735 CHKERR m_field.
get_moab().tag_get_data(pcomm->pstatus_tag(), &e, 1,
738 std::vector<int> shprocs(MAX_SHARING_PROCS, 0);
739 std::vector<EntityHandle> shhandles(MAX_SHARING_PROCS, 0);
741 CHKERR m_field.
get_moab().tag_get_data(pcomm->sharedp_tag(), &e, 1,
743 CHKERR m_field.
get_moab().tag_get_data(pcomm->sharedh_tag(), &e, 1,
745 if (pstatus & PSTATUS_MULTISHARED) {
746 CHKERR m_field.
get_moab().tag_get_data(pcomm->sharedps_tag(), &e, 1,
748 CHKERR m_field.
get_moab().tag_get_data(pcomm->sharedhs_tag(), &e, 1,
752 std::ostringstream ss;
755 if (!(pstatus & PSTATUS_NOT_OWNED))
757 if (pstatus & PSTATUS_SHARED)
758 ss <<
"PSTATUS_SHARED ";
759 if (pstatus & PSTATUS_MULTISHARED)
760 ss <<
"PSTATUS_MULTISHARED ";
762 ss <<
"owner " << moab_owner_proc <<
" (" << owner_proc <<
") ";
766 ss << shprocs[r] <<
" ";
770 ss << shhandles[r] <<
" ";
773 MOFEM_LOG(
"SYNC", Sev::noisy) << ss.str();
779 for (
auto e : all_ents_range)
869 const int adj_dim,
const int n_parts,
870 Tag *th_vertex_weights,
Tag *th_edge_weights,
871 Tag *th_part_weights,
int verb,
const bool debug) {
876 int rstart, rend, nb_elems;
880 CHKERR PetscLayoutSetBlockSize(layout, 1);
881 CHKERR PetscLayoutSetSize(layout, ents.size());
882 CHKERR PetscLayoutSetUp(layout);
883 CHKERR PetscLayoutGetSize(layout, &nb_elems);
884 CHKERR PetscLayoutGetRange(layout, &rstart, &rend);
885 CHKERR PetscLayoutDestroy(&layout);
888 <<
"Finite elements in problem: row lower " << rstart <<
" row upper "
889 << rend <<
" nb. elems " << nb_elems <<
" ( " << ents.size() <<
" )";
894 std::vector<EntityHandle> weight_ents;
895 weight_ents.reserve(rend - rstart + 1);
899 std::vector<int> adj;
900 AdjBridge(
const EntityHandle ent, std::vector<int> &adj)
901 : ent(ent), adj(adj) {}
904 typedef multi_index_container<
908 hashed_unique<member<AdjBridge, EntityHandle, &AdjBridge::ent>>
913 auto get_it = [&](
auto i) {
914 auto it = ents.begin();
916 if (it == ents.end())
924 proc_ents.insert(get_it(rstart), get_it(rend));
925 if (proc_ents.size() != rend - rstart)
927 "Wrong number of elements in range %ld != %d", proc_ents.size(),
932 proc_ents, adj_dim,
true, all_dim_ents, moab::Interface::UNION);
934 AdjBridgeMap adj_bridge_map;
935 auto hint = adj_bridge_map.begin();
936 std::vector<int> adj;
937 for (
auto ent : all_dim_ents) {
939 CHKERR m_field.
get_moab().get_adjacencies(&ent, 1, dim,
false, adj_ents);
940 adj_ents = intersect(adj_ents, ents);
942 adj.reserve(adj_ents.size());
943 for (
auto a : adj_ents)
944 adj.emplace_back(ents.index(
a));
945 hint = adj_bridge_map.emplace_hint(hint, ent, adj);
951 const int nb_loc_elements = rend - rstart;
952 std::vector<int>
i(nb_loc_elements + 1, 0),
j;
954 std::vector<int> row_adj;
955 Range::iterator fe_it;
958 for (fe_it = proc_ents.begin(), ii = rstart, jj = 0, max_row_size = 0;
959 fe_it != proc_ents.end(); ++fe_it, ++ii) {
963 "not yet implemented, don't know what to do for meshset "
968 CHKERR m_field.
get_moab().get_adjacencies(&*fe_it, 1, adj_dim,
false,
970 dim_ents = intersect(dim_ents, all_dim_ents);
973 for (
auto e : dim_ents) {
974 auto adj_it = adj_bridge_map.find(e);
975 if (adj_it != adj_bridge_map.end()) {
977 for (
const auto idx : adj_it->adj)
978 row_adj.push_back(idx);
985 std::sort(row_adj.begin(), row_adj.end());
986 auto end = std::unique(row_adj.begin(), row_adj.end());
988 size_t row_size = std::distance(row_adj.begin(), end);
989 max_row_size = std::max(max_row_size, row_size);
990 if (
j.capacity() < (nb_loc_elements - jj) * max_row_size)
991 j.reserve(nb_loc_elements * max_row_size);
994 auto diag = ents.index(*fe_it);
995 for (
auto it = row_adj.begin(); it != end; ++it)
1002 if (th_vertex_weights != NULL)
1003 weight_ents.push_back(*fe_it);
1009 CHKERR PetscMalloc(
i.size() *
sizeof(
int), &_i);
1010 CHKERR PetscMalloc(
j.size() *
sizeof(
int), &_j);
1011 copy(
i.begin(),
i.end(), _i);
1012 copy(
j.begin(),
j.end(), _j);
1016 int *vertex_weights = NULL;
1017 if (th_vertex_weights != NULL) {
1018 CHKERR PetscMalloc(weight_ents.size() *
sizeof(
int), &vertex_weights);
1020 &*weight_ents.begin(),
1021 weight_ents.size(), vertex_weights);
1027 CHKERR MatCreateMPIAdj(m_field.
get_comm(), rend - rstart, nb_elems, _i, _j,
1028 PETSC_NULLPTR, &Adj);
1029 CHKERR MatSetOption(Adj, MAT_STRUCTURALLY_SYMMETRIC, PETSC_TRUE);
1033 CHKERR MatConvert(Adj, MATMPIAIJ, MAT_INITIAL_MATRIX, &A);
1034 CHKERR MatView(A, PETSC_VIEWER_DRAW_WORLD);
1041 MOFEM_LOG(
"WORLD", Sev::verbose) <<
"Start";
1043 MatPartitioning part;
1047 CHKERR MatPartitioningSetAdjacency(part, Adj);
1048 CHKERR MatPartitioningSetFromOptions(part);
1049 CHKERR MatPartitioningSetNParts(part, n_parts);
1050 if (th_vertex_weights != NULL) {
1051 CHKERR MatPartitioningSetVertexWeights(part, vertex_weights);
1053 CHKERR MatPartitioningApply(part, &is);
1055 MOFEM_LOG(
"WORLD", Sev::verbose) <<
"End";
1058 IS is_gather, is_num, is_gather_num;
1059 CHKERR ISAllGather(is, &is_gather);
1060 CHKERR ISPartitioningToNumbering(is, &is_num);
1061 CHKERR ISAllGather(is_num, &is_gather_num);
1063 const int *part_number, *gids;
1064 CHKERR ISGetIndices(is_gather, &part_number);
1065 CHKERR ISGetIndices(is_gather_num, &gids);
1068 ParallelComm *pcomm = ParallelComm::get_pcomm(
1070 Tag part_tag = pcomm->part_tag();
1071 CHKERR m_field.
get_moab().tag_set_data(part_tag, ents, part_number);
1074 std::map<int, Range> parts_ents;
1077 Range::iterator eit = ents.begin();
1078 for (
int ii = 0; eit != ents.end(); eit++, ii++) {
1079 parts_ents[part_number[ii]].insert(*eit);
1083 0, MBENTITYSET, &part_tag, NULL, 1, tagged_sets,
1084 moab::Interface::UNION);
1085 if (!tagged_sets.empty())
1086 CHKERR m_field.
get_moab().tag_delete_data(part_tag, tagged_sets);
1088 if (n_parts > (
int)tagged_sets.size()) {
1090 int num_new = n_parts - tagged_sets.size();
1091 for (
int i = 0;
i < num_new;
i++) {
1094 tagged_sets.insert(new_set);
1096 }
else if (n_parts < (
int)tagged_sets.size()) {
1098 int num_del = tagged_sets.size() - n_parts;
1099 for (
int i = 0;
i < num_del;
i++) {
1106 std::vector<int> dum_ids(n_parts);
1107 for (
int i = 0;
i < n_parts;
i++)
1114 for (
int pp = 0; pp != n_parts; pp++) {
1115 Range dim_ents = parts_ents[pp].subset_by_dimension(dim);
1116 for (
int dd = dim - 1; dd >= 0; dd--) {
1119 dim_ents, dd,
false, adj_ents, moab::Interface::UNION);
1120 parts_ents[pp].merge(adj_ents);
1123 for (
int pp = 1; pp != n_parts; pp++) {
1124 for (
int ppp = 0; ppp != pp; ppp++) {
1125 parts_ents[pp] = subtract(parts_ents[pp], parts_ents[ppp]);
1129 for (
int pp = 0; pp != n_parts; pp++) {
1130 CHKERR m_field.
get_moab().add_entities(tagged_sets[pp], parts_ents[pp]);
1133 auto set_part = [&]() {
1135 for (EntityType
t = MBEDGE;
t != MBENTITYSET; ++
t) {
1136 for (
int pp = 0; pp != n_parts; pp++) {
1137 Range type_ents = parts_ents[pp].subset_by_type(
t);
1138 CHKERR m_field.
get_moab().tag_clear_data(part_tag, type_ents, &pp);
1144 auto set_gid = [&]() {
1146 for (
int d = 0; d != 4; ++d) {
1152 for (
int pp = 0; pp != n_parts; pp++) {
1153 Range type_ents = parts_ents[pp].subset_by_dimension(d);
1155 auto eit = type_ents.begin();
1156 for (; eit != type_ents.end();) {
1158 gid_tag, eit, type_ents.end(), count, ptr);
1159 auto gid_tag_ptr =
static_cast<int *
>(ptr);
1160 for (; count > 0; --count) {
1179 for (
int rr = 0; rr != n_parts; rr++) {
1181 ss <<
"out_part_" << rr <<
".vtk";
1182 MOFEM_LOG(
"SELF", Sev::inform) <<
"Save debug part mesh " << ss.str();
1186 CHKERR m_field.
get_moab().write_file(ss.str().c_str(),
"VTK",
"",
1193 CHKERR ISRestoreIndices(is_gather, &part_number);
1194 CHKERR ISRestoreIndices(is_gather_num, &gids);
1195 CHKERR ISDestroy(&is_num);
1196 CHKERR ISDestroy(&is_gather_num);
1197 CHKERR ISDestroy(&is_gather);
1199 CHKERR MatPartitioningDestroy(&part);
1207 moab::Interface &moab,
const char *file_name,
int dim,
1215 CHKERR moab.load_file(file_name, 0, options);
1220 ParallelComm *pcomm = ParallelComm::get_pcomm(&moab,
MYPCOMM_INDEX);
1222 pcomm =
new ParallelComm(&moab, PETSC_COMM_WORLD);
1223 if (pcomm->size() == 1)
1226 Skinner skin(&moab);
1228 auto print_range_on_procs = [&](
const Range &range, std::string name) {
1229 MOFEM_LOG(
"SYNC",
sev) << name <<
" on proc [" << pcomm->rank()
1230 <<
"] : " << range.size();
1234 auto save_range_to_file = [&](
const Range range, std::string name =
"part") {
1237 int rr = pcomm->rank();
1239 ss <<
"out_" << name <<
"_" << rr <<
".vtk";
1240 MOFEM_LOG(
"SYNC",
sev) <<
"Save debug part mesh " << ss.str();
1242 CHKERR moab.create_meshset(MESHSET_SET, meshset);
1243 CHKERR moab.add_entities(meshset, range);
1245 CHKERR moab.write_file(ss.str().c_str(),
"VTK",
"", &meshset, 1);
1247 CHKERR moab.delete_entities(&meshset, 1);
1251 auto get_skin = [&](
auto e) {
1253 CHKERR skin.find_skin(0, e.subset_by_dimension(dim),
false, s);
1257 auto get_adj = [&](
auto ents,
auto dim) {
1259 CHKERR moab.get_adjacencies(ents, dim,
false, adj, moab::Interface::UNION);
1264 CHKERR moab.get_entities_by_handle(0, all_ents,
false);
1265 save_range_to_file(all_ents,
"all_ents");
1267 Tag part_tag = pcomm->part_tag();
1268 Range tagged_sets, proc_ents, off_proc_ents;
1270 CHKERR moab.get_entities_by_type_and_tag(0, MBENTITYSET, &part_tag, NULL, 1,
1271 tagged_sets, moab::Interface::UNION);
1272 print_range_on_procs(tagged_sets,
"tagged_sets");
1273 for (
auto &mit : tagged_sets) {
1275 CHKERR moab.tag_get_data(part_tag, &mit, 1, &part);
1276 if (part == pcomm->rank()) {
1277 CHKERR moab.get_entities_by_handle(mit, proc_ents,
true);
1279 CHKERR moab.get_entities_by_handle(mit, off_proc_ents,
true);
1284 for (
auto &mit : tagged_sets) {
1286 CHKERR moab.tag_get_data(part_tag, &mit, 1, &part);
1288 CHKERR moab.get_entities_by_handle(mit, ents,
true);
1289 CHKERR moab.tag_clear_data(part_tag, ents, &part);
1292 print_range_on_procs(proc_ents,
"proc_ents");
1293 save_range_to_file(proc_ents,
"proc_ents");
1295 auto get_proc_ents_skin = [&]() {
1296 std::array<Range, 4>
1298 auto all_skin = get_skin(all_ents.subset_by_dimension(dim));
1299 auto proc_skin = get_skin(proc_ents.subset_by_dimension(dim));
1301 proc_ents_skin[2] = subtract(proc_skin, all_skin);
1302 proc_ents_skin[1] = get_adj(proc_ents_skin[dim - 1], 1);
1305 proc_ents_skin[1] = subtract(proc_skin, all_skin);
1307 CHKERR moab.get_connectivity(proc_ents_skin[dim - 1], proc_ents_skin[0],
1309 return proc_ents_skin;
1312 auto add_post_proc_skin = [&](
auto &&proc_ents_skin) {
1315 Tag nsTag, ssTag, nsTag_data, ssTag_data, bhTag, bhTag_header;
1317 bhTag, bhTag_header,
QUIET);
1320 CHKERR moab.get_entities_by_type(0, MBENTITYSET, meshsets,
false);
1321 for (
auto m : meshsets) {
1326 auto p = cubit_meshsets_index.insert(block);
1331 CHKERR moab.get_entities_by_handle(
m, ents,
true);
1332 CHKERR moab.add_entities(p.first->getMeshset(), ents);
1333 CHKERR moab.delete_entities(&
m, 1);
1338 for (
auto &
m : cubit_meshsets_index) {
1343 std::vector<const CubitMeshSets *> vec_ptr;
1344 for (
auto &
m : cubit_meshsets_index) {
1345 vec_ptr.push_back(&
m);
1349 return proc_skin_fun(std::move(proc_ents_skin), std::move(vec_ptr));
1352 auto remove_off_proc_ents = [&](
auto &all_ents,
auto &off_proc_ents,
1353 auto &proc_ents_skin) {
1354 auto to_remove = off_proc_ents;
1355 for (
auto d = dim - 1; d >= 0; --d) {
1356 to_remove = subtract(to_remove, proc_ents_skin[d]);
1360 CHKERR moab.get_entities_by_type(0, MBENTITYSET, meshsets,
true);
1361 for (
auto m : meshsets) {
1362 CHKERR moab.remove_entities(
m, to_remove);
1364 for (
int dd = dim; dd > 0; --dd) {
1365 CHKERR moab.delete_entities(to_remove.subset_by_dimension(dd));
1369 auto proc_ents_skin = add_post_proc_skin(get_proc_ents_skin());
1370 for (
auto d = dim - 1; d >= 0; --d) {
1371 save_range_to_file(proc_ents_skin[d],
"proc_ents_skin" + std::to_string(d));
1374 if (pcomm->rank()) {
1375 remove_off_proc_ents(all_ents, off_proc_ents, proc_ents_skin);
1378 for (
auto d = dim - 1; d >= 0; --d) {
1379 print_range_on_procs(proc_ents_skin[d],
1380 "proc_ents_skin by dim" + std::to_string(d));
1383 CHKERR pcomm->resolve_shared_ents(0, proc_ents, dim, -1,
1384 proc_ents_skin.data());
1389 auto gid_tag = moab.globalId_tag();
1393 CHKERR moab.tag_get_handle(
"TestGather", 1, MB_TYPE_INTEGER, tag_handle,
1394 MB_TAG_DENSE | MB_TAG_CREAT, &def_val);
1396 if (pcomm->rank() > 0) {
1397 gather_ents = proc_ents.subset_by_dimension(3);
1398 std::vector<int> vals(gather_ents.size());
1399 std::for_each(vals.begin(), vals.end(), [](
auto &
v) { v = std::rand(); });
1400 CHKERR moab.tag_set_data(tag_handle, gather_ents, vals.data());
1401 CHKERR pcomm->gather_data(gather_ents, tag_handle, gid_tag, 0, 0);
1403 gather_ents = proc_ents.subset_by_dimension(3);
1404 CHKERR pcomm->gather_data(gather_ents, tag_handle, gid_tag, 0, 0);
1408 Range all_ents_after;
1409 CHKERR moab.get_entities_by_handle(0, all_ents_after);
1410 save_range_to_file(all_ents_after.subset_by_dimension(3),
1411 "all_ents_part_after");
1462 int dim,
const int nb_coeffs,
Sev sev,
1466 Range r, ghost_ents;
1472 Tag part_tag = pcomm->part_tag();
1473 Range tagged_sets, proc_ents, off_proc_ents;
1474 CHKERR moab.get_entities_by_type_and_tag(0, MBENTITYSET, &part_tag, NULL, 1,
1476 moab::Interface::UNION);
1479 for (
auto &mit : tagged_sets) {
1481 CHKERR moab.tag_get_data(part_tag, &mit, 1, &part);
1482 if (part == pcomm->rank()) {
1483 CHKERR moab.get_entities_by_handle(mit, proc_ents,
true);
1485 CHKERR moab.get_entities_by_handle(mit, off_proc_ents,
true);
1487 proc_ents = subtract(proc_ents, off_proc_ents);
1490 std::map<int, Range> parts_ents;
1491 for (
auto &mit : tagged_sets) {
1493 CHKERR moab.tag_get_data(part_tag, &mit, 1, &part);
1494 CHKERR moab.get_entities_by_handle(mit, parts_ents[part],
true);
1497 CHKERR moab.get_connectivity(parts_ents[pcomm->rank()],
v);
1499 for (; p != pcomm->rank(); ++p) {
1501 CHKERR moab.get_connectivity(parts_ents[p], vv);
1502 v = subtract(
v, vv);
1503 off_proc_ents.merge(vv);
1505 for (; p != pcomm->size(); ++p) {
1507 CHKERR moab.get_connectivity(parts_ents[p], vv);
1508 vv = subtract(vv,
v);
1509 off_proc_ents.merge(vv);
1514 r = proc_ents.subset_by_dimension(dim);
1515 auto o = off_proc_ents.subset_by_dimension(dim);
1517 auto set_ghosts = [&](
Range &ents) {
1518 auto gid_tag = moab.globalId_tag();
1519 std::vector<int> ghosts(ents.size());
1520 CHKERR moab.tag_get_data(gid_tag, ents, ghosts.data());
1521 std::vector<int> ghosts_nb(nb_coeffs * ents.size());
1522 for (
int i = 0;
i < ents.size(); ++
i)
1523 for (
int j = 0;
j < nb_coeffs; ++
j)
1524 ghosts_nb[
i * nb_coeffs +
j] = nb_coeffs * (ghosts[
i] - 1) +
j;
1528 std::vector<int> ghosts;
1529 if (pcomm->rank() == root_rank) {
1530 ghosts = set_ghosts(o);
1534 CHKERR moab.get_entities_by_dimension(0, dim, ents);
1535 ents = subtract(ents, r);
1536 ghosts = set_ghosts(ents);
1540 auto create_vec = [&]() {
1543 ghosts.size(), ghosts.data());
1553 auto out = std::make_pair(std::make_pair(r, ghost_ents), vec);
1558 auto check = [&](
auto &ents,
auto &id,
auto &idx) {
1561 for (
int i = 0;
i < ents.size(); ++
i) {
1562 for (
int j = 0;
j < nb_coeffs; ++
j) {
1563 if (idx[
i * nb_coeffs +
j] != nb_coeffs * (
id[
i] - 1) +
j) {
1566 <<
"indexes not equal: " << idx[
i * nb_coeffs +
j]
1567 <<
" != " << nb_coeffs * (idx[
i] - 1) +
j;
1577 std::vector<double> def_val(nb_coeffs, 0.0);
1579 CHKERR moab.tag_get_handle(
"TestGather", nb_coeffs, MB_TYPE_DOUBLE, tag,
1580 MB_TAG_SPARSE | MB_TAG_CREAT, def_val.data());
1581 auto gid_tag = moab.globalId_tag();
1582 std::vector<int> id(r.size());
1583 CHKERR moab.tag_get_data(gid_tag, r,
id.data());
1584 std::vector<double> idx(nb_coeffs * r.size());
1585 for (
int i = 0;
i < r.size(); ++
i) {
1586 for (
int j = 0;
j < nb_coeffs; ++
j) {
1587 idx[
i * nb_coeffs +
j] = nb_coeffs * (
id[
i] - 1) +
j;
1590 CHKERR moab.tag_set_data(tag, r, idx.data());
1592 idx.resize(nb_coeffs * r.size());
1593 CHKERR moab.tag_get_data(tag, r, idx.data());
1595 id.resize(ghost_ents.size());
1596 CHKERR moab.tag_get_data(gid_tag, ghost_ents,
id.data());
1597 idx.resize(nb_coeffs * ghost_ents.size());
1598 CHKERR moab.tag_get_data(tag, ghost_ents, idx.data());
1600 CHKERR moab.tag_delete(tag);
multi_index_container< CubitMeshSets, indexed_by< hashed_unique< tag< Meshset_mi_tag >, member< CubitMeshSets, EntityHandle, &CubitMeshSets::meshset > >, ordered_non_unique< tag< CubitMeshsetType_mi_tag >, const_mem_fun< CubitMeshSets, unsigned long int, &CubitMeshSets::getBcTypeULong > >, ordered_non_unique< tag< CubitMeshsetMaskedType_mi_tag >, const_mem_fun< CubitMeshSets, unsigned long int, &CubitMeshSets::getMaskedBcTypeULong > >, ordered_non_unique< tag< CubitMeshsets_name >, const_mem_fun< CubitMeshSets, std::string, &CubitMeshSets::getName > >, hashed_unique< tag< Composite_Cubit_msId_And_MeshsetType_mi_tag >, composite_key< CubitMeshSets, const_mem_fun< CubitMeshSets, int, &CubitMeshSets::getMeshsetId >, const_mem_fun< CubitMeshSets, unsigned long int, &CubitMeshSets::getMaskedBcTypeULong > > > > > CubitMeshSet_multiIndex
Stores data about meshsets (see CubitMeshSets) storing data about boundary conditions,...