v0.15.0
Loading...
Searching...
No Matches
CommInterface.cpp
Go to the documentation of this file.
1/** \file CommInterface.cpp
2 * \brief Functions for interprocess communications
3 * \mofem_comm
4 */
5
6namespace MoFEM {
7
9CommInterface::query_interface(boost::typeindex::type_index type_index,
10 UnknownInterface **iface) const {
12 *iface = const_cast<CommInterface *>(this);
14}
15
17 : cOre(const_cast<MoFEM::Core &>(core)), dEbug(false) {}
18
20 Range &ents, std::map<int, Range> *received_ents, int verb) {
21 MoFEM::Interface &m_field = cOre;
22 ParallelComm *pcomm = ParallelComm::get_pcomm(
23 &m_field.get_moab(), m_field.get_basic_entity_data_ptr()->pcommID);
25
26 auto get_pstatus = [&](const auto ent) {
27 unsigned char pstatus;
28 CHK_MOAB_THROW(m_field.get_moab().tag_get_data(pcomm->pstatus_tag(), &ent,
29 1, &pstatus),
30 "can not get pstatus");
31 return pstatus;
32 };
33
34 auto get_sharing_procs = [&](const auto ent, const auto pstatus) {
35 std::vector<int> sharing_procs(MAX_SHARING_PROCS, -1);
36 if (pstatus & PSTATUS_MULTISHARED) {
37 // entity is multi shared
38 CHK_MOAB_THROW(m_field.get_moab().tag_get_data(
39 pcomm->sharedps_tag(), &ent, 1, &sharing_procs[0]),
40 "can not ger sharing_procs_ptr");
41 } else if (pstatus & PSTATUS_SHARED) {
42 // shared
43 CHK_MOAB_THROW(m_field.get_moab().tag_get_data(pcomm->sharedp_tag(), &ent,
44 1, &sharing_procs[0]),
45 "can not get sharing proc");
46 }
47 return sharing_procs;
48 };
49
50 auto get_sharing_handles = [&](const auto ent, const auto pstatus) {
51 std::vector<EntityHandle> sharing_handles(MAX_SHARING_PROCS, 0);
52 if (pstatus & PSTATUS_MULTISHARED) {
53 // entity is multi shared
54 CHK_MOAB_THROW(m_field.get_moab().tag_get_data(
55 pcomm->sharedhs_tag(), &ent, 1, &sharing_handles[0]),
56 "get shared handles");
57 } else if (pstatus & PSTATUS_SHARED) {
58 // shared
59 CHK_MOAB_THROW(m_field.get_moab().tag_get_data(pcomm->sharedh_tag(), &ent,
60 1, &sharing_handles[0]),
61 "get sharing handle");
62 }
63 return sharing_handles;
64 };
65
66 // make a buffer entities to send
67 std::vector<std::vector<EntityHandle>> sbuffer(m_field.get_comm_size());
68
69 for (auto ent : ents) {
70
71 auto pstatus = get_pstatus(ent);
72 if (pstatus) {
73 auto sharing_procs = get_sharing_procs(ent, pstatus);
74 auto sharing_handles = get_sharing_handles(ent, pstatus);
75
76 if (verb >= NOISY) {
77 MOFEM_LOG("SYNC", Sev::noisy) << "pstatus " << std::bitset<8>(pstatus);
78 }
79
80 for (int proc = 0; proc < MAX_SHARING_PROCS && -1 != sharing_procs[proc];
81 proc++) {
82 if (sharing_procs[proc] == -1)
83 SETERRQ(PETSC_COMM_SELF, MOFEM_IMPOSSIBLE_CASE,
84 "sharing processor not set");
85
86 if (sharing_procs[proc] == m_field.get_comm_rank())
87 continue;
88
89 const auto handle_on_sharing_proc = sharing_handles[proc];
90 // add entity to send, handle on the other side
91 sbuffer[sharing_procs[proc]].push_back(handle_on_sharing_proc);
92 if (verb >= NOISY) {
93 MOFEM_LOG_C("SYNC", Sev::noisy, "send %lu (%lu) to %d at %d\n", ent,
94 handle_on_sharing_proc, sharing_procs[proc],
95 m_field.get_comm_rank());
96 }
97
98 if (!(pstatus & PSTATUS_MULTISHARED))
99 break;
100 }
101 }
102 }
103 if (verb >= NOISY) {
104 MOFEM_LOG_SEVERITY_SYNC(m_field.get_comm(), Sev::noisy);
105 }
106
107 int nsends = 0; // number of messages to send
108 std::vector<int> sbuffer_lengths(
109 m_field.get_comm_size()); // length of the message to proc
110 const size_t block_size =
111 sizeof(EntityHandle) /
112 sizeof(int); // FIXME check if that works on given architecture!
113 for (int proc = 0; proc < m_field.get_comm_size(); proc++) {
114
115 if (!sbuffer[proc].empty()) {
116
117 sbuffer_lengths[proc] = sbuffer[proc].size() * block_size;
118 nsends++;
119
120 } else {
121
122 sbuffer_lengths[proc] = 0;
123 }
124 }
125
126 // Make sure it is a PETSc m_field.get_comm()
127 MPI_Comm comm;
128 CHKERR PetscCommDuplicate(m_field.get_comm(), &comm, NULL);
129
130 std::vector<MPI_Status> status(m_field.get_comm_size());
131
132 // Computes the number of messages a node expects to receive
133 int nrecvs; // number of messages received
134 CHKERR PetscGatherNumberOfMessages(comm, NULL, &sbuffer_lengths[0], &nrecvs);
135
136 // Computes info about messages that a MPI-node will receive, including
137 // (from-id,length) pairs for each message.
138 int *onodes; // list of node-ids from which messages are expected
139 int *olengths; // corresponding message lengths
140 CHKERR PetscGatherMessageLengths(comm, nsends, nrecvs, &sbuffer_lengths[0],
141 &onodes, &olengths);
142
143 // Gets a unique new tag from a PETSc communicator. All processors that share
144 // the communicator MUST call this routine EXACTLY the same number of times.
145 // This tag should only be used with the current objects communicator; do NOT
146 // use it with any other MPI communicator.
147 int tag;
148 CHKERR PetscCommGetNewTag(comm, &tag);
149
150 // Allocate a buffer sufficient to hold messages of size specified in
151 // olengths. And post Irecvs on these buffers using node info from onodes
152 int **rbuf; // must bee freed by user
153 MPI_Request *r_waits; // must bee freed by user
154 // rbuf has a pointers to messages. It has size of of nrecvs (number of
155 // messages) +1. In the first index a block is allocated,
156 // such that rbuf[i] = rbuf[i-1]+olengths[i-1].
157 CHKERR PetscPostIrecvInt(comm, tag, nrecvs, onodes, olengths, &rbuf,
158 &r_waits);
159
160 MPI_Request *s_waits; // status of sens messages
161 CHKERR PetscMalloc1(nsends, &s_waits);
162
163 // Send messages
164 for (int proc = 0, kk = 0; proc < m_field.get_comm_size(); proc++) {
165 if (!sbuffer_lengths[proc])
166 continue; // no message to send to this proc
167 CHKERR MPI_Isend(&(sbuffer[proc])[0], // buffer to send
168 sbuffer_lengths[proc], // message length
169 MPIU_INT, proc, // to proc
170 tag, comm, s_waits + kk);
171 kk++;
172 }
173
174 // Wait for received
175 if (nrecvs)
176 CHKERR MPI_Waitall(nrecvs, r_waits, &status[0]);
177
178 // Wait for send messages
179 if (nsends)
180 CHKERR MPI_Waitall(nsends, s_waits, &status[0]);
181
182 if (verb >= VERBOSE) {
183 MOFEM_LOG_C("SYNC", Sev::verbose, "Rank %d nb. before ents %u\n",
184 m_field.get_comm_rank(), ents.size());
185 MOFEM_LOG_SEVERITY_SYNC(m_field.get_comm(), Sev::verbose);
186 }
187
188 // synchronise range
189 for (int kk = 0; kk < nrecvs; kk++) {
190
191 int len = olengths[kk];
192 int *data_from_proc = rbuf[kk];
193
194 for (int ee = 0; ee < len;) {
195 EntityHandle ent;
196 bcopy(&data_from_proc[ee], &ent, sizeof(EntityHandle));
197 ents.insert(ent);
198
199 if (received_ents) {
200 (*received_ents)[onodes[kk]].insert(ent);
201 }
202
203 ee += block_size;
204
205 if (verb >= VERBOSE) {
206 MOFEM_LOG_C("SYNC", Sev::noisy, "received %lu from %d at %d\n", ent,
207 onodes[kk], m_field.get_comm_rank());
208 }
209 }
210 }
211 if (verb >= VERBOSE) {
212 MOFEM_LOG_SEVERITY_SYNC(m_field.get_comm(), Sev::verbose);
213 }
214
215 if (verb >= VERBOSE) {
216 MOFEM_LOG_C("SYNC", Sev::verbose, "Rank %d nb. after ents %u",
217 m_field.get_comm_rank(), ents.size());
218 MOFEM_LOG_SEVERITY_SYNC(m_field.get_comm(), Sev::verbose);
219 }
220
221 // Cleaning
222 CHKERR PetscFree(s_waits);
223 CHKERR PetscFree(rbuf[0]);
224 CHKERR PetscFree(rbuf);
225 CHKERR PetscFree(r_waits);
226 CHKERR PetscFree(onodes);
227 CHKERR PetscFree(olengths);
228 CHKERR PetscCommDestroy(&comm);
229
231}
232
234 return synchroniseEntities(ents, nullptr, verb);
235}
236
238 int verb) {
239 MoFEM::Interface &m_field = cOre;
241 EntityHandle idm = m_field.get_field_meshset(name);
242 Range ents;
243 CHKERR m_field.get_moab().get_entities_by_handle(idm, ents, false);
244 CHKERR synchroniseEntities(ents, nullptr, verb);
245 CHKERR m_field.get_moab().add_entities(idm, ents);
247}
248
250 int verb) {
251 MoFEM::Interface &m_field = cOre;
252 ParallelComm *pcomm = ParallelComm::get_pcomm(
253 &m_field.get_moab(), m_field.get_basic_entity_data_ptr()->pcommID);
254
256
257 Range shared = ents;
258 CHKERR pcomm->filter_pstatus(shared, PSTATUS_NOT_OWNED, PSTATUS_NOT, -1,
259 nullptr);
260 CHKERR pcomm->filter_pstatus(shared, PSTATUS_SHARED | PSTATUS_MULTISHARED,
261 PSTATUS_OR, -1, nullptr);
262
263 auto th_RefParentHandle = cOre.get_th_RefParentHandle();
264 auto th_RefBitLevel = cOre.get_th_RefBitLevel();
265
266 auto get_pstatus = [&](const auto ent) {
267 unsigned char pstatus;
268 CHK_MOAB_THROW(m_field.get_moab().tag_get_data(pcomm->pstatus_tag(), &ent,
269 1, &pstatus),
270 "can not get pstatus");
271 return pstatus;
272 };
273
274 auto get_sharing_procs = [&](const auto ent, const auto pstatus) {
275 std::vector<int> sharing_procs(MAX_SHARING_PROCS, -1);
276 if (pstatus & PSTATUS_MULTISHARED) {
277 // entity is multi shared
278 CHK_MOAB_THROW(m_field.get_moab().tag_get_data(
279 pcomm->sharedps_tag(), &ent, 1, &sharing_procs[0]),
280 "can not ger sharing_procs_ptr");
281 } else if (pstatus & PSTATUS_SHARED) {
282 // shared
283 CHK_MOAB_THROW(m_field.get_moab().tag_get_data(pcomm->sharedp_tag(), &ent,
284 1, &sharing_procs[0]),
285 "can not get sharing proc");
286 }
287 return sharing_procs;
288 };
289
290 auto get_sharing_handles = [&](const auto ent, const auto pstatus) {
291 std::vector<EntityHandle> sharing_handles(MAX_SHARING_PROCS, 0);
292 if (pstatus & PSTATUS_MULTISHARED) {
293 // entity is multi shared
294 CHK_MOAB_THROW(m_field.get_moab().tag_get_data(
295 pcomm->sharedhs_tag(), &ent, 1, &sharing_handles[0]),
296 "get shared handles");
297 } else if (pstatus & PSTATUS_SHARED) {
298 // shared
299 CHK_MOAB_THROW(m_field.get_moab().tag_get_data(pcomm->sharedh_tag(), &ent,
300 1, &sharing_handles[0]),
301 "get sharing handle");
302 }
303 return sharing_handles;
304 };
305
306 auto get_parent_and_bit = [&](const auto ent) {
307 EntityHandle parent;
309 m_field.get_moab().tag_get_data(th_RefParentHandle, &ent, 1, &parent),
310 "get parent");
313 m_field.get_moab().tag_get_data(th_RefBitLevel, &ent, 1, &bit),
314 "get parent");
315 return std::make_pair(parent, bit);
316 };
317
318 auto set_parent = [&](const auto ent, const auto parent) {
319 return m_field.get_moab().tag_set_data(th_RefParentHandle, &ent, 1,
320 &parent);
321 };
322
323 auto set_bit = [&](const auto ent, const auto bit) {
324 return m_field.get_moab().tag_set_data(th_RefBitLevel, &ent, 1, &bit);
325 };
326
327 // make a buffer
328 std::vector<std::vector<unsigned long long>> sbuffer(m_field.get_comm_size());
329
330 for (auto ent : shared) {
331
332 auto pstatus = get_pstatus(ent);
333 auto sharing_procs = get_sharing_procs(ent, pstatus);
334 auto sharing_handles = get_sharing_handles(ent, pstatus);
335 auto [parent, bit] = get_parent_and_bit(ent);
336
337 if (verb >= NOISY)
338 MOFEM_LOG("SYNC", Sev::noisy) << "pstatus " << std::bitset<8>(pstatus);
339
340 if (parent) {
341 auto pstatus_parent = get_pstatus(parent);
342 auto sharing_procs_parent = get_sharing_procs(parent, pstatus_parent);
343 auto sharing_handles_parent = get_sharing_handles(parent, pstatus_parent);
344
345 for (int proc = 0; proc < MAX_SHARING_PROCS && -1 != sharing_procs[proc];
346 proc++) {
347 if (sharing_procs[proc] == -1)
348 SETERRQ(PETSC_COMM_SELF, MOFEM_IMPOSSIBLE_CASE,
349 "sharing processor not set");
350
351 if (sharing_procs[proc] != m_field.get_comm_rank()) {
352
353 auto it = std::find(sharing_procs_parent.begin(),
354 sharing_procs_parent.end(), sharing_procs[proc]);
355 if (it == sharing_procs_parent.end()) {
356 SETERRQ(
357 PETSC_COMM_SELF, MOFEM_DATA_INCONSISTENCY,
358 "Sharing proc for parent entity can not be found proc = %u",
359 sharing_procs[proc]);
360 }
361
362 auto handle_on_sharing_proc = sharing_handles[proc];
363 auto parent_handle_on_sharing_proc =
364 sharing_handles_parent[std::distance(sharing_procs_parent.begin(),
365 it)];
366 sbuffer[sharing_procs[proc]].push_back(handle_on_sharing_proc);
367 sbuffer[sharing_procs[proc]].push_back(parent_handle_on_sharing_proc);
368 try {
369 sbuffer[sharing_procs[proc]].push_back(bit.to_ullong());
370 } catch (std::exception &ex) {
371 MOFEM_LOG("SELF", Sev::warning) << ex.what();
372 MOFEM_LOG("SELF", Sev::warning)
373 << "On " << ent << " "
374 << moab::CN::EntityTypeName(type_from_handle(ent));
375 MOFEM_LOG("SELF", Sev::warning) << "For bit ref " << bit;
376 }
377 if (verb >= NOISY)
378 MOFEM_LOG_C("SYNC", Sev::noisy, "send %lu (%lu) to %d at %d\n", ent,
379 handle_on_sharing_proc, sharing_procs[proc],
380 m_field.get_comm_rank());
381
382 if (!(pstatus & PSTATUS_MULTISHARED))
383 break;
384 }
385 }
386 } else {
387 for (int proc = 0; proc < MAX_SHARING_PROCS && -1 != sharing_procs[proc];
388 proc++) {
389 if (sharing_procs[proc] == -1)
390 SETERRQ(PETSC_COMM_SELF, MOFEM_IMPOSSIBLE_CASE,
391 "sharing processor not set");
392
393 if (sharing_procs[proc] != m_field.get_comm_rank()) {
394 auto handle_on_sharing_proc = sharing_handles[proc];
395 sbuffer[sharing_procs[proc]].push_back(handle_on_sharing_proc);
396 sbuffer[sharing_procs[proc]].push_back(parent);
397
398 try {
399 sbuffer[sharing_procs[proc]].push_back(bit.to_ullong());
400 } catch (std::exception &ex) {
401 MOFEM_LOG("SELF", Sev::warning) << ex.what();
402 MOFEM_LOG("SELF", Sev::warning)
403 << "On " << ent << " "
404 << moab::CN::EntityTypeName(type_from_handle(ent));
405 MOFEM_LOG("SELF", Sev::warning) << "For bit ref " << bit;
406 }
407
408 if (verb >= NOISY)
409 MOFEM_LOG_C("SYNC", Sev::noisy, "send %lu (%lu) to %d at %d\n", ent,
410 handle_on_sharing_proc, sharing_procs[proc],
411 m_field.get_comm_rank());
412
413 if (!(pstatus & PSTATUS_MULTISHARED))
414 break;
415 }
416 }
417 }
418 }
419
420 int nsends = 0; // number of messages to send
421 std::vector<int> sbuffer_lengths(
422 m_field.get_comm_size()); // length of the message to proc
423
424 const size_t block_size = sizeof(unsigned long long) / sizeof(int);
425 for (int proc = 0; proc < m_field.get_comm_size(); proc++) {
426
427 if (!sbuffer[proc].empty()) {
428
429 sbuffer_lengths[proc] = sbuffer[proc].size() * block_size;
430 nsends++;
431
432 } else {
433
434 sbuffer_lengths[proc] = 0;
435 }
436 }
437
438 // Make sure it is a PETSc m_field.get_comm()
439 MPI_Comm comm;
440 CHKERR PetscCommDuplicate(m_field.get_comm(), &comm, NULL);
441
442 std::vector<MPI_Status> status(m_field.get_comm_size());
443
444 // Computes the number of messages a node expects to receive
445 int nrecvs; // number of messages received
446 CHKERR PetscGatherNumberOfMessages(comm, NULL, &sbuffer_lengths[0], &nrecvs);
447
448 // Computes info about messages that a MPI-node will receive, including
449 // (from-id,length) pairs for each message.
450 int *onodes; // list of node-ids from which messages are expected
451 int *olengths; // corresponding message lengths
452 CHKERR PetscGatherMessageLengths(comm, nsends, nrecvs, &sbuffer_lengths[0],
453 &onodes, &olengths);
454
455 // Gets a unique new tag from a PETSc communicator. All processors that share
456 // the communicator MUST call this routine EXACTLY the same number of times.
457 // This tag should only be used with the current objects communicator; do NOT
458 // use it with any other MPI communicator.
459 int tag;
460 CHKERR PetscCommGetNewTag(comm, &tag);
461
462 // Allocate a buffer sufficient to hold messages of size specified in
463 // olengths. And post Irecvs on these buffers using node info from onodes
464 int **rbuf; // must bee freed by user
465 MPI_Request *r_waits; // must bee freed by user
466 // rbuf has a pointers to messages. It has size of of nrecvs (number of
467 // messages) +1. In the first index a block is allocated,
468 // such that rbuf[i] = rbuf[i-1]+olengths[i-1].
469 CHKERR PetscPostIrecvInt(comm, tag, nrecvs, onodes, olengths, &rbuf,
470 &r_waits);
471
472 MPI_Request *s_waits; // status of sens messages
473 CHKERR PetscMalloc1(nsends, &s_waits);
474
475 // Send messages
476 for (int proc = 0, kk = 0; proc < m_field.get_comm_size(); proc++) {
477 if (!sbuffer_lengths[proc])
478 continue; // no message to send to this proc
479 CHKERR MPI_Isend(&(sbuffer[proc])[0], // buffer to send
480 sbuffer_lengths[proc], // message length
481 MPIU_INT, proc, // to proc
482 tag, comm, s_waits + kk);
483 kk++;
484 }
485
486 // Wait for received
487 if (nrecvs)
488 CHKERR MPI_Waitall(nrecvs, r_waits, &status[0]);
489
490 // Wait for send messages
491 if (nsends)
492 CHKERR MPI_Waitall(nsends, s_waits, &status[0]);
493
494 if (verb >= VERBOSE) {
495 MOFEM_LOG_C("SYNC", Sev::verbose,
496 "Rank %d nb. shared to synchronise parents ents %u\n",
497 m_field.get_comm_rank(), shared.size());
498 }
499
500 // synchronise range
501 for (int kk = 0; kk < nrecvs; kk++) {
502
503 int len = olengths[kk];
504 int *data_from_proc = rbuf[kk];
505
506 for (int ee = 0; ee < len;) {
507 EntityHandle ent;
508 EntityHandle parent;
509 unsigned long long uulong_bit;
510 bcopy(&data_from_proc[ee], &ent, sizeof(EntityHandle));
511 ee += block_size;
512 bcopy(&data_from_proc[ee], &parent, sizeof(EntityHandle));
513 ee += block_size;
514 bcopy(&data_from_proc[ee], &uulong_bit, sizeof(unsigned long long));
515 ee += block_size;
516
517 CHKERR set_parent(ent, parent);
518 CHKERR set_bit(ent, BitRefLevel(uulong_bit));
519
520 if (verb >= VERBOSE) {
521 MOFEM_LOG_C("SYNC", Sev::noisy, "received %lu (%lu) from %d at %d\n",
522 ent, parent, onodes[kk], m_field.get_comm_rank());
523 MOFEM_LOG("SYNC", Sev::noisy) << "Bit " << BitRefLevel(uulong_bit);
524 }
525 }
526 }
527
528 // Cleaning
529 CHKERR PetscFree(s_waits);
530 CHKERR PetscFree(rbuf[0]);
531 CHKERR PetscFree(rbuf);
532 CHKERR PetscFree(r_waits);
533 CHKERR PetscFree(onodes);
534 CHKERR PetscFree(olengths);
535 CHKERR PetscCommDestroy(&comm);
536
537 if (verb >= VERBOSE)
539
541}
542
544 const Problem *problem_ptr, const std::string &fe_name, int verb) {
545 MoFEM::Interface &m_field = cOre;
547 ParallelComm *pcomm = ParallelComm::get_pcomm(
548 &m_field.get_moab(), m_field.get_basic_entity_data_ptr()->pcommID);
549 std::vector<int> shprocs(MAX_SHARING_PROCS, 0);
550 std::vector<EntityHandle> shhandles(MAX_SHARING_PROCS, 0);
551 Range ents;
552 Tag th_gid = m_field.get_moab().globalId_tag();
553 PetscLayout layout;
554 CHKERR problem_ptr->getNumberOfElementsByNameAndPart(m_field.get_comm(),
555 fe_name, &layout);
556 int gid, last_gid;
557 CHKERR PetscLayoutGetRange(layout, &gid, &last_gid);
558 CHKERR PetscLayoutDestroy(&layout);
559
560 const FiniteElement_multiIndex *fes_ptr;
561 CHKERR m_field.get_finite_elements(&fes_ptr);
562
563 auto fe_miit = fes_ptr->get<FiniteElement_name_mi_tag>().find(fe_name);
564 if (fe_miit != fes_ptr->get<FiniteElement_name_mi_tag>().end()) {
565 auto fit =
566 problem_ptr->numeredFiniteElementsPtr->get<Unique_mi_tag>().lower_bound(
568 0, (*fe_miit)->getFEUId()));
569 auto hi_fe_it =
570 problem_ptr->numeredFiniteElementsPtr->get<Unique_mi_tag>().upper_bound(
572 get_id_for_max_type<MBENTITYSET>(), (*fe_miit)->getFEUId()));
573 for (; fit != hi_fe_it; ++fit) {
574
575 auto ent = (*fit)->getEnt();
576 auto part = (*fit)->getPart();
577
578 ents.insert(ent);
579 CHKERR m_field.get_moab().tag_set_data(pcomm->part_tag(), &ent, 1, &part);
580 if (part == pcomm->rank()) {
581 CHKERR m_field.get_moab().tag_set_data(th_gid, &ent, 1, &gid);
582 gid++;
583 }
584 shprocs.clear();
585 shhandles.clear();
586
587 if (pcomm->size() > 1) {
588
589 unsigned char pstatus = 0;
590 if (pcomm->rank() != part) {
591 pstatus = PSTATUS_NOT_OWNED;
592 pstatus |= PSTATUS_GHOST;
593 }
594
595 if (pcomm->size() > 2) {
596 pstatus |= PSTATUS_SHARED;
597 pstatus |= PSTATUS_MULTISHARED;
598 } else {
599 pstatus |= PSTATUS_SHARED;
600 }
601
602 size_t rrr = 0;
603 for (size_t rr = 0; rr < pcomm->size(); ++rr) {
604 if (rr != pcomm->rank()) {
605 shhandles[rrr] = ent;
606 shprocs[rrr] = rr;
607 ++rrr;
608 }
609 }
610 for (; rrr != pcomm->size(); ++rrr)
611 shprocs[rrr] = -1;
612
613 if (pstatus & PSTATUS_SHARED) {
614 CHKERR m_field.get_moab().tag_set_data(pcomm->sharedp_tag(), &ent, 1,
615 &shprocs[0]);
616 CHKERR m_field.get_moab().tag_set_data(pcomm->sharedh_tag(), &ent, 1,
617 &shhandles[0]);
618 }
619
620 if (pstatus & PSTATUS_MULTISHARED) {
621 CHKERR m_field.get_moab().tag_set_data(pcomm->sharedps_tag(), &ent, 1,
622 &shprocs[0]);
623 CHKERR m_field.get_moab().tag_set_data(pcomm->sharedhs_tag(), &ent, 1,
624 &shhandles[0]);
625 }
626 CHKERR m_field.get_moab().tag_set_data(pcomm->pstatus_tag(), &ent, 1,
627 &pstatus);
628 }
629 }
630 }
631
632 CHKERR pcomm->exchange_tags(th_gid, ents);
634}
635
637 const std::string name, const std::string &fe_name, int verb) {
638 MoFEM::Interface &m_field = cOre;
640 const Problem *problem_ptr;
641 CHKERR m_field.get_problem(name, &problem_ptr);
642 CHKERR resolveSharedFiniteElements(problem_ptr, fe_name, verb);
644}
645
648 const int num_entities,
649 const int owner_proc, int verb) {
650 MoFEM::Interface &m_field = cOre;
652
653 if (m_field.get_comm_size() > 1) {
654
655 ParallelComm *pcomm = ParallelComm::get_pcomm(
656 &m_field.get_moab(), m_field.get_basic_entity_data_ptr()->pcommID);
657
658 Range all_ents_range;
659 all_ents_range.insert_list(entities, entities + num_entities);
660
661 auto get_tag = [&]() {
662 const int negone = -1;
663 Tag th;
664 m_field.get_moab().tag_get_handle("TMP_GLOBAL_ID_TAG_NAME", 1,
665 MB_TYPE_INTEGER, th,
666 MB_TAG_CREAT | MB_TAG_SPARSE, &negone);
667 return th;
668 };
669
670 auto delete_tag = [&](auto &&th_gid) {
672 CHKERR m_field.get_moab().tag_delete(th_gid);
674 };
675
676 auto resolve_shared_ents = [&](auto &&th_gid, auto &all_ents_range) {
677 auto set_gid = [&](auto &th_gid) {
678 std::vector<int> gids(num_entities);
679 for (size_t g = 0; g != all_ents_range.size(); ++g)
680 gids[g] = g + 1;
681 CHKERR m_field.get_moab().tag_set_data(th_gid, all_ents_range,
682 &*gids.begin());
683
684 return &th_gid;
685 };
686
687 auto get_skin_ents = [&](auto &all_ents_range) {
688 std::array<Range, 4> proc_ents_skin;
689 proc_ents_skin[3] = all_ents_range.subset_by_dimension(3);
690 proc_ents_skin[2] = all_ents_range.subset_by_dimension(2);
691 proc_ents_skin[1] = all_ents_range.subset_by_dimension(1);
692 proc_ents_skin[0] = all_ents_range.subset_by_dimension(0);
693 return proc_ents_skin;
694 };
695
696 auto resolve_dim = [&](auto &all_ents_range) {
697 for (int resolve_dim = 3; resolve_dim >= 0; --resolve_dim) {
698 if (all_ents_range.num_of_dimension(resolve_dim))
699 return resolve_dim;
700 }
701 return -1;
702 };
703
704 auto get_proc_ent = [&](auto &all_ents_range) {
705 Range proc_ent;
706 if (m_field.get_comm_rank() == owner_proc)
707 proc_ent = all_ents_range;
708 return proc_ent;
709 };
710
711 auto resolve_shared_ents = [&](auto &&proc_ents, auto &&skin_ents) {
712 return pcomm->resolve_shared_ents(
713 0, proc_ents, resolve_dim(all_ents_range),
714 resolve_dim(all_ents_range), skin_ents.data(), set_gid(th_gid));
715 };
716
717 CHKERR resolve_shared_ents(get_proc_ent(all_ents_range),
718 get_skin_ents(all_ents_range));
719
720 return th_gid;
721 };
722
723 CHKERR delete_tag(resolve_shared_ents(get_tag(), all_ents_range));
724
725 if (verb >= NOISY) {
726
727 auto print_owner = [&](const EntityHandle e) {
729 int moab_owner_proc;
730 EntityHandle moab_owner_handle;
731 CHKERR pcomm->get_owner_handle(e, moab_owner_proc, moab_owner_handle);
732
733 unsigned char pstatus = 0;
734
735 CHKERR m_field.get_moab().tag_get_data(pcomm->pstatus_tag(), &e, 1,
736 &pstatus);
737
738 std::vector<int> shprocs(MAX_SHARING_PROCS, 0);
739 std::vector<EntityHandle> shhandles(MAX_SHARING_PROCS, 0);
740
741 CHKERR m_field.get_moab().tag_get_data(pcomm->sharedp_tag(), &e, 1,
742 &shprocs[0]);
743 CHKERR m_field.get_moab().tag_get_data(pcomm->sharedh_tag(), &e, 1,
744 &shhandles[0]);
745 if (pstatus & PSTATUS_MULTISHARED) {
746 CHKERR m_field.get_moab().tag_get_data(pcomm->sharedps_tag(), &e, 1,
747 &shprocs[0]);
748 CHKERR m_field.get_moab().tag_get_data(pcomm->sharedhs_tag(), &e, 1,
749 &shhandles[0]);
750 }
751
752 std::ostringstream ss;
753
754 ss << "Rank " << m_field.get_comm_rank() << " ";
755 if (!(pstatus & PSTATUS_NOT_OWNED))
756 ss << "OWNER ";
757 if (pstatus & PSTATUS_SHARED)
758 ss << "PSTATUS_SHARED ";
759 if (pstatus & PSTATUS_MULTISHARED)
760 ss << "PSTATUS_MULTISHARED ";
761
762 ss << "owner " << moab_owner_proc << " (" << owner_proc << ") ";
763
764 ss << "shprocs: ";
765 for (size_t r = 0; r != m_field.get_comm_size() + 1; ++r)
766 ss << shprocs[r] << " ";
767
768 ss << "shhandles: ";
769 for (size_t r = 0; r != m_field.get_comm_size() + 1; ++r)
770 ss << shhandles[r] << " ";
771
772 ss << std::endl;
773 MOFEM_LOG("SYNC", Sev::noisy) << ss.str();
775
777 };
778
779 for (auto e : all_ents_range)
780 CHKERR print_owner(e);
781 }
782 }
783
785}
786
788 const int owner_proc,
789 int verb) {
790 MoFEM::Interface &m_field = cOre;
792 if (m_field.get_comm_size() > 1) {
793 const int num_ents = entities.size();
794 std::vector<EntityHandle> vec_ents(num_ents);
795 std::copy(entities.begin(), entities.end(), vec_ents.begin());
796 CHKERR makeEntitiesMultishared(&*vec_ents.begin(), num_ents, owner_proc,
797 verb);
798 }
800}
801
804 const int owner_proc, int verb) {
805 MoFEM::Interface &m_field = cOre;
807 if (m_field.get_comm_size() > 1) {
808 EntityHandle field_meshset = m_field.get_field_meshset(field_name);
809 std::vector<EntityHandle> field_ents;
810 CHKERR m_field.get_moab().get_entities_by_handle(field_meshset, field_ents,
811 true);
812 CHKERR makeEntitiesMultishared(&*field_ents.begin(), field_ents.size(),
813 owner_proc, verb);
814 }
816}
817
819 int verb) {
820 MoFEM::Interface &m_field = cOre;
822 if (m_field.get_comm_size() > 1) {
823
824 Range exchange_ents_data_verts, exchange_ents_data;
825
826 auto *field_ents = m_field.get_field_ents();
827 auto field_bit_number = m_field.get_field_bit_number(field_name);
828 auto lo = field_ents->get<Unique_mi_tag>().lower_bound(
829 FieldEntity::getLoBitNumberUId(field_bit_number));
830 auto hi = field_ents->get<Unique_mi_tag>().lower_bound(
831 FieldEntity::getHiBitNumberUId(field_bit_number));
832
833 for (auto it = lo; it != hi; ++it)
834 if (
835
836 ((*it)->getPStatus()) &&
837
838 (*it)->getNbDofsOnEnt()
839
840 ) {
841 if ((*it)->getEntType() == MBVERTEX)
842 exchange_ents_data_verts.insert((*it)->getEnt());
843 else
844 exchange_ents_data.insert((*it)->getEnt());
845 }
846
847 auto field_ptr = m_field.get_field_structure(field_name);
848 ParallelComm *pcomm = ParallelComm::get_pcomm(
849 &m_field.get_moab(), m_field.get_basic_entity_data_ptr()->pcommID);
850
851 auto exchange = [&](const Range &ents, Tag th) {
853 if (!ents.empty()) {
854 std::vector<Tag> tags;
855 tags.push_back(th);
856 CHKERR pcomm->exchange_tags(tags, tags, ents);
857 }
859 };
860
861 CHKERR exchange(exchange_ents_data_verts, field_ptr->th_FieldDataVerts);
862 CHKERR exchange(exchange_ents_data, field_ptr->th_FieldData);
863 }
865}
866
868CommInterface::partitionMesh(const Range &ents, const int dim,
869 const int adj_dim, const int n_parts,
870 Tag *th_vertex_weights, Tag *th_edge_weights,
871 Tag *th_part_weights, int verb, const bool debug) {
872 MoFEM::Interface &m_field = cOre;
874
875 // get layout
876 int rstart, rend, nb_elems;
877 {
878 PetscLayout layout;
879 CHKERR PetscLayoutCreate(m_field.get_comm(), &layout);
880 CHKERR PetscLayoutSetBlockSize(layout, 1);
881 CHKERR PetscLayoutSetSize(layout, ents.size());
882 CHKERR PetscLayoutSetUp(layout);
883 CHKERR PetscLayoutGetSize(layout, &nb_elems);
884 CHKERR PetscLayoutGetRange(layout, &rstart, &rend);
885 CHKERR PetscLayoutDestroy(&layout);
886 if (verb >= VERBOSE) {
887 MOFEM_LOG("SYNC", Sev::inform)
888 << "Finite elements in problem: row lower " << rstart << " row upper "
889 << rend << " nb. elems " << nb_elems << " ( " << ents.size() << " )";
891 }
892 }
893
894 std::vector<EntityHandle> weight_ents;
895 weight_ents.reserve(rend - rstart + 1);
896
897 struct AdjBridge {
898 EntityHandle ent;
899 std::vector<int> adj;
900 AdjBridge(const EntityHandle ent, std::vector<int> &adj)
901 : ent(ent), adj(adj) {}
902 };
903
904 typedef multi_index_container<
905 AdjBridge,
906 indexed_by<
907
908 hashed_unique<member<AdjBridge, EntityHandle, &AdjBridge::ent>>
909
910 >>
911 AdjBridgeMap;
912
913 auto get_it = [&](auto i) {
914 auto it = ents.begin();
915 for (; i > 0; --i) {
916 if (it == ents.end())
917 break;
918 ++it;
919 }
920 return it;
921 };
922
923 Range proc_ents;
924 proc_ents.insert(get_it(rstart), get_it(rend));
925 if (proc_ents.size() != rend - rstart)
926 SETERRQ(PETSC_COMM_WORLD, MOFEM_DATA_INCONSISTENCY,
927 "Wrong number of elements in range %ld != %d", proc_ents.size(),
928 rend - rstart);
929
930 Range all_dim_ents;
931 CHKERR m_field.get_moab().get_adjacencies(
932 proc_ents, adj_dim, true, all_dim_ents, moab::Interface::UNION);
933
934 AdjBridgeMap adj_bridge_map;
935 auto hint = adj_bridge_map.begin();
936 std::vector<int> adj;
937 for (auto ent : all_dim_ents) {
938 Range adj_ents;
939 CHKERR m_field.get_moab().get_adjacencies(&ent, 1, dim, false, adj_ents);
940 adj_ents = intersect(adj_ents, ents);
941 adj.clear();
942 adj.reserve(adj_ents.size());
943 for (auto a : adj_ents)
944 adj.emplace_back(ents.index(a));
945 hint = adj_bridge_map.emplace_hint(hint, ent, adj);
946 }
947
948 int *_i;
949 int *_j;
950 {
951 const int nb_loc_elements = rend - rstart;
952 std::vector<int> i(nb_loc_elements + 1, 0), j;
953 {
954 std::vector<int> row_adj;
955 Range::iterator fe_it;
956 int ii, jj;
957 size_t max_row_size;
958 for (fe_it = proc_ents.begin(), ii = rstart, jj = 0, max_row_size = 0;
959 fe_it != proc_ents.end(); ++fe_it, ++ii) {
960
961 if (type_from_handle(*fe_it) == MBENTITYSET) {
962 SETERRQ(PETSC_COMM_SELF, MOFEM_NOT_IMPLEMENTED,
963 "not yet implemented, don't know what to do for meshset "
964 "element");
965 } else {
966
967 Range dim_ents;
968 CHKERR m_field.get_moab().get_adjacencies(&*fe_it, 1, adj_dim, false,
969 dim_ents);
970 dim_ents = intersect(dim_ents, all_dim_ents);
971
972 row_adj.clear();
973 for (auto e : dim_ents) {
974 auto adj_it = adj_bridge_map.find(e);
975 if (adj_it != adj_bridge_map.end()) {
976
977 for (const auto idx : adj_it->adj)
978 row_adj.push_back(idx);
979
980 } else
981 SETERRQ(PETSC_COMM_SELF, MOFEM_DATA_INCONSISTENCY,
982 "Entity not found");
983 }
984
985 std::sort(row_adj.begin(), row_adj.end());
986 auto end = std::unique(row_adj.begin(), row_adj.end());
987
988 size_t row_size = std::distance(row_adj.begin(), end);
989 max_row_size = std::max(max_row_size, row_size);
990 if (j.capacity() < (nb_loc_elements - jj) * max_row_size)
991 j.reserve(nb_loc_elements * max_row_size);
992
993 i[jj] = j.size();
994 auto diag = ents.index(*fe_it);
995 for (auto it = row_adj.begin(); it != end; ++it)
996 if (*it != diag)
997 j.push_back(*it);
998 }
999
1000 ++jj;
1001
1002 if (th_vertex_weights != NULL)
1003 weight_ents.push_back(*fe_it);
1004 }
1005
1006 i[jj] = j.size();
1007 }
1008
1009 CHKERR PetscMalloc(i.size() * sizeof(int), &_i);
1010 CHKERR PetscMalloc(j.size() * sizeof(int), &_j);
1011 copy(i.begin(), i.end(), _i);
1012 copy(j.begin(), j.end(), _j);
1013 }
1014
1015 // get weights
1016 int *vertex_weights = NULL;
1017 if (th_vertex_weights != NULL) {
1018 CHKERR PetscMalloc(weight_ents.size() * sizeof(int), &vertex_weights);
1019 CHKERR m_field.get_moab().tag_get_data(*th_vertex_weights,
1020 &*weight_ents.begin(),
1021 weight_ents.size(), vertex_weights);
1022 }
1023
1024 {
1025 Mat Adj;
1026 // Adjacency matrix used to partition problems, f.e. METIS
1027 CHKERR MatCreateMPIAdj(m_field.get_comm(), rend - rstart, nb_elems, _i, _j,
1028 PETSC_NULLPTR, &Adj);
1029 CHKERR MatSetOption(Adj, MAT_STRUCTURALLY_SYMMETRIC, PETSC_TRUE);
1030
1031 if (debug) {
1032 Mat A;
1033 CHKERR MatConvert(Adj, MATMPIAIJ, MAT_INITIAL_MATRIX, &A);
1034 CHKERR MatView(A, PETSC_VIEWER_DRAW_WORLD);
1035 std::string wait;
1036 std::cin >> wait;
1037 CHKERR MatDestroy(&A);
1038 }
1039
1040 // run pets to do partitioning
1041 MOFEM_LOG("WORLD", Sev::verbose) << "Start";
1042
1043 MatPartitioning part;
1044 IS is;
1045 CHKERR MatPartitioningCreate(m_field.get_comm(), &part);
1046
1047 CHKERR MatPartitioningSetAdjacency(part, Adj);
1048 CHKERR MatPartitioningSetFromOptions(part);
1049 CHKERR MatPartitioningSetNParts(part, n_parts);
1050 if (th_vertex_weights != NULL) {
1051 CHKERR MatPartitioningSetVertexWeights(part, vertex_weights);
1052 }
1053 CHKERR MatPartitioningApply(part, &is);
1054
1055 MOFEM_LOG("WORLD", Sev::verbose) << "End";
1056
1057 // gather
1058 IS is_gather, is_num, is_gather_num;
1059 CHKERR ISAllGather(is, &is_gather);
1060 CHKERR ISPartitioningToNumbering(is, &is_num);
1061 CHKERR ISAllGather(is_num, &is_gather_num);
1062
1063 const int *part_number, *gids;
1064 CHKERR ISGetIndices(is_gather, &part_number);
1065 CHKERR ISGetIndices(is_gather_num, &gids);
1066
1067 // set partition tag and gid tag to entities
1068 ParallelComm *pcomm = ParallelComm::get_pcomm(
1069 &m_field.get_moab(), m_field.get_basic_entity_data_ptr()->pcommID);
1070 Tag part_tag = pcomm->part_tag();
1071 CHKERR m_field.get_moab().tag_set_data(part_tag, ents, part_number);
1072 Tag gid_tag = m_field.get_moab().globalId_tag();
1073
1074 std::map<int, Range> parts_ents;
1075 {
1076 // get entities on each part
1077 Range::iterator eit = ents.begin();
1078 for (int ii = 0; eit != ents.end(); eit++, ii++) {
1079 parts_ents[part_number[ii]].insert(*eit);
1080 }
1081 Range tagged_sets;
1082 CHKERR m_field.get_moab().get_entities_by_type_and_tag(
1083 0, MBENTITYSET, &part_tag, NULL, 1, tagged_sets,
1084 moab::Interface::UNION);
1085 if (!tagged_sets.empty())
1086 CHKERR m_field.get_moab().tag_delete_data(part_tag, tagged_sets);
1087
1088 if (n_parts > (int)tagged_sets.size()) {
1089 // too few partition sets - create missing ones
1090 int num_new = n_parts - tagged_sets.size();
1091 for (int i = 0; i < num_new; i++) {
1092 EntityHandle new_set;
1093 CHKERR m_field.get_moab().create_meshset(MESHSET_SET, new_set);
1094 tagged_sets.insert(new_set);
1095 }
1096 } else if (n_parts < (int)tagged_sets.size()) {
1097 // too many partition sets - delete extras
1098 int num_del = tagged_sets.size() - n_parts;
1099 for (int i = 0; i < num_del; i++) {
1100 EntityHandle old_set = tagged_sets.pop_back();
1101 CHKERR m_field.get_moab().delete_entities(&old_set, 1);
1102 }
1103 }
1104 // write a tag to those sets denoting they're partition sets, with a
1105 // value of the proc number
1106 std::vector<int> dum_ids(n_parts);
1107 for (int i = 0; i < n_parts; i++)
1108 dum_ids[i] = i;
1109 CHKERR m_field.get_moab().tag_set_data(part_tag, tagged_sets,
1110 &*dum_ids.begin());
1111 CHKERR m_field.get_moab().clear_meshset(tagged_sets);
1112
1113 // get lower dimension entities on each part
1114 for (int pp = 0; pp != n_parts; pp++) {
1115 Range dim_ents = parts_ents[pp].subset_by_dimension(dim);
1116 for (int dd = dim - 1; dd >= 0; dd--) {
1117 Range adj_ents;
1118 CHKERR m_field.get_moab().get_adjacencies(
1119 dim_ents, dd, false, adj_ents, moab::Interface::UNION);
1120 parts_ents[pp].merge(adj_ents);
1121 }
1122 }
1123 for (int pp = 1; pp != n_parts; pp++) {
1124 for (int ppp = 0; ppp != pp; ppp++) {
1125 parts_ents[pp] = subtract(parts_ents[pp], parts_ents[ppp]);
1126 }
1127 }
1128
1129 for (int pp = 0; pp != n_parts; pp++) {
1130 CHKERR m_field.get_moab().add_entities(tagged_sets[pp], parts_ents[pp]);
1131 }
1132
1133 auto set_part = [&]() {
1135 for (EntityType t = MBEDGE; t != MBENTITYSET; ++t) {
1136 for (int pp = 0; pp != n_parts; pp++) {
1137 Range type_ents = parts_ents[pp].subset_by_type(t);
1138 CHKERR m_field.get_moab().tag_clear_data(part_tag, type_ents, &pp);
1139 }
1140 }
1142 };
1143
1144 auto set_gid = [&]() {
1146 for (int d = 0; d != 4; ++d) {
1147
1148 void *ptr;
1149 int count;
1150
1151 int gid = 1; // moab indexing from 1
1152 for (int pp = 0; pp != n_parts; pp++) {
1153 Range type_ents = parts_ents[pp].subset_by_dimension(d);
1154
1155 auto eit = type_ents.begin();
1156 for (; eit != type_ents.end();) {
1157 CHKERR m_field.get_moab().tag_iterate(
1158 gid_tag, eit, type_ents.end(), count, ptr);
1159 auto gid_tag_ptr = static_cast<int *>(ptr);
1160 for (; count > 0; --count) {
1161 *gid_tag_ptr = gid;
1162 ++eit;
1163 ++gid;
1164 ++gid_tag_ptr;
1165 }
1166 }
1167 }
1168 }
1169
1171 };
1172
1173 CHKERR set_part();
1174 CHKERR set_gid();
1175 }
1176
1177 if (debug) {
1178 if (m_field.get_comm_rank() == 0) {
1179 for (int rr = 0; rr != n_parts; rr++) {
1180 ostringstream ss;
1181 ss << "out_part_" << rr << ".vtk";
1182 MOFEM_LOG("SELF", Sev::inform) << "Save debug part mesh " << ss.str();
1183 EntityHandle meshset;
1184 CHKERR m_field.get_moab().create_meshset(MESHSET_SET, meshset);
1185 CHKERR m_field.get_moab().add_entities(meshset, parts_ents[rr]);
1186 CHKERR m_field.get_moab().write_file(ss.str().c_str(), "VTK", "",
1187 &meshset, 1);
1188 CHKERR m_field.get_moab().delete_entities(&meshset, 1);
1189 }
1190 }
1191 }
1192
1193 CHKERR ISRestoreIndices(is_gather, &part_number);
1194 CHKERR ISRestoreIndices(is_gather_num, &gids);
1195 CHKERR ISDestroy(&is_num);
1196 CHKERR ISDestroy(&is_gather_num);
1197 CHKERR ISDestroy(&is_gather);
1198 CHKERR ISDestroy(&is);
1199 CHKERR MatPartitioningDestroy(&part);
1200 CHKERR MatDestroy(&Adj);
1201 }
1202
1204}
1205
1207 moab::Interface &moab, const char *file_name, int dim,
1208 LoadFileFun proc_skin_fun, const char *options) {
1210
1211 MOFEM_LOG_CHANNEL("WORLD");
1212 MOFEM_LOG_CHANNEL("SYNC");
1213
1214 MOFEM_LOG("SYNC", sev) << "Load file " << file_name;
1215 CHKERR moab.load_file(file_name, 0, options);
1217 MOFEM_LOG("SYNC", sev) << "Load file <- done";
1218 MOFEM_LOG_SEVERITY_SYNC(PETSC_COMM_WORLD, sev);
1219
1220 ParallelComm *pcomm = ParallelComm::get_pcomm(&moab, MYPCOMM_INDEX);
1221 if (pcomm == NULL)
1222 pcomm = new ParallelComm(&moab, PETSC_COMM_WORLD);
1223 if (pcomm->size() == 1)
1225
1226 Skinner skin(&moab);
1227
1228 auto print_range_on_procs = [&](const Range &range, std::string name) {
1229 MOFEM_LOG("SYNC", sev) << name << " on proc [" << pcomm->rank()
1230 << "] : " << range.size();
1231 MOFEM_LOG_SEVERITY_SYNC(PETSC_COMM_WORLD, sev);
1232 };
1233
1234 auto save_range_to_file = [&](const Range range, std::string name = "part") {
1235 if (!debug)
1236 return;
1237 int rr = pcomm->rank();
1238 ostringstream ss;
1239 ss << "out_" << name << "_" << rr << ".vtk";
1240 MOFEM_LOG("SYNC", sev) << "Save debug part mesh " << ss.str();
1241 EntityHandle meshset;
1242 CHKERR moab.create_meshset(MESHSET_SET, meshset);
1243 CHKERR moab.add_entities(meshset, range);
1244 if (!range.empty())
1245 CHKERR moab.write_file(ss.str().c_str(), "VTK", "", &meshset, 1);
1246 MOFEM_LOG("SYNC", sev) << "Empty range";
1247 CHKERR moab.delete_entities(&meshset, 1);
1248 MOFEM_LOG_SEVERITY_SYNC(PETSC_COMM_WORLD, sev);
1249 };
1250
1251 auto get_skin = [&](auto e) {
1252 Range s;
1253 CHKERR skin.find_skin(0, e.subset_by_dimension(dim), false, s);
1254 return s;
1255 };
1256
1257 auto get_adj = [&](auto ents, auto dim) {
1258 Range adj;
1259 CHKERR moab.get_adjacencies(ents, dim, false, adj, moab::Interface::UNION);
1260 return adj;
1261 };
1262
1263 Range all_ents;
1264 CHKERR moab.get_entities_by_handle(0, all_ents, false);
1265 save_range_to_file(all_ents, "all_ents");
1266
1267 Tag part_tag = pcomm->part_tag();
1268 Range tagged_sets, proc_ents, off_proc_ents;
1269
1270 CHKERR moab.get_entities_by_type_and_tag(0, MBENTITYSET, &part_tag, NULL, 1,
1271 tagged_sets, moab::Interface::UNION);
1272 print_range_on_procs(tagged_sets, "tagged_sets");
1273 for (auto &mit : tagged_sets) {
1274 int part;
1275 CHKERR moab.tag_get_data(part_tag, &mit, 1, &part);
1276 if (part == pcomm->rank()) {
1277 CHKERR moab.get_entities_by_handle(mit, proc_ents, true);
1278 } else {
1279 CHKERR moab.get_entities_by_handle(mit, off_proc_ents, true);
1280 }
1281 }
1282
1283 // set part tags to entities, they might be not set
1284 for (auto &mit : tagged_sets) {
1285 int part;
1286 CHKERR moab.tag_get_data(part_tag, &mit, 1, &part);
1287 Range ents;
1288 CHKERR moab.get_entities_by_handle(mit, ents, true);
1289 CHKERR moab.tag_clear_data(part_tag, ents, &part);
1290 }
1291
1292 print_range_on_procs(proc_ents, "proc_ents");
1293 save_range_to_file(proc_ents, "proc_ents");
1294
1295 auto get_proc_ents_skin = [&]() {
1296 std::array<Range, 4>
1297 proc_ents_skin; // stores only entities shared with other processors
1298 auto all_skin = get_skin(all_ents.subset_by_dimension(dim));
1299 auto proc_skin = get_skin(proc_ents.subset_by_dimension(dim));
1300 if (dim == 3) {
1301 proc_ents_skin[2] = subtract(proc_skin, all_skin); // faces
1302 proc_ents_skin[1] = get_adj(proc_ents_skin[dim - 1], 1); // edges
1303 }
1304 if (dim == 2) {
1305 proc_ents_skin[1] = subtract(proc_skin, all_skin); // edges
1306 }
1307 CHKERR moab.get_connectivity(proc_ents_skin[dim - 1], proc_ents_skin[0],
1308 false); // vertices
1309 return proc_ents_skin;
1310 };
1311
1312 auto add_post_proc_skin = [&](auto &&proc_ents_skin) {
1313 CubitMeshSet_multiIndex cubit_meshsets_index; ///< cubit meshsets
1314
1315 Tag nsTag, ssTag, nsTag_data, ssTag_data, bhTag, bhTag_header;
1316 CHKERR MeshsetsManager::getTags(moab, nsTag, ssTag, nsTag_data, ssTag_data,
1317 bhTag, bhTag_header, QUIET);
1318
1319 Range meshsets;
1320 CHKERR moab.get_entities_by_type(0, MBENTITYSET, meshsets, false);
1321 for (auto m : meshsets) {
1322 // check if meshset is cubit meshset
1323 CubitMeshSets block(moab, m);
1325 .any()) {
1326 auto p = cubit_meshsets_index.insert(block);
1327 if (!p.second) {
1328 // blockset/nodeset/sideset set exist, could be created on other
1329 // processor.
1330 Range ents;
1331 CHKERR moab.get_entities_by_handle(m, ents, true);
1332 CHKERR moab.add_entities(p.first->getMeshset(), ents);
1333 CHKERR moab.delete_entities(&m, 1);
1334 }
1335 }
1336 }
1337
1338 for (auto &m : cubit_meshsets_index) {
1339 MOFEM_LOG("WORLD", sev) << "LoadFile read " << m;
1340 }
1341 MOFEM_LOG_SEVERITY_SYNC(PETSC_COMM_WORLD, sev);
1342
1343 std::vector<const CubitMeshSets *> vec_ptr;
1344 for (auto &m : cubit_meshsets_index) {
1345 vec_ptr.push_back(&m);
1346 }
1348
1349 return proc_skin_fun(std::move(proc_ents_skin), std::move(vec_ptr));
1350 };
1351
1352 auto remove_off_proc_ents = [&](auto &all_ents, auto &off_proc_ents,
1353 auto &proc_ents_skin) {
1354 auto to_remove = off_proc_ents;
1355 for (auto d = dim - 1; d >= 0; --d) {
1356 to_remove = subtract(to_remove, proc_ents_skin[d]);
1357 }
1358
1359 Range meshsets;
1360 CHKERR moab.get_entities_by_type(0, MBENTITYSET, meshsets, true);
1361 for (auto m : meshsets) {
1362 CHKERR moab.remove_entities(m, to_remove);
1363 }
1364 for (int dd = dim; dd > 0; --dd) {
1365 CHKERR moab.delete_entities(to_remove.subset_by_dimension(dd));
1366 }
1367 };
1368
1369 auto proc_ents_skin = add_post_proc_skin(get_proc_ents_skin());
1370 for (auto d = dim - 1; d >= 0; --d) {
1371 save_range_to_file(proc_ents_skin[d], "proc_ents_skin" + std::to_string(d));
1372 }
1373
1374 if (pcomm->rank()) {
1375 remove_off_proc_ents(all_ents, off_proc_ents, proc_ents_skin);
1376 }
1377
1378 for (auto d = dim - 1; d >= 0; --d) {
1379 print_range_on_procs(proc_ents_skin[d],
1380 "proc_ents_skin by dim" + std::to_string(d));
1381 }
1382
1383 CHKERR pcomm->resolve_shared_ents(0, proc_ents, dim, -1,
1384 proc_ents_skin.data());
1385
1386 if (debug) {
1387
1388 // Create a tag
1389 auto gid_tag = moab.globalId_tag();
1390
1391 int def_val = -1;
1392 Tag tag_handle;
1393 CHKERR moab.tag_get_handle("TestGather", 1, MB_TYPE_INTEGER, tag_handle,
1394 MB_TAG_DENSE | MB_TAG_CREAT, &def_val);
1395 Range gather_ents;
1396 if (pcomm->rank() > 0) {
1397 gather_ents = proc_ents.subset_by_dimension(3);
1398 std::vector<int> vals(gather_ents.size());
1399 std::for_each(vals.begin(), vals.end(), [](auto &v) { v = std::rand(); });
1400 CHKERR moab.tag_set_data(tag_handle, gather_ents, vals.data());
1401 CHKERR pcomm->gather_data(gather_ents, tag_handle, gid_tag, 0, 0);
1402 } else {
1403 gather_ents = proc_ents.subset_by_dimension(3);
1404 CHKERR pcomm->gather_data(gather_ents, tag_handle, gid_tag, 0, 0);
1405 }
1406 }
1407
1408 Range all_ents_after;
1409 CHKERR moab.get_entities_by_handle(0, all_ents_after);
1410 save_range_to_file(all_ents_after.subset_by_dimension(3),
1411 "all_ents_part_after");
1412
1413 MOFEM_LOG("SYNC", sev) << "Resolved entities <- done";
1414 MOFEM_LOG_SEVERITY_SYNC(PETSC_COMM_WORLD, sev);
1415
1417}
1418
1419Range CommInterface::getPartEntities(moab::Interface &moab, int part) {
1420
1421 ParallelComm *pcomm = ParallelComm::get_pcomm(&moab, MYPCOMM_INDEX);
1422 if (pcomm == NULL)
1423 pcomm = new ParallelComm(&moab, PETSC_COMM_WORLD);
1424 if (pcomm->size() == 1) {
1425 Range ents;
1426 CHK_MOAB_THROW(moab.get_entities_by_handle(0, ents, false),
1427 "get_entities_by_handle failed");
1428 return subtract(ents, ents.subset_by_type(MBENTITYSET));
1429 }
1430
1431 Range proc_ents;
1432
1433 auto get_proc_ents = [&]() {
1435
1436 Tag part_tag = pcomm->part_tag();
1437
1438 Range tagged_sets;
1439 CHKERR moab.get_entities_by_type_and_tag(0, MBENTITYSET, &part_tag, NULL, 1,
1440 tagged_sets,
1441 moab::Interface::UNION);
1442 for (auto &mit : tagged_sets) {
1443 int part_set;
1444 CHKERR moab.tag_get_data(part_tag, &mit, 1, &part_set);
1445 if (part_set == part) {
1446 CHKERR moab.get_entities_by_handle(mit, proc_ents, true);
1448 }
1449 }
1450 SETERRQ(PETSC_COMM_SELF, MOFEM_DATA_INCONSISTENCY,
1451 "Part not found in partitioned mesh");
1453 };
1454
1455 CHK_THROW_MESSAGE(get_proc_ents(), "get_proc_ents failed");
1456
1457 return proc_ents;
1458}
1459
1461CommInterface::createEntitiesPetscVector(MPI_Comm comm, moab::Interface &moab,
1462 int dim, const int nb_coeffs, Sev sev,
1463 int root_rank) {
1464
1466 Range r, ghost_ents;
1467
1468 auto fun = [&]() {
1470 auto pcomm = ParallelComm::get_pcomm(&moab, MYPCOMM_INDEX);
1471
1472 Tag part_tag = pcomm->part_tag();
1473 Range tagged_sets, proc_ents, off_proc_ents;
1474 CHKERR moab.get_entities_by_type_and_tag(0, MBENTITYSET, &part_tag, NULL, 1,
1475 tagged_sets,
1476 moab::Interface::UNION);
1477
1478 if (dim > 0) {
1479 for (auto &mit : tagged_sets) {
1480 int part;
1481 CHKERR moab.tag_get_data(part_tag, &mit, 1, &part);
1482 if (part == pcomm->rank()) {
1483 CHKERR moab.get_entities_by_handle(mit, proc_ents, true);
1484 } else {
1485 CHKERR moab.get_entities_by_handle(mit, off_proc_ents, true);
1486 }
1487 proc_ents = subtract(proc_ents, off_proc_ents);
1488 }
1489 } else {
1490 std::map<int, Range> parts_ents;
1491 for (auto &mit : tagged_sets) {
1492 int part;
1493 CHKERR moab.tag_get_data(part_tag, &mit, 1, &part);
1494 CHKERR moab.get_entities_by_handle(mit, parts_ents[part], true);
1495 }
1496 Range v;
1497 CHKERR moab.get_connectivity(parts_ents[pcomm->rank()], v);
1498 int p = 0;
1499 for (; p != pcomm->rank(); ++p) {
1500 Range vv;
1501 CHKERR moab.get_connectivity(parts_ents[p], vv);
1502 v = subtract(v, vv);
1503 off_proc_ents.merge(vv);
1504 }
1505 for (; p != pcomm->size(); ++p) {
1506 Range vv;
1507 CHKERR moab.get_connectivity(parts_ents[p], vv);
1508 vv = subtract(vv, v);
1509 off_proc_ents.merge(vv);
1510 }
1511 proc_ents = v;
1512 }
1513
1514 r = proc_ents.subset_by_dimension(dim);
1515 auto o = off_proc_ents.subset_by_dimension(dim);
1516
1517 auto set_ghosts = [&](Range &ents) {
1518 auto gid_tag = moab.globalId_tag();
1519 std::vector<int> ghosts(ents.size());
1520 CHKERR moab.tag_get_data(gid_tag, ents, ghosts.data());
1521 std::vector<int> ghosts_nb(nb_coeffs * ents.size());
1522 for (int i = 0; i < ents.size(); ++i)
1523 for (int j = 0; j < nb_coeffs; ++j)
1524 ghosts_nb[i * nb_coeffs + j] = nb_coeffs * (ghosts[i] - 1) + j;
1525 return ghosts_nb;
1526 };
1527
1528 std::vector<int> ghosts;
1529 if (pcomm->rank() == root_rank) {
1530 ghosts = set_ghosts(o);
1531 ghost_ents = o;
1532 } else {
1533 Range ents;
1534 CHKERR moab.get_entities_by_dimension(0, dim, ents);
1535 ents = subtract(ents, r);
1536 ghosts = set_ghosts(ents);
1537 ghost_ents = ents;
1538 }
1539
1540 auto create_vec = [&]() {
1542 vec = createGhostVector(comm, nb_coeffs * r.size(), PETSC_DETERMINE,
1543 ghosts.size(), ghosts.data());
1545 };
1546 CHKERR create_vec();
1547
1549 };
1550
1551 CHKERR fun();
1552
1553 auto out = std::make_pair(std::make_pair(r, ghost_ents), vec);
1554
1555#ifndef NDEBUG
1556 {
1557
1558 auto check = [&](auto &ents, auto &id, auto &idx) {
1560 bool error = false;
1561 for (int i = 0; i < ents.size(); ++i) {
1562 for (int j = 0; j < nb_coeffs; ++j) {
1563 if (idx[i * nb_coeffs + j] != nb_coeffs * (id[i] - 1) + j) {
1564 error = true;
1565 MOFEM_LOG("SYNC", Sev::error)
1566 << "indexes not equal: " << idx[i * nb_coeffs + j]
1567 << " != " << nb_coeffs * (idx[i] - 1) + j;
1568 }
1569 }
1570 }
1571 MOFEM_LOG_SEVERITY_SYNC(PETSC_COMM_WORLD, Sev::error);
1572 if (error)
1573 CHK_THROW_MESSAGE(MOFEM_DATA_INCONSISTENCY, "indexes do not match");
1575 };
1576
1577 std::vector<double> def_val(nb_coeffs, 0.0);
1578 Tag tag;
1579 CHKERR moab.tag_get_handle("TestGather", nb_coeffs, MB_TYPE_DOUBLE, tag,
1580 MB_TAG_SPARSE | MB_TAG_CREAT, def_val.data());
1581 auto gid_tag = moab.globalId_tag();
1582 std::vector<int> id(r.size());
1583 CHKERR moab.tag_get_data(gid_tag, r, id.data());
1584 std::vector<double> idx(nb_coeffs * r.size());
1585 for (int i = 0; i < r.size(); ++i) {
1586 for (int j = 0; j < nb_coeffs; ++j) {
1587 idx[i * nb_coeffs + j] = nb_coeffs * (id[i] - 1) + j;
1588 }
1589 }
1590 CHKERR moab.tag_set_data(tag, r, idx.data());
1591 CHKERR updateEntitiesPetscVector(moab, out, tag);
1592 idx.resize(nb_coeffs * r.size());
1593 CHKERR moab.tag_get_data(tag, r, idx.data());
1594 CHK_THROW_MESSAGE(check(r, id, idx), "indexes do not match")
1595 id.resize(ghost_ents.size());
1596 CHKERR moab.tag_get_data(gid_tag, ghost_ents, id.data());
1597 idx.resize(nb_coeffs * ghost_ents.size());
1598 CHKERR moab.tag_get_data(tag, ghost_ents, idx.data());
1599 CHK_THROW_MESSAGE(check(ghost_ents, id, idx), "ghost indexes do not match")
1600 CHKERR moab.tag_delete(tag);
1601 }
1602#endif // NDEBUG
1603
1604 return out;
1605}
1606
1609 EntitiesPetscVector &vec, Tag tag,
1610 UpdateGhosts update_gosts) {
1612
1613 int local_size;
1614 CHKERR VecGetLocalSize(vec.second, &local_size);
1615 int nb_coeffs;
1616 CHKERR moab.tag_get_length(tag, nb_coeffs);
1617 if (vec.first.first.size() * nb_coeffs != local_size) {
1618 SETERRQ(PETSC_COMM_SELF, MOFEM_DATA_INCONSISTENCY,
1619 "Local size of vector does not match number of entities");
1620 }
1621
1622 auto set_vec_from_tags = [&]() {
1624 double *v_array;
1625 // set vector
1626 CHKERR VecGetArray(vec.second, &v_array);
1627 CHKERR moab.tag_get_data(tag, vec.first.first, v_array);
1628 CHKERR moab.tag_get_data(tag, vec.first.second, &v_array[local_size]);
1629 CHKERR VecRestoreArray(vec.second, &v_array);
1631 };
1632
1633 auto set_tags_from_vec = [&]() {
1635 double *v_array;
1636 CHKERR VecGetArray(vec.second, &v_array);
1637 CHKERR moab.tag_set_data(tag, vec.first.first, v_array);
1638 CHKERR moab.tag_set_data(tag, vec.first.second, &v_array[local_size]);
1639 CHKERR VecRestoreArray(vec.second, &v_array);
1641 };
1642
1643 CHKERR set_vec_from_tags();
1644 CHKERR update_gosts(vec.second);
1645 CHKERR set_tags_from_vec();
1646
1648}
1649
1650} // namespace MoFEM
#define MOFEM_LOG_SEVERITY_SYNC(comm, severity)
Synchronise "SYNC" on curtain severity level.
#define MOFEM_LOG_SYNCHRONISE(comm)
Synchronise "SYNC" channel.
#define MOFEM_LOG_C(channel, severity, format,...)
constexpr double a
@ QUIET
@ NOISY
@ VERBOSE
#define CHK_THROW_MESSAGE(err, msg)
Check and throw MoFEM exception.
#define MoFEMFunctionReturnHot(a)
Last executable line of each PETSc function used for error handling. Replaces return()
#define MYPCOMM_INDEX
default communicator number PCOMM
#define MoFEMFunctionBegin
First executable line of each MoFEM function, used for error handling. Final line of MoFEM functions ...
#define CHK_MOAB_THROW(err, msg)
Check error code of MoAB function and throw MoFEM exception.
@ NODESET
@ SIDESET
@ BLOCKSET
@ MOFEM_IMPOSSIBLE_CASE
Definition definitions.h:35
@ MOFEM_DATA_INCONSISTENCY
Definition definitions.h:31
@ MOFEM_NOT_IMPLEMENTED
Definition definitions.h:32
#define MoFEMFunctionReturn(a)
Last executable line of each PETSc function used for error handling. Replaces return()
#define CHKERR
Inline error check.
#define MoFEMFunctionBeginHot
First executable line of each MoFEM function, used for error handling. Final line of MoFEM functions ...
auto fun
Function to approximate.
multi_index_container< boost::shared_ptr< FiniteElement >, indexed_by< hashed_unique< tag< FiniteElement_Meshset_mi_tag >, member< FiniteElement, EntityHandle, &FiniteElement::meshset > >, hashed_unique< tag< BitFEId_mi_tag >, const_mem_fun< FiniteElement, BitFEId, &FiniteElement::getId >, HashBit< BitFEId >, EqBit< BitFEId > >, ordered_unique< tag< FiniteElement_name_mi_tag >, const_mem_fun< FiniteElement, boost::string_ref, &FiniteElement::getNameRef > > > > FiniteElement_multiIndex
MultiIndex for entities for FiniteElement.
virtual const Problem * get_problem(const std::string problem_name) const =0
Get the problem object.
virtual const FieldEntity_multiIndex * get_field_ents() const =0
Get the field ents object.
virtual const FiniteElement_multiIndex * get_finite_elements() const =0
Get the finite elements object.
virtual const Field * get_field_structure(const std::string &name, enum MoFEMTypes bh=MF_EXIST) const =0
get field structure
MoFEMErrorCode synchroniseFieldEntities(const std::string name, int verb=DEFAULT_VERBOSITY)
#define MOFEM_LOG(channel, severity)
Log.
SeverityLevel
Severity levels.
#define MOFEM_LOG_CHANNEL(channel)
Set and reset channel.
MoFEMErrorCode partitionMesh(const Range &ents, const int dim, const int adj_dim, const int n_parts, Tag *th_vertex_weights=nullptr, Tag *th_edge_weights=nullptr, Tag *th_part_weights=nullptr, int verb=VERBOSE, const bool debug=false)
Set partition tag to each finite element in the problem.
MoFEMErrorCode resolveSharedFiniteElements(const Problem *problem_ptr, const std::string &fe_name, int verb=DEFAULT_VERBOSITY)
resolve shared entities for finite elements in the problem
auto bit
set bit
FTensor::Index< 'i', SPACE_DIM > i
const double v
phase velocity of light in medium (cm/ns)
FTensor::Index< 'j', 3 > j
PetscErrorCode MoFEMErrorCode
MoFEM/PETSc error code.
std::bitset< BITREFLEVEL_SIZE > BitRefLevel
Bit structure attached to each entity identifying to what mesh entity is attached.
Definition Types.hpp:40
std::bitset< 32 > CubitBCType
Definition Types.hpp:52
implementation of Data Operators for Forces and Sources
Definition Common.hpp:10
auto type_from_handle(const EntityHandle h)
get type from entity handle
EntityHandle get_id_for_max_type()
multi_index_container< CubitMeshSets, indexed_by< hashed_unique< tag< Meshset_mi_tag >, member< CubitMeshSets, EntityHandle, &CubitMeshSets::meshset > >, ordered_non_unique< tag< CubitMeshsetType_mi_tag >, const_mem_fun< CubitMeshSets, unsigned long int, &CubitMeshSets::getBcTypeULong > >, ordered_non_unique< tag< CubitMeshsetMaskedType_mi_tag >, const_mem_fun< CubitMeshSets, unsigned long int, &CubitMeshSets::getMaskedBcTypeULong > >, ordered_non_unique< tag< CubitMeshsets_name >, const_mem_fun< CubitMeshSets, std::string, &CubitMeshSets::getName > >, hashed_unique< tag< Composite_Cubit_msId_And_MeshsetType_mi_tag >, composite_key< CubitMeshSets, const_mem_fun< CubitMeshSets, int, &CubitMeshSets::getMeshsetId >, const_mem_fun< CubitMeshSets, unsigned long int, &CubitMeshSets::getMaskedBcTypeULong > > > > > CubitMeshSet_multiIndex
Stores data about meshsets (see CubitMeshSets) storing data about boundary conditions,...
static const bool debug
auto createGhostVector(MPI_Comm comm, PetscInt n, PetscInt N, PetscInt nghost, const PetscInt ghosts[])
Create smart ghost vector.
constexpr AssemblyType A
constexpr double t
plate stiffness
Definition plate.cpp:58
constexpr auto field_name
constexpr double g
FTensor::Index< 'm', 3 > m
Managing BitRefLevels.
static MoFEMErrorCode updateEntitiesPetscVector(moab::Interface &moab, EntitiesPetscVector &vec, Tag tag, UpdateGhosts update_gosts=defaultUpdateGhosts)
Exchange data between vector and data.
MoFEMErrorCode query_interface(boost::typeindex::type_index type_index, UnknownInterface **iface) const
std::function< MoFEMErrorCode(Vec vec)> UpdateGhosts
static Range getPartEntities(moab::Interface &moab, int part)
static EntitiesPetscVector createEntitiesPetscVector(MPI_Comm comm, moab::Interface &moab, int dim, const int nb_coeffs, Sev sev=Sev::verbose, int root_rank=0)
Create a ghost vector for exchanging data.
CommInterface(const MoFEM::Core &core)
MoFEMErrorCode makeEntitiesMultishared(const EntityHandle *entities, const int num_entities, const int owner_proc=0, int verb=DEFAULT_VERBOSITY)
make entities from proc 0 shared on all proc
std::pair< std::pair< Range, Range >, SmartPetscObj< Vec > > EntitiesPetscVector
static MoFEMErrorCode loadFileRootProcAllRestDistributed(moab::Interface &moab, const char *file_name, int dim, LoadFileFun proc_skin_fun=defaultProcSkinFun, const char *options="PARALLEL=BCAST;PARTITION=")
Root proc has whole mesh, other procs only part of it.
MoFEMErrorCode synchroniseEntities(Range &ent, std::map< int, Range > *received_ents, int verb=DEFAULT_VERBOSITY)
synchronize entity range on processors (collective)
MoFEMErrorCode makeFieldEntitiesMultishared(const std::string field_name, const int owner_proc=0, int verb=DEFAULT_VERBOSITY)
make field entities multi shared
MoFEMErrorCode resolveParentEntities(const Range &ent, int verb=DEFAULT_VERBOSITY)
Synchronise parent entities.
std::function< std::array< Range, 4 >( std::array< Range, 4 > &&, std::vector< const CubitMeshSets * > &&)> LoadFileFun
MoFEMErrorCode exchangeFieldData(const std::string field_name, int verb=DEFAULT_VERBOSITY)
Exchange field data.
virtual int get_comm_size() const =0
virtual FieldBitNumber get_field_bit_number(const std::string name) const =0
get field bit number
virtual moab::Interface & get_moab()=0
virtual EntityHandle get_field_meshset(const std::string name) const =0
get field meshset
virtual MPI_Comm & get_comm() const =0
virtual int get_comm_rank() const =0
virtual boost::shared_ptr< BasicEntityData > & get_basic_entity_data_ptr()=0
Get pointer to basic entity data.
Core (interface) class.
Definition Core.hpp:82
Tag get_th_RefBitLevel() const
Definition Core.hpp:198
Tag get_th_RefParentHandle() const
Definition Core.hpp:197
this struct keeps basic methods for moab meshset about material and boundary conditions
Deprecated interface functions.
UId getLocalUniqueIdCalculate() const
Generate UId for finite element entity.
static UId getHiBitNumberUId(const FieldBitNumber bit_number)
static UId getLoBitNumberUId(const FieldBitNumber bit_number)
static void sortMeshsets(std::vector< const CubitMeshSets * > &vec_ptr)
MoFEMErrorCode getTags(int verb=-1)
get tags handlers used on meshsets
keeps basic data about problem
MoFEMErrorCode getNumberOfElementsByNameAndPart(MPI_Comm comm, const std::string name, PetscLayout *layout) const
Get number of finite elements by name on processors.
boost::shared_ptr< NumeredEntFiniteElement_multiIndex > numeredFiniteElementsPtr
store finite elements
intrusive_ptr for managing petsc objects
base class for all interface classes