161 ProblemsByName::iterator p_miit,
const MatType type,
162 std::vector<PetscInt> &
i, std::vector<PetscInt> &
j,
const bool no_diagonals,
168 auto cache = boost::make_shared<std::vector<EntityCacheNumeredDofs>>(
174 const auto uid = (*it)->getLocalUniqueId();
176 for (
auto lo = r.first; lo != r.second; ++lo) {
178 if ((lo->getBitFEId() & p_miit->getBitFEId()).any()) {
180 const BitRefLevel &prb_bit = p_miit->getBitRefLevel();
181 const BitRefLevel &prb_mask = p_miit->getBitRefLevelMask();
182 const BitRefLevel &fe_bit = lo->entFePtr->getBitRefLevel();
185 if ((fe_bit & prb_mask) != fe_bit)
187 if ((fe_bit & prb_bit).none())
190 auto dit = p_miit->numeredColDofsPtr->lower_bound(uid);
192 decltype(dit) hi_dit;
193 if (dit != p_miit->numeredColDofsPtr->end())
194 hi_dit = p_miit->numeredColDofsPtr->upper_bound(
199 (*it)->entityCacheColDofs =
200 boost::shared_ptr<EntityCacheNumeredDofs>(cache, &((*cache)[idx]));
201 (*cache)[idx].loHi = {dit, hi_dit};
208 using NumeredDofEntitysByIdx =
213 const NumeredDofEntitysByIdx &dofs_row_by_idx =
214 p_miit->numeredRowDofsPtr->get<TAG>();
215 int nb_dofs_row = p_miit->getNbDofsRow();
216 if (nb_dofs_row == 0) {
218 p_miit->getName().c_str());
222 std::map<int, std::vector<int>> adjacent_dofs_on_other_parts;
229 TAG>::type::iterator miit_row,
232 if (TAG::IamNotPartitioned) {
237 CHKERR PetscLayoutSetBlockSize(layout, 1);
238 CHKERR PetscLayoutSetSize(layout, nb_dofs_row);
239 CHKERR PetscLayoutSetUp(layout);
240 PetscInt rstart, rend;
241 CHKERR PetscLayoutGetRange(layout, &rstart, &rend);
242 CHKERR PetscLayoutDestroy(&layout);
246 <<
"row lower " << rstart <<
" row upper " << rend;
250 miit_row = dofs_row_by_idx.lower_bound(rstart);
251 hi_miit_row = dofs_row_by_idx.lower_bound(rend);
252 if (std::distance(miit_row, hi_miit_row) != rend - rstart) {
255 "data inconsistency, std::distance(miit_row,hi_miit_row) != rend - "
256 "rstart (%ld != %d - %d = %d) ",
257 std::distance(miit_row, hi_miit_row), rend, rstart, rend - rstart);
267 std::vector<std::vector<int>> dofs_vec(
sIze);
269 boost::shared_ptr<FieldEntity> mofem_ent_ptr;
270 std::vector<int> dofs_col_view;
273 TAG>::type::iterator mit_row,
276 mit_row = dofs_row_by_idx.begin();
277 hi_mit_row = dofs_row_by_idx.end();
278 for (; mit_row != hi_mit_row; mit_row++) {
286 unsigned char pstatus = (*mit_row)->getPStatus();
287 if ((pstatus & PSTATUS_NOT_OWNED) &&
288 (pstatus & (PSTATUS_SHARED | PSTATUS_MULTISHARED))) {
290 bool get_adj_col =
true;
292 if (mofem_ent_ptr->getLocalUniqueId() ==
293 (*mit_row)->getFieldEntityPtr()->getLocalUniqueId()) {
300 mofem_ent_ptr = (*mit_row)->getFieldEntityPtr();
301 CHKERR getEntityAdjacencies<TAG>(p_miit, mit_row, mofem_ent_ptr,
302 dofs_col_view, verb);
305 std::sort(dofs_col_view.begin(), dofs_col_view.end());
306 std::vector<int>::iterator new_end =
307 std::unique(dofs_col_view.begin(), dofs_col_view.end());
308 int new_size = std::distance(dofs_col_view.begin(), new_end);
309 dofs_col_view.resize(new_size);
313 int owner = (*mit_row)->getOwnerProc();
314 dofs_vec[owner].emplace_back(TAG::get_index(mit_row));
315 dofs_vec[owner].emplace_back(
316 dofs_col_view.size());
318 dofs_vec[owner].insert(dofs_vec[owner].end(), dofs_col_view.begin(),
319 dofs_col_view.end());
325 std::vector<int> dofs_vec_length(
sIze);
326 for (
int proc = 0; proc <
sIze; proc++) {
328 if (!dofs_vec[proc].empty()) {
330 dofs_vec_length[proc] = dofs_vec[proc].size();
335 dofs_vec_length[proc] = 0;
339 std::vector<MPI_Status> status(
sIze);
343 CHKERR PetscGatherNumberOfMessages(comm, NULL, &dofs_vec_length[0],
350 CHKERR PetscGatherMessageLengths(comm, nsends, nrecvs, &dofs_vec_length[0],
355 CHKERR PetscCommGetNewTag(comm, &tag);
360 MPI_Request *r_waits;
365 CHKERR PetscPostIrecvInt(comm, tag, nrecvs, onodes, olengths, &rbuf,
368 MPI_Request *s_waits;
369 CHKERR PetscMalloc1(nsends, &s_waits);
372 for (
int proc = 0, kk = 0; proc <
sIze; proc++) {
373 if (!dofs_vec_length[proc])
375 CHKERR MPI_Isend(&(dofs_vec[proc])[0],
376 dofs_vec_length[proc],
378 tag, comm, s_waits + kk);
384 CHKERR MPI_Waitall(nrecvs, r_waits, &status[0]);
388 CHKERR MPI_Waitall(nsends, s_waits, &status[0]);
391 for (
int kk = 0; kk < nrecvs; kk++) {
393 int len = olengths[kk];
394 int *data_from_proc = rbuf[kk];
396 for (
int ii = 0; ii < len;) {
398 int row_idx = data_from_proc[ii++];
399 int nb_adj_dofs = data_from_proc[ii++];
403 DofByGlobalPetscIndex::iterator dit;
409 "dof %d can not be found in problem", row_idx);
413 for (
int jj = 0; jj < nb_adj_dofs; jj++) {
414 adjacent_dofs_on_other_parts[row_idx].push_back(data_from_proc[ii++]);
420 CHKERR PetscFree(s_waits);
421 CHKERR PetscFree(rbuf[0]);
423 CHKERR PetscFree(r_waits);
425 CHKERR PetscFree(olengths);
427 miit_row = dofs_row_by_idx.begin();
428 hi_miit_row = dofs_row_by_idx.end();
430 CHKERR PetscCommDestroy(&comm);
433 boost::shared_ptr<FieldEntity> mofem_ent_ptr;
434 int row_last_evaluated_idx = -1;
436 std::vector<int> dofs_vec;
437 std::vector<int> dofs_col_view;
440 int nb_loc_row_from_iterators = 0;
441 unsigned int rows_to_fill = p_miit->getNbLocalDofsRow();
442 i.reserve(rows_to_fill + 1);
443 for (; miit_row != hi_miit_row; miit_row++) {
445 if (!TAG::IamNotPartitioned) {
446 if (
static_cast<int>((*miit_row)->getPart()) !=
rAnk)
450 nb_loc_row_from_iterators++;
453 i.push_back(
j.size());
460 : (mofem_ent_ptr->getLocalUniqueId() !=
461 (*miit_row)->getFieldEntityPtr()->getLocalUniqueId())) {
464 mofem_ent_ptr = (*miit_row)->getFieldEntityPtr();
465 CHKERR getEntityAdjacencies<TAG>(p_miit, miit_row, mofem_ent_ptr,
466 dofs_col_view, verb);
467 row_last_evaluated_idx = TAG::get_index(miit_row);
471 dofs_vec.insert(dofs_vec.end(), dofs_col_view.begin(),
472 dofs_col_view.end());
474 unsigned char pstatus = (*miit_row)->getPStatus();
476 std::map<int, std::vector<int>>::iterator mit;
477 mit = adjacent_dofs_on_other_parts.find(row_last_evaluated_idx);
478 if (mit == adjacent_dofs_on_other_parts.end()) {
486 dofs_vec.insert(dofs_vec.end(), mit->second.begin(),
492 sort(dofs_vec.begin(), dofs_vec.end());
493 std::vector<int>::iterator new_end =
494 unique(dofs_vec.begin(), dofs_vec.end());
495 int new_size = std::distance(dofs_vec.begin(), new_end);
496 dofs_vec.resize(new_size);
500 if (
j.capacity() <
j.size() + dofs_vec.size()) {
503 unsigned int nb_nonzero =
j.size() + dofs_vec.size();
504 unsigned int average_row_fill = nb_nonzero /
i.size() + 1;
505 j.reserve(rows_to_fill * average_row_fill);
508 auto hi_diit = dofs_vec.end();
509 for (
auto diit = dofs_vec.begin(); diit != hi_diit; diit++) {
512 if (*diit == TAG::get_index(miit_row)) {
521 i.push_back(
j.size());
523 if (strcmp(type, MATMPIADJ) == 0) {
526 if (
i.size() - 1 != (
unsigned int)nb_loc_row_from_iterators) {
529 "Number of rows from iterator is different than size of rows in "
531 "matrix (unsigned int)nb_local_dofs_row != i.size() - 1, i.e. %d != "
533 (
unsigned int)nb_loc_row_from_iterators,
i.size() - 1);
536 }
else if (strcmp(type, MATMPIAIJ) == 0) {
539 if (
i.size() - 1 != (
unsigned int)nb_loc_row_from_iterators) {
542 "Number of rows from iterator is different than size of rows in "
544 "matrix (unsigned int)nb_local_dofs_row != i.size() - 1, i.e. %d != "
546 (
unsigned int)nb_loc_row_from_iterators,
i.size() - 1);
548 PetscInt nb_local_dofs_row = p_miit->getNbLocalDofsRow();
549 if ((
unsigned int)nb_local_dofs_row !=
i.size() - 1) {
552 "Number of rows is different than size of rows in compressed row "
553 "matrix (unsigned int)nb_local_dofs_row != i.size() - 1, i.e. %d != "
555 (
unsigned int)nb_local_dofs_row,
i.size() - 1);
558 }
else if (strcmp(type, MATAIJ) == 0) {
561 if (
i.size() - 1 != (
unsigned int)nb_loc_row_from_iterators) {
564 "Number of rows form iterator is different than size of rows in "
566 "matrix (unsigned int)nb_local_dofs_row != i.size() - 1, i.e. %d != "
568 (
unsigned int)nb_loc_row_from_iterators,
i.size() - 1);
570 PetscInt nb_local_dofs_row = p_miit->getNbLocalDofsRow();
571 if ((
unsigned int)nb_local_dofs_row !=
i.size() - 1) {
574 "Number of rows is different than size of rows in compressed row "
575 "matrix (unsigned int)nb_local_dofs_row != i.size() - 1, i.e. %d != "
577 (
unsigned int)nb_local_dofs_row,
i.size() - 1);
580 }
else if (strcmp(type, MATAIJCUSPARSE) == 0) {
582 if (
i.size() - 1 != (
unsigned int)nb_loc_row_from_iterators) {
583 SETERRQ(
get_comm(), PETSC_ERR_ARG_SIZ,
"data inconsistency");
585 PetscInt nb_local_dofs_row = p_miit->getNbLocalDofsRow();
586 if ((
unsigned int)nb_local_dofs_row !=
i.size() - 1) {
587 SETERRQ(
get_comm(), PETSC_ERR_ARG_SIZ,
"data inconsistency");
592 SETERRQ(
get_comm(), PETSC_ERR_ARG_NULL,
"not implemented");
900 int row_print,
int col_print,
927 if (refinedFiniteElementsPtr->find(
928 numeredEntFiniteElementPtr->getEnt()) ==
929 refinedFiniteElementsPtr->end()) {
931 "data inconsistency");
939 if (refinedEntitiesPtr->find((*cit)->getEnt()) ==
940 refinedEntitiesPtr->end()) {
942 "data inconsistency");
944 if (!(*cit)->getActive()) {
946 "data inconsistency");
949 FieldEntityEntFiniteElementAdjacencyMap_multiIndex::index<
952 boost::make_tuple((*cit)->getFieldEntityPtr()->getLocalUniqueId(),
953 numeredEntFiniteElementPtr->getLocalUniqueId()));
954 if (
ait == adjacenciesPtr->end()) {
956 "adjacencies data inconsistency");
959 if (entitiesPtr->find(
uid) == entitiesPtr->end()) {
961 "data inconsistency");
963 if (dofsPtr->find((*cit)->getLocalUniqueId()) == dofsPtr->end()) {
965 "data inconsistency");
969 if ((*cit)->getEntType() !=
MBVERTEX) {
976 if ((*cit)->getNbOfCoeffs() * (*cit)->getOrderNbDofs(
max_order) !=
984 <<
"Warning: Number of Dofs in Col different than number "
985 "of dofs for given entity order "
986 << (*cit)->getNbOfCoeffs() * (*cit)->getOrderNbDofs(
max_order)
994 if (refinedEntitiesPtr->find((*rit)->getEnt()) ==
995 refinedEntitiesPtr->end()) {
997 "data inconsistency");
999 if (!(*rit)->getActive()) {
1001 "data inconsistency");
1004 FieldEntityEntFiniteElementAdjacencyMap_multiIndex::index<
1007 boost::make_tuple((*rit)->getFieldEntityPtr()->getLocalUniqueId(),
1008 numeredEntFiniteElementPtr->getLocalUniqueId()));
1009 if (
ait == adjacenciesPtr->end()) {
1011 MOFEM_LOG(
"SELF", Sev::error) << *(*rit);
1012 MOFEM_LOG(
"SELF", Sev::error) << *(*rit);
1013 MOFEM_LOG(
"SELF", Sev::error) << *numeredEntFiniteElementPtr;
1014 MOFEM_LOG(
"SELF", Sev::error) <<
"dof: " << (*rit)->getBitRefLevel();
1016 <<
"fe: " << numeredEntFiniteElementPtr->getBitRefLevel();
1018 <<
"problem: " << problemPtr->getBitRefLevel();
1020 <<
"problem mask: " << problemPtr->getBitRefLevelMask();
1022 "adjacencies data inconsistency");
1025 if (entitiesPtr->find(
uid) == entitiesPtr->end()) {
1027 "data inconsistency");
1029 if (dofsPtr->find((*rit)->getLocalUniqueId()) == dofsPtr->end()) {
1031 "data inconsistency");
1034 int row = (*rit)->getPetscGlobalDofIdx();
1039 int col = (*cit)->getPetscGlobalDofIdx();
1042 MOFEM_LOG(
"SELF", Sev::noisy) <<
"fe:\n"
1043 << *numeredEntFiniteElementPtr;
1044 MOFEM_LOG(
"SELF", Sev::noisy) <<
"row:\n" << *(*rit);
1045 MOFEM_LOG(
"SELF", Sev::noisy) <<
"col:\n" << *(*cit);
1048 << numeredEntFiniteElementPtr->getBitRefLevel();
1049 MOFEM_LOG(
"SELF", Sev::noisy) <<
"row:\n"
1050 << (*rit)->getBitRefLevel();
1051 MOFEM_LOG(
"SELF", Sev::noisy) <<
"col:\n"
1052 << (*cit)->getBitRefLevel();
1058 if ((*rit)->getEntType() !=
MBVERTEX) {
1065 if ((*rit)->getNbOfCoeffs() * (*rit)->getOrderNbDofs(
max_order) !=
1073 <<
"Warning: Number of Dofs in Row different than number "
1074 "of dofs for given entity order "
1075 << (*rit)->getNbOfCoeffs() * (*rit)->getOrderNbDofs(
max_order)
1102 if (verb >=
NOISY) {
1116 "problem < %s > not found (top tip: check spelling)",
1118 MOFEM_LOG_C(
"WORLD", Sev::inform,
"check problem < %s >",
1123 for (
auto &fe : *
fe_ptr) {
1124 MOFEM_LOG_C(
"WORLD", Sev::verbose,
"\tcheck element %s",
1125 fe->getName().c_str());