DOF counts are checked against the entity counts and the per-entity DOF numbers of the bases; every stage is a TestingInterface test. Options: -file_name, -nb_levels, -order (BROKEN order; U and HYBRID one less), -dm_is_partitioned, -incremental (default true; false uses Simple's all-entities field/element builders), -local_from <l> (from level l on, refine only the tetrahedra with an edge in the star of one vertex, which keeps the mesh conforming), -rebuild_structures (from level 1 on, update the block and nested structures of the previous level with rebuildBlockMatStructure and rebuildSchurNestedMatrixStruture, and check them against new ones made by createBlockMatStructure and createSchurNestedMatrixStruture). Hybrid preallocation is checked against the couplings of skeleton faces sharing an active tetrahedron, then filled with new allocations forbidden and checked for a diagonal.
#include <unordered_map>
static char help[] =
"Hybridised broken H(div) problem on refined levels: "
"problem, sub-problem and matrix timings.\n"
"-file_name <mesh> -nb_levels <n> -order <p> "
"[-incremental true|false] [-local_from <l>] "
"[-rebuild_structures]\n\n";
namespace {
struct Timings {
double registration = 0, problem = 0, schurSub = 0, blockSub = 0,
blockMat = 0, nestedSchur = 0, hybridMat = 0;
};
PetscLogDouble t0, t1;
seconds = t1 - t0;
}
bool local, int &refined) {
CHKERR bit_mng->getEntitiesByDimAndRefLevel(
bit(
l - 1), allBits(), 3,
level_tets);
Range tets = level_tets, edges;
if (local) {
CHKERR moab.get_connectivity(level_tets, verts,
true);
const EntityHandle vertex = verts.front();
CHKERR moab.get_adjacencies(&vertex, 1, 3,
false, star);
star = intersect(star, level_tets);
CHKERR moab.get_adjacencies(star, 1,
true, edges, moab::Interface::UNION);
CHKERR moab.get_adjacencies(edges, 3,
false, touched,
moab::Interface::UNION);
tets = intersect(touched, level_tets);
} else {
CHKERR moab.get_adjacencies(tets, 1,
true, edges, moab::Interface::UNION);
}
refined = tets.size();
if (local) {
const Range unchanged = subtract(level_tets, tets);
CHKERR moab.get_connectivity(unchanged, adj,
true);
CHKERR moab.get_adjacencies(unchanged,
d,
false, adj,
moab::Interface::UNION);
}
CHKERR bit_mng->updateMeshsetByEntitiesChildren(
}
int &nb_skeleton_faces) {
CHKERR bit_mng->getEntitiesByDimAndRefLevel(
bit(
l), allBits(), 3, tets);
CHKERR moab.get_adjacencies(tets, 2,
true,
faces, moab::Interface::UNION);
Skinner skinner(&moab);
const EntityHandle meshset =
simple.getSkeletonMeshSet();
CHKERR moab.clear_meshset(&meshset, 1);
CHKERR moab.add_entities(meshset, skeleton);
nb_skeleton_faces = skeleton.size();
}
int order,
bool incremental) {
if (!incremental) {
}
bit(
l), allBits(), 3, tets);
registered);
tets = subtract(tets, registered);
registered.clear();
registered);
for (
const auto &
field : {
"BROKEN",
"U"}) {
}
CHKERR comm_ptr->synchroniseFieldEntities(
"HYBRID", 0);
}
const std::string &
tag,
double &nonzeros) {
CHKERR DMMoFEMGetProblemPtr(dm, &problem_ptr);
std::unordered_map<EntityHandle, std::vector<PetscInt>> face_dofs;
if (dof->getPetscGlobalDofIdx() >= 0)
face_dofs[dof->getEnt()].push_back(dof->getPetscGlobalDofIdx());
std::unordered_map<EntityHandle, std::vector<PetscInt>> neighbors;
bit(
l), allBits(), 3, tets);
for (auto tet : tets) {
std::vector<PetscInt> columns;
for (
auto face :
faces) {
auto it = face_dofs.find(face);
if (
it != face_dofs.end())
columns.insert(columns.end(),
it->second.begin(),
it->second.end());
}
if (face_dofs.count(face)) {
auto &adj = neighbors[face];
adj.insert(adj.end(), columns.begin(), columns.end());
}
}
PetscInt first_row, last_row;
CHKERR MatGetOwnershipRange(matrix, &first_row, &last_row);
MatInfo before;
CHKERR MatGetInfo(matrix, MAT_GLOBAL_SUM, &before);
CHKERR MatSetOption(matrix, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_TRUE);
double local_nnz = 0;
for (auto &entry : neighbors) {
auto &columns = entry.second;
std::sort(columns.begin(), columns.end());
columns.erase(std::unique(columns.begin(), columns.end()), columns.end());
const PetscInt nb_cols = columns.size();
const std::vector<PetscScalar>
values(nb_cols, 1.);
for (
auto row : face_dofs.at(entry.first))
if (
row >= first_row &&
row < last_row) {
local_nnz += nb_cols;
CHKERR MatSetValues(matrix, 1, &
row, nb_cols, columns.data(),
values.data(), INSERT_VALUES);
}
}
double expected_nnz;
CHKERR MPI_Allreduce(&local_nnz, &expected_nnz, 1, MPI_DOUBLE, MPI_SUM,
tag +
" hybrid allocation equals topology non-zeros", before.nz_allocated,
expected_nnz, zero);
CHKERR MatAssemblyBegin(matrix, MAT_FINAL_ASSEMBLY);
CHKERR MatAssemblyEnd(matrix, MAT_FINAL_ASSEMBLY);
MatInfo after;
CHKERR MatGetInfo(matrix, MAT_GLOBAL_SUM, &after);
nonzeros = after.nz_used;
tag +
" hybrid filled non-zeros equal topology", nonzeros, expected_nnz,
zero);
tag +
" hybrid fill without reallocations", after.mallocs, before.mallocs,
zero);
PetscBool missing;
PetscInt missing_row;
CHKERR MatMissingDiagonal(matrix, &missing, &missing_row);
tag +
" hybrid diagonal exists in every row", missing ? 1 : 0, 0, zero);
}
const std::vector<std::string> &fields,
dm = createDM(m_field.
get_comm(),
"DMMOFEM");
CHKERR DMMoFEMSetSquareProblem(dm, PETSC_TRUE);
for (
auto &
field : fields) {
}
}
}
}
int main(
int argc,
char *argv[]) {
try {
moab::Core mb_instance;
moab::Interface &moab = mb_instance;
CHKERR DMRegister_MoFEM(
"DMMOFEM");
PetscBool is_partitioned = PETSC_FALSE, incremental = PETSC_TRUE;
PetscBool rebuild_structures = PETSC_FALSE;
PetscInt local_from = -1;
PETSC_NULLPTR);
CHKERR PetscOptionsGetInt(PETSC_NULLPTR,
"",
"-order", &
order,
PETSC_NULLPTR);
CHKERR PetscOptionsGetBool(PETSC_NULLPTR,
"",
"-dm_is_partitioned",
&is_partitioned, PETSC_NULLPTR);
CHKERR PetscOptionsGetBool(PETSC_NULLPTR,
"",
"-incremental", &incremental,
PETSC_NULLPTR);
CHKERR PetscOptionsGetBool(PETSC_NULLPTR,
"",
"-rebuild_structures",
&rebuild_structures, PETSC_NULLPTR);
CHKERR PetscOptionsGetInt(PETSC_NULLPTR,
"",
"-local_from", &local_from,
PETSC_NULLPTR);
const int u_order =
order - 1;
const int hybrid_order =
order - 1;
simple->getAddBoundaryFE() =
false;
"Tetrahedral meshes only");
<< ", U and HYBRID order " << u_order << ", "
<< (is_partitioned ? "distributed" : "replicated") << " builder, "
<< (incremental ? "incremental" : "full")
<< " field/element registration"
<< (local_from >= 0 ? ", local refinement from level " +
std::to_string(local_from)
<< (rebuild_structures ? ", block and nested structures rebuilt"
: "");
const int dofs_hybrid_per_face =
NBFACETRI_L2(hybrid_order);
auto count_tets = [&](
int l) {
bit(
l), allBits(), 3, tets),
"count tets");
return static_cast<int>(tets.size());
};
{{"BROKEN", "BROKEN"},
{"U", "U"},
{"BROKEN", "U"},
{"U", "BROKEN"}}},
{{"BROKEN", "HYBRID"}, {"HYBRID", "BROKEN"}}}};
boost::shared_ptr<BlockStructure> kept_block;
boost::shared_ptr<NestSchurData> kept_nested;
struct Level {
boost::shared_ptr<BlockStructure> blockData;
boost::shared_ptr<NestSchurData> nestedData;
int tets = 0, skeletonFaces = 0,
rows = 0, schurRows = 0, blockRows = 0;
double hybridNnz = 0;
};
const auto tag =
"level " + std::to_string(
l);
const bool local =
l > 0 && local_from >= 0 &&
l >= local_from;
int refined = 0;
[&]() {
return registerLevel(m_field, *
simple,
l,
order, incremental);
},
lv.t.registration);
tag +
" tets are eight per parent", lv.tets,
8. * levels[
l - 1].tets, zero);
if (local) {
<<
"Level " <<
l <<
" local refinement of " << refined <<
" of "
<< levels[
l - 1].tets <<
" tets";
tag +
" local refinement leaves tets unchanged",
refined < levels[
l - 1].tets ? 1 : 0, 1, zero);
}
});
CHKERR time([&]() {
return simple->reSetUp(
true); }, lv.t.problem);
lv.rows = problemRows(m_field,
simple->getDM());
const double expected =
double(lv.tets) * (dofs_broken_per_tet + dofs_u_per_tet) +
double(lv.skeletonFaces) * dofs_hybrid_per_face;
tag +
" parent dofs = BROKEN + U + HYBRID", lv.rows, expected,
zero);
});
[&]() {
return subProblem(m_field, *
simple,
"SCHUR", {
"HYBRID"},
lv.schurDM);
},
lv.t.schurSub);
[&]() {
return subProblem(m_field, *
simple,
"BLOCK", {
"BROKEN",
"U"},
lv.blockDM);
},
lv.t.blockSub);
lv.schurRows = problemRows(m_field, lv.schurDM);
lv.blockRows = problemRows(m_field, lv.blockDM);
tag +
" SCHUR dofs = HYBRID", lv.schurRows,
double(lv.skeletonFaces) * dofs_hybrid_per_face, zero);
tag +
" BLOCK + SCHUR dofs = parent", lv.blockRows + lv.schurRows,
lv.rows, zero);
});
"matrices",
const bool rebuild = rebuild_structures && kept_block && kept_nested;
[&]() {
if (rebuild) {
kept_block);
lv.blockData = kept_block;
} else {
lv.blockData =
}
lv.blockMat = block.first;
},
lv.t.blockMat);
[&]() {
if (rebuild) {
{lv.schurDM, lv.blockDM}, kept_nested);
lv.nestedData = kept_nested;
} else {
{lv.schurDM, lv.blockDM}, lv.blockData, {"BROKEN", "U"},
{nullptr, nullptr}, true);
}
lv.nestedMat = nested.first;
},
lv.t.nestedSchur);
if (rebuild_structures) {
kept_block = lv.blockData;
kept_nested = lv.nestedData;
}
if (rebuild) {
boost::shared_ptr<BlockStructure> fresh_block;
boost::shared_ptr<NestSchurData> fresh_nested;
double fresh_block_seconds, fresh_nested_seconds;
[&]() {
fresh_block =
},
fresh_block_seconds);
[&]() {
{lv.schurDM, lv.blockDM}, fresh_block, {"BROKEN", "U"},
{nullptr, nullptr}, true);
},
fresh_nested_seconds);
<<
"Level " <<
l <<
" structures [s]: rebuilt block "
<< lv.t.blockMat << ", rebuilt nested " << lv.t.nestedSchur
<< "; new block " << fresh_block_seconds << ", new nested "
<< fresh_nested_seconds;
auto compare = [&](const std::string &what, auto rebuilt,
auto fresh) {
tag +
" rebuilt " + what +
" blocks",
r.nbBlocks,
f.nbBlocks,
zero);
tag +
" rebuilt " + what +
" A00 elements",
r.nbElems,
tag +
" rebuilt " + what +
" block index equal",
r.indexHash ==
f.indexHash ? 0 : 1, 0, zero);
tag +
" rebuilt " + what +
" blocks equal",
r.blocksHash ==
f.blocksHash ? 0 : 1, 0, zero);
tag +
" rebuilt " + what +
" A00 elements equal",
r.elemsHash ==
f.elemsHash ? 0 : 1, 0, zero);
};
CHKERR compare(
"block matrix", lv.blockData, fresh_block);
const auto &rebuilt_parts = std::get<1>(*lv.nestedData);
const auto &fresh_parts = std::get<1>(*fresh_nested);
for (
int q = 0;
q != 4; ++
q)
CHKERR compare(
"nested quadrant " + std::to_string(
q),
rebuilt_parts[
q], fresh_parts[
q]);
}
tag +
" block matrix rows = parent",
m, lv.rows, zero);
CHKERR MatGetSize(lv.nestedMat, &
m, &
n);
tag +
" nested matrix rows = parent",
m, lv.rows, zero);
});
[&]() {
},
lv.t.hybridMat);
CHKERR MatGetSize(lv.hybridMat, &
m, &
n);
tag +
" hybrid matrix rows = SCHUR dofs",
m, lv.schurRows, zero);
tag +
" hybrid matrix columns = SCHUR dofs",
n, lv.schurRows, zero);
CHKERR checkHybridMatrix(m_field, lv.schurDM,
l, lv.hybridMat, *
testing,
<<
"Level " <<
l <<
": tets " << lv.tets <<
", skeleton faces "
<< lv.skeletonFaces << ", dofs " << lv.rows << " (SCHUR "
<< lv.schurRows << ", BLOCK " << lv.blockRows
<< "), hybrid non-zeros " << lv.hybridNnz;
<<
"Level " <<
l <<
" times [s]: registration "
<< lv.t.registration << ", problem " << lv.t.problem
<< ", sub SCHUR " << lv.t.schurSub << ", sub BLOCK "
<< lv.t.blockSub << ", block matrix " << lv.t.blockMat
<< ", nested Schur " << lv.t.nestedSchur << ", hybrid matrix "
<< lv.t.hybridMat;
lv.hybridMat.reset();
lv.nestedMat.reset();
lv.nestedData.reset();
lv.blockMat.reset();
lv.blockData.reset();
lv.blockDM.reset();
lv.schurDM.reset();
});
}
}
return 0;
}
std::vector< std::array< EntityHandle, 3 > > faces
Sorted keys.
Vec t
Removal: the ring edge.
std::vector< FieldBitNumber > row
SmartPetscObj< Vec > values
void simple(double P1[], double P2[], double P3[], double c[], const int N)
#define CATCH_ERRORS
Catch errors.
#define CHK_THROW_MESSAGE(err, msg)
Check and throw MoFEM exception.
#define MoFEMFunctionReturnHot(a)
Last executable line of each PETSc function used for error handling. Replaces return()
@ L2
field with C-1 continuity
@ HDIV
field with continuous normal traction
#define MoFEMFunctionBegin
First executable line of each MoFEM function, used for error handling. Final line of MoFEM functions ...
#define MoFEMFunctionReturn(a)
Last executable line of each PETSc function used for error handling. Replaces return()
#define CHKERR
Inline error check.
MoFEMErrorCode getEntitiesByDimAndRefLevel(const BitRefLevel bit, const BitRefLevel mask, const int dim, const EntityHandle meshset, int verb=0) const
add all ents from ref level given by bit to meshset
MoFEMErrorCode addBitRefLevel(const Range &ents, const BitRefLevel bit, int verb=QUIET) const
Add bit ref level to ref entity.
virtual MoFEMErrorCode add_ents_to_finite_element_by_dim(const EntityHandle entities, const int dim, const std::string name, const bool recursive=true)=0
add entities to finite element
virtual MoFEMErrorCode build_finite_elements(int verb=DEFAULT_VERBOSITY)=0
Build finite elements.
virtual MoFEMErrorCode build_fields(int verb=DEFAULT_VERBOSITY)=0
virtual MoFEMErrorCode add_ents_to_field_by_dim(const Range &ents, const int dim, const std::string &name, int verb=DEFAULT_VERBOSITY)=0
Add entities to field meshset.
virtual MoFEMErrorCode set_field_order(const EntityHandle meshset, const EntityType type, const std::string &name, const ApproximationOrder order, int verb=DEFAULT_VERBOSITY)=0
Set order approximation of the entities in the field.
#define MOFEM_LOG(channel, severity)
Log.
#define MOFEM_LOG_TAG(channel, tag)
Tag channel.
#define MOFEM_LOG_CHANNEL(channel)
Set and reset channel.
#define NBFACETRI_L2(P)
Number of base functions on triangle for L2 space.
#define NBVOLUMETET_DEMKOWICZ_HDIV(P)
#define NBFACETRI_DEMKOWICZ_HDIV(P)
#define NBVOLUMETET_L2(P)
Number of base functions on tetrahedron for L2 space.
FTensor::Index< 'l', 3 > l
PetscErrorCode MoFEMErrorCode
MoFEM/PETSc error code.
std::bitset< BITREFLEVEL_SIZE > BitRefLevel
Bit structure attached to each entity identifying to what mesh entity is attached.
implementation of Data Operators for Forces and Sources
SchurBlockStructureDigest schurBlockStructureDigest(boost::shared_ptr< BlockStructure > data_ptr)
std::pair< SmartPetscObj< Mat >, boost::shared_ptr< NestSchurData > > createSchurNestedMatrix(boost::shared_ptr< NestSchurData > schur_net_data_ptr)
Create a Mat Diag Blocks object.
auto createDMHybridisedL2Matrix(DM dm)
Get smart hybridised L2 matrix from DM.
boost::shared_ptr< BlockStructure > createBlockMatStructure(DM dm, SchurFEOpsFEandFields schur_fe_op_vec)
Create block matrix structure.
boost::shared_ptr< NestSchurData > createSchurNestedMatrixStruture(std::pair< SmartPetscObj< DM >, SmartPetscObj< DM > > dms, boost::shared_ptr< BlockStructure > block_mat_data_ptr, std::vector< std::string > fields_names, std::vector< boost::shared_ptr< Range > > field_ents, bool add_preconditioner_block)
Create structure of nested matrix [A11 A10; A01 A00].
MoFEMErrorCode rebuildSchurNestedMatrixStruture(std::pair< SmartPetscObj< DM >, SmartPetscObj< DM > > dms, boost::shared_ptr< NestSchurData > nest_data_ptr)
Rebuild a nested Schur structure after rebuildBlockMatStructure.
MoFEMErrorCode rebuildBlockMatStructure(DM dm, SchurFEOpsFEandFields schur_fe_op_vec, boost::shared_ptr< BlockStructure > data_ptr)
Rebuild a block matrix structure after its problem changed.
std::vector< std::pair< std::string, std::vector< SchurFieldPair > > > SchurFEOpsFEandFields
SchurShellMatData createBlockMat(DM dm, boost::shared_ptr< BlockStructure > data)
Create a Schur Mat object.
FTensor::Index< 'm', 3 > m
virtual moab::Interface & get_moab()=0
virtual EntityHandle get_field_meshset(const std::string name) const =0
get field meshset
virtual MPI_Comm & get_comm() const =0
static MoFEMErrorCode Initialize(int *argc, char ***args, const char file[], const char help[])
Initializes the MoFEM database PETSc, MOAB and MPI.
static MoFEMErrorCode Finalize()
Checks for options to be called at the conclusion of the program.
Deprecated interface functions.
Mesh refinement interface.
MoFEMErrorCode addVerticesInTheMiddleOfEdges(const EntityHandle meshset, const BitRefLevel &bit, const bool recursive=false, int verb=QUIET, EntityHandle start_v=0)
make vertices in the middle of edges in meshset and add them to refinement levels defined by bit
keeps basic data about problem
DofIdx getNbDofsRow() const
auto & getNumeredColDofsPtr() const
get access to numeredColDofsPtr storing DOFs on cols
Simple interface for fast problem set-up.
intrusive_ptr for managing petsc objects
MoFEMErrorCode getInterface(IFACE *&iface) const
Get interface reference to pointer of interface.