42 void Foam::faMeshDecomposition::distributeFaces()
44 Info<<
"\nCalculating distribution of faces" <<
endl;
48 for (label procI = 0; procI <
nProcs(); procI++)
62 processorDb.timeName(),
75 if (hasGlobalFaceZones_)
93 const label ownerSize =
111 for (
int i = 0; i < ownerSize; ++i)
118 if (faceProcAddressingHash.found(
faceLabels()[faceI] + 1))
120 faceToProc_[faceI] = procI;
132 "faceProcAddressing",
144 if (faceProcAddressingHash.found(
faceLabels()[faceI] + 1))
146 faceToProc_[faceI] = procI;
152 Info<<
"\nFinished decomposition in "
153 << decompositionTime.elapsedCpuTime()
163 const fileName& decompDictFile
167 decompDictFile_(decompDictFile),
170 decompositionMethod::nDomains
172 decompositionModel::
New
180 hasGlobalFaceZones_(false),
181 faceToProc_(nFaces()),
182 procFaceLabels_(nProcs_),
183 procMeshEdgesMap_(nProcs_),
184 procNInternalEdges_(nProcs_,
Zero),
185 procPatchEdgeLabels_(nProcs_),
186 procPatchPointAddressing_(nProcs_),
187 procPatchEdgeAddressing_(nProcs_),
188 procEdgeAddressing_(nProcs_),
189 procFaceAddressing_(nProcs_),
190 procBoundaryAddressing_(nProcs_),
191 procPatchSize_(nProcs_),
192 procPatchStartIndex_(nProcs_),
193 procNeighbourProcessors_(nProcs_),
194 procProcessorPatchSize_(nProcs_),
195 procProcessorPatchStartIndex_(nProcs_),
196 globallySharedPoints_(0),
197 cyclicParallel_(false)
205 model.readIfPresent(
"distributed", distributed_);
206 hasGlobalFaceZones_ = model.found(
"globalFaceZones");
217 Info<<
"\nDistributing faces to processors" <<
endl;
221 List<SLList<label>> procFaceList(nProcs());
223 forAll(faceToProc_, faceI)
225 if (faceToProc_[faceI] >= nProcs())
228 <<
"Impossible processor label " << faceToProc_[faceI]
229 <<
"for face " << faceI
234 procFaceList[faceToProc_[faceI]].append(faceI);
239 forAll(procFaceList, procI)
241 procFaceAddressing_[procI] = procFaceList[procI];
248 for (label procI = 0; procI < nProcs(); procI++)
254 time().caseName()/(
"processor" +
Foam::name(procI))
262 processorDb.timeName(),
271 "pointProcAddressing",
273 procFvMesh.meshSubDir,
280 Map<label> fvFaceProcAddressingHash;
287 "faceProcAddressing",
289 procFvMesh.meshSubDir,
296 forAll(fvFaceProcAddressing, faceI)
298 fvFaceProcAddressingHash.insert
300 fvFaceProcAddressing[faceI], faceI
305 const labelList& curProcFaceAddressing = procFaceAddressing_[procI];
307 labelList& curFaceLabels = procFaceLabels_[procI];
309 curFaceLabels =
labelList(curProcFaceAddressing.size(), -1);
311 forAll(curProcFaceAddressing, faceI)
313 curFaceLabels[faceI] =
314 fvFaceProcAddressingHash.find
316 faceLabels()[curProcFaceAddressing[faceI]] + 1
324 procFaceLabels_[procI]
328 const Map<label>& map =
patch.meshPointMap();
330 EdgeMap<label> edgesHash;
334 const label nIntEdges =
patch.nInternalEdges();
336 for (label curEdge = 0; curEdge < nIntEdges; curEdge++)
338 edgesHash.insert(
patch.edges()[curEdge], ++edgeI);
345 const label size =
boundary()[patchI].labelList::size();
347 for(
int eI=0; eI<size; eI++)
359 const vectorField& procPoints = procPatch.localPoints();
360 const labelList& procMeshPoints = procPatch.meshPoints();
361 const edgeList& procEdges = procPatch.edges();
363 labelList& curPatchPointAddressing = procPatchPointAddressing_[procI];
364 curPatchPointAddressing.
setSize(procPoints.size(), -1);
366 forAll(procPoints, pointI)
368 curPatchPointAddressing[pointI] =
369 map[fvPointProcAddressing[procMeshPoints[pointI]]];
372 labelList& curPatchEdgeAddressing = procPatchEdgeAddressing_[procI];
373 curPatchEdgeAddressing.
setSize(procEdges.size(), -1);
377 edge curGlobalEdge = procEdges[edgeI];
378 curGlobalEdge[0] = curPatchPointAddressing[curGlobalEdge[0]];
379 curGlobalEdge[1] = curPatchPointAddressing[curGlobalEdge[1]];
381 curPatchEdgeAddressing[edgeI] = edgesHash.find(curGlobalEdge)();
384 Map<label>& curMap = procMeshEdgesMap_[procI];
386 curMap = Map<label>(2*procEdges.size());
388 forAll(curPatchEdgeAddressing, edgeI)
390 curMap.insert(curPatchEdgeAddressing[edgeI], edgeI);
393 procNInternalEdges_[procI] = procPatch.nInternalEdges();
397 Info <<
"\nDistributing edges to processors" <<
endl;
404 const edgeList& edges = this->edges();
406 const labelList& neighbour = edgeNeighbour();
410 List<SLList<label>> procEdgeList(nProcs());
412 forAll(procEdgeList, procI)
414 for(label i=0; i<procNInternalEdges_[procI]; i++)
416 procEdgeList[procI].append(procPatchEdgeAddressing_[procI][i]);
424 List<SLList<label>> interProcBoundaries(nProcs());
427 List<SLList<SLList<label>>> interProcBEdges(nProcs());
429 List<SLList<label>> procPatchIndex(nProcs());
433 if (faceToProc_[owner[edgeI]] != faceToProc_[neighbour[edgeI]])
439 bool interProcBouFound =
false;
441 const label ownProc = faceToProc_[owner[edgeI]];
442 const label neiProc = faceToProc_[neighbour[edgeI]];
444 auto curInterProcBdrsOwnIter =
445 interProcBoundaries[ownProc].cbegin();
447 auto curInterProcBEdgesOwnIter =
448 interProcBEdges[ownProc].begin();
455 curInterProcBdrsOwnIter.good()
456 && curInterProcBEdgesOwnIter.good();
457 ++curInterProcBdrsOwnIter,
458 ++curInterProcBEdgesOwnIter
461 if (curInterProcBdrsOwnIter() == neiProc)
464 interProcBouFound =
true;
466 bool neighbourFound =
false;
468 curInterProcBEdgesOwnIter().append(edgeI);
470 auto curInterProcBdrsNeiIter =
471 interProcBoundaries[neiProc].cbegin();
473 auto curInterProcBEdgesNeiIter =
474 interProcBEdges[neiProc].begin();
481 curInterProcBdrsNeiIter.good()
482 && curInterProcBEdgesNeiIter.good();
483 ++curInterProcBdrsNeiIter,
484 ++curInterProcBEdgesNeiIter
487 if (curInterProcBdrsNeiIter() == ownProc)
490 neighbourFound =
true;
492 curInterProcBEdgesNeiIter().append(edgeI);
495 if (neighbourFound)
break;
498 if (interProcBouFound && !neighbourFound)
501 (
"faDomainDecomposition::decomposeMesh()")
502 <<
"Inconsistency in inter - "
503 <<
"processor boundary lists for processors "
504 << ownProc <<
" and " << neiProc
509 if (interProcBouFound)
break;
512 if (!interProcBouFound)
520 interProcBoundaries[ownProc].append(neiProc);
521 interProcBEdges[ownProc].append(SLList<label>(edgeI));
524 interProcBoundaries[neiProc].append(ownProc);
525 interProcBEdges[neiProc]
541 forAll(procPatchSize_, procI)
543 procPatchSize_[procI].setSize(
patches.size());
544 procPatchStartIndex_[procI].setSize(
patches.size());
550 forAll(procPatchSize_, procI)
552 procPatchSize_[procI][patchI] = 0;
553 procPatchStartIndex_[procI][patchI] =
554 procEdgeList[procI].size();
557 const label patchStart =
patches[patchI].start();
567 const label size =
patches[patchI].labelList::size();
571 for(
int eI=0; eI<size; eI++)
573 patchEdgeFaces[eI] = eF[
patches[patchI][eI]][0];
576 forAll(patchEdgeFaces, edgeI)
578 const label curProc = faceToProc_[patchEdgeFaces[edgeI]];
581 procEdgeList[curProc].append(patchStart + edgeI);
584 procPatchSize_[curProc][patchI]++;
591 const faPatch& cPatch =
patches[patchI];
593 const label cycOffset = cPatch.size()/2;
609 forAll(firstEdgeFaces, edgeI)
613 faceToProc_[firstEdgeFaces[edgeI]]
614 != faceToProc_[secondEdgeFaces[edgeI]]
623 cyclicParallel_ =
true;
625 bool interProcBouFound =
false;
627 const label ownProc =
628 faceToProc_[firstEdgeFaces[edgeI]];
629 const label neiProc =
630 faceToProc_[secondEdgeFaces[edgeI]];
632 auto curInterProcBdrsOwnIter =
633 interProcBoundaries[ownProc].cbegin();
635 auto curInterProcBEdgesOwnIter =
636 interProcBEdges[ownProc].begin();
643 curInterProcBdrsOwnIter.good()
644 && curInterProcBEdgesOwnIter.good();
645 ++curInterProcBdrsOwnIter,
646 ++curInterProcBEdgesOwnIter
649 if (curInterProcBdrsOwnIter() == neiProc)
653 interProcBouFound =
true;
655 bool neighbourFound =
false;
657 curInterProcBEdgesOwnIter()
658 .append(patchStart + edgeI);
660 auto curInterProcBdrsNeiIter
661 = interProcBoundaries[neiProc].cbegin();
663 auto curInterProcBEdgesNeiIter =
664 interProcBEdges[neiProc].begin();
671 curInterProcBdrsNeiIter.good()
672 && curInterProcBEdgesNeiIter.good();
673 ++curInterProcBdrsNeiIter,
674 ++curInterProcBEdgesNeiIter
677 if (curInterProcBdrsNeiIter() == ownProc)
680 neighbourFound =
true;
682 curInterProcBEdgesNeiIter()
691 if (neighbourFound)
break;
694 if (interProcBouFound && !neighbourFound)
698 "faDomainDecomposition::decomposeMesh()"
699 ) <<
"Inconsistency in inter-processor "
700 <<
"boundary lists for processors "
701 << ownProc <<
" and " << neiProc
702 <<
" in cyclic boundary matching"
707 if (interProcBouFound)
break;
710 if (!interProcBouFound)
718 interProcBoundaries[ownProc].append(neiProc);
719 interProcBEdges[ownProc]
720 .append(SLList<label>(patchStart + edgeI));
723 interProcBoundaries[neiProc].append(ownProc);
724 interProcBEdges[neiProc]
739 label ownProc = faceToProc_[firstEdgeFaces[edgeI]];
742 procEdgeList[ownProc].append(patchStart + edgeI);
745 procPatchSize_[ownProc][patchI]++;
756 forAll(secondEdgeFaces, edgeI)
760 faceToProc_[firstEdgeFaces[edgeI]]
761 == faceToProc_[secondEdgeFaces[edgeI]]
765 label ownProc = faceToProc_[firstEdgeFaces[edgeI]];
768 procEdgeList[ownProc].append
769 (patchStart + cycOffset + edgeI);
772 procPatchSize_[ownProc][patchI]++;
780 forAll(procEdgeList, procI)
783 SLList<label>& curProcEdges = procEdgeList[procI];
786 labelList& curProcEdgeAddressing = procEdgeAddressing_[procI];
789 procNeighbourProcessors_[procI];
792 procProcessorPatchSize_[procI];
794 labelList& curProcProcessorPatchStartIndex =
795 procProcessorPatchStartIndex_[procI];
798 label nEdgesOnProcessor = curProcEdges.size();
800 for (
const auto& bedges : interProcBEdges[procI])
802 nEdgesOnProcessor += bedges.size();
805 curProcEdgeAddressing.
setSize(nEdgesOnProcessor);
818 for (
const label procEdgei : curProcEdges)
820 curProcEdgeAddressing[nEdges] = procEdgei;
828 curProcNeighbourProcessors.setSize
830 interProcBoundaries[procI].size()
833 curProcProcessorPatchSize.setSize
835 interProcBoundaries[procI].size()
838 curProcProcessorPatchStartIndex.setSize
840 interProcBoundaries[procI].size()
845 auto curInterProcBdrsIter =
846 interProcBoundaries[procI].cbegin();
848 auto curInterProcBEdgesIter =
849 interProcBEdges[procI].cbegin();
854 curInterProcBdrsIter.good()
855 && curInterProcBEdgesIter.good();
856 ++curInterProcBdrsIter,
857 ++curInterProcBEdgesIter
861 curInterProcBdrsIter();
873 for (
const label edgei : *curInterProcBEdgesIter)
879 if (faceToProc_[owner[edgei]] == procI)
881 curProcEdgeAddressing[nEdges] = edgei;
887 curProcEdgeAddressing[nEdges] = edgei;
902 Info <<
"\nCalculating processor boundary addressing" <<
endl;
908 forAll(procPatchSize_, procI)
911 const labelList oldPatchSizes = procPatchSize_[procI];
913 const labelList oldPatchStarts = procPatchStartIndex_[procI];
915 labelList& curPatchSizes = procPatchSize_[procI];
917 labelList& curPatchStarts = procPatchStartIndex_[procI];
919 const labelList& curProcessorPatchSizes =
920 procProcessorPatchSize_[procI];
922 labelList& curBoundaryAddressing = procBoundaryAddressing_[procI];
927 + curProcessorPatchSizes.size()
932 forAll(oldPatchSizes, patchI)
938 curBoundaryAddressing[
nPatches] = patchI;
940 curPatchSizes[
nPatches] = oldPatchSizes[patchI];
942 curPatchStarts[
nPatches] = oldPatchStarts[patchI];
952 forAll(curProcessorPatchSizes, procPatchI)
954 curBoundaryAddressing[
nPatches] = -1;
959 curBoundaryAddressing.setSize(
nPatches);
982 for (label procI = 0; procI < nProcs(); procI++)
985 const labelList& curEdgeLabels = procEdgeAddressing_[procI];
988 const labelList& curProcessorPatchStarts =
989 procProcessorPatchStartIndex_[procI];
991 const labelList& curProcessorPatchSizes =
992 procProcessorPatchSize_[procI];
997 forAll(curProcessorPatchStarts, patchI)
999 const label curStart = curProcessorPatchStarts[patchI];
1000 const label curEnd = curStart + curProcessorPatchSizes[patchI];
1004 label edgeI = curStart;
1011 const label curE = curEdgeLabels[edgeI];
1013 const edge&
e = edges[curE];
1017 if (pointsUsage[
e[pointI]] == 0)
1020 pointsUsage[
e[pointI]] = patchI + 1;
1022 else if (pointsUsage[
e[pointI]] != patchI + 1)
1025 gSharedPoints.insert(
e[pointI]);
1033 globallySharedPoints_ = gSharedPoints.sortedToc();
1039 for (label procI = 0; procI < nProcs(); procI++)
1041 fileName processorCasePath
1043 time().caseName()/(
"processor" +
Foam::name(procI))
1061 processorDb.timeName(),
1070 procFaceLabels_[procI]
1074 const labelList& curEdgeAddressing = procEdgeAddressing_[procI];
1076 const labelList& curPatchStartIndex = procPatchStartIndex_[procI];
1077 const labelList& curPatchSize = procPatchSize_[procI];
1079 const labelList& curProcessorPatchStartIndex =
1080 procProcessorPatchStartIndex_[procI];
1082 const labelList& curProcessorPatchSize =
1083 procProcessorPatchSize_[procI];
1085 labelListList& curPatchEdgeLabels = procPatchEdgeLabels_[procI];
1086 curPatchEdgeLabels =
1090 + curProcessorPatchSize.size()
1093 forAll(curPatchSize, patchI)
1095 labelList& curEdgeLabels = curPatchEdgeLabels[patchI];
1096 curEdgeLabels.
setSize(curPatchSize[patchI], -1);
1102 int i=curPatchStartIndex[patchI];
1103 i<(curPatchStartIndex[patchI]+curPatchSize[patchI]);
1107 curEdgeLabels[edgeI] =
1108 procMeshEdgesMap_[procI][curEdgeAddressing[i]];
1113 forAll(curProcessorPatchSize, patchI)
1116 curPatchEdgeLabels[curPatchSize.size() + patchI];
1117 curEdgeLabels.
setSize(curProcessorPatchSize[patchI], -1);
1123 int i=curProcessorPatchStartIndex[patchI];
1124 i<(curProcessorPatchStartIndex[patchI]
1125 +curProcessorPatchSize[patchI]);
1129 curEdgeLabels[edgeI] =
1130 procMeshEdgesMap_[procI][curEdgeAddressing[i]];
1140 Info<<
"\nConstructing processor FA meshes" <<
endl;
1144 Map<label> sharedPointLookup(2*globallySharedPoints_.size());
1146 forAll(globallySharedPoints_, pointi)
1148 sharedPointLookup.insert(globallySharedPoints_[pointi], pointi);
1151 label totProcEdges = 0;
1152 label maxProcPatches = 0;
1153 label maxProcEdges = 0;
1156 for (label procI = 0; procI < nProcs(); procI++)
1160 fileName processorCasePath
1162 time().caseName()/(
"processor" +
Foam::name(procI))
1179 processorDb.timeName(),
1188 "boundaryProcAddressing",
1190 procFvMesh.meshSubDir,
1202 procFaceLabels_[procI]
1206 const labelList& curBoundaryAddressing =
1207 procBoundaryAddressing_[procI];
1209 const labelList& curPatchSizes = procPatchSize_[procI];
1211 const labelList& curNeighbourProcessors =
1212 procNeighbourProcessors_[procI];
1214 const labelList& curProcessorPatchSizes =
1215 procProcessorPatchSize_[procI];
1218 procPatchEdgeLabels_[procI];
1222 List<faPatch*> procPatches
1224 curPatchSizes.size()
1225 + curProcessorPatchSizes.size(),
1226 reinterpret_cast<faPatch*
>(NULL)
1231 forAll(curPatchSizes, patchi)
1235 const label ngbPolyPatchIndex =
1236 fvBoundaryProcAddressing.find
1238 meshPatches[curBoundaryAddressing[patchi]]
1239 .ngbPolyPatchIndex()
1243 meshPatches[curBoundaryAddressing[patchi]].
clone
1245 procMesh.boundary(),
1254 forAll(curProcessorPatchSizes, procPatchI)
1259 new processorFaPatch
1263 +
Foam::name(curNeighbourProcessors[procPatchI]),
1266 procMesh.boundary(),
1269 curNeighbourProcessors[procPatchI]
1276 procMesh.addFaPatches(procPatches);
1284 <<
"Processor " << procI <<
nl
1285 <<
" Number of faces = " << procMesh.nFaces()
1288 label nBoundaryEdges = 0;
1290 label nProcEdges = 0;
1292 forAll(procMesh.boundary(), patchi)
1296 procMesh.boundary()[patchi].type()
1297 == processorFaPatch::typeName
1300 const processorFaPatch& ppp =
1301 refCast<const processorFaPatch>
1303 procMesh.boundary()[patchi]
1306 Info<<
" Number of edges shared with processor "
1307 << ppp.neighbProcNo() <<
" = " << ppp.size() <<
endl;
1310 nProcEdges += ppp.size();
1314 nBoundaryEdges += procMesh.boundary()[patchi].size();
1319 <<
" Number of processor edges = " << nProcEdges <<
nl
1320 <<
" Number of boundary edges = " << nBoundaryEdges <<
endl;
1322 totProcEdges += nProcEdges;
1324 maxProcEdges =
max(maxProcEdges, nProcEdges);
1331 "pointProcAddressing",
1333 procMesh.meshSubDir,
1338 procPatchPointAddressing_[procI]
1340 pointProcAddressing.write();
1346 "edgeProcAddressing",
1348 procMesh.meshSubDir,
1353 procEdgeAddressing_[procI]
1355 edgeProcAddressing.write();
1361 "faceProcAddressing",
1363 procMesh.meshSubDir,
1368 procFaceAddressing_[procI]
1376 "boundaryProcAddressing",
1378 procMesh.meshSubDir,
1383 procBoundaryAddressing_[procI]
1385 boundaryProcAddressing.write();
1389 <<
"Number of processor edges = " << totProcEdges/2 <<
nl
1390 <<
"Max number of processor patches = " << maxProcPatches <<
nl
1391 <<
"Max number of faces between processors = " << maxProcEdges