24 #if !defined(FEELPP_DOFTABLE_MPI_HPP)
25 #define FEELPP_DOFTABLE_MPI_HPP 1
30 template<
typename MeshType,
typename FEType,
typename PeriodicityType>
32 DofTable<MeshType, FEType, PeriodicityType>::buildGhostDofMap( mesh_type& mesh )
37 std::map<size_type,boost::tuple<size_type,size_type> > mapInterProcessDof;
39 std::map<size_type,boost::tuple<size_type,size_type> > setInterProcessDofNotPresent;
43 DVLOG(2) <<
"[buildGhostDofMap] call buildGhostInterProcessDofMap() with god rank "<< this->worldComm().godRank() <<
"\n";
44 buildGhostInterProcessDofMap( mesh,mapInterProcessDof );
46 DVLOG(2) <<
"[buildGhostDofMap] call buildDofNotPresent() with rank "<< this->worldComm().rank() <<
"\n";
47 buildDofNotPresent( mapInterProcessDof,setInterProcessDofNotPresent );
50 DVLOG(2) <<
"[buildGhostDofMap] call buildGlobalProcessToGlobalClusterDofMap() with rank "<< this->worldComm().rank() <<
"\n";
51 buildGlobalProcessToGlobalClusterDofMap( mesh,setInterProcessDofNotPresent );
57 DVLOG(2) <<
"[buildGhostDofMap] call buildGlobalProcessToGlobalClusterDofMapContinuous() with god rank "<< this->worldComm().godRank() <<
"\n";
58 buildGlobalProcessToGlobalClusterDofMapContinuous( mesh );
62 DVLOG(2) <<
"[buildGhostDofMap] call buildGlobalProcessToGlobalClusterDofMapDiscontinuous() with rank "<< this->worldComm().rank() <<
"\n";
63 buildGlobalProcessToGlobalClusterDofMapDiscontinuous();
69 DVLOG(2) <<
"[buildGhostDofMap] call localtoglobalOnCluster() with rank "<< this->worldComm().rank() <<
"\n";
70 auto it_elt = mesh.beginElementWithProcessId( this->comm().rank() );
71 auto en_elt = mesh.endElementWithProcessId( this->comm().rank() );
73 for ( ; it_elt != en_elt; ++it_elt )
77 for (
int i = 0; i < FEType::nLocalDof; ++i )
79 int nc1 = ( is_product?nComponents1:1 );
81 for (
int c1 =0; c1 < nc1; ++c1 )
83 int ind = FEType::nLocalDof*c1+i;
84 auto const& dof = localToGlobalOnCluster( elid, i, c1 );
86 M_locglobOnCluster_indices[elid][ind] = dof.index();
87 M_locglobOnCluster_signs[elid][ind] = dof.sign();
92 DVLOG(2) <<
"[buildGhostDofMap] finish () with god rank "<< this->worldComm().godRank() <<
"\n";
103 template<
typename MeshType,
typename FEType,
typename PeriodicityType>
105 DofTable<MeshType, FEType, PeriodicityType>::buildGlobalProcessToGlobalClusterDofMapContinuous( mesh_type& mesh )
109 if ( !fe_type::is_modal )
110 nbFaceDof = ( face_type::numVertices * fe_type::nDofPerVertex +
111 face_type::numEdges * fe_type::nDofPerEdge +
112 face_type::numFaces * fe_type::nDofPerFace );
114 nbFaceDof = face_type::numVertices * fe_type::nDofPerVertex;
116 if ( nbFaceDof == 0 )
return;
119 std::vector< std::map<size_type,std::set<boost::tuple<size_type,uint16_type> > > > listToSend(this->worldComm().size());
120 this->buildGlobalProcessToGlobalClusterDofMapContinuousActifDof(mesh,listToSend);
123 this->buildGlobalProcessToGlobalClusterDofMapContinuousGhostDof(mesh,listToSend);
134 template <
typename MeshType>
135 boost::tuple<int,size_type>
136 updateDofOnVertices( MeshType
const& mesh,
typename MeshType::face_type
const& theface,
137 const int IdProcessOfGhost,
const size_type idFaceInPartition,
typename MeshType::element_type
const& eltOnProc,
138 const uint16_type locDof )
140 int procMin = IdProcessOfGhost;
143 uint16_type iFaEl = ( theface.processId() == theface.proc_first() )? theface.pos_first():theface.pos_second();
145 uint16_type iPtEl = MeshType::element_type::fToP( iFaEl, locDof );
150 for ( uint16_type p=0;p<MeshType::face_type::numVertices && !hasFind; ++p)
152 auto const& thept = theface.point(p);
154 for (uint16_type d=0;d<MeshType::nRealDim;++d)
156 find = find && ( std::abs( thept(d)-thedofnode[d] )<1e-9 );
161 auto const theptId = thept.id();
162 if (eltOnProc.point(iPtEl).id()!=theptId) std::cout <<
"COOOOYIOKLK"<< std::endl;
164 auto iteltghost=thept.elementsGhost().begin();
165 auto const eneltghost=thept.elementsGhost().end();
167 for ( ; iteltghost!=eneltghost ; ++iteltghost)
169 if (procMin>iteltghost->template get<0>())
171 procMin=iteltghost->template get<0>();
172 auto const& eltGhost = mesh.element(iteltghost->template get<1>(),procMin);
174 for ( uint16_type f = 0; f < MeshType::element_type::numTopologicalFaces && !findFace; ++f )
176 auto const& faceOnGhost = eltGhost.face(f);
178 for ( uint16_type vv = 0; vv < MeshType::face_type::numVertices && !findFace ; ++vv )
180 if (faceOnGhost.point(vv).id()==theptId) {findFace=
true;idFaceMin = faceOnGhost.idInOthersPartitions(procMin); }
183 if (!findFace) std::cout <<
"YYYYYYY" << std::endl;
189 if (!hasFind) std::cout <<
"JKJKJKJKJKJKJ" << std::endl;
191 auto const& thept = eltOnProc.point(iPtEl);
192 auto const theptId = thept.id();
194 auto iteltghost=thept.elementsGhost().begin();
195 auto const eneltghost=thept.elementsGhost().end();
196 for ( ; iteltghost!=eneltghost ; ++iteltghost)
198 if (procMin>iteltghost->template get<0>())
200 procMin=iteltghost->template get<0>();
201 auto const& eltGhost = mesh.element(iteltghost->template get<1>(),procMin);
203 for ( uint16_type f = 0; f < MeshType::element_type::numTopologicalFaces && !findFace; ++f )
205 auto const& faceOnGhost = eltGhost.face(f);
206 for ( uint16_type vv = 0; vv < MeshType::face_type::numVertices && !findFace ; ++vv )
208 if (faceOnGhost.point(vv).id()==theptId) {findFace=
true;idFaceMin = faceOnGhost.idInOthersPartitions(procMin); }
211 CHECK( findFace ) <<
"\nPROBLEM with parallel dof table construction \n";
215 return boost::make_tuple(procMin,idFaceMin);
221 template <
typename MeshType>
222 boost::tuple<int,size_type>
223 updateDofOnEdges( MeshType
const& mesh,
typename MeshType::face_type
const& theface,
224 const int IdProcessOfGhost,
const size_type idFaceInPartition,
typename MeshType::element_type
const& eltOnProc,
225 const uint16_type idEdgesInFace, mpl::int_<0> )
227 return boost::make_tuple(0,0);
229 template <
typename MeshType>
230 boost::tuple<int,size_type>
231 updateDofOnEdges( MeshType
const& mesh,
typename MeshType::face_type
const& theface,
232 const int IdProcessOfGhost,
const size_type idFaceInPartition,
typename MeshType::element_type
const& eltOnProc,
233 const uint16_type idEdgesInFace, mpl::int_<1> )
235 return boost::make_tuple(0,0);
237 template <
typename MeshType>
238 boost::tuple<int,size_type>
239 updateDofOnEdges( MeshType
const& mesh,
typename MeshType::face_type
const& theface,
240 const int IdProcessOfGhost,
const size_type idFaceInPartition,
typename MeshType::element_type
const& eltOnProc,
241 const uint16_type idEdgesInFace, mpl::int_<2> )
243 return boost::make_tuple(0,0);
245 template <
typename MeshType>
246 boost::tuple<int,size_type>
247 updateDofOnEdges( MeshType
const& mesh,
typename MeshType::face_type
const& theface,
248 const int IdProcessOfGhost,
const size_type idFaceInPartition,
typename MeshType::element_type
const& eltOnProc,
249 const uint16_type idEdgesInFace, mpl::int_<3> )
251 int procMin = IdProcessOfGhost;
254 uint16_type iFaEl = ( theface.processId() == theface.proc_first() )? theface.pos_first():theface.pos_second();
256 uint16_type iEdEl = MeshType::element_type::fToE( iFaEl, idEdgesInFace );
259 auto const& theedge = eltOnProc.edge(iEdEl);
260 auto const theedgeId = theedge.id();
261 auto iteltghost=theedge.elementsGhost().begin();
262 auto const eneltghost=theedge.elementsGhost().end();
264 for ( ; iteltghost!=eneltghost ; ++iteltghost)
266 if (procMin>iteltghost->template get<0>())
268 procMin=iteltghost->template get<0>();
269 auto const& eltGhost = mesh.element(iteltghost->template get<1>(),procMin);
271 for ( uint16_type f = 0; f < MeshType::element_type::numTopologicalFaces && !findFace; ++f )
273 auto const& faceOnGhost = eltGhost.face(f);
274 for ( uint16_type vv = 0; vv < MeshType::face_type::numEdges && !findFace ; ++vv )
276 if (faceOnGhost.edge(vv).id()==theedgeId) { findFace=
true;idFaceMin = faceOnGhost.idInOthersPartitions(procMin); }
279 CHECK( findFace ) <<
"\nPROBLEM with parallel dof table construction \n";
283 return boost::make_tuple(procMin,idFaceMin);
288 template <
typename MeshType>
289 boost::tuple<int,size_type>
290 updateDofOnEdges( MeshType
const& mesh,
typename MeshType::face_type
const& theface,
291 const int IdProcessOfGhost,
const size_type idFaceInPartition,
typename MeshType::element_type
const& eltOnProc,
292 const uint16_type idEdgesInFace )
294 return updateDofOnEdges(mesh,theface,IdProcessOfGhost,idFaceInPartition,eltOnProc,idEdgesInFace,mpl::int_<MeshType::nDim>() );
304 template<
typename MeshType,
typename FEType,
typename PeriodicityType>
306 DofTable<MeshType, FEType, PeriodicityType>::buildGlobalProcessToGlobalClusterDofMapContinuousActifDof( mesh_type& mesh,
307 std::vector< std::map<
size_type,std::set<boost::tuple<size_type,uint16_type> > > > & listToSend )
311 const int myRank = this->worldComm().rank();
315 if ( !fe_type::is_modal )
316 nbFaceDof = ( face_type::numVertices * fe_type::nDofPerVertex +
317 face_type::numEdges * fe_type::nDofPerEdge +
318 face_type::numFaces * fe_type::nDofPerFace );
320 nbFaceDof = face_type::numVertices * fe_type::nDofPerVertex;
322 DVLOG(2) <<
"[buildGhostInterProcessDofMap] nbFaceDof " << nbFaceDof <<
"\n";
324 if ( nbFaceDof == 0 )
return;
326 const uint16_type ncdof = is_product?nComponents:1;
327 DVLOG(2) <<
"[buildGhostInterProcessDofMap] ncdof " << ncdof <<
"\n";
330 std::vector<bool> dofdone(this->M_n_localWithGhost_df[myRank],
false);
334 auto face_it = mesh.interProcessFaces().first;
335 auto const face_en = mesh.interProcessFaces().second;
336 DVLOG(2) <<
"[buildGhostInterProcessDofMap] nb interprocess faces: " << std::distance( face_it, face_en ) <<
"\n";
337 for ( ; face_it != face_en ; ++face_it )
339 DVLOG(2) <<
"[buildGhostInterProcessDofMap] face id: " << face_it->id() <<
"\n";
340 auto const& elt0 = face_it->element0();
341 auto const& elt1 = face_it->element1();
342 const bool elt0isGhost = elt0.isGhostCell();
343 auto const& eltOnProc = (elt0isGhost)?elt1:elt0;
344 auto const& eltOffProc = (elt0isGhost)?elt0:elt1;
345 DVLOG(2) <<
"[buildGhostInterProcessDofMap] (myRank:" << myRank <<
") eltOnProc id: " << eltOnProc.id() <<
"G(): " << eltOnProc.G() <<
"\n";
346 DVLOG(2) <<
"[buildGhostInterProcessDofMap] (myRank:" << myRank <<
") eltOffProc id: " << eltOffProc.id() <<
"G(): " << eltOffProc.G() <<
"\n";
350 const int IdProcessOfGhostIP = eltOffProc.processId();
351 const size_type idFaceInPartitionIP = face_it->idInOthersPartitions( IdProcessOfGhostIP );
352 int IdProcessOfGhost = IdProcessOfGhostIP;
353 size_type idFaceInPartition = idFaceInPartitionIP;
357 for ( uint16_type locDof = 0; locDof < nbFaceDof; ++locDof )
359 IdProcessOfGhost = IdProcessOfGhostIP;
360 idFaceInPartition = idFaceInPartitionIP;
361 if ( locDof < face_type::numVertices*fe_type::nDofPerVertex)
363 const int nDofPerVertexTemp = mpl::if_<boost::is_same<mpl::int_<fe_type::nDofPerVertex>,mpl::int_<0> >,
365 mpl::int_<fe_type::nDofPerVertex> >::type::value;
366 int pointGetLocDof = locDof / nDofPerVertexTemp;
368 boost::tie( IdProcessOfGhost, idFaceInPartition ) = detail::updateDofOnVertices<mesh_type>(mesh,*face_it, IdProcessOfGhost, idFaceInPartition, eltOnProc, pointGetLocDof );
370 else if ( nDim == 3 && locDof < (face_type::numVertices*fe_type::nDofPerVertex + face_type::numEdges*fe_type::nDofPerEdge) )
372 int locDofInEgde = locDof - face_type::numVertices*fe_type::nDofPerVertex;
373 const int nDofPerEdgeTemp = mpl::if_<boost::is_same<mpl::int_<fe_type::nDofPerEdge>,mpl::int_<0> >,
375 mpl::int_<fe_type::nDofPerEdge> >::type::value;
376 int edgeGetLocDof = locDofInEgde / nDofPerEdgeTemp;
378 boost::tie( IdProcessOfGhost, idFaceInPartition ) = detail::updateDofOnEdges<mesh_type>(mesh,*face_it, IdProcessOfGhost, idFaceInPartition, eltOnProc, edgeGetLocDof );
381 if (IdProcessOfGhost<myRank)
383 for ( uint16_type c = 0; c < ncdof; ++c )
386 const size_type theglobdof = faceLocalToGlobal( face_it->id(),locDof,c ).
template get<0>();
387 if (!dofdone[theglobdof])
389 listToSend[IdProcessOfGhost][idFaceInPartition].insert(boost::make_tuple(theglobdof,c));
390 dofdone[theglobdof]=
true;
406 const size_type mynDofWithoutGhost = this->M_n_localWithGhost_df[myRank] - nDofNotPresent;
407 mpi::all_gather( this->worldComm(),
409 this->M_n_localWithoutGhost_df );
412 for (
int proc=0; proc<this->worldComm().size(); ++proc )
414 this->M_n_dofs+=this->M_n_localWithoutGhost_df[proc];
417 this->M_first_df_globalcluster=this->M_first_df;
418 this->M_last_df_globalcluster=this->M_last_df;
419 for (
int i=1; i<this->worldComm().size(); ++i )
421 if ( this->M_n_localWithoutGhost_df[i-1] >0 )
422 this->M_first_df_globalcluster[i]=this->M_last_df_globalcluster[i-1]+1;
424 this->M_first_df_globalcluster[i]=this->M_last_df_globalcluster[i-1];
426 if ( this->M_n_localWithoutGhost_df[i] >0 )
427 this->M_last_df_globalcluster[i]=this->M_first_df_globalcluster[i]+this->M_n_localWithoutGhost_df[i]-1;
429 this->M_last_df_globalcluster[i]=this->M_first_df_globalcluster[i];
434 this->M_mapGlobalClusterToGlobalProcess.resize( this->M_n_localWithoutGhost_df[myRank],
invalid_size_type_value );
437 size_type firstGlobIndex = this->M_first_df_globalcluster[myRank];
438 size_type nextGlobIndex = firstGlobIndex;
439 for (
size_type i=0; i< this->M_n_localWithGhost_df[myRank]; ++i )
444 this->M_mapGlobalProcessToGlobalCluster[i]=nextGlobIndex;
445 this->M_mapGlobalClusterToGlobalProcess[nextGlobIndex-firstGlobIndex]=i;
457 template<
typename MeshType,
typename FEType,
typename PeriodicityType>
459 DofTable<MeshType, FEType, PeriodicityType>::buildGlobalProcessToGlobalClusterDofMapContinuousGhostDof( mesh_type& mesh,
460 std::vector< std::map<
size_type,std::set<boost::tuple<size_type,uint16_type> > > >
const& listToSend )
462 const int myRank = this->worldComm().rank();
465 if ( !fe_type::is_modal )
466 nbFaceDof = ( face_type::numVertices * fe_type::nDofPerVertex +
467 face_type::numEdges * fe_type::nDofPerEdge +
468 face_type::numFaces * fe_type::nDofPerFace );
471 nbFaceDof = face_type::numVertices * fe_type::nDofPerVertex;
474 std::vector<int> nbMsgToSend( this->worldComm().size(), 0 );
476 std::vector< std::vector< std::vector<size_type> > > memoryInitialRequest( this->worldComm().size() );
478 typedef std::vector< boost::tuple<uint16_type, ublas::vector<double> > > dofs_in_face_subcontainer_type;
479 typedef boost::tuple<size_type, dofs_in_face_subcontainer_type > dofs_in_face_container_type;
481 for (
int proc=0; proc<this->worldComm().size(); ++proc )
483 auto itFaces = listToSend[proc].begin();
484 auto const enFaces = listToSend[proc].end();
485 memoryInitialRequest[proc].resize(std::distance(itFaces,enFaces));
486 for (
int cptFaces=0 ; itFaces!=enFaces ; ++itFaces, ++cptFaces)
488 auto itDof = itFaces->second.begin();
489 auto const enDof = itFaces->second.end();
490 auto const nDofsInFace = std::distance(itDof,enDof);
491 dofs_in_face_subcontainer_type dofsInFaceContainer(nDofsInFace);
492 memoryInitialRequest[proc][cptFaces].resize(nDofsInFace);
493 for (
int cptDof=0 ; itDof!=enDof ; ++itDof,++cptDof)
495 auto const theglobdof = itDof->get<0>();
496 auto const comp = itDof->get<1>();
498 memoryInitialRequest[proc][cptFaces][cptDof] = theglobdof;
501 ublas::vector<double> nodeDofToSend( nRealDim );
502 nodeDofToSend[0]=dofPoint( theglobdof ).template get<0>()[0];
504 nodeDofToSend[1]=dofPoint( theglobdof ).template get<0>()[1];
506 nodeDofToSend[2]=dofPoint( theglobdof ).template get<0>()[2];
508 dofsInFaceContainer[cptDof] = boost::make_tuple(comp,nodeDofToSend);
512 this->worldComm().send( proc , nbMsgToSend[proc], boost::make_tuple(itFaces->first,dofsInFaceContainer) );
519 std::vector<int> nbMsgToRecv;
520 mpi::all_to_all( this->worldComm(),
525 for (
int proc=0; proc<this->worldComm().size(); ++proc )
527 for (
int cpt=0; cpt<nbMsgToRecv[proc]; ++cpt )
529 dofs_in_face_container_type dataToRecvVec;
530 this->worldComm().recv( proc, cpt, dataToRecvVec );
532 auto const idFaceInMyPartition = dataToRecvVec.template get<0>();
533 DVLOG(2) <<
"[buildGhostInterProcessDofMap] (myRank:" << myRank <<
") "
534 <<
"idFaceInMyPartition: " << idFaceInMyPartition <<
"\n";
536 auto itDofInFace = dataToRecvVec.template get<1>().begin();
537 auto const enDofInFace = dataToRecvVec.template get<1>().end();
538 std::vector< size_type > resAskedWithMultiProcess(std::distance(itDofInFace,enDofInFace));
539 for (
int cptDofInFace=0 ; itDofInFace != enDofInFace ; ++itDofInFace,++cptDofInFace )
541 auto const comp = itDofInFace->template get<0>();
542 auto const nodeDofRecv = itDofInFace->template get<1>();
546 auto const& theface = mesh.face( idFaceInMyPartition );
547 auto const& elt0 = theface.element0();
548 auto const& elt1 = theface.element1();
550 const bool elt0isGhost = elt0.isGhostCell();
551 auto const& eltOnProc = (elt0isGhost)?elt1:elt0;
552 auto const& eltOffProc = (elt0isGhost)?elt0:elt1;
556 int locDof = nbFaceDof;
561 auto itdofpt = this->dofPointBegin();
562 auto const endofpt = this->dofPointEnd();
563 for ( ; itdofpt!=endofpt && !find ; ++itdofpt )
565 const auto thedofPt = itdofpt->template get<0>();
566 if ( itdofpt->template get<2>() != comp )
continue;
568 DVLOG(3) <<
"[buildGhostInterProcessDofMap] (myRank:" << myRank <<
") "
569 <<
"thedofPt: " << thedofPt <<
"nodeDofRecv: " << nodeDofRecv <<
"\n";
573 for (uint16_type d=0;d<nRealDim;++d)
575 find2 = find2 && (std::abs( thedofPt[d]-nodeDofRecv[d] )<1e-9);
578 if (find2) { locDof = itdofpt->template get<1>();find=
true; }
581 CHECK( find ) <<
"\nPROBLEM with parallel dof table construction : Dof point not find on interprocess face " << nodeDofRecv <<
"\n";
584 const auto dofGlobAsked = locDof;
586 resAskedWithMultiProcess[cptDofInFace] = this->M_mapGlobalProcessToGlobalCluster[dofGlobAsked];
590 for ( uint16_type l = 0; ( l < nbFaceDof && !find ) ; ++l )
593 const auto thedofPtInFace = dofPoint(faceLocalToGlobal( idFaceInMyPartition, l, comp ).
template get<0>()).template get<0>();
594 DVLOG(3) <<
"[buildGhostInterProcessDofMap] (myRank:" << myRank <<
") "
595 <<
"thedofPtInFace: " << thedofPtInFace <<
"nodeDofRecv: " << nodeDofRecv <<
"\n";
599 for (uint16_type d=0;d<nRealDim;++d)
601 find2 = find2 && (std::abs( thedofPtInFace[d]-nodeDofRecv[d] )<1e-9);
612 CHECK( find ) <<
"\nPROBLEM with parallel dof table construction : Dof point not find on interprocess face " << nodeDofRecv <<
"\n";
615 const auto thedof = faceLocalToGlobal( idFaceInMyPartition, locDof, comp );
616 const auto dofGlobAsked = thedof.template get<0>();
618 resAskedWithMultiProcess[cptDofInFace] = this->M_mapGlobalProcessToGlobalCluster[dofGlobAsked];
622 this->worldComm().send( proc, cpt, resAskedWithMultiProcess );
629 for (
int proc=0; proc<this->worldComm().size(); ++proc )
631 for (
int cpt=0; cpt<nbMsgToSend[proc]; ++cpt )
635 std::vector< size_type > resultRecvWithMultiProcess;
636 this->worldComm().recv( proc, cpt, resultRecvWithMultiProcess );
639 auto itDofRes = resultRecvWithMultiProcess.begin();
640 auto const enDofRes = resultRecvWithMultiProcess.end();
641 for (
int cptDofRes=0 ; itDofRes != enDofRes ; ++itDofRes,++cptDofRes )
643 auto const myGlobProcessDof = memoryInitialRequest[proc][cpt][cptDofRes];
644 const auto dofGlobRecv = *itDofRes;
646 this->M_mapGlobalProcessToGlobalCluster[myGlobProcessDof]=dofGlobRecv;
660 template<
typename MeshType,
typename FEType,
typename PeriodicityType>
662 DofTable<MeshType, FEType, PeriodicityType>::buildGlobalProcessToGlobalClusterDofMapDiscontinuous()
664 const int myRank = this->worldComm().rank();
667 const size_type mynDofWithoutGhost = this->M_last_df[myRank] - this->M_first_df[myRank] + 1;
668 mpi::all_gather( this->worldComm(),
670 this->M_n_localWithoutGhost_df );
673 for (
int proc=0; proc<this->worldComm().size(); ++proc )
675 this->M_n_dofs+=this->M_n_localWithoutGhost_df[proc];
678 this->M_first_df_globalcluster=this->M_first_df;
679 this->M_last_df_globalcluster=this->M_last_df;
680 for (
int i=1; i<this->worldComm().size(); ++i )
682 this->M_first_df_globalcluster[i]=this->M_last_df_globalcluster[i-1]+1;
683 this->M_last_df_globalcluster[i]=this->M_first_df_globalcluster[i]+this->M_n_localWithoutGhost_df[i]-1;
688 this->M_mapGlobalClusterToGlobalProcess.resize( this->M_n_localWithoutGhost_df[myRank],
invalid_size_type_value );
691 size_type firstGlobIndex = this->M_first_df_globalcluster[myRank];
692 size_type nextGlobIndex = firstGlobIndex;
693 for (
size_type i=0; i< this->M_n_localWithGhost_df[myRank]; ++i )
697 this->M_mapGlobalProcessToGlobalCluster[i]=nextGlobIndex;
698 this->M_mapGlobalClusterToGlobalProcess[nextGlobIndex-firstGlobIndex]=i;
720 template<
typename MeshType,
typename FEType,
typename PeriodicityType>
722 DofTable<MeshType, FEType, PeriodicityType>::buildGhostInterProcessDofMap( mesh_type& mesh,
723 std::map<
size_type,boost::tuple<size_type,size_type> > & mapInterProcessDof )
728 if ( !fe_type::is_modal )
729 nbFaceDof = ( face_type::numVertices * fe_type::nDofPerVertex +
730 face_type::numEdges * fe_type::nDofPerEdge +
731 face_type::numFaces * fe_type::nDofPerFace );
733 nbFaceDof = face_type::numVertices * fe_type::nDofPerVertex;
735 if ( nbFaceDof == 0 )
return;
739 std::vector< std::map<size_type,std::set<boost::tuple<size_type,uint16_type> > > > listToSend(this->worldComm().size());
740 this->buildGhostInterProcessDofMapInit(mesh,listToSend);
745 std::vector< std::set<size_type > > memoryFace(this->worldComm().size());
746 bool allDofAreFind =
false;
747 while (!allDofAreFind)
749 auto const& res = this->buildGhostInterProcessDofMapRecursive(mesh,listToSend,
753 allDofAreFind = res.template get<0>();
754 if (!allDofAreFind) listToSend = res.template get<1>();
765 template<
typename MeshType,
typename FEType,
typename PeriodicityType>
767 DofTable<MeshType, FEType, PeriodicityType>::buildGhostInterProcessDofMapInit( mesh_type& mesh,
768 std::vector< std::map<
size_type,std::set<boost::tuple<size_type,uint16_type> > > > & listToSend )
773 const int myRank = this->worldComm().rank();
777 if ( !fe_type::is_modal )
778 nbFaceDof = ( face_type::numVertices * fe_type::nDofPerVertex +
779 face_type::numEdges * fe_type::nDofPerEdge +
780 face_type::numFaces * fe_type::nDofPerFace );
782 nbFaceDof = face_type::numVertices * fe_type::nDofPerVertex;
784 if ( nbFaceDof == 0 )
return;
786 std::vector<bool> dofdone(this->M_n_localWithGhost_df[myRank],
false);
788 auto face_it = mesh.interProcessFaces().first;
789 auto const face_en = mesh.interProcessFaces().second;
790 DVLOG(2) <<
"[buildGhostInterProcessDofMap] nb interprocess faces: " << std::distance( face_it, face_en ) <<
"\n";
791 for ( ; face_it != face_en ; ++face_it )
793 DVLOG(2) <<
"[buildGhostInterProcessDofMap] face id: " << face_it->id() <<
"\n";
794 auto const& elt0 = face_it->element0();
795 auto const& elt1 = face_it->element1();
796 const bool elt0isGhost = elt0.isGhostCell();
797 auto const& eltOnProc = (elt0isGhost)?elt1:elt0;
798 auto const& eltOffProc = (elt0isGhost)?elt0:elt1;
801 element_type eltOnProc,eltOffProc;
802 if ( elt0.processId()!=myRank )
807 else if ( elt1.processId()!=myRank )
812 else std::cout <<
"\n[buildGhostInterProcessDofMapInit] PROBLEME1!!!!" << std::endl;
816 const int IdProcessOfGhost = eltOffProc.processId();
817 const size_type idFaceInPartition = face_it->idInOthersPartitions( IdProcessOfGhost );
820 const uint16_type ncdof = is_product?nComponents:1;
822 std::set<boost::tuple<size_type,uint16_type> > dofSetAdd;
824 for ( uint16_type l = 0; l < nbFaceDof; ++l )
826 for ( uint16_type c = 0; c < ncdof; ++c )
829 const size_type theglobdof = faceLocalToGlobal( face_it->id(),l,c ).
template get<0>();
830 if (!dofdone[theglobdof])
832 dofSetAdd.insert(boost::make_tuple(theglobdof,c));
833 dofdone[theglobdof]=
true;
838 listToSend[IdProcessOfGhost].insert( std::make_pair( idFaceInPartition, dofSetAdd ) );
851 template<
typename element_type>
853 searchPartitionAroundNode(
const int myRank, ublas::vector<double>
const& nodeSearched,
854 element_type
const& eltOnProc, std::set<size_type> & memoryIdsFaces,
855 std::set<boost::tuple<int,size_type> > & multiProcessRes )
857 for ( uint16_type f = 0; f < element_type::numTopologicalFaces; ++f )
859 auto const& theFace = eltOnProc.face(f);
861 if (memoryIdsFaces.find(theFaceId) != memoryIdsFaces.end())
continue;
864 memoryIdsFaces.insert( theFaceId );
867 auto const& facevertices = theFace.vertices();
868 for (uint16_type v=0;v<element_type::face_type::numVertices && !find ;++v)
871 for (uint16_type d=0;d<element_type::nDim;++d)
873 find2 = find2 && ( std::abs( facevertices(d,v)-nodeSearched[d] )<1e-9 );
875 if (find2) find=
true;
880 if ( theFace.isConnectedTo0() )
882 auto const& eltOnProc0 = theFace.element0();
883 if (eltOnProc0.processId()!=myRank)
884 multiProcessRes.insert( boost::make_tuple(eltOnProc0.processId(),theFace.idInOthersPartitions(eltOnProc0.processId()) ));
885 else if (eltOnProc.id()!=eltOnProc0.id())
886 searchPartitionAroundNode(myRank,nodeSearched,eltOnProc0,memoryIdsFaces,multiProcessRes );
888 if ( theFace.isConnectedTo1() )
890 auto const& eltOnProc1 = theFace.element1();
891 if (eltOnProc1.processId()!=myRank)
892 multiProcessRes.insert( boost::make_tuple(eltOnProc1.processId(),theFace.idInOthersPartitions(eltOnProc1.processId()) ));
893 else if (eltOnProc.id()!=eltOnProc1.id())
894 searchPartitionAroundNode(myRank,nodeSearched,eltOnProc1,memoryIdsFaces,multiProcessRes );
903 template<
typename mesh_type>
905 searchPartitionAroundEdge(
const int myRank,
906 typename mesh_type::face_type
const& faceContainEdge,
const uint16_type idEdgesInFace,
907 typename mesh_type::element_type
const& eltOnProc, std::set<size_type> & memoryIdsFaces,
908 std::set<boost::tuple<int,size_type> > & multiProcessRes,
bool firstExp, mpl::int_<0> )
911 template<
typename mesh_type>
913 searchPartitionAroundEdge(
const int myRank,
914 typename mesh_type::face_type
const& faceContainEdge,
const uint16_type idEdgesInFace,
915 typename mesh_type::element_type
const& eltOnProc, std::set<size_type> & memoryIdsFaces,
916 std::set<boost::tuple<int,size_type> > & multiProcessRes,
bool firstExp, mpl::int_<1> )
919 template<
typename mesh_type>
921 searchPartitionAroundEdge(
const int myRank,
922 typename mesh_type::face_type
const& faceContainEdge,
const uint16_type idEdgesInFace,
923 typename mesh_type::element_type
const& eltOnProc, std::set<size_type> & memoryIdsFaces,
924 std::set<boost::tuple<int,size_type> > & multiProcessRes,
bool firstExp, mpl::int_<2> )
927 template<
typename mesh_type>
929 searchPartitionAroundEdge(
const int myRank,
930 typename mesh_type::face_type
const& faceContainEdge,
const uint16_type idEdgesInFace,
931 typename mesh_type::element_type
const& eltOnProc, std::set<size_type> & memoryIdsFaces,
932 std::set<boost::tuple<int,size_type> > & multiProcessRes,
bool firstExp, mpl::int_<3> )
936 uint16_type iFaEl=0,iEdEl=0;
939 iFaEl = ( faceContainEdge.processId() == faceContainEdge.proc_first() )? faceContainEdge.pos_first():faceContainEdge.pos_second();
941 iEdEl = mesh_type::element_type::fToE( iFaEl, idEdgesInFace );
944 auto const& theedge = (firstExp)?eltOnProc.edge(iEdEl):faceContainEdge.edge(idEdgesInFace);
947 for ( uint16_type f = 0; f < mesh_type::element_type::numTopologicalFaces; ++f )
949 auto const& theFace = eltOnProc.face(f);
954 if (memoryIdsFaces.find(theFaceId) != memoryIdsFaces.end())
continue;
957 memoryIdsFaces.insert( theFaceId );
960 bool find=
false;uint16_type newEdgeIdInFace=0;
961 for (uint16_type ed=0;ed<mesh_type::element_type::face_type::numEdges && !find ;++ed)
964 if (theFace.edge(ed).id() == theedge.id())
966 find=
true; newEdgeIdInFace=ed;
969 auto const& newEdgesVertices = theFace.edge(ed).vertices();
972 for (uint16_type d=0;d<mesh_type::element_type::nDim;++d)
974 find2 = find2 && ( std::abs( newEdgesVertices(d,0)-theEdgesVertices(d,0) )<1e-9 );
976 if (!find2) std::cout <<
"\n IAOIOIOAIOIO"<<std::endl;
984 if ( theFace.isConnectedTo0() )
986 auto const& eltOnProc0 = theFace.element0();
987 if (eltOnProc0.processId()!=myRank)
988 multiProcessRes.insert( boost::make_tuple(eltOnProc0.processId(),theFace.idInOthersPartitions(eltOnProc0.processId()) ));
989 else if (eltOnProc.id()!=eltOnProc0.id())
990 searchPartitionAroundEdge<mesh_type>(myRank,theFace,newEdgeIdInFace,eltOnProc0,memoryIdsFaces,multiProcessRes,
false,mpl::int_<3>());
992 if ( theFace.isConnectedTo1() )
994 auto const& eltOnProc1 = theFace.element1();
995 if (eltOnProc1.processId()!=myRank)
996 multiProcessRes.insert( boost::make_tuple(eltOnProc1.processId(),theFace.idInOthersPartitions(eltOnProc1.processId()) ));
997 else if (eltOnProc.id()!=eltOnProc1.id())
998 searchPartitionAroundEdge<mesh_type>(myRank,theFace,newEdgeIdInFace,eltOnProc1,memoryIdsFaces,multiProcessRes,
false,mpl::int_<3>());
1006 template<
typename mesh_type>
1008 searchPartitionAroundEdge(
const int myRank,
1009 typename mesh_type::face_type
const& faceContainEdge,
const uint16_type idEdgesInFace,
1010 typename mesh_type::element_type
const& eltOnProc, std::set<size_type> & memoryIdsFaces,
1011 std::set<boost::tuple<int,size_type> > & multiProcessRes )
1013 searchPartitionAroundEdge<mesh_type>(myRank,faceContainEdge,idEdgesInFace,eltOnProc,memoryIdsFaces,multiProcessRes,
true,mpl::int_<mesh_type::nDim>());
1018 template<
typename MeshType,
typename FEType,
typename PeriodicityType>
1019 boost::tuple<bool, std::vector< std::map<size_type,std::set<boost::tuple<size_type,uint16_type> > > > >
1020 DofTable<MeshType, FEType, PeriodicityType>::buildGhostInterProcessDofMapRecursive( mesh_type& mesh,
1021 std::vector< std::map<
size_type,std::set<boost::tuple<size_type,uint16_type> > > >
const& listToSend,
1022 std::map<
size_type,boost::tuple<size_type,size_type> > & mapInterProcessDof,
1023 std::vector< std::set<size_type > > & memoryFace )
1025 const int myRank = this->worldComm().rank();
1028 if ( !fe_type::is_modal )
1029 nbFaceDof = ( face_type::numVertices * fe_type::nDofPerVertex +
1030 face_type::numEdges * fe_type::nDofPerEdge +
1031 face_type::numFaces * fe_type::nDofPerFace );
1034 nbFaceDof = face_type::numVertices * fe_type::nDofPerVertex;
1037 std::vector<int> nbMsgToSend( this->worldComm().size(), 0 );
1039 std::vector< std::map<size_type,std::set<boost::tuple<size_type,uint16_type> > > > listNeedMoreSearch(this->worldComm().size());
1040 int counterListNeedMoreSearch = 0, counterListNeedMoreSearchLoc = 0;
1042 std::vector< std::vector< std::vector<boost::tuple<uint16_type,int> > > > memoryInitialRequest( this->worldComm().size() );
1044 typedef std::vector< boost::tuple<uint16_type, ublas::vector<double> > > dofs_in_face_subcontainer_type;
1045 typedef boost::tuple<size_type, dofs_in_face_subcontainer_type > dofs_in_face_container_type;
1047 for (
int proc=0; proc<this->worldComm().size(); ++proc )
1049 auto itFaces = listToSend[proc].begin();
1050 auto const enFaces = listToSend[proc].end();
1051 memoryInitialRequest[proc].resize(std::distance(itFaces,enFaces));
1052 for (
int cptFaces=0 ; itFaces!=enFaces ; ++itFaces, ++cptFaces)
1054 auto itDof = itFaces->second.begin();
1055 auto const enDof = itFaces->second.end();
1056 auto const nDofsInFace = std::distance(itDof,enDof);
1057 dofs_in_face_subcontainer_type dofsInFaceContainer(nDofsInFace);
1058 memoryInitialRequest[proc][cptFaces].resize(nDofsInFace);
1059 for (
int cptDof=0 ; itDof!=enDof ; ++itDof,++cptDof)
1061 auto const theglobdof = itDof->get<0>();
1062 auto const comp = itDof->get<1>();
1064 memoryInitialRequest[proc][cptFaces][cptDof] = boost::make_tuple(comp,theglobdof);
1067 ublas::vector<double> nodeDofToSend( nDim );
1068 nodeDofToSend[0]=dofPoint( theglobdof ).template get<0>()[0];
1070 nodeDofToSend[1]=dofPoint( theglobdof ).template get<0>()[1];
1072 nodeDofToSend[2]=dofPoint( theglobdof ).template get<0>()[2];
1074 dofsInFaceContainer[cptDof] = boost::make_tuple(comp,nodeDofToSend);
1078 this->worldComm().send( proc , nbMsgToSend[proc], boost::make_tuple(itFaces->first,dofsInFaceContainer) );
1079 ++nbMsgToSend[proc];
1085 std::vector<int> nbMsgToRecv;
1086 mpi::all_to_all( this->worldComm(),
1091 for (
int proc=0; proc<this->worldComm().size(); ++proc )
1093 for (
int cpt=0; cpt<nbMsgToRecv[proc]; ++cpt )
1095 dofs_in_face_container_type dataToRecvVec;
1096 this->worldComm().recv( proc, cpt, dataToRecvVec );
1098 auto const idFaceInMyPartition = dataToRecvVec.template get<0>();
1100 auto itDofInFace = dataToRecvVec.template get<1>().begin();
1101 auto const enDofInFace = dataToRecvVec.template get<1>().end();
1102 std::vector< boost::tuple<size_type,std::vector<boost::tuple<int,size_type> > > > resAskedWithMultiProcess(std::distance(itDofInFace,enDofInFace));
1103 for (
int cptDofInFace=0 ; itDofInFace != enDofInFace ; ++itDofInFace,++cptDofInFace )
1105 auto const comp = itDofInFace->template get<0>();
1106 auto const nodeDofRecv = itDofInFace->template get<1>();
1110 auto const& theface = mesh.face( idFaceInMyPartition );
1111 auto const& elt0 = theface.element0();
1112 auto const& elt1 = theface.element1();
1114 const bool elt0isGhost = elt0.isGhostCell();
1115 auto const& eltOnProc = (elt0isGhost)?elt1:elt0;
1116 auto const& eltOffProc = (elt0isGhost)?elt0:elt1;
1119 element_type eltOnProc, eltOffProc;
1121 if ( elt0.processId()!=myRank )
1125 faceIdInEltOnProc = theface.pos_second();
1127 else if ( elt1.processId()!=myRank )
1131 faceIdInEltOnProc = theface.pos_first();
1135 CHECK( (elt0.processId()==myRank) ||
1136 (elt1.processId()==myRank) )
1137 <<
"\nPROBLEM with parallel dof table construction\n"
1138 <<
" elt0.processId() " << elt0.processId()
1139 <<
" elt1.processId() " << elt1.processId()
1146 int locDof = nbFaceDof;
1149 for ( uint16_type l = 0; ( l < nbFaceDof && !find ) ; ++l )
1152 const auto thedofPtInFace = dofPoint(faceLocalToGlobal( idFaceInMyPartition, l, comp ).
template get<0>()).template get<0>();
1155 for (uint16_type d=0;d<nDim;++d)
1157 find2 = find2 && (std::abs( thedofPtInFace[d]-nodeDofRecv[d] )<1e-9);
1169 CHECK( find ) <<
"\nPROBLEM with parallel dof table construction : Dof point not find on interprocess face " << nodeDofRecv <<
"\n";
1172 const auto thedof = faceLocalToGlobal( idFaceInMyPartition, locDof, comp );
1173 const auto dofGlobAsked = thedof.template get<0>();
1179 std::set<size_type> memoryIdsFaces;
1180 std::set<boost::tuple<int,size_type> > multiProcessRes;
1181 if ( locDof < face_type::numVertices*fe_type::nDofPerVertex)
1183 detail::searchPartitionAroundNode( myRank, nodeDofRecv, eltOnProc, memoryIdsFaces, multiProcessRes );
1185 else if ( nDim == 3 && locDof < (face_type::numVertices*fe_type::nDofPerVertex + face_type::numEdges*fe_type::nDofPerEdge) )
1187 int locDofInEgde = locDof - face_type::numVertices*fe_type::nDofPerVertex;
1189 const int nDofPerEdgeTemp = mpl::if_<boost::is_same<mpl::int_<fe_type::nDofPerEdge>,mpl::int_<0> >,
1191 mpl::int_<fe_type::nDofPerEdge> >::type::value;
1193 int edgeGetLocDof = locDofInEgde / nDofPerEdgeTemp;
1195 detail::searchPartitionAroundEdge<mesh_type>( myRank, theface, edgeGetLocDof, eltOnProc, memoryIdsFaces, multiProcessRes );
1200 auto const nbMultiProcessRes = multiProcessRes.size();
1201 std::vector<boost::tuple<int,size_type> > multiProcessResVec( nbMultiProcessRes );
1202 std::copy( multiProcessRes.begin(),multiProcessRes.end(),
1203 multiProcessResVec.begin() );
1208 resAskedWithMultiProcess[cptDofInFace] = boost::make_tuple(dofGlobAsked,multiProcessResVec);
1211 this->worldComm().send( proc, cpt, resAskedWithMultiProcess );
1218 for (
int proc=0; proc<this->worldComm().size(); ++proc )
1220 for (
int cpt=0; cpt<nbMsgToSend[proc]; ++cpt )
1224 std::vector< boost::tuple<size_type,std::vector<boost::tuple<int,size_type> > > > resultRecvWithMultiProcess;
1225 this->worldComm().recv( proc, cpt, resultRecvWithMultiProcess );
1228 auto itDofRes = resultRecvWithMultiProcess.begin();
1229 auto const enDofRes = resultRecvWithMultiProcess.end();
1230 for (
int cptDofRes=0 ; itDofRes != enDofRes ; ++itDofRes,++cptDofRes )
1233 auto const mycomp = memoryInitialRequest[proc][cpt][cptDofRes].template get<0>();
1234 auto const myGlobProcessDof = memoryInitialRequest[proc][cpt][cptDofRes].template get<1>();
1235 const auto dofGlobRecv = itDofRes->template get<0>();
1236 const auto multiProcessRes = itDofRes->template get<1>();
1240 if ( mapInterProcessDof.find( myGlobProcessDof ) == mapInterProcessDof.end() )
1242 mapInterProcessDof.insert( std::make_pair( myGlobProcessDof, boost::make_tuple( proc,dofGlobRecv ) ) );
1244 else if ( (
int )mapInterProcessDof.find( myGlobProcessDof )->second.template get<0>() > proc )
1246 mapInterProcessDof[ myGlobProcessDof ] = boost::make_tuple( proc,dofGlobRecv );
1250 memoryFace[proc].insert(myGlobProcessDof);
1253 auto itMP = multiProcessRes.begin();
1254 auto const enMP = multiProcessRes.end();
1255 for ( ; itMP!=enMP ;++itMP )
1257 auto const eltMultiProcessProcId = itMP->template get<0>();
1258 auto const eltMultiProcessFaceId = itMP->template get<1>();
1259 if ( memoryFace[eltMultiProcessProcId].find( myGlobProcessDof ) == memoryFace[eltMultiProcessProcId].end() )
1261 listNeedMoreSearch[eltMultiProcessProcId][eltMultiProcessFaceId].insert(boost::make_tuple(myGlobProcessDof,mycomp));
1262 ++counterListNeedMoreSearchLoc;
1271 mpi::all_reduce( this->worldComm().globalComm(),
1272 counterListNeedMoreSearchLoc,
1273 counterListNeedMoreSearch,
1277 return boost::make_tuple( (
bool) (counterListNeedMoreSearch == 0),
1278 listNeedMoreSearch);
1288 template<
typename MeshType,
typename FEType,
typename PeriodicityType>
1290 DofTable<MeshType, FEType, PeriodicityType>::buildDofNotPresent( std::map<
size_type,boost::tuple<size_type,size_type> >
const & mapInterProcessDof,
1291 std::map<
size_type,boost::tuple<size_type,size_type> > & setInterProcessDofNotPresent )
1294 const int myRank = this->worldComm().rank();
1296 auto it_mapIPDof = mapInterProcessDof.begin();
1297 auto const en_mapIPDof = mapInterProcessDof.end();
1298 for ( ; it_mapIPDof!=en_mapIPDof; ++it_mapIPDof )
1300 if ( it_mapIPDof->second.template get<0>() < myRank )
1301 setInterProcessDofNotPresent[it_mapIPDof->first] = it_mapIPDof->second;
1310 template<
typename MeshType,
typename FEType,
typename PeriodicityType>
1312 DofTable<MeshType, FEType, PeriodicityType>::buildGlobalProcessToGlobalClusterDofMap( mesh_type& mesh,
1313 std::map<
size_type,boost::tuple<size_type,size_type> >
const& setInterProcessDofNotPresent )
1315 const int myRank = this->worldComm().rank();
1320 const size_type mynDofWithoutGhost = this->M_n_localWithGhost_df[myRank] - setInterProcessDofNotPresent.size();
1321 mpi::all_gather( this->worldComm(),
1323 this->M_n_localWithoutGhost_df );
1326 for (
int proc=0; proc<this->worldComm().size(); ++proc )
1328 this->M_n_dofs+=this->M_n_localWithoutGhost_df[proc];
1331 this->M_first_df_globalcluster=this->M_first_df;
1332 this->M_last_df_globalcluster=this->M_last_df;
1333 for (
int i=1; i<this->worldComm().size(); ++i )
1335 if ( this->M_n_localWithoutGhost_df[i-1] >0 )
1336 this->M_first_df_globalcluster[i]=this->M_last_df_globalcluster[i-1]+1;
1338 this->M_first_df_globalcluster[i]=this->M_last_df_globalcluster[i-1];
1340 if ( this->M_n_localWithoutGhost_df[i] >0 )
1341 this->M_last_df_globalcluster[i]=this->M_first_df_globalcluster[i]+this->M_n_localWithoutGhost_df[i]-1;
1343 this->M_last_df_globalcluster[i]=this->M_first_df_globalcluster[i];
1348 this->M_mapGlobalProcessToGlobalCluster.resize( this->M_n_localWithGhost_df[myRank],
invalid_size_type_value );
1349 this->M_mapGlobalClusterToGlobalProcess.resize( this->M_n_localWithoutGhost_df[myRank],
invalid_size_type_value );
1353 size_type firstGlobIndex = this->M_first_df_globalcluster[myRank];
1354 size_type nextGlobIndex = firstGlobIndex;
1355 for (
size_type i=0; i< this->M_n_localWithGhost_df[myRank]; ++i )
1357 if ( setInterProcessDofNotPresent.find( i )==setInterProcessDofNotPresent.end() )
1359 this->M_mapGlobalProcessToGlobalCluster[i]=nextGlobIndex;
1360 this->M_mapGlobalClusterToGlobalProcess[nextGlobIndex-firstGlobIndex]=i;
1367 this->updateGhostGlobalDof( setInterProcessDofNotPresent );
1376 template<
typename MeshType,
typename FEType,
typename PeriodicityType>
1378 DofTable<MeshType, FEType, PeriodicityType>::updateGhostGlobalDof( std::map<
size_type,boost::tuple<size_type,size_type> >
const& setInterProcessDofNotPresent )
1380 std::vector<int> nbMsgToSend2( this->worldComm().size(), 0 );
1381 std::vector< std::map<int,size_type> > mapMsg2( this->worldComm().size() );
1383 auto dofNP_it = setInterProcessDofNotPresent.begin();
1384 auto dofNP_en = setInterProcessDofNotPresent.end();
1385 for ( ; dofNP_it!=dofNP_en ; ++dofNP_it )
1387 auto const globalProcessDof = dofNP_it->first;
1388 auto const IdProcessOfGhost = dofNP_it->second.template get<0>();
1389 auto const dofFaceInPartition = dofNP_it->second.template get<1>();
1391 this->worldComm().send( IdProcessOfGhost , nbMsgToSend2[IdProcessOfGhost], dofFaceInPartition );
1392 mapMsg2[IdProcessOfGhost].insert( std::make_pair( nbMsgToSend2[IdProcessOfGhost],globalProcessDof ) );
1393 ++nbMsgToSend2[IdProcessOfGhost];
1397 std::vector<int> nbMsgToRecv2;
1398 mpi::all_to_all( this->worldComm(),
1403 for (
int proc=0; proc<this->worldComm().size(); ++proc )
1405 for (
int cpt=0; cpt<nbMsgToRecv2[proc]; ++cpt )
1409 this->worldComm().recv( proc, cpt, dofRecv );
1411 size_type const dofToSend= this->M_mapGlobalProcessToGlobalCluster[dofRecv];
1412 this->worldComm().send( proc, cpt,dofToSend );
1417 for (
int proc=0; proc<this->worldComm().size(); ++proc )
1419 for (
int cpt=0; cpt<nbMsgToSend2[proc]; ++cpt )
1423 this->worldComm().recv( proc, cpt, dofGlobClusterRecv );
1425 this->M_mapGlobalProcessToGlobalCluster[mapMsg2[proc][cpt]]=dofGlobClusterRecv;