Back to index

salome-med  6.5.0
ElementLocator.cxx
Go to the documentation of this file.
00001 // Copyright (C) 2007-2012  CEA/DEN, EDF R&D
00002 //
00003 // This library is free software; you can redistribute it and/or
00004 // modify it under the terms of the GNU Lesser General Public
00005 // License as published by the Free Software Foundation; either
00006 // version 2.1 of the License.
00007 //
00008 // This library is distributed in the hope that it will be useful,
00009 // but WITHOUT ANY WARRANTY; without even the implied warranty of
00010 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00011 // Lesser General Public License for more details.
00012 //
00013 // You should have received a copy of the GNU Lesser General Public
00014 // License along with this library; if not, write to the Free Software
00015 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
00016 //
00017 // See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
00018 //
00019 
00020 #include <mpi.h>
00021 #include "CommInterface.hxx"
00022 #include "ElementLocator.hxx"
00023 #include "Topology.hxx"
00024 #include "BlockTopology.hxx"
00025 #include "ParaFIELD.hxx"
00026 #include "ParaMESH.hxx"
00027 #include "ProcessorGroup.hxx"
00028 #include "MPIProcessorGroup.hxx"
00029 #include "MEDCouplingFieldDouble.hxx"
00030 #include "DirectedBoundingBox.hxx"
00031 
00032 #include <map>
00033 #include <set>
00034 #include <limits>
00035 
00036 using namespace std;
00037 
00038 //#define USE_DIRECTED_BB
00039 
00040 namespace ParaMEDMEM 
00041 { 
00042   ElementLocator::ElementLocator(const ParaFIELD& sourceField,
00043                                  const ProcessorGroup& distant_group,
00044                                  const ProcessorGroup& local_group)
00045     : _local_para_field(sourceField),
00046       _local_cell_mesh(sourceField.getSupport()->getCellMesh()),
00047       _local_face_mesh(sourceField.getSupport()->getFaceMesh()),
00048       _distant_group(distant_group),
00049       _local_group(local_group)
00050   { 
00051     _union_group = _local_group.fuse(distant_group);
00052     _computeBoundingBoxes();
00053     _comm=getCommunicator();
00054   }
00055 
00056   ElementLocator::~ElementLocator()
00057   {
00058     delete _union_group;
00059     delete [] _domain_bounding_boxes;
00060   }
00061 
00062   const MPI_Comm *ElementLocator::getCommunicator() const
00063   {
00064     MPIProcessorGroup* group=static_cast<MPIProcessorGroup*> (_union_group);
00065     return group->getComm();
00066   }
00067 
00068   NatureOfField ElementLocator::getLocalNature() const
00069   {
00070     return _local_para_field.getField()->getNature();
00071   }
00072 
00073   // ==========================================================================
00074   // Procedure for exchanging mesh between a distant proc and a local processor
00075   // param idistantrank  proc id on distant group
00076   // param distant_mesh on return , points to a local reconstruction of
00077   //  the distant mesh
00078   // param distant_ids on return, contains a vector defining a correspondence
00079   // between the distant ids and the ids of the local reconstruction 
00080   // ==========================================================================
00081   void ElementLocator::exchangeMesh(int idistantrank,
00082                                     MEDCouplingPointSet*& distant_mesh,
00083                                     int*& distant_ids)
00084   {
00085     int rank = _union_group->translateRank(&_distant_group,idistantrank);
00086 
00087     if (find(_distant_proc_ids.begin(), _distant_proc_ids.end(),rank)==_distant_proc_ids.end())
00088       return;
00089    
00090     vector<int> elems;
00091 #ifdef USE_DIRECTED_BB
00092     INTERP_KERNEL::DirectedBoundingBox dbb;
00093     double* distant_bb = _domain_bounding_boxes+rank*dbb.dataSize(_local_cell_mesh_space_dim);
00094     dbb.setData(distant_bb);
00095     _local_cell_mesh->getCellsInBoundingBox(dbb,getBoundingBoxAdjustment(),elems);
00096 #else
00097     double* distant_bb = _domain_bounding_boxes+rank*2*_local_cell_mesh_space_dim;
00098     _local_cell_mesh->getCellsInBoundingBox(distant_bb,getBoundingBoxAdjustment(),elems);
00099 #endif
00100     
00101     DataArrayInt *distant_ids_send;
00102     MEDCouplingPointSet *send_mesh = (MEDCouplingPointSet *)_local_para_field.getField()->buildSubMeshData(&elems[0],&elems[elems.size()],distant_ids_send);
00103     _exchangeMesh(send_mesh, distant_mesh, idistantrank, distant_ids_send, distant_ids);
00104     distant_ids_send->decrRef();
00105     
00106     if(send_mesh)
00107       send_mesh->decrRef();
00108   }
00109 
00110   void ElementLocator::exchangeMethod(const std::string& sourceMeth, int idistantrank, std::string& targetMeth)
00111   {
00112     CommInterface comm_interface=_union_group->getCommInterface();
00113     MPIProcessorGroup* group=static_cast<MPIProcessorGroup*> (_union_group);
00114     const MPI_Comm* comm=(group->getComm());
00115     MPI_Status status;
00116     // it must be converted to union numbering before communication
00117     int idistRankInUnion = group->translateRank(&_distant_group,idistantrank);
00118     char *recv_buffer=new char[4];
00119     std::vector<char> send_buffer(4);
00120     std::copy(sourceMeth.begin(),sourceMeth.end(),send_buffer.begin());
00121     comm_interface.sendRecv(&send_buffer[0], 4, MPI_CHAR,idistRankInUnion, 1112,
00122                             recv_buffer, 4, MPI_CHAR,idistRankInUnion, 1112,
00123                             *comm, &status);
00124     targetMeth=recv_buffer;
00125     delete [] recv_buffer;
00126   }
00127 
00128 
00129   // ======================
00130   // Compute bounding boxes
00131   // ======================
00132 
00133   void ElementLocator::_computeBoundingBoxes()
00134   {
00135     CommInterface comm_interface =_union_group->getCommInterface();
00136     MPIProcessorGroup* group=static_cast<MPIProcessorGroup*> (_union_group);
00137     const MPI_Comm* comm = group->getComm();
00138     _local_cell_mesh_space_dim = -1;
00139     if(_local_cell_mesh->getMeshDimension() != -1)
00140       _local_cell_mesh_space_dim=_local_cell_mesh->getSpaceDimension();
00141     int *spaceDimForAll=new int[_union_group->size()];
00142     comm_interface.allGather(&_local_cell_mesh_space_dim, 1, MPI_INT,
00143                              spaceDimForAll,1, MPI_INT, 
00144                              *comm);
00145     _local_cell_mesh_space_dim=*std::max_element(spaceDimForAll,spaceDimForAll+_union_group->size());
00146     _is_m1d_corr=((*std::min_element(spaceDimForAll,spaceDimForAll+_union_group->size()))==-1);
00147     for(int i=0;i<_union_group->size();i++)
00148       if(spaceDimForAll[i]!=_local_cell_mesh_space_dim && spaceDimForAll[i]!=-1)
00149         throw INTERP_KERNEL::Exception("Spacedim not matches !");
00150     delete [] spaceDimForAll;
00151 #ifdef USE_DIRECTED_BB
00152     INTERP_KERNEL::DirectedBoundingBox dbb;
00153     int bbSize = dbb.dataSize(_local_cell_mesh_space_dim);
00154     _domain_bounding_boxes = new double[bbSize*_union_group->size()];
00155     if(_local_cell_mesh->getMeshDimension() != -1)
00156       dbb = INTERP_KERNEL::DirectedBoundingBox(_local_cell_mesh->getCoords()->getPointer(),
00157                                                _local_cell_mesh->getNumberOfNodes(),
00158                                                _local_cell_mesh_space_dim);
00159     std::vector<double> dbbData = dbb.getData();
00160     if ( dbbData.size() < bbSize ) dbbData.resize(bbSize,0);
00161     double * minmax= &dbbData[0];
00162 #else
00163     int bbSize = 2*_local_cell_mesh_space_dim;
00164     _domain_bounding_boxes = new double[bbSize*_union_group->size()];
00165     double * minmax=new double [bbSize];
00166     if(_local_cell_mesh->getMeshDimension() != -1)
00167       _local_cell_mesh->getBoundingBox(minmax);
00168     else
00169       for(int i=0;i<_local_cell_mesh_space_dim;i++)
00170         {
00171           minmax[i*2]=-std::numeric_limits<double>::max();
00172           minmax[i*2+1]=std::numeric_limits<double>::max();
00173         }
00174 #endif
00175 
00176     comm_interface.allGather(minmax, bbSize, MPI_DOUBLE,
00177                              _domain_bounding_boxes,bbSize, MPI_DOUBLE, 
00178                              *comm);
00179   
00180     for (int i=0; i< _distant_group.size(); i++)
00181       {
00182         int rank=_union_group->translateRank(&_distant_group,i);
00183 
00184         if (_intersectsBoundingBox(rank))
00185           {
00186             _distant_proc_ids.push_back(rank);
00187           }
00188       }
00189 #ifdef USE_DIRECTED_BB
00190 #else
00191     delete [] minmax;
00192 #endif
00193   }
00194 
00195 
00196   // =============================================
00197   // Intersect Bounding Box (with a given "irank")
00198   // =============================================
00199   bool ElementLocator::_intersectsBoundingBox(int irank)
00200   {
00201 #ifdef USE_DIRECTED_BB
00202     INTERP_KERNEL::DirectedBoundingBox local_dbb, distant_dbb;
00203     local_dbb.setData( _domain_bounding_boxes+_union_group->myRank()*local_dbb.dataSize( _local_cell_mesh_space_dim ));
00204     distant_dbb.setData( _domain_bounding_boxes+irank*distant_dbb.dataSize( _local_cell_mesh_space_dim ));
00205     return !local_dbb.isDisjointWith( distant_dbb );
00206 #else
00207     double*  local_bb = _domain_bounding_boxes+_union_group->myRank()*2*_local_cell_mesh_space_dim;
00208     double*  distant_bb =  _domain_bounding_boxes+irank*2*_local_cell_mesh_space_dim;
00209 
00210     for (int idim=0; idim < _local_cell_mesh_space_dim; idim++)
00211       {
00212         const double eps =  1e-12;
00213         bool intersects = (distant_bb[idim*2]<local_bb[idim*2+1]+eps)
00214           && (local_bb[idim*2]<distant_bb[idim*2+1]+eps);
00215         if (!intersects) return false; 
00216       }
00217     return true;
00218 #endif
00219   } 
00220 
00221   // ======================
00222   // Exchanging meshes data
00223   // ======================
00224   void ElementLocator::_exchangeMesh( MEDCouplingPointSet* local_mesh,
00225                                       MEDCouplingPointSet*& distant_mesh,
00226                                       int iproc_distant,
00227                                       const DataArrayInt* distant_ids_send,
00228                                       int*& distant_ids_recv)
00229   {
00230     CommInterface comm_interface=_union_group->getCommInterface();
00231   
00232     // First stage : exchanging sizes
00233     // ------------------------------
00234     vector<double> tinyInfoLocalD,tinyInfoDistantD(1);//not used for the moment
00235     vector<int> tinyInfoLocal,tinyInfoDistant;
00236     vector<string> tinyInfoLocalS;
00237     //Getting tiny info of local mesh to allow the distant proc to initialize and allocate
00238     //the transmitted mesh.
00239     local_mesh->getTinySerializationInformation(tinyInfoLocalD,tinyInfoLocal,tinyInfoLocalS);
00240     tinyInfoLocal.push_back(distant_ids_send->getNumberOfTuples());
00241     tinyInfoDistant.resize(tinyInfoLocal.size());
00242     std::fill(tinyInfoDistant.begin(),tinyInfoDistant.end(),0);
00243     MPIProcessorGroup* group=static_cast<MPIProcessorGroup*> (_union_group);
00244     const MPI_Comm* comm=group->getComm();
00245     MPI_Status status; 
00246     
00247     // iproc_distant is the number of proc in distant group
00248     // it must be converted to union numbering before communication
00249     int iprocdistant_in_union = group->translateRank(&_distant_group,
00250                                                      iproc_distant);
00251     
00252     comm_interface.sendRecv(&tinyInfoLocal[0], tinyInfoLocal.size(), MPI_INT, iprocdistant_in_union, 1112,
00253                             &tinyInfoDistant[0], tinyInfoDistant.size(), MPI_INT,iprocdistant_in_union,1112,
00254                             *comm, &status);
00255     DataArrayInt *v1Local=0;
00256     DataArrayDouble *v2Local=0;
00257     DataArrayInt *v1Distant=DataArrayInt::New();
00258     DataArrayDouble *v2Distant=DataArrayDouble::New();
00259     //serialization of local mesh to send data to distant proc.
00260     local_mesh->serialize(v1Local,v2Local);
00261     //Building the right instance of copy of distant mesh.
00262     MEDCouplingPointSet *distant_mesh_tmp=MEDCouplingPointSet::BuildInstanceFromMeshType((MEDCouplingMeshType)tinyInfoDistant[0]);
00263     std::vector<std::string> unusedTinyDistantSts;
00264     distant_mesh_tmp->resizeForUnserialization(tinyInfoDistant,v1Distant,v2Distant,unusedTinyDistantSts);
00265     int nbLocalElems=0;
00266     int nbDistElem=0;
00267     int *ptLocal=0;
00268     int *ptDist=0;
00269     if(v1Local)
00270       {
00271         nbLocalElems=v1Local->getNbOfElems();
00272         ptLocal=v1Local->getPointer();
00273       }
00274     if(v1Distant)
00275       {
00276         nbDistElem=v1Distant->getNbOfElems();
00277         ptDist=v1Distant->getPointer();
00278       }
00279     comm_interface.sendRecv(ptLocal, nbLocalElems, MPI_INT,
00280                             iprocdistant_in_union, 1111,
00281                             ptDist, nbDistElem, MPI_INT,
00282                             iprocdistant_in_union,1111,
00283                             *comm, &status);
00284     nbLocalElems=0;
00285     double *ptLocal2=0;
00286     double *ptDist2=0;
00287     if(v2Local)
00288       {
00289         nbLocalElems=v2Local->getNbOfElems();
00290         ptLocal2=v2Local->getPointer();
00291       }
00292     nbDistElem=0;
00293     if(v2Distant)
00294       {
00295         nbDistElem=v2Distant->getNbOfElems();
00296         ptDist2=v2Distant->getPointer();
00297       }
00298     comm_interface.sendRecv(ptLocal2, nbLocalElems, MPI_DOUBLE,
00299                             iprocdistant_in_union, 1112,
00300                             ptDist2, nbDistElem, MPI_DOUBLE,
00301                             iprocdistant_in_union, 1112, 
00302                             *comm, &status);
00303     //
00304     distant_mesh=distant_mesh_tmp;
00305     //finish unserialization
00306     distant_mesh->unserialization(tinyInfoDistantD,tinyInfoDistant,v1Distant,v2Distant,unusedTinyDistantSts);
00307     //
00308     distant_ids_recv=new int[tinyInfoDistant.back()];
00309     comm_interface.sendRecv((void *)distant_ids_send->getConstPointer(),tinyInfoLocal.back(), MPI_INT,
00310                             iprocdistant_in_union, 1113,
00311                             distant_ids_recv,tinyInfoDistant.back(), MPI_INT,
00312                             iprocdistant_in_union,1113,
00313                             *comm, &status);
00314     if(v1Local)
00315       v1Local->decrRef();
00316     if(v2Local)
00317       v2Local->decrRef();
00318     if(v1Distant)
00319       v1Distant->decrRef();
00320     if(v2Distant)
00321       v2Distant->decrRef();
00322   }
00323   
00327   void ElementLocator::recvPolicyFromLazySideW(std::vector<int>& policy)
00328   {
00329     policy.resize(_distant_proc_ids.size());
00330     int procId=0;
00331     CommInterface comm;
00332     MPI_Status status;
00333     for(vector<int>::const_iterator iter=_distant_proc_ids.begin();iter!=_distant_proc_ids.end();iter++,procId++)
00334       {
00335         int toRecv;
00336         comm.recv((void *)&toRecv,1,MPI_INT,*iter,1120,*_comm,&status);
00337         policy[procId]=toRecv;
00338       }
00339   }
00340 
00344   void ElementLocator::sendSumToLazySideW(const std::vector< std::vector<int> >& distantLocEltIds, const std::vector< std::vector<double> >& partialSumRelToDistantIds)
00345   {
00346     int procId=0;
00347     CommInterface comm;
00348     for(vector<int>::const_iterator iter=_distant_proc_ids.begin();iter!=_distant_proc_ids.end();iter++,procId++)
00349       {
00350         const vector<int>& eltIds=distantLocEltIds[procId];
00351         const vector<double>& valued=partialSumRelToDistantIds[procId];
00352         int lgth=eltIds.size();
00353         comm.send(&lgth,1,MPI_INT,*iter,1114,*_comm);
00354         comm.send((void *)&eltIds[0],lgth,MPI_INT,*iter,1115,*_comm);
00355         comm.send((void *)&valued[0],lgth,MPI_DOUBLE,*iter,1116,*_comm);
00356       }
00357   }
00358 
00362   void ElementLocator::recvSumFromLazySideW(std::vector< std::vector<double> >& globalSumRelToDistantIds)
00363   {
00364     int procId=0;
00365     CommInterface comm;
00366     MPI_Status status;
00367     for(vector<int>::const_iterator iter=_distant_proc_ids.begin();iter!=_distant_proc_ids.end();iter++,procId++)
00368       {
00369         std::vector<double>& vec=globalSumRelToDistantIds[procId];
00370         comm.recv(&vec[0],vec.size(),MPI_DOUBLE,*iter,1117,*_comm,&status);
00371       }
00372   }
00373 
00377   void ElementLocator::sendLocalIdsToLazyProcsW(const std::vector< std::vector<int> >& distantLocEltIds)
00378   {
00379     int procId=0;
00380     CommInterface comm;
00381     for(vector<int>::const_iterator iter=_distant_proc_ids.begin();iter!=_distant_proc_ids.end();iter++,procId++)
00382       {
00383         const vector<int>& eltIds=distantLocEltIds[procId];
00384         int lgth=eltIds.size();
00385         comm.send(&lgth,1,MPI_INT,*iter,1121,*_comm);
00386         comm.send((void *)&eltIds[0],lgth,MPI_INT,*iter,1122,*_comm);
00387       }
00388   }
00389 
00393   void ElementLocator::recvGlobalIdsFromLazyProcsW(const std::vector< std::vector<int> >& distantLocEltIds, std::vector< std::vector<int> >& globalIds)
00394   {
00395     int procId=0;
00396     CommInterface comm;
00397     MPI_Status status;
00398     globalIds.resize(_distant_proc_ids.size());
00399     for(vector<int>::const_iterator iter=_distant_proc_ids.begin();iter!=_distant_proc_ids.end();iter++,procId++)
00400       {
00401         const std::vector<int>& vec=distantLocEltIds[procId];
00402         std::vector<int>& global=globalIds[procId];
00403         global.resize(vec.size());
00404         comm.recv(&global[0],vec.size(),MPI_INT,*iter,1123,*_comm,&status);
00405       }
00406   }
00407   
00411   void ElementLocator::recvCandidatesGlobalIdsFromLazyProcsW(std::vector< std::vector<int> >& globalIds)
00412   {
00413     int procId=0;
00414     CommInterface comm;
00415     MPI_Status status;
00416     globalIds.resize(_distant_proc_ids.size());
00417     for(vector<int>::const_iterator iter=_distant_proc_ids.begin();iter!=_distant_proc_ids.end();iter++,procId++)
00418       {
00419         std::vector<int>& global=globalIds[procId];
00420         int lgth;
00421         comm.recv(&lgth,1,MPI_INT,*iter,1132,*_comm,&status);
00422         global.resize(lgth);
00423         comm.recv(&global[0],lgth,MPI_INT,*iter,1133,*_comm,&status);
00424       }
00425   }
00426   
00430   void ElementLocator::sendPartialSumToLazyProcsW(const std::vector<int>& distantGlobIds, const std::vector<double>& sum)
00431   {
00432     int procId=0;
00433     CommInterface comm;
00434     int lgth=distantGlobIds.size();
00435     for(vector<int>::const_iterator iter=_distant_proc_ids.begin();iter!=_distant_proc_ids.end();iter++,procId++)
00436       {
00437         comm.send(&lgth,1,MPI_INT,*iter,1124,*_comm);
00438         comm.send((void*)&distantGlobIds[0],lgth,MPI_INT,*iter,1125,*_comm);
00439         comm.send((void*)&sum[0],lgth,MPI_DOUBLE,*iter,1126,*_comm);
00440       }
00441   }
00442 
00446   void ElementLocator::sendCandidatesForAddElementsW(const std::vector<int>& distantGlobIds)
00447   {
00448     int procId=0;
00449     CommInterface comm;
00450     int lgth=distantGlobIds.size();
00451     for(vector<int>::const_iterator iter=_distant_proc_ids.begin();iter!=_distant_proc_ids.end();iter++,procId++)
00452       {
00453         comm.send(&lgth,1,MPI_INT,*iter,1128,*_comm);
00454         comm.send((void*)&distantGlobIds[0],lgth,MPI_INT,*iter,1129,*_comm);
00455       }
00456   }
00457   
00461   void ElementLocator::recvAddElementsFromLazyProcsW(std::vector<std::vector<int> >& elementsToAdd)
00462   {
00463     int procId=0;
00464     CommInterface comm;
00465     MPI_Status status;
00466     int lgth=_distant_proc_ids.size();
00467     elementsToAdd.resize(lgth);
00468     for(vector<int>::const_iterator iter=_distant_proc_ids.begin();iter!=_distant_proc_ids.end();iter++,procId++)
00469       {
00470         int locLgth;
00471         std::vector<int>& eltToFeed=elementsToAdd[procId];
00472         comm.recv(&locLgth,1,MPI_INT,*iter,1130,*_comm,&status);
00473         eltToFeed.resize(locLgth);
00474         comm.recv(&eltToFeed[0],locLgth,MPI_INT,*iter,1131,*_comm,&status);
00475       }
00476   }
00477 
00481   int ElementLocator::sendPolicyToWorkingSideL()
00482   {
00483     CommInterface comm;
00484     int toSend;
00485     DataArrayInt *isCumulative=_local_para_field.returnCumulativeGlobalNumbering();
00486     if(isCumulative)
00487       {
00488         toSend=CUMULATIVE_POLICY;
00489         isCumulative->decrRef();
00490       }
00491     else
00492       toSend=NO_POST_TREATMENT_POLICY;
00493     for(vector<int>::const_iterator iter=_distant_proc_ids.begin();iter!=_distant_proc_ids.end();iter++)
00494       comm.send(&toSend,1,MPI_INT,*iter,1120,*_comm);
00495     return toSend;
00496   }
00497 
00501   void ElementLocator::recvFromWorkingSideL()
00502   {
00503     _values_added.resize(_local_para_field.getField()->getNumberOfTuples());
00504     int procId=0;
00505     CommInterface comm;
00506     _ids_per_working_proc.resize(_distant_proc_ids.size());
00507     MPI_Status status;
00508     for(vector<int>::const_iterator iter=_distant_proc_ids.begin();iter!=_distant_proc_ids.end();iter++,procId++)
00509       {
00510         int lgth;
00511         comm.recv(&lgth,1,MPI_INT,*iter,1114,*_comm,&status);
00512         vector<int>& ids=_ids_per_working_proc[procId];
00513         ids.resize(lgth);
00514         vector<double> values(lgth);
00515         comm.recv(&ids[0],lgth,MPI_INT,*iter,1115,*_comm,&status);
00516         comm.recv(&values[0],lgth,MPI_DOUBLE,*iter,1116,*_comm,&status);
00517         for(int i=0;i<lgth;i++)
00518           _values_added[ids[i]]+=values[i];
00519       }
00520   }
00521 
00525   void ElementLocator::sendToWorkingSideL()
00526   {
00527     int procId=0;
00528     CommInterface comm;
00529     for(vector<int>::const_iterator iter=_distant_proc_ids.begin();iter!=_distant_proc_ids.end();iter++,procId++)
00530       {
00531         vector<int>& ids=_ids_per_working_proc[procId];
00532         vector<double> valsToSend(ids.size());
00533         vector<double>::iterator iter3=valsToSend.begin();
00534         for(vector<int>::const_iterator iter2=ids.begin();iter2!=ids.end();iter2++,iter3++)
00535           *iter3=_values_added[*iter2];
00536         comm.send(&valsToSend[0],ids.size(),MPI_DOUBLE,*iter,1117,*_comm);
00537         //ids.clear();
00538       }
00539     //_ids_per_working_proc.clear();
00540   }
00541 
00545   void ElementLocator::recvLocalIdsFromWorkingSideL()
00546   {
00547     int procId=0;
00548     CommInterface comm;
00549     _ids_per_working_proc.resize(_distant_proc_ids.size());
00550     MPI_Status status;
00551     for(vector<int>::const_iterator iter=_distant_proc_ids.begin();iter!=_distant_proc_ids.end();iter++,procId++)
00552       {
00553         int lgth;
00554         vector<int>& ids=_ids_per_working_proc[procId];
00555         comm.recv(&lgth,1,MPI_INT,*iter,1121,*_comm,&status);
00556         ids.resize(lgth);
00557         comm.recv(&ids[0],lgth,MPI_INT,*iter,1122,*_comm,&status);
00558       }
00559   }
00560 
00564   void ElementLocator::sendGlobalIdsToWorkingSideL()
00565   {
00566     int procId=0;
00567     CommInterface comm;
00568     DataArrayInt *globalIds=_local_para_field.returnGlobalNumbering();
00569     const int *globalIdsC=globalIds->getConstPointer();
00570     for(vector<int>::const_iterator iter=_distant_proc_ids.begin();iter!=_distant_proc_ids.end();iter++,procId++)
00571       {
00572         const vector<int>& ids=_ids_per_working_proc[procId];
00573         vector<int> valsToSend(ids.size());
00574         vector<int>::iterator iter1=valsToSend.begin();
00575         for(vector<int>::const_iterator iter2=ids.begin();iter2!=ids.end();iter2++,iter1++)
00576           *iter1=globalIdsC[*iter2];
00577         comm.send(&valsToSend[0],ids.size(),MPI_INT,*iter,1123,*_comm);
00578       }
00579     if(globalIds)
00580       globalIds->decrRef();
00581   }
00582 
00586   void ElementLocator::recvSumFromWorkingSideL()
00587   {
00588     int procId=0;
00589     int wProcSize=_distant_proc_ids.size();
00590     CommInterface comm;
00591     _ids_per_working_proc.resize(wProcSize);
00592     _values_per_working_proc.resize(wProcSize);
00593     MPI_Status status;
00594     std::map<int,double> sums;
00595     for(vector<int>::const_iterator iter=_distant_proc_ids.begin();iter!=_distant_proc_ids.end();iter++,procId++)
00596       {
00597         int lgth;
00598         comm.recv(&lgth,1,MPI_INT,*iter,1124,*_comm,&status);
00599         vector<int>& ids=_ids_per_working_proc[procId];
00600         vector<double>& vals=_values_per_working_proc[procId];
00601         ids.resize(lgth);
00602         vals.resize(lgth);
00603         comm.recv(&ids[0],lgth,MPI_INT,*iter,1125,*_comm,&status);
00604         comm.recv(&vals[0],lgth,MPI_DOUBLE,*iter,1126,*_comm,&status);
00605         vector<int>::const_iterator iter1=ids.begin();
00606         vector<double>::const_iterator iter2=vals.begin();
00607         for(;iter1!=ids.end();iter1++,iter2++)
00608           sums[*iter1]+=*iter2;
00609       }
00610     //assign sum to prepare sending to working side
00611     for(procId=0;procId<wProcSize;procId++)
00612       {
00613         vector<int>& ids=_ids_per_working_proc[procId];
00614         vector<double>& vals=_values_per_working_proc[procId];
00615         vector<int>::const_iterator iter1=ids.begin();
00616         vector<double>::iterator iter2=vals.begin();
00617         for(;iter1!=ids.end();iter1++,iter2++)
00618           *iter2=sums[*iter1];
00619         ids.clear();
00620       }
00621   }
00622 
00631   void ElementLocator::recvCandidatesForAddElementsL()
00632   {
00633     int procId=0;
00634     int wProcSize=_distant_proc_ids.size();
00635     CommInterface comm;
00636     _ids_per_working_proc3.resize(wProcSize);
00637     MPI_Status status;
00638     std::map<int,double> sums;
00639     DataArrayInt *globalIds=_local_para_field.returnGlobalNumbering();
00640     const int *globalIdsC=globalIds->getConstPointer();
00641     int nbElts=globalIds->getNumberOfTuples();
00642     std::set<int> globalIdsS(globalIdsC,globalIdsC+nbElts);
00643     for(vector<int>::const_iterator iter=_distant_proc_ids.begin();iter!=_distant_proc_ids.end();iter++,procId++)
00644       {
00645         const std::vector<int>& ids0=_ids_per_working_proc[procId];
00646         int lgth0=ids0.size();
00647         std::set<int> elts0;
00648         for(int i=0;i<lgth0;i++)
00649           elts0.insert(globalIdsC[ids0[i]]);
00650         int lgth;
00651         comm.recv(&lgth,1,MPI_INT,*iter,1128,*_comm,&status);
00652         vector<int> ids(lgth);
00653         comm.recv(&ids[0],lgth,MPI_INT,*iter,1129,*_comm,&status);
00654         set<int> ids1(ids.begin(),ids.end());
00655         ids.clear();
00656         set<int> tmp5,tmp6;
00657         set_intersection(globalIdsS.begin(),globalIdsS.end(),ids1.begin(),ids1.end(),inserter(tmp5,tmp5.begin()));
00658         set_difference(tmp5.begin(),tmp5.end(),elts0.begin(),elts0.end(),inserter(tmp6,tmp6.begin()));
00659         std::vector<int>& ids2=_ids_per_working_proc3[procId];
00660         ids2.resize(tmp6.size());
00661         std::copy(tmp6.begin(),tmp6.end(),ids2.begin());
00662         //global->local
00663         for(std::vector<int>::iterator iter2=ids2.begin();iter2!=ids2.end();iter2++)
00664           *iter2=std::find(globalIdsC,globalIdsC+nbElts,*iter2)-globalIdsC;
00665       }
00666     if(globalIds)
00667       globalIds->decrRef();
00668   }
00669 
00673   void ElementLocator::sendAddElementsToWorkingSideL()
00674   {
00675     int procId=0;
00676     CommInterface comm;
00677     for(vector<int>::const_iterator iter=_distant_proc_ids.begin();iter!=_distant_proc_ids.end();iter++,procId++)
00678       {
00679         const std::vector<int>& vals=_ids_per_working_proc3[procId];
00680         int size=vals.size();
00681         comm.send((void *)&size,1,MPI_INT,*iter,1130,*_comm);
00682         comm.send((void *)&vals[0],size,MPI_INT,*iter,1131,*_comm);
00683       }
00684   }
00685 
00690   void ElementLocator::sendCandidatesGlobalIdsToWorkingSideL()
00691   { 
00692     int procId=0;
00693     CommInterface comm;
00694     DataArrayInt *globalIds=_local_para_field.returnGlobalNumbering();
00695     const int *globalIdsC=globalIds->getConstPointer();
00696     std::vector<int> candidates;
00697     _local_para_field.getSupport()->getCellMesh()->findBoundaryNodes(candidates);
00698     for(std::vector<int>::iterator iter1=candidates.begin();iter1!=candidates.end();iter1++)
00699       (*iter1)=globalIdsC[*iter1];
00700     std::set<int> candidatesS(candidates.begin(),candidates.end());
00701     for(vector<int>::const_iterator iter=_distant_proc_ids.begin();iter!=_distant_proc_ids.end();iter++,procId++)
00702       {
00703         const vector<int>& ids=_ids_per_working_proc[procId];
00704         vector<int> valsToSend(ids.size());
00705         vector<int>::iterator iter1=valsToSend.begin();
00706         for(vector<int>::const_iterator iter2=ids.begin();iter2!=ids.end();iter2++,iter1++)
00707           *iter1=globalIdsC[*iter2];
00708         std::set<int> tmp2(valsToSend.begin(),valsToSend.end());
00709         std::vector<int> tmp3;
00710         set_intersection(candidatesS.begin(),candidatesS.end(),tmp2.begin(),tmp2.end(),std::back_insert_iterator< std::vector<int> >(tmp3));
00711         int lgth=tmp3.size();
00712         comm.send(&lgth,1,MPI_INT,*iter,1132,*_comm);
00713         comm.send(&tmp3[0],lgth,MPI_INT,*iter,1133,*_comm);
00714       }
00715     if(globalIds)
00716       globalIds->decrRef();
00717   }
00718 }