Back to index

salome-med  6.5.0
MPIAccess.hxx
Go to the documentation of this file.
00001 // Copyright (C) 2007-2012  CEA/DEN, EDF R&D
00002 //
00003 // This library is free software; you can redistribute it and/or
00004 // modify it under the terms of the GNU Lesser General Public
00005 // License as published by the Free Software Foundation; either
00006 // version 2.1 of the License.
00007 //
00008 // This library is distributed in the hope that it will be useful,
00009 // but WITHOUT ANY WARRANTY; without even the implied warranty of
00010 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00011 // Lesser General Public License for more details.
00012 //
00013 // You should have received a copy of the GNU Lesser General Public
00014 // License along with this library; if not, write to the Free Software
00015 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
00016 //
00017 // See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
00018 //
00019 
00020 #ifndef __MPIACCESS_HXX__
00021 #define __MPIACCESS_HXX__
00022 
00023 #include "CommInterface.hxx"
00024 #include "ProcessorGroup.hxx"
00025 #include "MPIProcessorGroup.hxx"
00026 
00027 #include <map>
00028 #include <list>
00029 #include <vector>
00030 #include <iostream>
00031 
00032 namespace ParaMEDMEM
00033 {
00034   typedef struct
00035   {
00036     double time ;
00037     double deltatime ;
00038     int tag ;
00039   } TimeMessage;
00040   
00041   static MPI_Request mpirequestnull = MPI_REQUEST_NULL ;
00042   enum _MessageIdent { _message_unknown, _message_time, _message_int, _message_double } ;
00043 
00044   class MPIAccess
00045   {
00046   private:
00047     struct RequestStruct
00048     {
00049       int MPITarget ;
00050       bool MPIIsRecv ;
00051       int MPITag ;
00052       bool MPIAsynchronous ;
00053       bool MPICompleted ;
00054       MPI_Datatype MPIDatatype ;
00055       MPI_Request MPIRequest ;
00056       MPI_Status *MPIStatus ;
00057       int MPIOutCount ;
00058     };
00059   public:
00060     MPIAccess(MPIProcessorGroup * ProcessorGroup, int BaseTag=0, int MaxTag=0) ;
00061     virtual ~MPIAccess() ;
00062 
00063     void trace( bool trace = true ) ;
00064 
00065     void deleteRequest( int RequestId ) ;
00066     void deleteRequests(int size , int *ArrayOfSendRequests ) ;
00067 
00068     int sendMPITag(int destrank) ;
00069     int recvMPITag(int sourcerank) ;
00070 
00071     int sendRequestIdsSize() ;
00072     int sendRequestIds(int size, int *ArrayOfSendRequests) ;
00073     int recvRequestIdsSize() ;
00074     int recvRequestIds(int size, int *ArrayOfRecvRequests) ;
00075 
00076     int sendRequestIdsSize(int destrank) ;
00077     int sendRequestIds(int destrank, int size, int *ArrayOfSendRequests) ;
00078     int recvRequestIdsSize(int sourcerank) ;
00079     int recvRequestIds(int sourcerank, int size, int *ArrayOfRecvRequests) ;
00080 
00081     int send(void* buffer, int count, MPI_Datatype datatype, int target,
00082              int &RequestId) ;
00083     int ISend(void* buffer, int count, MPI_Datatype datatype, int target,
00084               int &RequestId) ;
00085     int recv(void* buffer, int count, MPI_Datatype datatype, int source,
00086              int &RequestId, int *OutCount=NULL) ;
00087     int IRecv(void* buffer, int count, MPI_Datatype datatype, int source,
00088               int &RequestId) ;
00089     int sendRecv(void* sendbuf, int sendcount, MPI_Datatype sendtype, int dest,
00090                  int &SendRequestId, void* recvbuf, int recvcount,
00091                  MPI_Datatype recvtype, int source,
00092                  int &RecvRequestId, int *OutCount=NULL) ;
00093     int ISendRecv(void* sendbuf, int sendcount, MPI_Datatype sendtype, int dest,
00094                   int &SendRequestId, void* recvbuf, int recvcount,
00095                   MPI_Datatype recvtype, int source, int &RecvRequestId) ;
00096 
00097     int wait(int RequestId) ;
00098     int test(int RequestId, int &flag) ;
00099     int waitAny(int count, int *array_of_RequestIds, int &RequestId) ;
00100     int testAny(int count, int *array_of_RequestIds, int &RequestId, int &flag)  ;
00101     int waitAll(int count, int *array_of_RequestIds) ;
00102     int testAll(int count, int *array_of_RequestIds, int &flag)  ;
00103     int waitSome(int count, int *array_of_RequestIds, int outcount,
00104                  int *outarray_of_RequestIds) ;
00105     int testSome(int count, int *array_of_RequestIds, int outcounts,
00106                  int *outarray_of_RequestIds) ;
00107     int probe(int FromSource, int &source, int &MPITag, MPI_Datatype &datatype,
00108               int &outcount) ;
00109     int IProbe(int FromSource, int &source, int &MPITag, MPI_Datatype &datatype,
00110                int &outcount, int &flag) ;
00111     int cancel( int RecvRequestId, int &flag ) ; 
00112     int cancel( int source, int MPITag, MPI_Datatype datatype, int outcount,
00113                 int &flag ) ;
00114     int cancelAll() ;
00115     int barrier() ;
00116     int errorString(int errorcode, char *string, int *resultlen) const ;
00117     int status(int RequestId, int &source, int &tag, int &error, int &outcount,
00118                bool keepRequestStruct=false) ;
00119     int requestFree( MPI_Request *request ) ;
00120 
00121     void check() const ;
00122 
00123     MPI_Datatype timeType() const ;
00124     bool isTimeMessage( int MPITag ) const ;
00125     MPI_Aint timeExtent() const ;
00126     MPI_Aint intExtent() const ;
00127     MPI_Aint doubleExtent() const ;
00128     MPI_Aint extent( MPI_Datatype datatype ) const ;
00129 
00130     int MPITag( int RequestId ) ;
00131     int MPITarget( int RequestId ) ;
00132     bool MPIIsRecv( int RequestId ) ;
00133     bool MPIAsynchronous( int RequestId ) ;
00134     bool MPICompleted( int RequestId ) ;
00135     MPI_Datatype MPIDatatype( int RequestId ) ;
00136     int MPIOutCount( int RequestId ) ;
00137 
00138   private:
00139     int newRequest( MPI_Datatype datatype, int tag , int destsourcerank ,
00140                     bool fromsourcerank , bool asynchronous ) ;
00141     int newSendTag( MPI_Datatype datatype, int destrank , int method ,
00142                     bool asynchronous, int &RequestId ) ;
00143     int newRecvTag( MPI_Datatype datatype, int sourcerank , int method ,
00144                     bool asynchronous, int &RequestId ) ;
00145     int incrTag( int prevtag ) ;
00146     int valTag( int tag, int method ) ;
00147 
00148     void deleteSendRecvRequest( int RequestId ) ;
00149 
00150     void deleteStatus( int RequestId ) ;
00151 
00152     MPI_Request *MPIRequest( int RequestId ) ;
00153     MPI_Status *MPIStatus( int RequestId ) ;
00154     void setMPICompleted( int RequestId , bool completed ) ;
00155     void setMPIOutCount( int RequestId , int outcount ) ;
00156     void clearMPIStatus( int RequestId ) ;
00157 
00158     _MessageIdent methodId( MPI_Datatype datatype ) const ;
00159     MPI_Datatype datatype( _MessageIdent aMethodIdent ) const ;
00160   private:
00161     const CommInterface &_comm_interface ;
00162     const MPI_Comm* _intra_communicator ;
00163     MPIProcessorGroup * _processor_group ;
00164     int _processor_group_size ;
00165     int _my_rank ;
00166     bool _trace ;
00167     int _base_request ;
00168     int _max_request ;
00169     int _request ;
00170     int * _send_request ;
00171     int * _recv_request ;
00172     std::vector< std::list< int > > _send_requests ;
00173     std::vector< std::list< int > > _recv_requests ;
00174     int _base_MPI_tag ;
00175     int _max_MPI_tag ;
00176     int * _send_MPI_tag ;
00177     int * _recv_MPI_Tag ;
00178     MPI_Datatype _MPI_TIME ;
00179     static const int MODULO_TAG=10;
00180     std::map< int , RequestStruct * > _map_of_request_struct ;
00181 
00182   };
00183 
00184   inline void MPIAccess::trace( bool trace )
00185   {
00186     _trace = trace ;
00187   }
00188 
00189   // Delete the structure Request corresponding to RequestId identifier after
00190   // the deletion of the structures MPI_Request * and MPI_Status *
00191   // remove it from _MapOfRequestStruct (erase)
00192   inline void MPIAccess::deleteRequest( int RequestId )
00193   {
00194     struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
00195     if ( aRequestStruct )
00196       {
00197         if ( _trace )
00198           std::cout << "MPIAccess::DeleteRequest" << _my_rank << "( " << RequestId << " ) "
00199                     << aRequestStruct << " MPIRequest " << aRequestStruct->MPIRequest
00200                     << " MPIIsRecv " << aRequestStruct->MPIIsRecv << std::endl ;
00201         if ( _map_of_request_struct[RequestId]->MPIRequest != MPI_REQUEST_NULL )
00202           requestFree( &_map_of_request_struct[RequestId]->MPIRequest ) ;
00203         deleteSendRecvRequest( RequestId ) ;
00204         deleteStatus( RequestId ) ;
00205         _map_of_request_struct.erase( RequestId ) ;
00206         delete aRequestStruct ;
00207       }
00208     else
00209       {
00210         if ( _trace )
00211           std::cout << "MPIAccess::DeleteRequest" << _my_rank << "( " << RequestId
00212                     << " ) Request not found" << std::endl ;
00213       }
00214   }
00215 
00216   // Delete all requests of the array ArrayOfSendRequests
00217   inline void MPIAccess::deleteRequests(int size , int *ArrayOfSendRequests )
00218   {
00219     for (int i = 0 ; i < size ; i++ )
00220       deleteRequest( ArrayOfSendRequests[i] ) ;
00221   }
00222 
00223   // Returns the last MPITag of the destination rank destrank
00224   inline int MPIAccess::sendMPITag(int destrank)
00225   {
00226     return _send_MPI_tag[destrank] ;
00227   }
00228 
00229   // Returns the last MPITag of the source rank sourcerank
00230   inline int MPIAccess::recvMPITag(int sourcerank)
00231   {
00232     return _recv_MPI_Tag[sourcerank] ;
00233   }
00234 
00235   // Returns the number of all SendRequestIds matching a destination rank. It may be
00236   // used to allocate ArrayOfSendRequests for the call to SendRequestIds
00237   inline int MPIAccess::sendRequestIdsSize(int destrank)
00238   {
00239     return _send_requests[destrank].size() ;
00240   }
00241 
00242   // Returns the number of all RecvRequestIds matching a source rank. It may be
00243   // used to allocate ArrayOfRecvRequests for the call to RecvRequestIds
00244   inline int MPIAccess::recvRequestIdsSize(int sourcerank)
00245   {
00246     return _recv_requests[sourcerank].size() ;
00247   }
00248 
00249   // Returns the MPI_Datatype (registered in MPI in the constructor with
00250   // MPI_Type_struct and MPI_Type_commit) for TimeMessages
00251   inline MPI_Datatype MPIAccess::timeType() const
00252   {
00253     return _MPI_TIME ;
00254   }
00255   
00256   // Returns true if the tag MPITag corresponds to a TimeMessage
00257   inline bool MPIAccess::isTimeMessage( int MPITag ) const
00258   {
00259     return ((MPITag%MODULO_TAG) == _message_time) ;
00260   }
00261 
00262   // Returns the MPI size of a TimeMessage
00263   inline MPI_Aint MPIAccess::timeExtent() const
00264   {
00265     MPI_Aint extent ;
00266     MPI_Type_extent( _MPI_TIME , &extent ) ;
00267     return extent ;
00268   }
00269 
00270   // Returns the MPI size of a MPI_INT
00271   inline MPI_Aint MPIAccess::intExtent() const
00272   {
00273     MPI_Aint extent ;
00274     MPI_Type_extent( MPI_INT , &extent ) ;
00275     return extent ;
00276   }
00277 
00278   // Returns the MPI size of a MPI_DOUBLE
00279   inline MPI_Aint MPIAccess::doubleExtent() const
00280   {
00281     MPI_Aint extent ;
00282     MPI_Type_extent( MPI_DOUBLE , &extent ) ;
00283     return extent ;
00284   }
00285 
00286   // Returns the MPI size of the MPI_Datatype datatype
00287   inline MPI_Aint MPIAccess::extent( MPI_Datatype datatype ) const
00288   {
00289     if ( datatype == _MPI_TIME )
00290       return timeExtent() ;
00291     if ( datatype == MPI_INT )
00292       return intExtent() ;
00293     if ( datatype == MPI_DOUBLE )
00294       return doubleExtent() ;
00295     return 0 ;
00296   }
00297   
00298   // Returns the MPITag of the request corresponding to RequestId identifier
00299   inline int MPIAccess::MPITag( int RequestId )
00300   {
00301     struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
00302     if ( aRequestStruct )
00303       return aRequestStruct->MPITag ;
00304     return -1 ;
00305   }
00306   
00307   // Returns the MPITarget of the request corresponding to RequestId identifier
00308   inline int MPIAccess::MPITarget( int RequestId )
00309   {
00310     struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
00311     if ( aRequestStruct )
00312       return aRequestStruct->MPITarget ;
00313     return -1 ;
00314   }
00315 
00316   // Returns true if the request corresponding to RequestId identifier was [I]Recv
00317   inline bool MPIAccess::MPIIsRecv( int RequestId )
00318   {
00319     struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
00320     if ( aRequestStruct )
00321       return aRequestStruct->MPIIsRecv ;
00322     return false ;
00323   }
00324 
00325   // Returns true if the request corresponding to RequestId identifier was asynchronous
00326   inline bool MPIAccess::MPIAsynchronous( int RequestId )
00327   {
00328     struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
00329     if ( aRequestStruct )
00330       return aRequestStruct->MPIAsynchronous ;
00331     return false ;
00332   }
00333   
00334   // Returns true if the request corresponding to RequestId identifier was completed
00335   inline bool MPIAccess::MPICompleted( int RequestId )
00336   {
00337     struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
00338     if ( aRequestStruct )
00339       return aRequestStruct->MPICompleted;
00340     return true ;
00341   }
00342 
00343   // Returns the MPI_datatype  of the request corresponding to RequestId identifier
00344   inline MPI_Datatype MPIAccess::MPIDatatype( int RequestId )
00345   {
00346     struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
00347     if ( aRequestStruct )
00348       return aRequestStruct->MPIDatatype;
00349     return (MPI_Datatype ) NULL ;
00350   }
00351 
00352   // Returns the size of the receiving message of the request corresponding to
00353   // RequestId identifier
00354   inline int MPIAccess::MPIOutCount( int RequestId )
00355   {
00356     struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
00357     if ( aRequestStruct )
00358       return aRequestStruct->MPIOutCount;
00359     return 0 ;
00360   }
00361 
00362   // Increments the previous tag value (cyclically)
00363   // Look at MPIAccess::NewSendTag/NewRecvTag in MPIAccess.cxx
00364   inline int MPIAccess::incrTag( int prevtag )
00365   {
00366     int tag;
00367     if ( (prevtag % MODULO_TAG) == _message_time )
00368       tag = ((prevtag/MODULO_TAG)*MODULO_TAG);
00369     else
00370       tag = ((prevtag/MODULO_TAG + 1)*MODULO_TAG);
00371     if ( tag > _max_MPI_tag )
00372       tag = _base_MPI_tag ;
00373     return tag ;
00374   }
00375 
00376   // Returns the MPITag with the method-type field
00377   // Look at MPIAccess::NewSendTag/NewRecvTag in MPIAccess.cxx
00378   inline int MPIAccess::valTag( int tag, int method )
00379   {
00380     return ((tag/MODULO_TAG)*MODULO_TAG) + method;
00381   }
00382   
00383   // Remove a Request identifier from the list _RecvRequests/_SendRequests for
00384   // the corresponding target.
00385   inline void MPIAccess::deleteSendRecvRequest( int RequestId )
00386   {
00387     if ( _trace )
00388       std::cout << "MPIAccess::DeleteSendRecvRequest" << _my_rank
00389                 << "( " << RequestId << " ) " << std::endl ;
00390     if ( MPIIsRecv( RequestId ) )
00391       _recv_requests[ MPITarget( RequestId ) ].remove( RequestId );
00392     else
00393       _send_requests[ MPITarget( RequestId ) ].remove( RequestId );
00394   }
00395 
00396   // Delete the MPI structure MPI_status * of a ReaquestId
00397   inline void MPIAccess::deleteStatus( int RequestId )
00398   {
00399     if ( _map_of_request_struct[RequestId]->MPIStatus != NULL )
00400       {
00401         delete _map_of_request_struct[RequestId]->MPIStatus ;
00402         clearMPIStatus( RequestId ) ;
00403       }
00404   }
00405 
00406   // Returns the MPI structure MPI_request * of a RequestId
00407   inline MPI_Request * MPIAccess::MPIRequest( int RequestId )
00408   {
00409     struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
00410     if ( aRequestStruct )
00411       return &aRequestStruct->MPIRequest;
00412     return &mpirequestnull ;
00413   }
00414   
00415   // Returns the MPI structure MPI_status * of a RequestId
00416   inline MPI_Status * MPIAccess::MPIStatus( int RequestId )
00417   {
00418     struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ];
00419     if ( aRequestStruct )
00420       return aRequestStruct->MPIStatus;
00421     return NULL ;
00422   }
00423 
00424   // Set the MPICompleted field of the structure Request corresponding to RequestId
00425   // identifier with the value completed
00426   inline void MPIAccess::setMPICompleted( int RequestId , bool completed )
00427   {
00428     struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
00429     if ( aRequestStruct )
00430       aRequestStruct->MPICompleted = completed;
00431   }
00432 
00433   // Set the MPIOutCount field of the structure Request corresponding to RequestId
00434   // identifier with the value outcount
00435   inline void MPIAccess::setMPIOutCount( int RequestId , int outcount )
00436   {
00437     struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
00438     if ( aRequestStruct )
00439       aRequestStruct->MPIOutCount = outcount;
00440   }
00441 
00442   // Nullify the MPIStatusfield of the structure Request corresponding to RequestId
00443   // identifier
00444   inline void MPIAccess::clearMPIStatus( int RequestId )
00445   {
00446     struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
00447     if ( aRequestStruct )
00448       aRequestStruct->MPIStatus = NULL ;
00449   }
00450 
00451   // Returns the _MessageIdent enum value corresponding to the MPI_Datatype datatype
00452   // Look at MPIAccess::NewSendTag/NewRecvTag in MPIAccess.cxx
00453   inline _MessageIdent MPIAccess::methodId( MPI_Datatype datatype ) const
00454   {
00455     _MessageIdent aMethodIdent ;
00456     if ( datatype == _MPI_TIME )
00457       aMethodIdent = _message_time;
00458     else if ( datatype == MPI_INT )
00459       aMethodIdent = _message_int ;
00460     else if ( datatype == MPI_DOUBLE )
00461       aMethodIdent = _message_double ;
00462     else
00463       aMethodIdent = _message_unknown ;
00464     return aMethodIdent ;
00465   }
00466   
00467   // Returns the MPI_Datatype corresponding to the _MessageIdent enum aMethodIdent
00468   inline MPI_Datatype MPIAccess::datatype( _MessageIdent aMethodIdent ) const
00469   {
00470     MPI_Datatype aDataType ;
00471     switch( aMethodIdent )
00472       {
00473       case _message_time :
00474         aDataType = _MPI_TIME ;
00475         break ;
00476       case _message_int :
00477         aDataType = MPI_INT ;
00478         break ;
00479       case _message_double :
00480         aDataType = MPI_DOUBLE ;
00481         break ;
00482       default :
00483         aDataType = (MPI_Datatype) -1 ;
00484         break ;
00485       }
00486     return aDataType ;
00487   }
00488 
00489   std::ostream & operator<< (std::ostream &,const _MessageIdent &);
00490 
00491   std::ostream & operator<< (std::ostream &,const TimeMessage &);
00492 
00493 }
00494 
00495 #endif