diff --git a/src/parallel/mpAccess_MPI.cc b/src/parallel/mpAccess_MPI.cc index 423a5dd25a1d5a0b92f009f0fece7f60933f6d67..53a703b2fab6d1cfb0675e7e86236a66bb3a63a0 100644 --- a/src/parallel/mpAccess_MPI.cc +++ b/src/parallel/mpAccess_MPI.cc @@ -2,6 +2,74 @@ #include "mpAccess_MPI.h" +#ifndef NDEBUG +#define MY_INT_TEST int test = +#else +#define MY_INT_TEST +#endif + +MPI_Comm mpiComm(void * mpiCommPtr) +{ + typedef MpAccessMPI :: Comm< MPI_Comm > MyComm; + MyComm& comm = (*((MyComm *) mpiCommPtr)); + return comm; +} + +template <> +MpAccessMPI :: Comm< MPI_Comm > :: Comm( MPI_Comm mpicomm ) +{ + // duplicate mpi communicator + MY_INT_TEST MPI_Comm_dup ( mpicomm, & _mpiComm ) ; + assert (test == MPI_SUCCESS) ; +} + +// workarround for old member variable +#define _mpiComm (mpiComm(_mpiCommPtr)) + +void MpAccessMPI :: initialize() +{ + { + MY_INT_TEST MPI_Comm_size ( _mpiComm, & _psize ); + assert (test == MPI_SUCCESS) ; + } + { + MY_INT_TEST MPI_Comm_rank ( _mpiComm, & _myrank ); + assert (test == MPI_SUCCESS) ; + } +} + +MpAccessMPI :: MpAccessMPI (const MpAccessMPI & a) +: _mpiCommPtr((void *) new Comm<MPI_Comm> (mpiComm(a._mpiCommPtr))), + _psize( 0 ) , _myrank( -1 ) +{ + initialize(); +} + +MpAccessMPI :: ~MpAccessMPI () +{ + typedef Comm<MPI_Comm> MyComm; + MyComm* comm = (MyComm *) _mpiCommPtr; + delete comm; + _mpiCommPtr = 0; + return ; +} + +int MpAccessMPI :: barrier () const { + return MPI_SUCCESS == MPI_Barrier (_mpiComm) ? psize () : 0 ; +} + +int MpAccessMPI :: mpi_allgather (int * i, int si, int * o, int so) const { + return MPI_Allgather (i, si, MPI_INT, o, so, MPI_INT, _mpiComm) ; +} + +int MpAccessMPI :: mpi_allgather (char * i, int si, char * o, int so) const { + return MPI_Allgather (i, si, MPI_BYTE, o, so, MPI_BYTE, _mpiComm) ; +} + +int MpAccessMPI :: mpi_allgather (double * i, int si, double * o, int so) const { + return MPI_Allgather (i, si, MPI_DOUBLE, o, so, MPI_DOUBLE, _mpiComm) ; +} + template < class A > vector < vector < A > > doGcollectV (const vector < A > & in, MPI_Datatype mpiType, MPI_Comm comm) { @@ -522,3 +590,4 @@ exchange (const vector < ObjectStream > & in, } return ; } +#undef _mpiComm diff --git a/src/parallel/mpAccess_MPI.h b/src/parallel/mpAccess_MPI.h index ba8f1bc568acbd00bbfcc1096907c1b0969ab3f1..c4e1c9d56205403d150b42bd49a60dc6fae98ae9 100644 --- a/src/parallel/mpAccess_MPI.h +++ b/src/parallel/mpAccess_MPI.h @@ -6,126 +6,98 @@ class MpAccessMPI : public MpAccessLocal { - MPI_Comm _mpiComm ; +public: + template <class MPICommunicator> + class Comm + { + // no copying or assigning + Comm( const Comm& ); + Comm& operator= (const Comm& ); + public: + // we don't want MPI types here to avoid include of mpi.h + mutable MPICommunicator _mpiComm; + Comm( MPICommunicator ); + operator MPICommunicator () const { return _mpiComm; } + }; + +protected: + void * _mpiCommPtr; + int _psize; + int _myrank; + + int mpi_allgather (int *, int , int *, int) const ; + int mpi_allgather (char *, int, char *, int) const ; + int mpi_allgather (double *, int, double *, int ) const ; +public : + template <class MPICommunicator> + inline MpAccessMPI (MPICommunicator i) + : _mpiCommPtr( (void *) new Comm<MPICommunicator> ( i ) ), + _psize( 0 ), _myrank( -1 ) + { + initialize(); + } + + MpAccessMPI (const MpAccessMPI &) ; + ~MpAccessMPI () ; +protected: + void initialize () ; +public: + inline int psize () const ; + inline int myrank () const ; + int barrier () const ; + int gmax (int) const ; + int gmin (int) const ; + int gsum (int) const ; + long gmax (long) const ; + long gmin (long) const ; + long gsum (long) const ; + double gmax (double) const ; + double gmin (double) const ; + double gsum (double) const ; + void gmax (double*,int,double*) const ; + void gmin (double*,int,double*) const ; + void gsum (double*,int,double*) const ; + pair<double,double> gmax (pair<double,double>) const ; + pair<double,double> gmin (pair<double,double>) const ; + pair<double,double> gsum (pair<double,double>) const ; + vector < int > gcollect (int) const ; + vector < double > gcollect (double) const ; + vector < vector < int > > gcollect (const vector < int > &) const ; + vector < vector < double > > gcollect (const vector < double > &) const ; + vector < ObjectStream > gcollect (const ObjectStream &) const ; + vector < vector < int > > exchange (const vector < vector < int > > &) const ; + vector < vector < double > > exchange (const vector < vector < double > > &) const ; + vector < vector < char > > exchange (const vector < vector < char > > &) const ; + + vector < ObjectStream > exchange (const vector < ObjectStream > &) const ; + + // symectric exchange with same buffer size + void exchange (const vector < ObjectStream > & in, + vector< ObjectStream > & out) const; - inline int mpi_allgather (int *, int , int *, int) const ; - inline int mpi_allgather (char *, int, char *, int) const ; - inline int mpi_allgather (double *, int, double *, int ) const ; - public : - inline MpAccessMPI (MPI_Comm) ; - inline MpAccessMPI (const MpAccessMPI &) ; - inline ~MpAccessMPI () ; - inline int psize () const ; - inline int myrank () const ; - inline int barrier () const ; - int gmax (int) const ; - int gmin (int) const ; - int gsum (int) const ; - long gmax (long) const ; - long gmin (long) const ; - long gsum (long) const ; - double gmax (double) const ; - double gmin (double) const ; - double gsum (double) const ; - void gmax (double*,int,double*) const ; - void gmin (double*,int,double*) const ; - void gsum (double*,int,double*) const ; - pair<double,double> gmax (pair<double,double>) const ; - pair<double,double> gmin (pair<double,double>) const ; - pair<double,double> gsum (pair<double,double>) const ; - vector < int > gcollect (int) const ; - vector < double > gcollect (double) const ; - vector < vector < int > > gcollect (const vector < int > &) const ; - vector < vector < double > > gcollect (const vector < double > &) const ; - vector < ObjectStream > gcollect (const ObjectStream &) const ; - vector < vector < int > > exchange (const vector < vector < int > > &) const ; - vector < vector < double > > exchange (const vector < vector < double > > &) const ; - vector < vector < char > > exchange (const vector < vector < char > > &) const ; - - vector < ObjectStream > exchange (const vector < ObjectStream > &) const ; - - // symectric exchange with same buffer size - void exchange (const vector < ObjectStream > & in, - vector< ObjectStream > & out) const; - - // return address of MPI communicator (dirty hack, but what can we do) - void* communicator() { return ((void *) &_mpiComm); } + // return address of MPI communicator (dirty hack, but what can we do) + void* communicator() { return _mpiCommPtr; } } ; - // - // # # # # # # # ###### - // # ## # # # ## # # - // # # # # # # # # # ##### - // # # # # # # # # # # - // # # ## # # # ## # - // # # # ###### # # # ###### - // -#ifndef NDEBUG -#define MY_INT_TEST int test = -#else -#define MY_INT_TEST -#endif - -#define USE_MPI_COMM_DUP - -inline MpAccessMPI :: MpAccessMPI (MPI_Comm i) { -#ifdef USE_MPI_COMM_DUP - MY_INT_TEST MPI_Comm_dup (i, &_mpiComm) ; - assert (test == MPI_SUCCESS) ; -#else - _mpiComm = i; -#endif - return ; -} - -inline MpAccessMPI :: MpAccessMPI (const MpAccessMPI & a) { -#ifdef USE_MPI_COMM_DUP - MY_INT_TEST MPI_Comm_dup (a._mpiComm, &_mpiComm) ; - assert (test == MPI_SUCCESS) ; -#else - _mpiComm = a._mpiComm; -#endif - return ; -} - -inline MpAccessMPI :: ~MpAccessMPI () { -#ifdef USE_MPI_COMM_DUP - MY_INT_TEST MPI_Comm_free (&_mpiComm) ; - assert (test == MPI_SUCCESS) ; -#endif - return ; -} - -inline int MpAccessMPI :: psize () const { - int i ; - MY_INT_TEST MPI_Comm_size (_mpiComm, & i) ; - assert (test == MPI_SUCCESS) ; - return i ; -} - -inline int MpAccessMPI :: myrank () const { - int i ; - MY_INT_TEST MPI_Comm_rank (_mpiComm, & i) ; - assert (test == MPI_SUCCESS) ; - return i ; -} - -inline int MpAccessMPI :: barrier () const { - return MPI_SUCCESS == MPI_Barrier (_mpiComm) ? psize () : 0 ; -} - -inline int MpAccessMPI :: mpi_allgather (int * i, int si, int * o, int so) const { - return MPI_Allgather (i, si, MPI_INT, o, so, MPI_INT, _mpiComm) ; -} - -inline int MpAccessMPI :: mpi_allgather (char * i, int si, char * o, int so) const { - return MPI_Allgather (i, si, MPI_BYTE, o, so, MPI_BYTE, _mpiComm) ; +// +// # # # # # # # ###### +// # ## # # # ## # # +// # # # # # # # # # ##### +// # # # # # # # # # # +// # # ## # # # ## # +// # # # ###### # # # ###### +// +inline int MpAccessMPI :: psize () const +{ + assert( _psize > 0 ); + //return _psize; + return _psize; } -inline int MpAccessMPI :: mpi_allgather (double * i, int si, double * o, int so) const { - return MPI_Allgather (i, si, MPI_DOUBLE, o, so, MPI_DOUBLE, _mpiComm) ; +inline int MpAccessMPI :: myrank () const +{ + assert( _myrank != -1 ); + return _myrank; } - -#undef USE_MPI_COMM_DUP #endif