diff --git a/dune/alugrid/src/alugrid_parallel.cc b/dune/alugrid/src/alugrid_parallel.cc index b822c7766384fb8b312d3bc43f34c632ab5e7015..ac6736a987f8cfdcd5dc75d2679ef10f339ed217 100644 --- a/dune/alugrid/src/alugrid_parallel.cc +++ b/dune/alugrid/src/alugrid_parallel.cc @@ -13,34 +13,6 @@ #warning -- Compiling ALUGrid code in debug mode!!! Use -DNDEBUG to get more optimized code!!! #endif -// avoid C++ bindings of MPI (-DMPIPP_H is not common enough) -// this is the only thing all MPI implementations have in common -// to do that we pretend that we are compiling C code -#if defined(__cplusplus) -#define rem__cplusplus __cplusplus -#undef __cplusplus -#endif - -#if defined(c_plusplus) -#define remc_plusplus c_plusplus -#undef c_plusplus -#endif - -extern "C" { - // the message passing interface (MPI) headers for C - #include <mpi.h> -} - -// restore defines -#if defined(rem__cplusplus) -#define __cplusplus rem__cplusplus -#undef rem__cplusplus -#endif - -#if defined(c_plusplus) -#define c_plusplus remc_plusplus -#undef remc_plusplus -#endif namespace ALUGridSpace { diff --git a/dune/alugrid/src/parallel/mpAccess_MPI.cc b/dune/alugrid/src/parallel/mpAccess_MPI.cc index 60d35c92b32918a67eeb346168175fed6304606b..4531ef82a820593adfa69171a612048a36fcd45d 100644 --- a/dune/alugrid/src/parallel/mpAccess_MPI.cc +++ b/dune/alugrid/src/parallel/mpAccess_MPI.cc @@ -1,55 +1,88 @@ +#include <iostream> + #include "mpAccess_MPI.h" +// avoid C++ bindings of MPI (-DMPIPP_H is not common enough) +// this is the only thing all MPI implementations have in common +// to do that we pretend that we are compiling C code +#if defined(__cplusplus) +#define rem__cplusplus __cplusplus +#undef __cplusplus +#endif + +#if defined(c_plusplus) +#define remc_plusplus c_plusplus +#undef c_plusplus +#endif + +extern "C" { + // the message passing interface (MPI) headers for C + #include <mpi.h> +} + +// restore defines +#if defined(rem__cplusplus) +#define __cplusplus rem__cplusplus +#undef rem__cplusplus +#endif + +#if defined(c_plusplus) +#define c_plusplus remc_plusplus +#undef remc_plusplus +#endif + + #ifndef NDEBUG #define MY_INT_TEST int test = #else #define MY_INT_TEST #endif -MPI_Comm getMPICommunicator(const MpAccessMPI :: CommIF* mpiCommPtr) + +MPI_Comm getMPICommunicator(const MpAccessMPI::CommIF* mpiCommPtr) { - typedef MpAccessMPI :: Comm< MPI_Comm > MyComm; + typedef MpAccessMPI::Comm< MPI_Comm > MyComm; return static_cast<const MyComm&> (*mpiCommPtr); } template <> -MpAccessMPI :: Comm< MPI_Comm > :: Comm( MPI_Comm mpicomm ) +MpAccessMPI::Comm< MPI_Comm >::Comm( MPI_Comm mpicomm ) { // duplicate mpi communicator - MY_INT_TEST MPI_Comm_dup ( mpicomm, &_mpiComm) ; - assert (test == MPI_SUCCESS) ; + MY_INT_TEST MPI_Comm_dup ( mpicomm, &_mpiComm); + assert (test == MPI_SUCCESS); } template <> -MpAccessMPI :: Comm< MPI_Comm > :: ~Comm( ) +MpAccessMPI::Comm< MPI_Comm >::~Comm( ) { // free mpi communicator - MY_INT_TEST MPI_Comm_free (&_mpiComm) ; - assert (test == MPI_SUCCESS) ; + MY_INT_TEST MPI_Comm_free (&_mpiComm); + assert (test == MPI_SUCCESS); } // workarround for old member variable #define _mpiComm (getMPICommunicator(_mpiCommPtr)) -int MpAccessMPI :: getSize() +int MpAccessMPI::getSize() { // get size from MPI int size = 0; MY_INT_TEST MPI_Comm_size ( _mpiComm, & size ); - assert (test == MPI_SUCCESS) ; + assert (test == MPI_SUCCESS); return size; } -int MpAccessMPI :: getRank() +int MpAccessMPI::getRank() { // get rank from MPI int rank = -1; MY_INT_TEST MPI_Comm_rank ( _mpiComm, & rank ); - assert (test == MPI_SUCCESS) ; + assert (test == MPI_SUCCESS); return rank; } -MpAccessMPI :: MpAccessMPI (const MpAccessMPI & a) +MpAccessMPI::MpAccessMPI (const MpAccessMPI & a) : _mpiCommPtr( a._mpiCommPtr->clone() ), _minmaxsum( 0 ), _psize( getSize() ) , _myrank( getRank() ) @@ -57,257 +90,257 @@ MpAccessMPI :: MpAccessMPI (const MpAccessMPI & a) initMinMaxSum(); } -MpAccessMPI :: ~MpAccessMPI () +MpAccessMPI::~MpAccessMPI () { delete _minmaxsum; delete _mpiCommPtr; _mpiCommPtr = 0; } -int MpAccessMPI :: barrier () const { - return MPI_SUCCESS == MPI_Barrier (_mpiComm) ? psize () : 0 ; +int MpAccessMPI::barrier () const { + return MPI_SUCCESS == MPI_Barrier (_mpiComm) ? psize () : 0; } -int MpAccessMPI :: mpi_allgather (int * i, int si, int * o, int so) const { - return MPI_Allgather (i, si, MPI_INT, o, so, MPI_INT, _mpiComm) ; +int MpAccessMPI::mpi_allgather (int * i, int si, int * o, int so) const { + return MPI_Allgather (i, si, MPI_INT, o, so, MPI_INT, _mpiComm); } -int MpAccessMPI :: mpi_allgather (char * i, int si, char * o, int so) const { - return MPI_Allgather (i, si, MPI_BYTE, o, so, MPI_BYTE, _mpiComm) ; +int MpAccessMPI::mpi_allgather (char * i, int si, char * o, int so) const { + return MPI_Allgather (i, si, MPI_BYTE, o, so, MPI_BYTE, _mpiComm); } -int MpAccessMPI :: mpi_allgather (double * i, int si, double * o, int so) const { - return MPI_Allgather (i, si, MPI_DOUBLE, o, so, MPI_DOUBLE, _mpiComm) ; +int MpAccessMPI::mpi_allgather (double * i, int si, double * o, int so) const { + return MPI_Allgather (i, si, MPI_DOUBLE, o, so, MPI_DOUBLE, _mpiComm); } -template < class A > vector < vector < A > > -doGcollectV (const vector < A > & in, MPI_Datatype mpiType, MPI_Comm comm) +template < class A > std::vector< std::vector< A > > +doGcollectV (const std::vector< A > & in, MPI_Datatype mpiType, MPI_Comm comm) { - int np, me, test ; + int np, me, test; - test = MPI_Comm_rank (comm, & me) ; + test = MPI_Comm_rank (comm, & me); if (test != MPI_SUCCESS) { - cerr << "Unable to obtain rank in MPI communicator." << endl; + std::cerr << "ERROR (fatal): Unable to obtain rank in MPI communicator." << std::endl; abort(); } - test = MPI_Comm_size (comm, & np) ; + test = MPI_Comm_size (comm, & np); if (test != MPI_SUCCESS) { - cerr << "Unable to obtain size of MPI communicator." << endl; + std::cerr << "ERROR (fatal): Unable to obtain size of MPI communicator." << std::endl; abort(); } - int * rcounts = new int [np] ; - int * displ = new int [np] ; - assert (rcounts) ; - vector < vector < A > > res (np) ; + int * rcounts = new int [np]; + int * displ = new int [np]; + assert (rcounts); + std::vector< std::vector< A > > res (np); { - int ln = in.size () ; - MY_INT_TEST MPI_Allgather (& ln, 1, MPI_INT, rcounts, 1, MPI_INT, comm) ; - assert (test == MPI_SUCCESS) ; - displ [0] = 0 ; - {for (int j = 1 ; j < np ; j ++) { + int ln = in.size (); + MY_INT_TEST MPI_Allgather (& ln, 1, MPI_INT, rcounts, 1, MPI_INT, comm); + assert (test == MPI_SUCCESS); + displ [0] = 0; + {for (int j = 1; j < np; j ++) { displ [j] = displ [j-1] + rcounts [j-1]; }} - const int xSize = displ [np-1] + rcounts [np-1] ; - A * x = new A [xSize] ; - A * y = new A [ln] ; - assert (x && y) ; - copy (in.begin(), in.end(), y) ; - test = MPI_Allgatherv (y, ln, mpiType, x, rcounts, displ, mpiType, comm) ; - delete [] y ; - y = 0 ; - assert (test == MPI_SUCCESS) ; - {for (int i = 0 ; i < np ; i ++ ) { - res [i].reserve (rcounts [i]) ; - copy (x + displ [i], x + displ [i] + rcounts [i], back_inserter(res [i])) ; + const int xSize = displ [np-1] + rcounts [np-1]; + A * x = new A [xSize]; + A * y = new A [ln]; + assert (x && y); + copy (in.begin(), in.end(), y); + test = MPI_Allgatherv (y, ln, mpiType, x, rcounts, displ, mpiType, comm); + delete [] y; + y = 0; + assert (test == MPI_SUCCESS); + {for (int i = 0; i < np; i ++ ) { + res [i].reserve (rcounts [i]); + copy (x + displ [i], x + displ [i] + rcounts [i], back_inserter(res [i])); }} - delete [] x ; + delete [] x; } - delete [] displ ; - delete [] rcounts ; - return res ; + delete [] displ; + delete [] rcounts; + return res; } template < class A > -vector < vector < A > > doExchange (const vector < vector < A > > & in, +std::vector< std::vector< A > > doExchange (const std::vector< std::vector< A > > & in, MPI_Datatype mpiType, MPI_Comm comm, - const vector < int > & d) + const std::vector< int > & d) { - const int messagetag = MpAccessMPI :: messagetag ; - assert (in.size() == d.size()) ; - int nl = d.size () ; - vector < vector < A > > out (nl) ; + const int messagetag = MpAccessMPI::messagetag; + assert (in.size() == d.size()); + int nl = d.size (); + std::vector< std::vector< A > > out (nl); // only do this if number of links is not zero if( nl > 0 ) { - A ** buf = new A * [nl] ; - assert (buf) ; - MPI_Request * req = new MPI_Request [nl] ; - assert (req) ; + A ** buf = new A * [nl]; + assert (buf); + MPI_Request * req = new MPI_Request [nl]; + assert (req); { - for (int link = 0 ; link < nl ; ++ link) + for (int link = 0; link < nl; ++ link) { - int size = in [link].size() ; - A * lne = new A [size] ; - assert (lne) ; - copy (in [link].begin (), in [link].end (), lne) ; - buf [link] = lne ; - MY_INT_TEST MPI_Isend (lne, size, mpiType, d [link], messagetag, comm, & req [link]) ; - assert (test == MPI_SUCCESS) ; + int size = in [link].size(); + A * lne = new A [size]; + assert (lne); + copy (in [link].begin (), in [link].end (), lne); + buf [link] = lne; + MY_INT_TEST MPI_Isend (lne, size, mpiType, d [link], messagetag, comm, & req [link]); + assert (test == MPI_SUCCESS); } } { - for (int link = 0 ; link < nl ; ++ link) + for (int link = 0; link < nl; ++ link) { - MPI_Status s ; - int cnt ; + MPI_Status s; + int cnt; { - MY_INT_TEST MPI_Probe (d [link], messagetag, comm, & s) ; - assert (test == MPI_SUCCESS) ; + MY_INT_TEST MPI_Probe (d [link], messagetag, comm, & s); + assert (test == MPI_SUCCESS); } { - MY_INT_TEST MPI_Get_count ( & s, mpiType, & cnt ) ; - assert (test == MPI_SUCCESS) ; + MY_INT_TEST MPI_Get_count ( & s, mpiType, & cnt ); + assert (test == MPI_SUCCESS); } - A * lne = new A [cnt] ; - assert (lne) ; + A * lne = new A [cnt]; + assert (lne); { - MY_INT_TEST MPI_Recv (lne, cnt, mpiType, d [link], messagetag, comm, & s) ; - assert (test == MPI_SUCCESS) ; + MY_INT_TEST MPI_Recv (lne, cnt, mpiType, d [link], messagetag, comm, & s); + assert (test == MPI_SUCCESS); } - copy (lne, lne + cnt, back_inserter (out[link])) ; - delete [] lne ; + copy (lne, lne + cnt, back_inserter (out[link])); + delete [] lne; } } { - MPI_Status * sta = new MPI_Status [nl] ; - assert (sta) ; - MY_INT_TEST MPI_Waitall (nl, req, sta) ; - assert (test == MPI_SUCCESS) ; - delete [] sta ; + MPI_Status * sta = new MPI_Status [nl]; + assert (sta); + MY_INT_TEST MPI_Waitall (nl, req, sta); + assert (test == MPI_SUCCESS); + delete [] sta; } { - for (int i = 0 ; i < nl ; ++i) delete [] buf [i] ; + for (int i = 0; i < nl; ++i) delete [] buf [i]; } - delete [] buf ; - delete [] req ; + delete [] buf; + delete [] req; } // end if( nl > 0 ) - return out ; + return out; } -bool MpAccessMPI :: gmax (bool i) const +bool MpAccessMPI::gmax (bool i) const { int j = int( i ); // call int method const int ret = gmax( j ); - return (ret == 1) ? true : false ; + return (ret == 1) ? true : false; } -int MpAccessMPI :: gmax (int i) const { - int j ; - MY_INT_TEST MPI_Allreduce (&i, &j, 1, MPI_INT, MPI_MAX, _mpiComm) ; - assert (test == MPI_SUCCESS) ; - return j ; +int MpAccessMPI::gmax (int i) const { + int j; + MY_INT_TEST MPI_Allreduce (&i, &j, 1, MPI_INT, MPI_MAX, _mpiComm); + assert (test == MPI_SUCCESS); + return j; } -int MpAccessMPI :: gmin (int i) const { - int j ; - MY_INT_TEST MPI_Allreduce (&i, &j, 1, MPI_INT, MPI_MIN, _mpiComm) ; - assert (test == MPI_SUCCESS) ; - return j ; +int MpAccessMPI::gmin (int i) const { + int j; + MY_INT_TEST MPI_Allreduce (&i, &j, 1, MPI_INT, MPI_MIN, _mpiComm); + assert (test == MPI_SUCCESS); + return j; } -int MpAccessMPI :: gsum (int i) const { - int j ; - MY_INT_TEST MPI_Allreduce (&i, &j, 1, MPI_INT, MPI_SUM, _mpiComm) ; - assert (test == MPI_SUCCESS) ; - return j ; +int MpAccessMPI::gsum (int i) const { + int j; + MY_INT_TEST MPI_Allreduce (&i, &j, 1, MPI_INT, MPI_SUM, _mpiComm); + assert (test == MPI_SUCCESS); + return j; } -long MpAccessMPI :: gmax (long i) const { - long j ; - MY_INT_TEST MPI_Allreduce (&i, &j, 1, MPI_LONG, MPI_MAX, _mpiComm) ; - assert (test == MPI_SUCCESS) ; - return j ; +long MpAccessMPI::gmax (long i) const { + long j; + MY_INT_TEST MPI_Allreduce (&i, &j, 1, MPI_LONG, MPI_MAX, _mpiComm); + assert (test == MPI_SUCCESS); + return j; } -long MpAccessMPI :: gmin (long i) const { - long j ; - MY_INT_TEST MPI_Allreduce (&i, &j, 1, MPI_LONG, MPI_MIN, _mpiComm) ; - assert (test == MPI_SUCCESS) ; - return j ; +long MpAccessMPI::gmin (long i) const { + long j; + MY_INT_TEST MPI_Allreduce (&i, &j, 1, MPI_LONG, MPI_MIN, _mpiComm); + assert (test == MPI_SUCCESS); + return j; } -long MpAccessMPI :: gsum (long i) const { - long j ; - MY_INT_TEST MPI_Allreduce (&i, &j, 1, MPI_LONG, MPI_SUM, _mpiComm) ; - assert (test == MPI_SUCCESS) ; - return j ; +long MpAccessMPI::gsum (long i) const { + long j; + MY_INT_TEST MPI_Allreduce (&i, &j, 1, MPI_LONG, MPI_SUM, _mpiComm); + assert (test == MPI_SUCCESS); + return j; } -double MpAccessMPI :: gmax (double a) const { - double x ; - MY_INT_TEST MPI_Allreduce (&a, &x, 1, MPI_DOUBLE, MPI_MAX, _mpiComm) ; - assert (test == MPI_SUCCESS) ; - return x ; +double MpAccessMPI::gmax (double a) const { + double x; + MY_INT_TEST MPI_Allreduce (&a, &x, 1, MPI_DOUBLE, MPI_MAX, _mpiComm); + assert (test == MPI_SUCCESS); + return x; } -double MpAccessMPI :: gmin (double a) const { - double x ; - MY_INT_TEST MPI_Allreduce (&a, &x, 1, MPI_DOUBLE, MPI_MIN, _mpiComm) ; - assert (test == MPI_SUCCESS) ; - return x ; +double MpAccessMPI::gmin (double a) const { + double x; + MY_INT_TEST MPI_Allreduce (&a, &x, 1, MPI_DOUBLE, MPI_MIN, _mpiComm); + assert (test == MPI_SUCCESS); + return x; } -double MpAccessMPI :: gsum (double a) const { - double x ; - MY_INT_TEST MPI_Allreduce (&a, &x, 1, MPI_DOUBLE, MPI_SUM, _mpiComm) ; - assert (test == MPI_SUCCESS) ; - return x ; +double MpAccessMPI::gsum (double a) const { + double x; + MY_INT_TEST MPI_Allreduce (&a, &x, 1, MPI_DOUBLE, MPI_SUM, _mpiComm); + assert (test == MPI_SUCCESS); + return x; } -void MpAccessMPI :: gmax (double* a,int size,double *x) const { - MY_INT_TEST MPI_Allreduce (a, x, size, MPI_DOUBLE, MPI_MAX, _mpiComm) ; - assert (test == MPI_SUCCESS) ; +void MpAccessMPI::gmax (double* a,int size,double *x) const { + MY_INT_TEST MPI_Allreduce (a, x, size, MPI_DOUBLE, MPI_MAX, _mpiComm); + assert (test == MPI_SUCCESS); } -void MpAccessMPI :: gmin (double* a,int size,double *x) const { - MY_INT_TEST MPI_Allreduce (a, x, size, MPI_DOUBLE, MPI_MIN, _mpiComm) ; - assert (test == MPI_SUCCESS) ; +void MpAccessMPI::gmin (double* a,int size,double *x) const { + MY_INT_TEST MPI_Allreduce (a, x, size, MPI_DOUBLE, MPI_MIN, _mpiComm); + assert (test == MPI_SUCCESS); } -void MpAccessMPI :: gsum (double* a,int size,double *x) const { - MY_INT_TEST MPI_Allreduce (a, x, size, MPI_DOUBLE, MPI_SUM, _mpiComm) ; - assert (test == MPI_SUCCESS) ; +void MpAccessMPI::gsum (double* a,int size,double *x) const { + MY_INT_TEST MPI_Allreduce (a, x, size, MPI_DOUBLE, MPI_SUM, _mpiComm); + assert (test == MPI_SUCCESS); } -void MpAccessMPI :: gmax (int* a,int size,int *x) const { - MY_INT_TEST MPI_Allreduce (a, x, size, MPI_INT, MPI_MAX, _mpiComm) ; - assert (test == MPI_SUCCESS) ; +void MpAccessMPI::gmax (int* a,int size,int *x) const { + MY_INT_TEST MPI_Allreduce (a, x, size, MPI_INT, MPI_MAX, _mpiComm); + assert (test == MPI_SUCCESS); } -void MpAccessMPI :: gmin (int* a,int size,int *x) const { - MY_INT_TEST MPI_Allreduce (a, x, size, MPI_INT, MPI_MIN, _mpiComm) ; - assert (test == MPI_SUCCESS) ; +void MpAccessMPI::gmin (int* a,int size,int *x) const { + MY_INT_TEST MPI_Allreduce (a, x, size, MPI_INT, MPI_MIN, _mpiComm); + assert (test == MPI_SUCCESS); } -void MpAccessMPI :: gsum (int* a,int size,int *x) const { - MY_INT_TEST MPI_Allreduce (a, x, size, MPI_INT, MPI_SUM, _mpiComm) ; - assert (test == MPI_SUCCESS) ; +void MpAccessMPI::gsum (int* a,int size,int *x) const { + MY_INT_TEST MPI_Allreduce (a, x, size, MPI_INT, MPI_SUM, _mpiComm); + assert (test == MPI_SUCCESS); } -struct MinMaxSumOp : public MpAccessMPI :: MinMaxSumIF +struct MinMaxSumOp : public MpAccessMPI::MinMaxSumIF { - typedef MpAccessMPI :: minmaxsum_t minmaxsum_t; + typedef MpAccessMPI::minmaxsum_t minmaxsum_t; MinMaxSumOp ( const MpAccessMPI& mpAccess ) : _mpAccess( mpAccess ) @@ -323,7 +356,7 @@ struct MinMaxSumOp : public MpAccessMPI :: MinMaxSumIF MPI_Type_free(&_mpi_minmaxsum_t); } - const MpAccessMPI& _mpAccess ; + const MpAccessMPI& _mpAccess; MPI_Op _op; MPI_Datatype _mpi_minmaxsum_t; @@ -332,7 +365,7 @@ struct MinMaxSumOp : public MpAccessMPI :: MinMaxSumIF min_max_sum( minmaxsum_t* in, minmaxsum_t* inout, int* len, MPI_Datatype* datatype ) { - const int size = *len ; + const int size = *len; for( int i = 0; i < size; ++i ) { // min @@ -346,12 +379,12 @@ struct MinMaxSumOp : public MpAccessMPI :: MinMaxSumIF minmaxsum_t minmaxsum( double value ) const { - const MpAccessMPI :: CommIF* _mpiCommPtr = _mpAccess.mpiCommPtr(); + const MpAccessMPI::CommIF* _mpiCommPtr = _mpAccess.mpiCommPtr(); // get mpi communicator (use define, see above) - MPI_Comm comm = _mpiComm ; + MPI_Comm comm = _mpiComm; // create send buf - minmaxsum_t sendbuf( value ) ; + minmaxsum_t sendbuf( value ); // create recv buf minmaxsum_t result( value ); @@ -361,116 +394,114 @@ struct MinMaxSumOp : public MpAccessMPI :: MinMaxSumIF } }; -void MpAccessMPI :: initMinMaxSum() +void MpAccessMPI::initMinMaxSum() { if( ! _minmaxsum ) _minmaxsum = new MinMaxSumOp( *this ); } -MpAccessMPI :: minmaxsum_t MpAccessMPI :: minmaxsum( double value ) const +MpAccessMPI::minmaxsum_t MpAccessMPI::minmaxsum( double value ) const { assert( _minmaxsum ); return _minmaxsum->minmaxsum( value ); } -pair<double,double> MpAccessMPI :: gmax (pair<double,double> p) const { - double x[2] ; +std::pair<double,double> MpAccessMPI::gmax (std::pair<double,double> p) const { + double x[2]; double a[2]={p.first,p.second}; - MY_INT_TEST MPI_Allreduce (a, x, 2, MPI_DOUBLE, MPI_MAX, _mpiComm) ; - assert (test == MPI_SUCCESS) ; - return pair<double,double>(x[0],x[1]) ; + MY_INT_TEST MPI_Allreduce (a, x, 2, MPI_DOUBLE, MPI_MAX, _mpiComm); + assert (test == MPI_SUCCESS); + return std::pair<double,double>(x[0],x[1]); } -pair<double,double> MpAccessMPI :: gmin (pair<double,double> p) const { - double x[2] ; +std::pair<double,double> MpAccessMPI::gmin (std::pair<double,double> p) const { + double x[2]; double a[2]={p.first,p.second}; - MY_INT_TEST MPI_Allreduce (a, x, 2, MPI_DOUBLE, MPI_MIN, _mpiComm) ; - assert (test == MPI_SUCCESS) ; - return pair<double,double>(x[0],x[1]) ; + MY_INT_TEST MPI_Allreduce (a, x, 2, MPI_DOUBLE, MPI_MIN, _mpiComm); + assert (test == MPI_SUCCESS); + return std::pair<double,double>(x[0],x[1]); } -pair<double,double> MpAccessMPI :: gsum (pair<double,double> p) const { - double x[2] ; +std::pair<double,double> MpAccessMPI::gsum (std::pair<double,double> p) const { + double x[2]; double a[2]={p.first,p.second}; - MY_INT_TEST MPI_Allreduce (a, x, 2, MPI_DOUBLE, MPI_SUM, _mpiComm) ; - assert (test == MPI_SUCCESS) ; - return pair<double,double>(x[0],x[1]) ; + MY_INT_TEST MPI_Allreduce (a, x, 2, MPI_DOUBLE, MPI_SUM, _mpiComm); + assert (test == MPI_SUCCESS); + return std::pair<double,double>(x[0],x[1]); } -void MpAccessMPI :: bcast (int* buff, int length, int root ) const +void MpAccessMPI::bcast (int* buff, int length, int root ) const { - MPI_Bcast(buff, length, MPI_INT, root, _mpiComm) ; + MPI_Bcast(buff, length, MPI_INT, root, _mpiComm); } -void MpAccessMPI :: bcast (char* buff, int length, int root ) const +void MpAccessMPI::bcast (char* buff, int length, int root ) const { - MPI_Bcast(buff, length, MPI_BYTE, root, _mpiComm) ; + MPI_Bcast(buff, length, MPI_BYTE, root, _mpiComm); } -void MpAccessMPI :: bcast (double* buff, int length, int root ) const +void MpAccessMPI::bcast (double* buff, int length, int root ) const { - MPI_Bcast(buff, length, MPI_DOUBLE, root, _mpiComm) ; + MPI_Bcast(buff, length, MPI_DOUBLE, root, _mpiComm); } -int MpAccessMPI :: exscan( int myvalue ) const +int MpAccessMPI::exscan( int myvalue ) const { - int sum = myvalue ; - MPI_Exscan(&myvalue, &sum, 1, MPI_INT, MPI_SUM, _mpiComm) ; + int sum = myvalue; + MPI_Exscan(&myvalue, &sum, 1, MPI_INT, MPI_SUM, _mpiComm); return sum; } -int MpAccessMPI :: scan( int myvalue ) const +int MpAccessMPI::scan( int myvalue ) const { - int sum = myvalue ; - MPI_Scan(&myvalue, &sum, 1, MPI_INT, MPI_SUM, _mpiComm) ; + int sum = myvalue; + MPI_Scan(&myvalue, &sum, 1, MPI_INT, MPI_SUM, _mpiComm); return sum; } -vector < int > MpAccessMPI :: gcollect (int i) const { - vector < int > r (psize (), 0L) ; - mpi_allgather (&i, 1, &r[ 0], 1) ; - return r ; +std::vector< int > MpAccessMPI::gcollect (int i) const { + std::vector< int > r (psize (), 0L); + mpi_allgather (&i, 1, &r[ 0], 1); + return r; } -vector < double > MpAccessMPI :: gcollect (double a) const +std::vector< double > MpAccessMPI::gcollect (double a) const { - vector < double > r (psize (),0.0) ; - mpi_allgather (& a, 1, &r[ 0 ], 1) ; - return r ; + std::vector< double > r (psize (),0.0); + mpi_allgather (& a, 1, &r[ 0 ], 1); + return r; } -vector < vector < int > > MpAccessMPI :: gcollect (const vector < int > & v) const { - return doGcollectV (v, MPI_INT, _mpiComm) ; +std::vector< std::vector< int > > MpAccessMPI::gcollect (const std::vector< int > & v) const { + return doGcollectV (v, MPI_INT, _mpiComm); } -vector < vector < double > > MpAccessMPI :: gcollect (const vector < double > & v) const { - return doGcollectV (v, MPI_DOUBLE, _mpiComm) ; +std::vector< std::vector< double > > MpAccessMPI::gcollect (const std::vector< double > & v) const { + return doGcollectV (v, MPI_DOUBLE, _mpiComm); } -vector < ObjectStream > MpAccessMPI :: -gcollect (const ObjectStream & in, const vector<int>& len ) const +std::vector< ObjectStream > MpAccessMPI:: +gcollect (const ObjectStream & in, const std::vector<int>& len ) const { // number of processes const int np = psize (); // size of buffer - const int snum = in._wb - in._rb ; + const int snum = in._wb - in._rb; // create empty objects streams - vector < ObjectStream > o (np) ; + std::vector< ObjectStream > o (np); #ifndef NDEBUG // check that given length values are correct { - vector< int > checkLength = gcollect( snum ); + std::vector< int > checkLength = gcollect( snum ); // check sizes assert( len.size() == checkLength.size() ); if( ! equal( checkLength.begin(), checkLength.end(), len.begin() ) ) { for(size_t i=0; i<checkLength.size(); ++i ) - { - cout << "comm " << checkLength[ i ] << " stored " << len[ i ] << endl; - } + std::cout << "comm " << checkLength[ i ] << " stored " << len[ i ] << std::endl; abort(); } // make sure the list is correct @@ -478,31 +509,31 @@ gcollect (const ObjectStream & in, const vector<int>& len ) const } #endif - int * rcounts = new int [np] ; - assert (rcounts) ; - copy (len.begin (), len.end (), rcounts) ; - int * const displ = new int [np] ; - assert (displ) ; + int * rcounts = new int [np]; + assert (rcounts); + copy (len.begin (), len.end (), rcounts); + int * const displ = new int [np]; + assert (displ); // set offsets - displ [0] = 0 ; - for (int j = 1 ; j < np ; ++j ) + displ [0] = 0; + for (int j = 1; j < np; ++j ) { - displ [j] = displ [j - 1] + rcounts [j - 1] ; + displ [j] = displ [j - 1] + rcounts [j - 1]; } // overall buffer size - const size_t bufSize = displ [np - 1] + rcounts [np - 1] ; + const size_t bufSize = displ [np - 1] + rcounts [np - 1]; { // allocate buffer - char * y = ObjectStream :: allocateBuffer(bufSize); - assert (y) ; + char * y = ObjectStream::allocateBuffer(bufSize); + assert (y); // gather all data - MY_INT_TEST MPI_Allgatherv (in._buf + in._rb, snum, MPI_BYTE, y, rcounts, displ, MPI_BYTE, _mpiComm) ; - assert (test == MPI_SUCCESS) ; + MY_INT_TEST MPI_Allgatherv (in._buf + in._rb, snum, MPI_BYTE, y, rcounts, displ, MPI_BYTE, _mpiComm); + assert (test == MPI_SUCCESS); // copy data to object streams - for (int i = 0 ; i < np ; ++ i ) + for (int i = 0; i < np; ++ i ) { // write data to stream if( rcounts [i] ) @@ -512,36 +543,36 @@ gcollect (const ObjectStream & in, const vector<int>& len ) const } // delete buffer - ObjectStream :: freeBuffer( y ); + ObjectStream::freeBuffer( y ); } // delete helper functions - delete [] displ ; - delete [] rcounts ; + delete [] displ; + delete [] rcounts; - return o ; + return o; } -vector < vector < int > > MpAccessMPI :: exchange (const vector < vector < int > > & in) const { - assert (static_cast<int> (in.size ()) == nlinks ()) ; - return doExchange (in, MPI_INT, _mpiComm, dest ()) ; +std::vector< std::vector< int > > MpAccessMPI::exchange (const std::vector< std::vector< int > > & in) const { + assert (static_cast<int> (in.size ()) == nlinks ()); + return doExchange (in, MPI_INT, _mpiComm, dest ()); } -vector < vector < double > > MpAccessMPI :: exchange (const vector < vector < double > > & in) const { - assert (static_cast<int> (in.size ()) == nlinks ()) ; - return doExchange (in, MPI_DOUBLE, _mpiComm, dest ()) ; +std::vector< std::vector< double > > MpAccessMPI::exchange (const std::vector< std::vector< double > > & in) const { + assert (static_cast<int> (in.size ()) == nlinks ()); + return doExchange (in, MPI_DOUBLE, _mpiComm, dest ()); } -vector < vector < char > > MpAccessMPI :: exchange (const vector < vector < char > > & in) const { - assert (static_cast<int> (in.size ()) == nlinks ()) ; - return doExchange (in, MPI_BYTE, _mpiComm, dest ()) ; +std::vector< std::vector< char > > MpAccessMPI::exchange (const std::vector< std::vector< char > > & in) const { + assert (static_cast<int> (in.size ()) == nlinks ()); + return doExchange (in, MPI_BYTE, _mpiComm, dest ()); } ////////////////////////////////////////////////////////////////////////// // non-blocking communication object // this class is defined here since it contains MPI information ////////////////////////////////////////////////////////////////////////// -class NonBlockingExchangeMPI : public MpAccessLocal :: NonBlockingExchange +class NonBlockingExchangeMPI : public MpAccessLocal::NonBlockingExchange { const MpAccessMPI& _mpAccess; @@ -565,7 +596,7 @@ public: NonBlockingExchangeMPI( const MpAccessMPI& mpAccess, const int tag, - const vector< ObjectStream > & in ) + const std::vector< ObjectStream > & in ) : _mpAccess( mpAccess ), _nLinks( _mpAccess.nlinks() ), _tag( tag ), @@ -588,62 +619,62 @@ public: } // virtual methods - void send( const vector< ObjectStream > & in ) { sendImpl( in ); } - vector< ObjectStream > receive() { return receiveImpl(); } + void send( const std::vector< ObjectStream > & in ) { sendImpl( in ); } + std::vector< ObjectStream > receive() { return receiveImpl(); } ////////////////////////////////////////// // implementation ////////////////////////////////////////// // send data implementation - void sendImpl( const vector< ObjectStream > & osv ) + void sendImpl( const std::vector< ObjectStream > & osv ) { - const MpAccessMPI :: CommIF* _mpiCommPtr = _mpAccess.mpiCommPtr(); + const MpAccessMPI::CommIF* _mpiCommPtr = _mpAccess.mpiCommPtr(); // get mpi communicator (use define, see above) - MPI_Comm comm = _mpiComm ; + MPI_Comm comm = _mpiComm; // get vector with destinations - const vector< int >& dest = _mpAccess.dest(); + const std::vector< int >& dest = _mpAccess.dest(); // send data - for (int link = 0 ; link < _nLinks ; ++link) + for (int link = 0; link < _nLinks; ++link) { // get send buffer from object stream - char* buffer = osv[ link ]._buf + osv[ link ]._rb ; + char* buffer = osv[ link ]._buf + osv[ link ]._rb; // get number of bytes to send - int bufferSize = osv[ link ]._wb - osv[ link ]._rb ; + int bufferSize = osv[ link ]._wb - osv[ link ]._rb; - MY_INT_TEST MPI_Isend ( buffer, bufferSize, MPI_BYTE, dest[ link ], _tag, comm, & _request[ link ] ) ; - assert (test == MPI_SUCCESS) ; + MY_INT_TEST MPI_Isend ( buffer, bufferSize, MPI_BYTE, dest[ link ], _tag, comm, & _request[ link ] ); + assert (test == MPI_SUCCESS); } } // receive data without buffer given - vector< ObjectStream > receiveImpl () + std::vector< ObjectStream > receiveImpl () { // create vector of empty streams - vector < ObjectStream > out ( _nLinks ) ; + std::vector< ObjectStream > out ( _nLinks ); receiveImpl( out ); - return out ; + return out; } // receive data implementation with given buffers - void receiveImpl ( vector< ObjectStream >& out ) + void receiveImpl ( std::vector< ObjectStream >& out ) { // do nothing if number of links is zero - if( _nLinks == 0 ) return ; + if( _nLinks == 0 ) return; - const MpAccessMPI :: CommIF* _mpiCommPtr = _mpAccess.mpiCommPtr(); + const MpAccessMPI::CommIF* _mpiCommPtr = _mpAccess.mpiCommPtr(); // get mpi communicator (use define, see above) - MPI_Comm comm = _mpiComm ; + MPI_Comm comm = _mpiComm; // get vector with destinations - const vector< int >& dest = _mpAccess.dest(); + const std::vector< int >& dest = _mpAccess.dest(); #ifndef NDEBUG // check for all links messages - for (int link = 0 ; link < _nLinks ; ++link ) + for (int link = 0; link < _nLinks; ++link ) { // contains no written data assert( out[ link ].notReceived() ); @@ -655,19 +686,19 @@ public: while( numReceived < _nLinks ) { // check for all links messages - for (int link = 0 ; link < _nLinks ; ++link ) + for (int link = 0; link < _nLinks; ++link ) { ObjectStream& objStream = out[ link ]; if( objStream.notReceived() ) { // corresponding MPI status - MPI_Status status ; + MPI_Status status; // received, 0 or 1 - int received = 0 ; + int received = 0; // check for any message with tag (nonblocking) - MPI_Iprobe( dest[ link ], _tag, comm, &received, &status ) ; + MPI_Iprobe( dest[ link ], _tag, comm, &received, &status ); // receive message of received flag is true if( received ) @@ -676,20 +707,20 @@ public: assert( dest[ link ] == status.MPI_SOURCE ); // receive message for link - bufferpair_t buff = receiveLink( comm, status ) ; + bufferpair_t buff = receiveLink( comm, status ); #ifndef NDEBUG - const int length = buff.second ; + const int length = buff.second; #endif // copy to buffers (buff is reset to (0,0)) // also sets objStream.notReceived() to false - objStream = buff ; + objStream = buff; // make sure buffer match assert( length == objStream.size() ); // increase number of received messages - ++ numReceived ; + ++ numReceived; } } } @@ -697,12 +728,12 @@ public: // wait until all processes are done with receiving { - MPI_Status * sta = new MPI_Status [ _nLinks ] ; + MPI_Status * sta = new MPI_Status [ _nLinks ]; assert( sta ); assert( _request ); - MY_INT_TEST MPI_Waitall ( _nLinks, _request, sta) ; - assert (test == MPI_SUCCESS) ; - delete [] sta ; + MY_INT_TEST MPI_Waitall ( _nLinks, _request, sta); + assert (test == MPI_SUCCESS); + delete [] sta; } } protected: @@ -716,20 +747,20 @@ protected: // get length of message { - MY_INT_TEST MPI_Get_count ( & status, MPI_BYTE, & bufferSize ) ; - assert (test == MPI_SUCCESS) ; + MY_INT_TEST MPI_Get_count ( & status, MPI_BYTE, & bufferSize ); + assert (test == MPI_SUCCESS); } assert( bufferSize >= 0 ); // use alloc from objects stream because this is the // buffer of the object stream - char* buffer = ObjectStream :: allocateBuffer( bufferSize ); + char* buffer = ObjectStream::allocateBuffer( bufferSize ); // MPI receive { - MY_INT_TEST MPI_Recv ( buffer, bufferSize, MPI_BYTE, status.MPI_SOURCE, _tag, comm, & status) ; - assert (test == MPI_SUCCESS) ; + MY_INT_TEST MPI_Recv ( buffer, bufferSize, MPI_BYTE, status.MPI_SOURCE, _tag, comm, & status); + assert (test == MPI_SUCCESS); } // copy buffer and count to object stream @@ -739,22 +770,22 @@ protected: }; -MpAccessMPI :: NonBlockingExchange* -MpAccessMPI :: nonBlockingExchange( const int tag, const vector < ObjectStream > & in ) const +MpAccessMPI::NonBlockingExchange* +MpAccessMPI::nonBlockingExchange( const int tag, const std::vector< ObjectStream > & in ) const { assert( tag > messagetag+1 ); return new NonBlockingExchangeMPI( *this, tag, in ); } -MpAccessMPI :: NonBlockingExchange* -MpAccessMPI :: nonBlockingExchange( const int tag ) const +MpAccessMPI::NonBlockingExchange* +MpAccessMPI::nonBlockingExchange( const int tag ) const { assert( tag > messagetag+1 ); return new NonBlockingExchangeMPI( *this, tag ); } // --exchange -vector < ObjectStream > MpAccessMPI :: exchange (const vector < ObjectStream > & in) const +std::vector< ObjectStream > MpAccessMPI::exchange (const std::vector< ObjectStream > & in) const { NonBlockingExchangeMPI nonBlockingExchange( *this, messagetag+1, in ); return nonBlockingExchange.receiveImpl(); diff --git a/dune/alugrid/src/parallel/mpAccess_MPI.h b/dune/alugrid/src/parallel/mpAccess_MPI.h index 1008f5e2853051f2300234bb22f2107610205b00..9692cb5827fbb3a6688b7bd7ea4db4c9caa91a25 100644 --- a/dune/alugrid/src/parallel/mpAccess_MPI.h +++ b/dune/alugrid/src/parallel/mpAccess_MPI.h @@ -6,7 +6,7 @@ class MpAccessMPI : public MpAccessLocal { public: - typedef MpAccessLocal :: minmaxsum_t minmaxsum_t; + typedef MpAccessLocal::minmaxsum_t minmaxsum_t; class MinMaxSumIF { @@ -17,7 +17,7 @@ public: virtual minmaxsum_t minmaxsum( double ) const = 0; }; - typedef MpAccessGlobal :: CommIF CommIF; + typedef MpAccessGlobal::CommIF CommIF; template <class MPICommunicator> class Comm : public CommIF @@ -39,7 +39,7 @@ public: virtual CommIF* clone() const { return new Comm< MPICommunicator > ( _mpiComm ); } }; - typedef MpAccessLocal :: NonBlockingExchange NonBlockingExchange; + typedef MpAccessLocal::NonBlockingExchange NonBlockingExchange; // MPI communication tag enum { messagetag = 123 }; @@ -57,11 +57,11 @@ protected: // my processor number const int _myrank; - int mpi_allgather (int *, int , int *, int) const ; - int mpi_allgather (char *, int, char *, int) const ; - int mpi_allgather (double *, int, double *, int ) const ; + int mpi_allgather (int *, int , int *, int) const; + int mpi_allgather (char *, int, char *, int) const; + int mpi_allgather (double *, int, double *, int ) const; - void initMinMaxSum() ; + void initMinMaxSum(); public : // constructor taking MPI_Comm // to avoid MPI types here this is a template constructor @@ -75,63 +75,63 @@ public : } // copy constructor - MpAccessMPI (const MpAccessMPI &) ; + MpAccessMPI (const MpAccessMPI &); // destructor - ~MpAccessMPI () ; + ~MpAccessMPI (); protected: - int getSize () ; - int getRank () ; + int getSize (); + int getRank (); public: - inline int psize () const ; - inline int myrank () const ; - int barrier () const ; - bool gmax (bool) const ; - int gmax (int) const ; - int gmin (int) const ; - int gsum (int) const ; - long gmax (long) const ; - long gmin (long) const ; - long gsum (long) const ; - double gmax (double) const ; - double gmin (double) const ; - double gsum (double) const ; - void gmax (double*,int,double*) const ; - void gmin (double*,int,double*) const ; - void gsum (double*,int,double*) const ; - void gmax (int*,int,int*) const ; - void gmin (int*,int,int*) const ; - void gsum (int*,int,int*) const ; - minmaxsum_t minmaxsum( double ) const ; - pair<double,double> gmax (pair<double,double>) const ; - pair<double,double> gmin (pair<double,double>) const ; - pair<double,double> gsum (pair<double,double>) const ; + inline int psize () const; + inline int myrank () const; + int barrier () const; + bool gmax (bool) const; + int gmax (int) const; + int gmin (int) const; + int gsum (int) const; + long gmax (long) const; + long gmin (long) const; + long gsum (long) const; + double gmax (double) const; + double gmin (double) const; + double gsum (double) const; + void gmax (double*,int,double*) const; + void gmin (double*,int,double*) const; + void gsum (double*,int,double*) const; + void gmax (int*,int,int*) const; + void gmin (int*,int,int*) const; + void gsum (int*,int,int*) const; + minmaxsum_t minmaxsum( double ) const; + std::pair<double,double> gmax (std::pair<double,double>) const; + std::pair<double,double> gmin (std::pair<double,double>) const; + std::pair<double,double> gsum (std::pair<double,double>) const; void bcast(int*, int, int ) const; void bcast(char*, int, int ) const; void bcast(double*, int, int ) const; - int exscan ( int ) const ; - int scan ( int ) const ; - - using MpAccessLocal :: gcollect ; - - vector < int > gcollect (int) const ; - vector < double > gcollect (double) const ; - vector < vector < int > > gcollect (const vector < int > &) const ; - vector < vector < double > > gcollect (const vector < double > &) const ; - vector < ObjectStream > gcollect (const ObjectStream &, const vector<int>& ) const ; - vector < vector < int > > exchange (const vector < vector < int > > &) const ; - vector < vector < double > > exchange (const vector < vector < double > > &) const ; - vector < vector < char > > exchange (const vector < vector < char > > &) const ; + int exscan ( int ) const; + int scan ( int ) const; + + using MpAccessLocal::gcollect; + + std::vector< int > gcollect (int) const; + std::vector< double > gcollect (double) const; + std::vector< std::vector< int > > gcollect (const std::vector< int > &) const; + std::vector< std::vector< double > > gcollect (const std::vector< double > &) const; + std::vector< ObjectStream > gcollect (const ObjectStream &, const std::vector<int>& ) const; + std::vector< std::vector< int > > exchange (const std::vector< std::vector< int > > &) const; + std::vector< std::vector< double > > exchange (const std::vector< std::vector< double > > &) const; + std::vector< std::vector< char > > exchange (const std::vector< std::vector< char > > &) const; - vector < ObjectStream > exchange (const vector < ObjectStream > &) const ; + std::vector< ObjectStream > exchange (const std::vector< ObjectStream > &) const; // return MPI communicator wrapper const CommIF* communicator() const { return _mpiCommPtr; } // return handle for non-blocking exchange and already do send operation - NonBlockingExchange* nonBlockingExchange( const int tag, const vector < ObjectStream > & ) const ; + NonBlockingExchange* nonBlockingExchange( const int tag, const std::vector< ObjectStream > & ) const; // return handle for non-blocking exchange - NonBlockingExchange* nonBlockingExchange( const int tag ) const ; -} ; + NonBlockingExchange* nonBlockingExchange( const int tag ) const; +}; // @@ -142,13 +142,13 @@ public: // # # ## # # # ## # // # # # ###### # # # ###### // -inline int MpAccessMPI :: psize () const +inline int MpAccessMPI::psize () const { assert( _psize > 0 ); return _psize; } -inline int MpAccessMPI :: myrank () const +inline int MpAccessMPI::myrank () const { assert( _myrank != -1 ); return _myrank; diff --git a/lib/Makefile.am b/lib/Makefile.am index 10aeec9ee60c29d5281c77876a3b12ce6284f0b3..74756079b3995595e326f878d3b14e6746d362f6 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -1,4 +1,4 @@ -lib_LTLIBRARIES = libalugrid_serial.la libdunealugrid.la +lib_LTLIBRARIES = libalugrid_serial.la libalugrid_mpi.la libdunealugrid.la libalugrid_serial_la_SOURCES = \ $(top_srcdir)/dune/alugrid/src/serial/ghost_elements.cc \ @@ -18,6 +18,13 @@ libalugrid_serial_la_CPPFLAGS = $(AM_CPPFLAGS) libalugrid_serial_la_LDFLAGS = $(AM_LDFLAGS) libalugrid_serial_la_LIBADD = +libalugrid_mpi_la_SOURCES = \ + $(top_srcdir)/dune/alugrid/src/parallel/mpAccess_MPI.cc + +libalugrid_mpi_la_CPPFLAGS = $(AM_CPPFLAGS) $(DUNEMPICPPFLAGS) +libalugrid_mpi_la_LDFLAGS = $(AM_LDFLAGS) $(DUNEMPILDFLAGS) +libalugrid_mpi_la_LIBADD = $(DUNEMPILIBS) + libdunealugrid_la_SOURCES = \ $(top_srcdir)/dune/alugrid/2d/intersection.cc \ $(top_srcdir)/dune/alugrid/2d/alu2dgridfactory.cc \