8#ifndef PARTICLE_SPATIAL_LAYOUT_H
9#define PARTICLE_SPATIAL_LAYOUT_H
54template <
class T,
unsigned Dim,
class Mesh,
class CachingPolicy>
62template <
class T,
unsigned Dim,
class Mesh=UniformCartesian<Dim,T>,
class CachingPolicy=BoxParticleCachingPolicy<T,Dim,Mesh > >
118 return RLayout.getFieldLayout();
177 "not supported with setOutsideToNearest(true)");
195 if (outsideToNearest &&
caching)
196 throw IpplException(
"ParticleSpatialLayout::setOutsideToNearest",
197 "not supported with caching of ghost particles");
245 for (
unsigned int j = 0; j <
Dim; j++)
247 const T lo = domain[j].min();
248 const T hi = domain[j].max();
249 const T eps = std::max((T)1e-7 * (hi - lo),
250 (T)16 * std::numeric_limits<T>::epsilon() *
251 std::max(std::abs(lo), std::abs(hi)));
253 if (std::isfinite(x)) {
256 else if (x >= hi - eps)
268 template <
class PB >
286 for (i=1; i < haveLocal; ++i)
288 for (d=0; d <
Dim; ++d)
290 if (PData.R[i][d] < minpos[d])
291 minpos[d] = PData.R[i][d];
292 if (PData.R[i][d] > maxpos[d])
293 maxpos[d] = PData.R[i][d];
324 size_t tmphaveLocal = 0;
328 while (unreceived > 0)
332 msg->
get(tmphaveLocal);
333 if (tmphaveLocal > 0)
337 for (i=0; i <
Dim; ++i)
339 if (tmpminpos[i] < minpos[i])
340 minpos[i] = tmpminpos[i];
341 if (tmpmaxpos[i] > maxpos[i])
342 maxpos[i] = tmpmaxpos[i];
354 for (i=0; i <
Dim; ++i)
356 if (minpos[i] >= 0.0)
357 minpos[i] = (int)(minpos[i]);
359 minpos[i] = (int)(minpos[i] - 1);
360 maxpos[i] = (int)(maxpos[i] + 1);
376 for (i=0; i <
Dim; ++i)
377 range[i] =
Index((
int)(minpos[i]), (
int)(maxpos[i]));
386 RLayout.changeDomain(range, vn);
396 template <
class PB >
480 for (d = 1; d <
Dim; ++d)
482 T first = (*(
RLayout.begin_iv())).second->getDomain()[d].first();
483 T last = (*(
RLayout.begin_iv())).second->getDomain()[d].last();
484 T mid = first + 0.5 * (last - first);
488 for (d = 0; d <
Dim; ++d)
498 for (i = 0; i < N; i++)
504 for (ip=0; ip<LocalNum; ++ip)
508 for (j = 0; j <= d; j++)
509 pLoc[j] =
PRegion<T>(PData.R[ip][j], PData.R[ip][j]);
512 bool foundit =
false;
517 localV != localEnd && !foundit; ++localV)
519 foundit= (((*localV).second)->getDomain())[d].touches(pLoc[d]);
530 if (touchingVN.first == touchingVN.second)
533 ERRORMSG(
"Local particle " << ip <<
" with ID=");
535 ERRORMSG(PData.R[ip] <<
" is outside of global domain ");
537 ERRORMSG(
"This occurred when searching for point " << pLoc);
545 unsigned node = (*(touchingVN.first)).second->getNode();
550 PData.destroy(1, ip);
560 for (i = 0; i < N; i++)
568 PData.putMessage(*(
SwapMsgList[i]), (
size_t) 0, (
size_t) 0);
583 LocalNum -= PData.getDestroyNum();
584 ADDIPPLSTAT(incParticlesSwapped, PData.getDestroyNum());
585 PData.performDestroy();
589 while (sendnum-- > 0)
594 while ((recvd = PData.getMessage(*recmsg)) > 0)
603 for (i = 0; i < N; ++i)
610 while ((recvd = PData.getMessage(*recmsg)) > 0)
622 msg <<
"case getEmptyNode(myN) " <<
endl;
624 for (i = 0; i < N; i++)
630 for (ip=0; ip<LocalNum; ++ip)
633 for (j = 0; j <
Dim; j++)
634 pLoc[j] =
PRegion<T>(PData.R[ip][j], PData.R[ip][j]);
641 if (touchingVN.first == touchingVN.second)
643 ERRORMSG(
"Local particle " << ip <<
" with ID=");
645 ERRORMSG(PData.R[ip] <<
" is outside of global domain ");
647 ERRORMSG(
"This occurred when searching for point " << pLoc);
654 unsigned node = (*(touchingVN.first)).second->getNode();
659 PData.destroy(1, ip);
664 for (i = 0; i < N; i++)
672 PData.putMessage(*(
SwapMsgList[i]), (
size_t) 0, (
size_t) 0);
683 LocalNum -= PData.getDestroyNum();
684 ADDIPPLSTAT(incParticlesSwapped, PData.getDestroyNum());
685 PData.performDestroy();
698 template <
class PB >
717 for (d = 1; d <
Dim; ++d)
719 T first = (*(
RLayout.begin_iv())).second->getDomain()[d].first();
720 T last = (*(
RLayout.begin_iv())).second->getDomain()[d].last();
721 T mid = first + 0.5 * (last - first);
725 for (d = 0; d <
Dim; ++d)
735 for (i = 0; i < N; i++)
741 for (ip=0; ip<LocalNum; ++ip)
745 for (j = 0; j <= d; j++)
746 pLoc[j] =
PRegion<T>(PData.R[ip][j], PData.R[ip][j]);
749 bool foundit =
false;
752 localV != localEnd && !foundit; ++localV)
754 foundit= (((*localV).second)->getDomain())[d].touches(pLoc[d]);
766 unsigned node = (*(touchingVN.first)).second->getNode();
771 PData.destroy(1, ip);
778 std::vector<MPI_Request> requests;
779 std::vector<MsgBuffer*> buffers;
782 for (i = 0; i < N; i++)
791 PData.putMessage(*(
SwapMsgList[i]), (
size_t) 0, (
size_t) 0);
804 LocalNum -= PData.getDestroyNum();
805 ADDIPPLSTAT(incParticlesSwapped, PData.getDestroyNum());
806 PData.performDestroy();
810 while (sendnum-- > 0)
815 while ((recvd = PData.getMessage(*recmsg)) > 0)
839 template <
class PB >
919 for (d = 1; d <
Dim; ++d)
921 T first = (*(
RLayout.begin_iv())).second->getDomain()[d].first();
922 T last = (*(
RLayout.begin_iv())).second->getDomain()[d].last();
923 T mid = first + 0.5 * (last - first);
927 for (d = 0; d <
Dim; ++d)
937 for (i = 0; i < N; i++)
943 for (ip=0; ip<LocalNum; ++ip)
945 if (!
bool(canSwap[ip]))
continue;
948 for (j = 0; j <= d; j++)
949 pLoc[j] =
PRegion<T>(PData.R[ip][j], PData.R[ip][j]);
952 bool foundit =
false;
957 localV != localEnd && !foundit; ++localV)
959 foundit= (((*localV).second)->getDomain())[d].touches(pLoc[d]);
970 if (touchingVN.first == touchingVN.second)
973 ERRORMSG(
"Local particle " << ip <<
" with ID=");
975 ERRORMSG(PData.R[ip] <<
" is outside of global domain ");
977 ERRORMSG(
"This occurred when searching for point " << pLoc);
984 unsigned node = (*(touchingVN.first)).second->getNode();
989 PData.destroy(1, ip);
999 for (i = 0; i < N; i++)
1007 PData.putMessage(*(
SwapMsgList[i]), (
size_t) 0, (
size_t) 0);
1022 LocalNum -= PData.getDestroyNum();
1023 ADDIPPLSTAT(incParticlesSwapped, PData.getDestroyNum());
1024 PData.performDestroy();
1028 while (sendnum-- > 0)
1033 while ((recvd = PData.getMessage(*recmsg)) > 0)
1042 for (i = 0; i < N; ++i)
1049 while ((recvd = PData.getMessage(*recmsg)) > 0)
1062 for (i = 0; i < N; i++)
1068 for (ip=0; ip<LocalNum; ++ip)
1070 if (!
bool(canSwap[ip]))
continue;
1072 for (j = 0; j <
Dim; j++)
1073 pLoc[j] =
PRegion<T>(PData.R[ip][j], PData.R[ip][j]);
1077 RLayout.touch_range_rdv(pLoc);
1080 if (touchingVN.first == touchingVN.second)
1082 ERRORMSG(
"Local particle " << ip <<
" with ID=");
1084 ERRORMSG(PData.R[ip] <<
" is outside of global domain ");
1086 ERRORMSG(
"This occurred when searching for point " << pLoc);
1093 unsigned node = (*(touchingVN.first)).second->getNode();
1098 PData.destroy(1, ip);
1103 for (i = 0; i < N; i++)
1111 PData.putMessage(*(
SwapMsgList[i]), (
size_t) 0, (
size_t) 0);
1122 LocalNum -= PData.getDestroyNum();
1123 ADDIPPLSTAT(incParticlesSwapped, PData.getDestroyNum());
1124 PData.performDestroy();
1139 template <
class PB >
1150 std::vector<int> msgsend(N, 0);
1151 std::vector<int> msgrecv(N, 0);
1155 std::multimap<unsigned, unsigned> p2n;
1157 bool responsibleNodeNotFound =
false;
1159 for (
unsigned int ip=0; ip<LocalNum; ++ip)
1161 for (
unsigned int j = 0; j <
Dim; j++)
1162 pLoc[j] =
PRegion<T>(PData.R[ip][j], PData.R[ip][j]);
1164 unsigned destination = myN;
1166 for (localV =
RLayout.
begin_iv(); localV != localEnd && !found; ++localV)
1168 if ((((*localV).second)->getDomain()).touches(pLoc))
1178 if (touchingVN.first == touchingVN.second) {
1180 responsibleNodeNotFound =
true;
1187 for (localV =
RLayout.
begin_iv(); localV != localEnd && !found; ++localV)
1189 if ((((*localV).second)->getDomain()).touches(pLoc))
1197 if (touchingVN.first == touchingVN.second) {
1198 responsibleNodeNotFound =
true;
1202 destination = (*(touchingVN.first)).second->getNode();
1204 msgsend[destination] = 1;
1206 p2n.insert(std::pair<unsigned, unsigned>(destination, ip));
1211 std::logical_or<bool>());
1213 if (responsibleNodeNotFound) {
1214 throw IpplException(
"ParticleSpatialLayout::new_swap_particles",
1215 "could not find node responsible for particle");
1219 allreduce(msgsend.data(), msgrecv.data(), N, std::plus<int>());
1223 typename std::multimap<unsigned, unsigned>::iterator i = p2n.begin();
1225 std::unique_ptr<Format> format(PData.getFormat());
1228 std::vector<MPI_Request> requests;
1229 std::vector<std::shared_ptr<MsgBuffer> > buffers;
1231 while (i!=p2n.end())
1233 unsigned cur_destination = i->first;
1235 std::shared_ptr<MsgBuffer> msgbuf(
new MsgBuffer(format.get(), p2n.count(i->first)));
1237 for (; i!=p2n.end() && i->first == cur_destination; ++i)
1240 PData.putMessage(msg, i->second);
1241 PData.destroy(1, i->second);
1245 MPI_Request request =
Ippl::Comm->
raw_isend( msgbuf->getBuffer(), msgbuf->getSize(), cur_destination, tag);
1248 requests.push_back(request);
1249 buffers.push_back(msgbuf);
1252 LocalNum -= PData.getDestroyNum();
1253 PData.performDestroy();
1256 for (
int k = 0; k<msgrecv[myN]; ++k)
1261 MsgBuffer recvbuf(format.get(), buffer, bufsize);
1266 LocalNum += PData.getSingleMessage(*msg);
1268 msg = recvbuf.
get();
1275 MPI_Request* requests_ptr = requests.
empty()?
static_cast<MPI_Request*
>(0): &(requests[0]);
1276 MPI_Waitall(requests.size(), requests_ptr, MPI_STATUSES_IGNORE);
1281 template <
class PB >
1293 std::vector<int> msgsend(N, 0);
1294 std::vector<int> msgrecv(N, 0);
1298 std::multimap<unsigned, unsigned> p2n;
1300 bool responsibleNodeNotFound =
false;
1302 for (
unsigned int ip=0; ip<LocalNum; ++ip)
1304 if (!
bool(canSwap[ip]))
1307 for (
unsigned int j = 0; j <
Dim; j++)
1308 pLoc[j] =
PRegion<T>(PData.R[ip][j], PData.R[ip][j]);
1310 unsigned destination = myN;
1312 for (localV =
RLayout.
begin_iv(); localV != localEnd && !found; ++localV)
1314 if ((((*localV).second)->getDomain()).touches(pLoc))
1324 if (touchingVN.first == touchingVN.second) {
1326 responsibleNodeNotFound =
true;
1333 for (localV =
RLayout.
begin_iv(); localV != localEnd && !found; ++localV)
1335 if ((((*localV).second)->getDomain()).touches(pLoc))
1343 if (touchingVN.first == touchingVN.second) {
1344 responsibleNodeNotFound =
true;
1348 destination = (*(touchingVN.first)).second->getNode();
1350 msgsend[destination] = 1;
1352 p2n.insert(std::pair<unsigned, unsigned>(destination, ip));
1357 std::logical_or<bool>());
1359 if (responsibleNodeNotFound) {
1360 throw IpplException(
"ParticleSpatialLayout::new_swap_particles",
1361 "could not find node responsible for particle");
1365 allreduce(msgsend.data(), msgrecv.data(), N, std::plus<int>());
1369 typename std::multimap<unsigned, unsigned>::iterator i = p2n.begin();
1371 std::unique_ptr<Format> format(PData.getFormat());
1373 std::vector<MPI_Request> requests;
1374 std::vector<std::shared_ptr<MsgBuffer> > buffers;
1376 while (i!=p2n.end())
1378 unsigned cur_destination = i->first;
1380 std::shared_ptr<MsgBuffer> msgbuf(
new MsgBuffer(format.get(), p2n.count(i->first)));
1382 for (; i!=p2n.end() && i->first == cur_destination; ++i)
1385 PData.putMessage(msg, i->second);
1386 PData.destroy(1, i->second);
1390 MPI_Request request =
Ippl::Comm->
raw_isend( msgbuf->getBuffer(), msgbuf->getSize(), cur_destination, tag);
1393 requests.push_back(request);
1394 buffers.push_back(msgbuf);
1397 LocalNum -= PData.getDestroyNum();
1398 PData.performDestroy();
1401 for (
int k = 0; k<msgrecv[myN]; ++k)
1406 MsgBuffer recvbuf(format.get(), buffer, bufsize);
1411 LocalNum += PData.getSingleMessage(*msg);
1413 msg = recvbuf.
get();
1418 MPI_Request* requests_ptr = requests.
empty()?
static_cast<MPI_Request*
>(0): &(requests[0]);
1419 MPI_Waitall(requests.size(), requests_ptr, 0);
PartBunchBase< T, Dim >::ConstIterator end(PartBunchBase< T, Dim > const &bunch)
PartBunchBase< T, Dim >::ConstIterator begin(PartBunchBase< T, Dim > const &bunch)
std::ostream & operator<<(std::ostream &, const ParticleSpatialLayout< T, Dim, Mesh, CachingPolicy > &)
#define P_SPATIAL_TRANSFER_TAG
#define P_SPATIAL_RANGE_TAG
#define P_SPATIAL_RETURN_TAG
void allreduce(const T *input, T *output, int count, Op op)
Inform & endl(Inform &inf)
#define ADDIPPLSTAT(stat, amount)
Message & getMessage(Message &m)
Message & putMessage(Message &m) const
virtual MPI_Request raw_isend(void *, int, int, int)
bool send(Message *, int node, int tag, bool delmsg=true)
virtual int raw_probe_receive(char *&, int &, int &)
virtual int broadcast_others(Message *, int, bool delmsg=true)
Message * receive_block(int &node, int &tag)
Message & put(const T &val)
Message & get(const T &cval)
int next_tag(int t, int s=1000)
int getNodeCount(unsigned i) const
void rebuild_layout(size_t haveLocal, PB &PData)
ParticleAttrib< Index_t > ParticleIndex_t
ParticleAttrib< SingleParticlePos_t > ParticlePos_t
const RegionLayout< T, Dim, Mesh > & getLayout() const
bool getEmptyNode(unsigned i) const
void setOutsideToNearest(bool outsideToNearest)
virtual void Repartition(UserList *)
RegionLayout< T, Dim, Mesh > & getLayout()
virtual void notifyUserOfDelete(UserList *)
ParticleLayout< T, Dim >::SingleParticlePos_t SingleParticlePos_t
bool getOutsideToNearest() const
void moveIntoDomain(NDRegion< T, Dim > &pLoc, const SingleParticlePos_t &pos) const
RegionLayout< T, Dim, Mesh > RLayout
void update(IpplParticleBase< ParticleSpatialLayout< T, Dim, Mesh, CachingPolicy > > &p, const ParticleAttrib< char > *canSwap=0)
size_t new_swap_particles(size_t LocalNum, PB &PData, const ParticleAttrib< char > &canSwap)
ParticleLayout< T, Dim >::Index_t Index_t
size_t new_swap_particles(size_t LocalNum, PB &PData)
std::vector< size_t > * PutList
size_t swap_particles(size_t LocalNum, PB &PData)
size_t short_swap_particles(size_t LocalNum, PB &PData)
FieldLayout< Dim > & getFieldLayout()
unsigned NeighborNodes[Dim]
RegionLayout< T, Dim, Mesh > RegionLayout_t
size_t getOutsideCount() const
size_t swap_particles(size_t LocalNum, PB &PData, const ParticleAttrib< char > &canSwap)
void printDebug(Inform &)
touch_range_dv touch_range_rdv(const NDRegion< T, Dim > &domain)
ac_id_vnodes::iterator iterator_iv
std::pair< touch_iterator_dv, touch_iterator_dv > touch_range_dv
static void abort(const char *=0)
static Communicate * Comm