OPAL (Object Oriented Parallel Accelerator Library) 2024.2
OPAL
ParticleSpatialLayout.h
Go to the documentation of this file.
1// -*- C++ -*-
2/***************************************************************************
3 *
4 * The IPPL Framework
5 *
6 ***************************************************************************/
7
8#ifndef PARTICLE_SPATIAL_LAYOUT_H
9#define PARTICLE_SPATIAL_LAYOUT_H
10
11/*
12 * ParticleSpatialLayout - particle layout based on spatial decomposition.
13 *
14 * This is a specialized version of ParticleLayout, which places particles
15 * on processors based on their spatial location relative to a fixed grid.
16 * In particular, this can maintain particles on processors based on a
17 * specified FieldLayout or RegionLayout, so that particles are always on
18 * the same node as the node containing the Field region to which they are
19 * local. This may also be used if there is no associated Field at all,
20 * in which case a grid is selected based on an even distribution of
21 * particles among processors.
22 */
23
24// include files
27#include "Region/RegionLayout.h"
28#include "Message/Message.h"
29#include "Message/Format.h"
30#include "Message/MsgBuffer.h"
33
34#include <algorithm>
35#include <cmath>
36#include <cstddef>
37
38#include <functional>
39#include <iostream>
40#include <limits>
41#include <map>
42#include <memory>
43#include <vector>
44
46
47#include <mpi.h>
48
49// forward declarations
50class UserList;
51template <class T> class ParticleAttrib;
52template <unsigned Dim, class T> class UniformCartesian;
53template <class T, unsigned Dim, class Mesh, class CachingPolicy> class ParticleSpatialLayout;
54template <class T, unsigned Dim, class Mesh, class CachingPolicy>
55std::ostream& operator<<(std::ostream&, const ParticleSpatialLayout<T,Dim,Mesh,CachingPolicy>&);
56
57// ParticleSpatialLayout class definition. Template parameters are the type
58// and dimension of the ParticlePos object used for the particles. The
59// dimension of the position must match the dimension of the FieldLayout
60// object used in this particle layout, if any.
61// Optional template parameter for the mesh type
62template < class T, unsigned Dim, class Mesh=UniformCartesian<Dim,T>, class CachingPolicy=BoxParticleCachingPolicy<T,Dim,Mesh > >
64 public FieldLayoutUser, public CachingPolicy
65{
66 /*
67 Enable cashing of particles. The size is
68 given in multiples of the mesh size in each dimension.
69 */
70
71public:
72 // pair iterator definition ... this layout does not allow for pairlists
73 typedef int pair_t;
78
79 // type of attributes this layout should use for position and ID
83
84public:
85 // constructor: The Field layout to which we match our particle's
86 // locations.
88
89 // constructor: this one also takes a Mesh
91
92 // a similar constructor, but this one takes a RegionLayout.
94
95 // a default constructor ... in this case, no layout will
96 // be assumed by this class. A layout may be given later via the
97 // 'setLayout' method, either as a FieldLayout or as a RegionLayout.
99
100 // destructor
102
103 //
104 // spatial decomposition layout information
105 //
106
107 // retrieve a reference to the FieldLayout object in use. This may be used,
108 // e.g., to construct a Field with the same layout as the Particles. Note
109 // that if this object was constructed by providing a RegionLayout in the
110 // constructor, then this generated FieldLayout will not necessarily match
111 // up with the Region (it will be offset by some amount). But, if this
112 // object was either 1) created with a FieldLayout to begin with, or 2)
113 // created with no layout, and one was generated internally, then the
114 // returned FieldLayout will match and can be used to make new Fields or
115 // Particles.
117 {
118 return RLayout.getFieldLayout();
119 }
120
121 // retrieve a reference to the RegionLayout object in use
123 {
124 return RLayout;
125 }
127 {
128 return RLayout;
129 }
130
131 // get number of particles on a physical node
132 int getNodeCount(unsigned i) const
133 {
134 PAssert_LT(i, (unsigned int) Ippl::getNodes());
135 return NodeCount[i];
136 }
137
138 // get flag for empty node domain
139 bool getEmptyNode(unsigned i) const
140 {
141 PAssert_LT(i, (unsigned int) Ippl::getNodes());
142 return EmptyNode[i];
143 }
144
145 //
146 // Particle swapping/update routines
147 //
148
149 // Update the location and indices of all atoms in the given IpplParticleBase
150 // object. This handles swapping particles among processors if
151 // needed, and handles create and destroy requests. When complete,
152 // all nodes have correct layout information.
154 const ParticleAttrib<char>* canSwap=0);
155
156
157 //
158 // I/O
159 //
160
161 // Print out information for debugging purposes.
162 void printDebug(Inform&);
163
164 //
165 // virtual functions for FieldLayoutUser's (and other UserList users)
166 //
167
168 // Repartition onto a new layout
169 virtual void Repartition(UserList *);
170
171 // Tell this object that an object is being deleted
172 virtual void notifyUserOfDelete(UserList *);
173
176 throw IpplException("ParticleSpatialLayout::enableCaching",
177 "not supported with setOutsideToNearest(true)");
178 caching = true;
179 }
180 void disableCaching() { caching = false; }
181
182 // What update() does with a particle that lies in no rnode, i.e. outside
183 // the global domain. false (the default): throw "could not find node
184 // responsible for particle". true: send it to the node owning the point of
185 // the global domain nearest to it. Only the point used for the lookup is
186 // moved, the particle position is not changed. A particle with a
187 // coordinate that is not finite is still refused. The particle boundary
188 // conditions are applied before the lookup, as always, so a periodic or
189 // reflective face still moves the particle back into the domain. The
190 // legacy swap_particles(), used by ParticleCashedLayout and
191 // ParticleInteractLayout, ignores this setting and still aborts.
192 // Caching is refused with it: the ghost particles are selected by the
193 // rnodes, so a particle outside all of them would miss its neighbours.
194 void setOutsideToNearest(bool outsideToNearest) {
195 if (outsideToNearest && caching)
196 throw IpplException("ParticleSpatialLayout::setOutsideToNearest",
197 "not supported with caching of ghost particles");
198 outsideToNearest_m = outsideToNearest;
199 }
201
202 // Number of local particles that the last particle swap found outside the
203 // global domain with setOutsideToNearest(true). The swap runs only on more
204 // than one node, and it skips the particles that update(canSwap) is told
205 // not to swap.
206 size_t getOutsideCount() const { return outsideCount_m; }
207
208protected:
209 // The RegionLayout which determines where our particles go.
211
212 // The number of particles located on each physical node.
213 size_t *NodeCount;
214
215 // Flag for which nodes have no local domain
217
218 // a list of Message pointers used in swapping particles, and flags
219 // for which nodes expect messages in each dimension
223 std::vector<size_t>* PutList;
224
226
227 // see setOutsideToNearest() and getOutsideCount()
230
231 // perform common constructor tasks
232 void setup();
233
234 // Move the lookup point pLoc of a particle at pos, which lies in no rnode,
235 // to the point of the global domain nearest to it, just inside the
236 // half-open domain [min, max). The shift is 1e-7 of the domain width or,
237 // far from the origin where that would round away, 16 to 32 units in the
238 // last place of the bounds. Both are far below one cell, so the lookup
239 // point lies in the rnode at the nearest point of the domain. A coordinate
240 // that is not finite (NaN or Inf) is not moved, so such a particle is
241 // still not found.
243 {
244 const NDRegion<T,Dim>& domain = RLayout.getDomain();
245 for (unsigned int j = 0; j < Dim; j++)
246 {
247 const T lo = domain[j].min();
248 const T hi = domain[j].max();
249 const T eps = std::max((T)1e-7 * (hi - lo),
250 (T)16 * std::numeric_limits<T>::epsilon() *
251 std::max(std::abs(lo), std::abs(hi)));
252 T x = pos[j];
253 if (std::isfinite(x)) {
254 if (x < lo + eps)
255 x = lo + eps;
256 else if (x >= hi - eps)
257 x = hi - eps;
258 }
259 pLoc[j] = PRegion<T>(x, x);
260 }
261 }
262
264 // Rebuild the RegionLayout entirely, by recalculating our min and max
265 // domains, adding a buffer region, and then giving this new Domain to
266 // our internal RegionLayout. When this is done, we must rebuild all
267 // our other data structures as well.
268 template < class PB >
269 void rebuild_layout(size_t haveLocal, PB& PData)
270 {
271 size_t i;
272 unsigned d; // loop variables
273
274 //~ Inform dbgmsg("SpatialLayout::rebuild_layout", INFORM_ALL_NODES);
275 //~ dbgmsg << "rebuild..." << endl;
276 SingleParticlePos_t minpos = 0;
277 SingleParticlePos_t maxpos = 0;
280
281 // if we have local particles, then find the min and max positions
282 if (haveLocal > 0)
283 {
284 minpos = PData.R[0];
285 maxpos = PData.R[0];
286 for (i=1; i < haveLocal; ++i)
287 {
288 for (d=0; d < Dim; ++d)
289 {
290 if (PData.R[i][d] < minpos[d])
291 minpos[d] = PData.R[i][d];
292 if (PData.R[i][d] > maxpos[d])
293 maxpos[d] = PData.R[i][d];
294 }
295 }
296 }
297
298 // if we're not on node 0, send data to node 0
299 if (Ippl::myNode() != 0)
300 {
301 Message *msg = new Message;
302 msg->put(haveLocal);
303 if (haveLocal > 0)
304 {
305 minpos.putMessage(*msg);
306 maxpos.putMessage(*msg);
307 }
308 Ippl::Comm->send(msg, 0, tag);
309
310 // now receive back min and max range as provided by the master node.
311 // These will include some buffer region, and will be integral values,
312 // so we can make a FieldLayout and use it to initialize the RegionLayout.
313 int node = 0;
314 msg = Ippl::Comm->receive_block(node, btag);
315 minpos.getMessage(*msg);
316 maxpos.getMessage(*msg);
317 delete msg;
318
319 }
320 else // on node 0, collect data and compute region
321 {
322 SingleParticlePos_t tmpminpos;
323 SingleParticlePos_t tmpmaxpos;
324 size_t tmphaveLocal = 0;
325 unsigned unreceived = Ippl::getNodes() - 1;
326
327 // collect data from other nodes
328 while (unreceived > 0)
329 {
330 int node = COMM_ANY_NODE;
331 Message *msg = Ippl::Comm->receive_block(node, tag);
332 msg->get(tmphaveLocal);
333 if (tmphaveLocal > 0)
334 {
335 tmpminpos.getMessage(*msg);
336 tmpmaxpos.getMessage(*msg);
337 for (i=0; i < Dim; ++i)
338 {
339 if (tmpminpos[i] < minpos[i])
340 minpos[i] = tmpminpos[i];
341 if (tmpmaxpos[i] > maxpos[i])
342 maxpos[i] = tmpmaxpos[i];
343 }
344 }
345 delete msg;
346 unreceived--;
347 }
348
349 // adjust min and max to include a buffer region and fall on integral
350 // values
351 SingleParticlePos_t extrapos = (maxpos - minpos) * ((T)0.125);
352 maxpos += extrapos;
353 minpos -= extrapos;
354 for (i=0; i < Dim; ++i)
355 {
356 if (minpos[i] >= 0.0)
357 minpos[i] = (int)(minpos[i]);
358 else
359 minpos[i] = (int)(minpos[i] - 1);
360 maxpos[i] = (int)(maxpos[i] + 1);
361 }
362
363 // send these values out to the other nodes
364 if (Ippl::getNodes() > 1)
365 {
366 Message *bmsg = new Message;
367 minpos.putMessage(*bmsg);
368 maxpos.putMessage(*bmsg);
369 Ippl::Comm->broadcast_others(bmsg, btag);
370 }
371 }
372
373 // determine the size of the new domain, and the number of blocks into
374 // which it should be broken
375 NDIndex<Dim> range;
376 for (i=0; i < Dim; ++i)
377 range[i] = Index((int)(minpos[i]), (int)(maxpos[i]));
378 int vn = -1;
379 if (RLayout.initialized())
380 vn = RLayout.size_iv() + RLayout.size_rdv();
381
382 // ask the RegionLayout to change the paritioning to match this size
383 // and block count. This will eventually end up by calling Repartition
384 // here, which will lead to rebuilding the neighbor data, etc., so we
385 // are done.
386 RLayout.changeDomain(range, vn);
387 }
388
389 // swap particles to neighboring nodes if they have moved too far
390 // PB is the type of IpplParticleBase which should have it's layout rebuilt.
391 //mwerks template<class PB>
392 //mwerks unsigned swap_particles(unsigned, PB&);
394 // go through all our local particles, and send particles which must
395 // be swapped to another node to that node.
396 template < class PB >
397 size_t swap_particles(size_t LocalNum, PB& PData)
398 {
399
400//~ Inform dbgmsg("SpatialLayout::swap_particles", INFORM_ALL_NODES);
401 //~ dbgmsg << "swap..." << endl;
402
403 Inform msg("ParticleSpatialLayout ERROR ", INFORM_ALL_NODES);
404
405 unsigned d, i, j; // loop variables
406 size_t ip;
407 unsigned N = Ippl::getNodes();
408 unsigned myN = Ippl::myNode();
409
410 // iterators used to search local domains
411 typename RegionLayout<T,Dim,Mesh>::iterator_iv localV, localEnd = RLayout.end_iv();
412
413 // iterators used to search remote domains
414 typename RegionLayout<T,Dim,Mesh>::iterator_dv remoteV; // remoteEnd = RLayout.end_rdv();
415
416 // JCC: This "nudge factor" stuff was added when we were experiencing
417 // problems with particles getting lost in between PRegions on
418 // neighboring nodes. This problem has since been resolved by
419 // fixing the way in which PRegion boundaries are computed, so I am
420 // commenting this out for now. We can bring it back later if the
421 // need arises.
422
423 /*
424
425 // Calculate a 'nudge factor', an amount that can get added to a
426 // particle position to determine where it should be located. The nudge
427 // factor equals 1/100th the smallest width of the rnodes in each dimension.
428 // When we try to find where a particle is located, we check what vnode
429 // contains this particle 'nudge region', a box around the particle's pos
430 // of the size of the nudge factor.
431 T pNudge[Dim];
432 for (d=0; d < Dim; ++d) {
433 // initialize to the first rnode's width
434 T minval = (*(RLayout.begin_iv())).second->getDomain()[d].length();
435
436 // check the local rnodes
437 for (localV = RLayout.begin_iv(); localV != localEnd; ++localV) {
438 T checkval = (*localV).second->getDomain()[d].length();
439 if (checkval < minval)
440 minval = checkval;
441 }
442
443 // check the remote rnodes
444 for (remoteV = RLayout.begin_rdv(); remoteV != remoteEnd; ++remoteV) {
445 T checkval = (*remoteV).second->getDomain()[d].length();
446 if (checkval < minval)
447 minval = checkval;
448 }
449
450 // now rescale the minval, and save it
451 pNudge[d] = 0.00001 * minval;
452 }
453
454 */
455
456 // An NDRegion object used to store a particle position.
457 NDRegion<T,Dim> pLoc;
458
459 // get new message tag for particle exchange with empty domains
461
462 if (!getEmptyNode(myN))
463 {
464
465 // Particles are swapped in multipple passes, one for each dimension.
466 // The tasks completed here for each dimension are the following:
467 // 1. For each local Vnode, find the remote Vnodes which exist along
468 // same axis as the current axis (i.e. all Vnodes along the x-axis).
469 // 2. From this list, determine which nodes we send messages to.
470 // 3. Go through all the particles, finding those which have moved to
471 // an off-processor vnode, and store index in an array for that node
472 // 4. Send off the particles to the nodes (if no particles are
473 // going to a node, send them a message with 0 in it)
474 // 5. Delete the send particles from our local list
475 // 6. Receive particles sent to us by other nodes (some messages may
476 // say that we're receiving 0 particles from that node).
477
478 // Initialize NDRegion with a position inside the first Vnode.
479 // We can skip dim 0, since it will be filled below.
480 for (d = 1; d < Dim; ++d)
481 {
482 T first = (*(RLayout.begin_iv())).second->getDomain()[d].first();
483 T last = (*(RLayout.begin_iv())).second->getDomain()[d].last();
484 T mid = first + 0.5 * (last - first);
485 pLoc[d] = PRegion<T>(mid, mid);
486 }
487
488 for (d = 0; d < Dim; ++d)
489 {
490
491 // get new message tag for particle exchange along this dimension
493
494 // we only need to do the rest if there are other nodes in this dim
495 if (NeighborNodes[d] > 0)
496 {
497 // create new messages to send to our neighbors
498 for (i = 0; i < N; i++)
499 if (SwapNodeList[d][i])
500 SwapMsgList[i] = new Message;
501
502 // Go through the particles and find those moving in the current dir.
503 // When one is found, copy it into outgoing message and delete it.
504 for (ip=0; ip<LocalNum; ++ip)
505 {
506 // get the position of particle ip, and find the closest grid pnt
507 // for just the dimensions 0 ... d
508 for (j = 0; j <= d; j++)
509 pLoc[j] = PRegion<T>(PData.R[ip][j], PData.R[ip][j]);
510
511 // first check local domains (in this dimension)
512 bool foundit = false;
513 // JCC: int nudged = 0;
514 while (!foundit)
515 {
516 for (localV = RLayout.begin_iv();
517 localV != localEnd && !foundit; ++localV)
518 {
519 foundit= (((*localV).second)->getDomain())[d].touches(pLoc[d]);
520 }
521
522 // if not found, it might be remote
523 if (!foundit)
524 {
525 // see which Vnode this postion is in
526 typename RegionLayout<T,Dim,Mesh>::touch_range_dv touchingVN =
527 RLayout.touch_range_rdv(pLoc);
528
529 // make sure we have a vnode to send it to
530 if (touchingVN.first == touchingVN.second)
531 {
532 // JCC: if (nudged >= Dim) {
533 ERRORMSG("Local particle " << ip << " with ID=");
534 ERRORMSG(PData.ID[ip] << " at ");
535 ERRORMSG(PData.R[ip] << " is outside of global domain ");
536 ERRORMSG(RLayout.getDomain() << endl);
537 ERRORMSG("This occurred when searching for point " << pLoc);
538 ERRORMSG(" in RegionLayout = " << RLayout << endl);
539 Ippl::abort();
540 }
541 else
542 {
543
544 // the node has been found - add index to put list
545 unsigned node = (*(touchingVN.first)).second->getNode();
546 PAssert_EQ(SwapNodeList[d][node], true);
547 PutList[node].push_back(ip);
548
549 // .. and then add to DestroyList
550 PData.destroy(1, ip);
551
552 // indicate we found it to quit this check
553 foundit = true;
554 }
555 }
556 }
557 }
558
559 // send the particles to their destination nodes
560 for (i = 0; i < N; i++)
561 {
562 if (SwapNodeList[d][i])
563 {
564 // put data for particles on this put list into message
565 PData.putMessage( *(SwapMsgList[i]), PutList[i] );
566
567 // add a final 'zero' number of particles to indicate the end
568 PData.putMessage(*(SwapMsgList[i]), (size_t) 0, (size_t) 0);
569
570 // send the message
571 // Inform dbgmsg("SpatialLayout", INFORM_ALL_NODES);
572 //dbgmsg << "Swapping "<<PutList[i].size() << " particles to node ";
573 //dbgmsg << i<<" with tag " << tag << " (" << 'x' + d << ")" << endl;
574 //dbgmsg << " ... msg = " << *(SwapMsgList[i]) << endl;
575 int node = i;
576 Ippl::Comm->send(SwapMsgList[i], node, tag);
577
578 // clear the list
579 PutList[i].erase(PutList[i].begin(), PutList[i].end());
580 }
581 }
582
583 LocalNum -= PData.getDestroyNum(); // update local num
584 ADDIPPLSTAT(incParticlesSwapped, PData.getDestroyNum());
585 PData.performDestroy();
586
587 // receive particles from neighbor nodes, and add them to our list
588 unsigned sendnum = NeighborNodes[d];
589 while (sendnum-- > 0)
590 {
592 Message *recmsg = Ippl::Comm->receive_block(node, tag);
593 size_t recvd;
594 while ((recvd = PData.getMessage(*recmsg)) > 0)
595 LocalNum += recvd;
596 delete recmsg;
597 }
598 } // end if (NeighborNodes[d] > 0)
599
600 if (d == 0)
601 {
602 // receive messages from any empty nodes
603 for (i = 0; i < N; ++i)
604 {
605 if (getEmptyNode(i))
606 {
607 int node = i;
608 Message *recmsg = Ippl::Comm->receive_block(node, etag);
609 size_t recvd;
610 while ((recvd = PData.getMessage(*recmsg)) > 0)
611 LocalNum += recvd;
612 delete recmsg;
613 }
614 }
615 }
616
617 } // end for (d=0; d<Dim; ++d)
618
619 }
620 else // empty node sends, but does not receive
621 {
622 msg << "case getEmptyNode(myN) " << endl;
623 // create new messages to send to our neighbors along dim 0
624 for (i = 0; i < N; i++)
625 if (SwapNodeList[0][i])
626 SwapMsgList[i] = new Message;
627
628 // Go through the particles and find those moving to other nodes.
629 // When one is found, copy it into outgoing message and delete it.
630 for (ip=0; ip<LocalNum; ++ip)
631 {
632 // get the position of particle ip, and find the closest grid pnt
633 for (j = 0; j < Dim; j++)
634 pLoc[j] = PRegion<T>(PData.R[ip][j], PData.R[ip][j]);
635
636 // see which remote Vnode this postion is in
637 typename RegionLayout<T,Dim,Mesh>::touch_range_dv touchingVN =
638 RLayout.touch_range_rdv(pLoc);
639
640 // make sure we have a vnode to send it to
641 if (touchingVN.first == touchingVN.second)
642 {
643 ERRORMSG("Local particle " << ip << " with ID=");
644 ERRORMSG(PData.ID[ip] << " at ");
645 ERRORMSG(PData.R[ip] << " is outside of global domain ");
646 ERRORMSG(RLayout.getDomain() << endl);
647 ERRORMSG("This occurred when searching for point " << pLoc);
648 ERRORMSG(" in RegionLayout = " << RLayout << endl);
649 Ippl::abort();
650 }
651 else
652 {
653 // the node has been found - add index to put list
654 unsigned node = (*(touchingVN.first)).second->getNode();
655 PAssert_EQ(SwapNodeList[0][node], true);
656 PutList[node].push_back(ip);
657
658 // .. and then add to DestroyList
659 PData.destroy(1, ip);
660 }
661 }
662
663 // send the particles to their destination nodes
664 for (i = 0; i < N; i++)
665 {
666 if (SwapNodeList[0][i])
667 {
668 // put data for particles on this put list into message
669 PData.putMessage( *(SwapMsgList[i]), PutList[i] );
670
671 // add a final 'zero' number of particles to indicate the end
672 PData.putMessage(*(SwapMsgList[i]), (size_t) 0, (size_t) 0);
673
674 // send the message
675 int node = i;
676 Ippl::Comm->send(SwapMsgList[i], node, etag);
677
678 // clear the list
679 PutList[i].erase(PutList[i].begin(), PutList[i].end());
680 }
681 }
682
683 LocalNum -= PData.getDestroyNum(); // update local num
684 ADDIPPLSTAT(incParticlesSwapped, PData.getDestroyNum());
685 PData.performDestroy();
686
687 }
688
689 // return how many particles we have now
690 return LocalNum;
691 }
692
693
694/*
695 * Simplified version for testing purposes.
696 */
697
698 template < class PB >
699 size_t short_swap_particles(size_t LocalNum, PB& PData)
700 {
701 unsigned d, i, j; // loop variables
702 size_t ip;
703 unsigned N = Ippl::getNodes();
704
705 // iterators used to search local domains
706 typename RegionLayout<T,Dim,Mesh>::iterator_iv localV, localEnd = RLayout.end_iv();
707
708 // iterators used to search remote domains
709 typename RegionLayout<T,Dim,Mesh>::iterator_dv remoteV; // remoteEnd = RLayout.end_rdv();
710
711
712 // An NDRegion object used to store a particle position.
713 NDRegion<T,Dim> pLoc;
714
715 // Initialize NDRegion with a position inside the first Vnode.
716 // We can skip dim 0, since it will be filled below.
717 for (d = 1; d < Dim; ++d)
718 {
719 T first = (*(RLayout.begin_iv())).second->getDomain()[d].first();
720 T last = (*(RLayout.begin_iv())).second->getDomain()[d].last();
721 T mid = first + 0.5 * (last - first);
722 pLoc[d] = PRegion<T>(mid, mid);
723 }
724
725 for (d = 0; d < Dim; ++d)
726 {
727
728 // get new message tag for particle exchange along this dimension
730
731 // we only need to do the rest if there are other nodes in this dim
732 if (NeighborNodes[d] > 0)
733 {
734 // create new messages to send to our neighbors
735 for (i = 0; i < N; i++)
736 if (SwapNodeList[d][i])
737 SwapMsgList[i] = new Message;
738
739 // Go through the particles and find those moving in the current dir.
740 // When one is found, copy it into outgoing message and delete it.
741 for (ip=0; ip<LocalNum; ++ip)
742 {
743 // get the position of particle ip, and find the closest grid pnt
744 // for just the dimensions 0 ... d
745 for (j = 0; j <= d; j++)
746 pLoc[j] = PRegion<T>(PData.R[ip][j], PData.R[ip][j]);
747
748 // first check local domains (in this dimension)
749 bool foundit = false;
750
751 for (localV = RLayout.begin_iv();
752 localV != localEnd && !foundit; ++localV)
753 {
754 foundit= (((*localV).second)->getDomain())[d].touches(pLoc[d]);
755 }
756
757 // if not found, it might be remote
758 if (!foundit)
759 {
760 // see which Vnode this postion is in
761 typename RegionLayout<T,Dim,Mesh>::touch_range_dv touchingVN =
762 RLayout.touch_range_rdv(pLoc);
763
764
765 // the node has been found - add index to put list
766 unsigned node = (*(touchingVN.first)).second->getNode();
767 PAssert_EQ(SwapNodeList[d][node], true);
768 PutList[node].push_back(ip);
769
770 // .. and then add to DestroyList
771 PData.destroy(1, ip);
772
773 // indicate we found it to quit this check
774 foundit = true;
775 }
776 }
777
778 std::vector<MPI_Request> requests;
779 std::vector<MsgBuffer*> buffers;
780
781 // send the particles to their destination nodes
782 for (i = 0; i < N; i++)
783 {
784 if (SwapNodeList[d][i])
785 {
786
787 // put data for particles on this put list into message
788 PData.putMessage( *(SwapMsgList[i]), PutList[i] );
789
790 // add a final 'zero' number of particles to indicate the end
791 PData.putMessage(*(SwapMsgList[i]), (size_t) 0, (size_t) 0);
792
793 int node = i;
794 Ippl::Comm->send(SwapMsgList[i], node, tag);
795
796 // clear the list
797 PutList[i].erase(PutList[i].begin(), PutList[i].end());
798
799
800
801 }
802 }
803
804 LocalNum -= PData.getDestroyNum(); // update local num
805 ADDIPPLSTAT(incParticlesSwapped, PData.getDestroyNum());
806 PData.performDestroy();
807
808 // receive particles from neighbor nodes, and add them to our list
809 unsigned sendnum = NeighborNodes[d];
810 while (sendnum-- > 0)
811 {
813 Message *recmsg = Ippl::Comm->receive_block(node, tag);
814 size_t recvd;
815 while ((recvd = PData.getMessage(*recmsg)) > 0)
816 LocalNum += recvd;
817 delete recmsg;
818 }
819
820 } // end if (NeighborNodes[d] > 0)
821
822 } // end for (d=0; d<Dim; ++d)
823
824
825 // return how many particles we have now
826 return LocalNum;
827 }
828
829
830
831
832
833 // PB is the type of IpplParticleBase which should have it's layout rebuilt.
834 //mwerks template<class PB>
835 //mwerks unsigned swap_particles(unsigned, PB&, const ParticleAttrib<char>&);
837 // go through all our local particles, and send particles which must
838 // be swapped to another node to that node.
839 template < class PB >
840 size_t swap_particles(size_t LocalNum, PB& PData,
841 const ParticleAttrib<char>& canSwap)
842 {
843
844 unsigned d, i, j; // loop variables
845 size_t ip;
846 unsigned N = Ippl::getNodes();
847 unsigned myN = Ippl::myNode();
848
849 // iterators used to search local domains
850 typename RegionLayout<T,Dim,Mesh>::iterator_iv localV, localEnd = RLayout.end_iv();
851
852 // iterators used to search remote domains
853 typename RegionLayout<T,Dim,Mesh>::iterator_dv remoteV; // remoteEnd = RLayout.end_rdv();
854
855 // JCC: This "nudge factor" stuff was added when we were experiencing
856 // problems with particles getting lost in between PRegions on
857 // neighboring nodes. This problem has since been resolved by
858 // fixing the way in which PRegion boundaries are computed, so I am
859 // commenting this out for now. We can bring it back later if the
860 // need arises.
861
862 /*
863
864 // Calculate a 'nudge factor', an amount that can get added to a
865 // particle position to determine where it should be located. The nudge
866 // factor equals 1/100th the smallest width of the rnodes in each dimension.
867 // When we try to find where a particle is located, we check what vnode
868 // contains this particle 'nudge region', a box around the particle's pos
869 // of the size of the nudge factor.
870 T pNudge[Dim];
871 for (d=0; d < Dim; ++d) {
872 // initialize to the first rnode's width
873 T minval = (*(RLayout.begin_iv())).second->getDomain()[d].length();
874
875 // check the local rnodes
876 for (localV = RLayout.begin_iv(); localV != localEnd; ++localV) {
877 T checkval = (*localV).second->getDomain()[d].length();
878 if (checkval < minval)
879 minval = checkval;
880 }
881
882 // check the remote rnodes
883 for (remoteV = RLayout.begin_rdv(); remoteV != remoteEnd; ++remoteV) {
884 T checkval = (*remoteV).second->getDomain()[d].length();
885 if (checkval < minval)
886 minval = checkval;
887 }
888
889 // now rescale the minval, and save it
890 pNudge[d] = 0.00001 * minval;
891 }
892
893 */
894
895 // An NDRegion object used to store a particle position.
896 NDRegion<T,Dim> pLoc;
897
898 // get new message tag for particle exchange with empty domains
900
901 if (!getEmptyNode(myN))
902 {
903
904 // Particles are swapped in multipple passes, one for each dimension.
905 // The tasks completed here for each dimension are the following:
906 // 1. For each local Vnode, find the remote Vnodes which exist along
907 // same axis as the current axis (i.e. all Vnodes along the x-axis).
908 // 2. From this list, determine which nodes we send messages to.
909 // 3. Go through all the particles, finding those which have moved to
910 // an off-processor vnode, and store index in an array for that node
911 // 4. Send off the particles to the nodes (if no particles are
912 // going to a node, send them a message with 0 in it)
913 // 5. Delete the send particles from our local list
914 // 6. Receive particles sent to us by other nodes (some messages may
915 // say that we're receiving 0 particles from that node).
916
917 // Initialize NDRegion with a position inside the first Vnode.
918 // We can skip dim 0, since it will be filled below.
919 for (d = 1; d < Dim; ++d)
920 {
921 T first = (*(RLayout.begin_iv())).second->getDomain()[d].first();
922 T last = (*(RLayout.begin_iv())).second->getDomain()[d].last();
923 T mid = first + 0.5 * (last - first);
924 pLoc[d] = PRegion<T>(mid, mid);
925 }
926
927 for (d = 0; d < Dim; ++d)
928 {
929
930 // get new message tag for particle exchange along this dimension
932
933 // we only need to do the rest if there are other nodes in this dim
934 if (NeighborNodes[d] > 0)
935 {
936 // create new messages to send to our neighbors
937 for (i = 0; i < N; i++)
938 if (SwapNodeList[d][i])
939 SwapMsgList[i] = new Message;
940
941 // Go through the particles and find those moving in the current dir.
942 // When one is found, copy it into outgoing message and delete it.
943 for (ip=0; ip<LocalNum; ++ip)
944 {
945 if (!bool(canSwap[ip])) continue; // skip if can't swap
946 // get the position of particle ip, and find the closest grid pnt
947 // for just the dimensions 0 ... d
948 for (j = 0; j <= d; j++)
949 pLoc[j] = PRegion<T>(PData.R[ip][j], PData.R[ip][j]);
950
951 // first check local domains (in this dimension)
952 bool foundit = false;
953 // JCC: int nudged = 0;
954 while (!foundit)
955 {
956 for (localV = RLayout.begin_iv();
957 localV != localEnd && !foundit; ++localV)
958 {
959 foundit= (((*localV).second)->getDomain())[d].touches(pLoc[d]);
960 }
961
962 // if not found, it might be remote
963 if (!foundit)
964 {
965 // see which Vnode this postion is in
966 typename RegionLayout<T,Dim,Mesh>::touch_range_dv touchingVN =
967 RLayout.touch_range_rdv(pLoc);
968
969 // make sure we have a vnode to send it to
970 if (touchingVN.first == touchingVN.second)
971 {
972 // JCC: if (nudged >= Dim) {
973 ERRORMSG("Local particle " << ip << " with ID=");
974 ERRORMSG(PData.ID[ip] << " at ");
975 ERRORMSG(PData.R[ip] << " is outside of global domain ");
976 ERRORMSG(RLayout.getDomain() << endl);
977 ERRORMSG("This occurred when searching for point " << pLoc);
978 ERRORMSG(" in RegionLayout = " << RLayout << endl);
979 Ippl::abort();
980 }
981 else
982 {
983 // the node has been found - add index to put list
984 unsigned node = (*(touchingVN.first)).second->getNode();
985 PAssert_EQ(SwapNodeList[d][node], true);
986 PutList[node].push_back(ip);
987
988 // .. and then add to DestroyList
989 PData.destroy(1, ip);
990
991 // indicate we found it to quit this check
992 foundit = true;
993 }
994 }
995 }
996 }
997
998 // send the particles to their destination nodes
999 for (i = 0; i < N; i++)
1000 {
1001 if (SwapNodeList[d][i])
1002 {
1003 // put data for particles on this put list into message
1004 PData.putMessage( *(SwapMsgList[i]), PutList[i] );
1005
1006 // add a final 'zero' number of particles to indicate the end
1007 PData.putMessage(*(SwapMsgList[i]), (size_t) 0, (size_t) 0);
1008
1009 // send the message
1010 //Inform dbgmsg("SpatialLayout", INFORM_ALL_NODES);
1011 //dbgmsg << "Swapping "<<PutList[i].size() << " particles to node ";
1012 //dbgmsg << i<<" with tag " << tag << " (" << 'x' + d << ")" << endl;
1013 //dbgmsg << " ... msg = " << *(SwapMsgList[i]) << endl;
1014 int node = i;
1015 Ippl::Comm->send(SwapMsgList[i], node, tag);
1016
1017 // clear the list
1018 PutList[i].erase(PutList[i].begin(), PutList[i].end());
1019 }
1020 }
1021
1022 LocalNum -= PData.getDestroyNum(); // update local num
1023 ADDIPPLSTAT(incParticlesSwapped, PData.getDestroyNum());
1024 PData.performDestroy();
1025
1026 // receive particles from neighbor nodes, and add them to our list
1027 unsigned sendnum = NeighborNodes[d];
1028 while (sendnum-- > 0)
1029 {
1030 int node = Communicate::COMM_ANY_NODE;
1031 Message *recmsg = Ippl::Comm->receive_block(node, tag);
1032 size_t recvd;
1033 while ((recvd = PData.getMessage(*recmsg)) > 0)
1034 LocalNum += recvd;
1035 delete recmsg;
1036 }
1037 } // end if (NeighborNodes[d] > 0)
1038
1039 if (d == 0)
1040 {
1041 // receive messages from any empty nodes
1042 for (i = 0; i < N; ++i)
1043 {
1044 if (getEmptyNode(i))
1045 {
1046 int node = i;
1047 Message *recmsg = Ippl::Comm->receive_block(node, etag);
1048 size_t recvd;
1049 while ((recvd = PData.getMessage(*recmsg)) > 0)
1050 LocalNum += recvd;
1051 delete recmsg;
1052 }
1053 }
1054 }
1055
1056 } // end for (d=0; d<Dim; ++d)
1057
1058 }
1059 else // empty node sends, but does not receive
1060 {
1061 // create new messages to send to our neighbors along dim 0
1062 for (i = 0; i < N; i++)
1063 if (SwapNodeList[0][i])
1064 SwapMsgList[i] = new Message;
1065
1066 // Go through the particles and find those moving to other nodes.
1067 // When one is found, copy it into outgoing message and delete it.
1068 for (ip=0; ip<LocalNum; ++ip)
1069 {
1070 if (!bool(canSwap[ip])) continue; // skip if can't swap
1071 // get the position of particle ip, and find the closest grid pnt
1072 for (j = 0; j < Dim; j++)
1073 pLoc[j] = PRegion<T>(PData.R[ip][j], PData.R[ip][j]);
1074
1075 // see which remote Vnode this postion is in
1076 typename RegionLayout<T,Dim,Mesh>::touch_range_dv touchingVN =
1077 RLayout.touch_range_rdv(pLoc);
1078
1079 // make sure we have a vnode to send it to
1080 if (touchingVN.first == touchingVN.second)
1081 {
1082 ERRORMSG("Local particle " << ip << " with ID=");
1083 ERRORMSG(PData.ID[ip] << " at ");
1084 ERRORMSG(PData.R[ip] << " is outside of global domain ");
1085 ERRORMSG(RLayout.getDomain() << endl);
1086 ERRORMSG("This occurred when searching for point " << pLoc);
1087 ERRORMSG(" in RegionLayout = " << RLayout << endl);
1088 Ippl::abort();
1089 }
1090 else
1091 {
1092 // the node has been found - add index to put list
1093 unsigned node = (*(touchingVN.first)).second->getNode();
1094 PAssert_EQ(SwapNodeList[0][node], true);
1095 PutList[node].push_back(ip);
1096
1097 // .. and then add to DestroyList
1098 PData.destroy(1, ip);
1099 }
1100 }
1101
1102 // send the particles to their destination nodes
1103 for (i = 0; i < N; i++)
1104 {
1105 if (SwapNodeList[0][i])
1106 {
1107 // put data for particles on this put list into message
1108 PData.putMessage( *(SwapMsgList[i]), PutList[i] );
1109
1110 // add a final 'zero' number of particles to indicate the end
1111 PData.putMessage(*(SwapMsgList[i]), (size_t) 0, (size_t) 0);
1112
1113 // send the message
1114 int node = i;
1115 Ippl::Comm->send(SwapMsgList[i], node, etag);
1116
1117 // clear the list
1118 PutList[i].erase(PutList[i].begin(), PutList[i].end());
1119 }
1120 }
1121
1122 LocalNum -= PData.getDestroyNum(); // update local num
1123 ADDIPPLSTAT(incParticlesSwapped, PData.getDestroyNum());
1124 PData.performDestroy();
1125
1126 }
1127
1128 // return how many particles we have now
1129 return LocalNum;
1130 }
1131
1132
1133
1134
1135/*
1136 * Newer (cleaner) version of swap particles that uses less bandwidth
1137 * and drastically lowers message counts for real cases.
1138 */
1139 template < class PB >
1140 size_t new_swap_particles(size_t LocalNum, PB& PData)
1141 {
1143
1144 unsigned N = Ippl::getNodes();
1145 unsigned myN = Ippl::myNode();
1146
1147 typename RegionLayout<T,Dim,Mesh>::iterator_iv localV, localEnd = RLayout.end_iv();
1149
1150 std::vector<int> msgsend(N, 0);
1151 std::vector<int> msgrecv(N, 0);
1152
1153 NDRegion<T,Dim> pLoc;
1154
1155 std::multimap<unsigned, unsigned> p2n; //<node ID, particle ID>
1156
1157 bool responsibleNodeNotFound = false;
1158 outsideCount_m = 0;
1159 for (unsigned int ip=0; ip<LocalNum; ++ip)
1160 {
1161 for (unsigned int j = 0; j < Dim; j++)
1162 pLoc[j] = PRegion<T>(PData.R[ip][j], PData.R[ip][j]);
1163
1164 unsigned destination = myN;
1165 bool found = false;
1166 for (localV = RLayout.begin_iv(); localV != localEnd && !found; ++localV)
1167 {
1168 if ((((*localV).second)->getDomain()).touches(pLoc))
1169 found = true; // particle is local and doesn't need to be sent anywhere
1170 }
1171
1172 if (found)
1173 continue;
1174
1175 typename RegionLayout<T,Dim,Mesh>::touch_range_dv touchingVN = RLayout.touch_range_rdv(pLoc);
1176
1177 //external location
1178 if (touchingVN.first == touchingVN.second) {
1179 if (!outsideToNearest_m) {
1180 responsibleNodeNotFound = true;
1181 break;
1182 }
1183
1184 // look up the owner of the nearest point of the domain instead
1186 moveIntoDomain(pLoc, PData.R[ip]);
1187 for (localV = RLayout.begin_iv(); localV != localEnd && !found; ++localV)
1188 {
1189 if ((((*localV).second)->getDomain()).touches(pLoc))
1190 found = true;
1191 }
1192
1193 if (found)
1194 continue;
1195
1196 touchingVN = RLayout.touch_range_rdv(pLoc);
1197 if (touchingVN.first == touchingVN.second) {
1198 responsibleNodeNotFound = true;
1199 break;
1200 }
1201 }
1202 destination = (*(touchingVN.first)).second->getNode();
1203
1204 msgsend[destination] = 1;
1205
1206 p2n.insert(std::pair<unsigned, unsigned>(destination, ip));
1207 }
1208
1209 allreduce(&responsibleNodeNotFound,
1210 1,
1211 std::logical_or<bool>());
1212
1213 if (responsibleNodeNotFound) {
1214 throw IpplException("ParticleSpatialLayout::new_swap_particles",
1215 "could not find node responsible for particle");
1216 }
1217
1218 //reduce message count so every node knows how many messages to receive
1219 allreduce(msgsend.data(), msgrecv.data(), N, std::plus<int>());
1220
1222
1223 typename std::multimap<unsigned, unsigned>::iterator i = p2n.begin();
1224
1225 std::unique_ptr<Format> format(PData.getFormat());
1226
1227
1228 std::vector<MPI_Request> requests;
1229 std::vector<std::shared_ptr<MsgBuffer> > buffers;
1230
1231 while (i!=p2n.end())
1232 {
1233 unsigned cur_destination = i->first;
1234
1235 std::shared_ptr<MsgBuffer> msgbuf(new MsgBuffer(format.get(), p2n.count(i->first)));
1236
1237 for (; i!=p2n.end() && i->first == cur_destination; ++i)
1238 {
1239 Message msg;
1240 PData.putMessage(msg, i->second);
1241 PData.destroy(1, i->second);
1242 msgbuf->add(&msg);
1243 }
1244
1245 MPI_Request request = Ippl::Comm->raw_isend( msgbuf->getBuffer(), msgbuf->getSize(), cur_destination, tag);
1246
1247 //remember request and buffer so we can delete them later
1248 requests.push_back(request);
1249 buffers.push_back(msgbuf);
1250 }
1251
1252 LocalNum -= PData.getDestroyNum(); // update local num
1253 PData.performDestroy();
1254
1255 //receive new particles
1256 for (int k = 0; k<msgrecv[myN]; ++k)
1257 {
1258 int node = Communicate::COMM_ANY_NODE;
1259 char *buffer = 0;
1260 int bufsize = Ippl::Comm->raw_probe_receive(buffer, node, tag);
1261 MsgBuffer recvbuf(format.get(), buffer, bufsize);
1262
1263 Message *msg = recvbuf.get();
1264 while (msg != 0)
1265 {
1266 LocalNum += PData.getSingleMessage(*msg);
1267 delete msg;
1268 msg = recvbuf.get();
1269 }
1270
1271
1272 }
1273
1274 //wait for communication to finish and clean up buffers
1275 MPI_Request* requests_ptr = requests.empty()? static_cast<MPI_Request*>(0): &(requests[0]);
1276 MPI_Waitall(requests.size(), requests_ptr, MPI_STATUSES_IGNORE);
1277
1278 return LocalNum;
1279 }
1280
1281 template < class PB >
1282 size_t new_swap_particles(size_t LocalNum, PB& PData,
1283 const ParticleAttrib<char>& canSwap)
1284 {
1286
1287 unsigned N = Ippl::getNodes();
1288 unsigned myN = Ippl::myNode();
1289
1290 typename RegionLayout<T,Dim,Mesh>::iterator_iv localV, localEnd = RLayout.end_iv();
1292
1293 std::vector<int> msgsend(N, 0);
1294 std::vector<int> msgrecv(N, 0);
1295
1296 NDRegion<T,Dim> pLoc;
1297
1298 std::multimap<unsigned, unsigned> p2n; //<node ID, particle ID>
1299
1300 bool responsibleNodeNotFound = false;
1301 outsideCount_m = 0;
1302 for (unsigned int ip=0; ip<LocalNum; ++ip)
1303 {
1304 if (!bool(canSwap[ip]))//skip if it can't be swapped
1305 continue;
1306
1307 for (unsigned int j = 0; j < Dim; j++)
1308 pLoc[j] = PRegion<T>(PData.R[ip][j], PData.R[ip][j]);
1309
1310 unsigned destination = myN;
1311 bool found = false;
1312 for (localV = RLayout.begin_iv(); localV != localEnd && !found; ++localV)
1313 {
1314 if ((((*localV).second)->getDomain()).touches(pLoc))
1315 found = true; // particle is local and doesn't need to be sent anywhere
1316 }
1317
1318 if (found)
1319 continue;
1320
1321 typename RegionLayout<T,Dim,Mesh>::touch_range_dv touchingVN = RLayout.touch_range_rdv(pLoc);
1322
1323 //external location
1324 if (touchingVN.first == touchingVN.second) {
1325 if (!outsideToNearest_m) {
1326 responsibleNodeNotFound = true;
1327 break;
1328 }
1329
1330 // look up the owner of the nearest point of the domain instead
1332 moveIntoDomain(pLoc, PData.R[ip]);
1333 for (localV = RLayout.begin_iv(); localV != localEnd && !found; ++localV)
1334 {
1335 if ((((*localV).second)->getDomain()).touches(pLoc))
1336 found = true;
1337 }
1338
1339 if (found)
1340 continue;
1341
1342 touchingVN = RLayout.touch_range_rdv(pLoc);
1343 if (touchingVN.first == touchingVN.second) {
1344 responsibleNodeNotFound = true;
1345 break;
1346 }
1347 }
1348 destination = (*(touchingVN.first)).second->getNode();
1349
1350 msgsend[destination] = 1;
1351
1352 p2n.insert(std::pair<unsigned, unsigned>(destination, ip));
1353 }
1354
1355 allreduce(&responsibleNodeNotFound,
1356 1,
1357 std::logical_or<bool>());
1358
1359 if (responsibleNodeNotFound) {
1360 throw IpplException("ParticleSpatialLayout::new_swap_particles",
1361 "could not find node responsible for particle");
1362 }
1363
1364 //reduce message count so every node knows how many messages to receive
1365 allreduce(msgsend.data(), msgrecv.data(), N, std::plus<int>());
1366
1368
1369 typename std::multimap<unsigned, unsigned>::iterator i = p2n.begin();
1370
1371 std::unique_ptr<Format> format(PData.getFormat());
1372
1373 std::vector<MPI_Request> requests;
1374 std::vector<std::shared_ptr<MsgBuffer> > buffers;
1375
1376 while (i!=p2n.end())
1377 {
1378 unsigned cur_destination = i->first;
1379
1380 std::shared_ptr<MsgBuffer> msgbuf(new MsgBuffer(format.get(), p2n.count(i->first)));
1381
1382 for (; i!=p2n.end() && i->first == cur_destination; ++i)
1383 {
1384 Message msg;
1385 PData.putMessage(msg, i->second);
1386 PData.destroy(1, i->second);
1387 msgbuf->add(&msg);
1388 }
1389
1390 MPI_Request request = Ippl::Comm->raw_isend( msgbuf->getBuffer(), msgbuf->getSize(), cur_destination, tag);
1391
1392 //remember request and buffer so we can delete them later
1393 requests.push_back(request);
1394 buffers.push_back(msgbuf);
1395 }
1396
1397 LocalNum -= PData.getDestroyNum(); // update local num
1398 PData.performDestroy();
1399
1400 //receive new particles
1401 for (int k = 0; k<msgrecv[myN]; ++k)
1402 {
1403 int node = Communicate::COMM_ANY_NODE;
1404 char *buffer = 0;
1405 int bufsize = Ippl::Comm->raw_probe_receive(buffer, node, tag);
1406 MsgBuffer recvbuf(format.get(), buffer, bufsize);
1407
1408 Message *msg = recvbuf.get();
1409 while (msg != 0)
1410 {
1411 LocalNum += PData.getSingleMessage(*msg);
1412 delete msg;
1413 msg = recvbuf.get();
1414 }
1415 }
1416
1417 //wait for communication to finish and clean up buffers
1418 MPI_Request* requests_ptr = requests.empty()? static_cast<MPI_Request*>(0): &(requests[0]);
1419 MPI_Waitall(requests.size(), requests_ptr, 0);
1420
1421 return LocalNum;
1422 }
1423
1424};
1425
1427
1428#endif // PARTICLE_SPATIAL_LAYOUT_H
const unsigned Dim
PartBunchBase< T, Dim >::ConstIterator end(PartBunchBase< T, Dim > const &bunch)
PartBunchBase< T, Dim >::ConstIterator begin(PartBunchBase< T, Dim > const &bunch)
std::ostream & operator<<(std::ostream &, const ParticleSpatialLayout< T, Dim, Mesh, CachingPolicy > &)
#define P_SPATIAL_TRANSFER_TAG
Definition Tags.h:82
#define P_SPATIAL_RANGE_TAG
Definition Tags.h:84
#define P_LAYOUT_CYCLE
Definition Tags.h:86
#define P_SPATIAL_RETURN_TAG
Definition Tags.h:81
const int COMM_ANY_NODE
Definition Communicate.h:40
void allreduce(const T *input, T *output, int count, Op op)
#define ERRORMSG(msg)
Definition IpplInfo.h:350
Inform & endl(Inform &inf)
Definition Inform.cpp:42
#define INFORM_ALL_NODES
Definition Inform.h:39
#define PAssert_LT(a, b)
Definition PAssert.h:106
#define PAssert_EQ(a, b)
Definition PAssert.h:104
#define ADDIPPLSTAT(stat, amount)
Definition IpplStats.h:237
Message & getMessage(Message &m)
Definition Vektor.h:168
Message & putMessage(Message &m) const
Definition Vektor.h:162
Definition Index.h:237
Definition Mesh.h:35
virtual MPI_Request raw_isend(void *, int, int, int)
bool send(Message *, int node, int tag, bool delmsg=true)
virtual int raw_probe_receive(char *&, int &, int &)
virtual int broadcast_others(Message *, int, bool delmsg=true)
Message * receive_block(int &node, int &tag)
void barrier(void)
Message & put(const T &val)
Definition Message.h:406
bool empty() const
Definition Message.h:300
Message & get(const T &cval)
Definition Message.h:476
Message * get()
Definition MsgBuffer.cpp:71
int next_tag(int t, int s=1000)
Definition TagMaker.h:39
int getNodeCount(unsigned i) const
void rebuild_layout(size_t haveLocal, PB &PData)
ParticleAttrib< Index_t > ParticleIndex_t
ParticleAttrib< SingleParticlePos_t > ParticlePos_t
const RegionLayout< T, Dim, Mesh > & getLayout() const
bool getEmptyNode(unsigned i) const
void setOutsideToNearest(bool outsideToNearest)
virtual void Repartition(UserList *)
RegionLayout< T, Dim, Mesh > & getLayout()
virtual void notifyUserOfDelete(UserList *)
ParticleLayout< T, Dim >::SingleParticlePos_t SingleParticlePos_t
void moveIntoDomain(NDRegion< T, Dim > &pLoc, const SingleParticlePos_t &pos) const
RegionLayout< T, Dim, Mesh > RLayout
void update(IpplParticleBase< ParticleSpatialLayout< T, Dim, Mesh, CachingPolicy > > &p, const ParticleAttrib< char > *canSwap=0)
size_t new_swap_particles(size_t LocalNum, PB &PData, const ParticleAttrib< char > &canSwap)
ParticleLayout< T, Dim >::Index_t Index_t
size_t new_swap_particles(size_t LocalNum, PB &PData)
std::vector< size_t > * PutList
size_t swap_particles(size_t LocalNum, PB &PData)
size_t short_swap_particles(size_t LocalNum, PB &PData)
FieldLayout< Dim > & getFieldLayout()
RegionLayout< T, Dim, Mesh > RegionLayout_t
size_t swap_particles(size_t LocalNum, PB &PData, const ParticleAttrib< char > &canSwap)
iterator_iv begin_iv()
touch_range_dv touch_range_rdv(const NDRegion< T, Dim > &domain)
ac_id_vnodes::iterator iterator_iv
std::pair< touch_iterator_dv, touch_iterator_dv > touch_range_dv
static void abort(const char *=0)
Definition IpplInfo.cpp:617
static int getNodes()
Definition IpplInfo.cpp:689
static int myNode()
Definition IpplInfo.cpp:710
static Communicate * Comm
Definition IpplInfo.h:84