OPAL (Object Oriented Parallel Accelerator Library) 2024.2
OPAL
ParticleBalancer.hpp
Go to the documentation of this file.
1// -*- C++ -*-
2/***************************************************************************
3 *
4 * The IPPL Framework
5 *
6 * This program was prepared by PSI.
7 * All rights in the program are reserved by PSI.
8 * Neither PSI nor the author(s)
9 * makes any warranty, express or implied, or assumes any liability or
10 * responsibility for the use of this software
11 *
12 * Visit www.amas.web.psi for more details
13 *
14 ***************************************************************************/
15
16// -*- C++ -*-
17/***************************************************************************
18 *
19 * The IPPL Framework
20 *
21 *
22 * Visit http://people.web.psi.ch/adelmann/ for more details
23 *
24 ***************************************************************************/
25
26// include files
32#include "Particle/IntNGP.h"
33#include "Region/RegionLayout.h"
34#include "Index/NDIndex.h"
37#include "Utility/IpplInfo.h"
38
39
40
42// number density scatter as scatter(f, pp, intop), but only of the particles
43// inside the given region. A particle outside it is not counted, and its
44// position is never looked up on the Field: beyond the guard cells, the lookup
45// would abort.
46template <class FT, unsigned Dim, class M, class C, class PT, class IntOp>
47void
49 const ParticleAttrib< Vektor<PT,Dim> >& pp,
50 const NDRegion<PT,Dim>& domain, const IntOp& /*intop*/) {
51
52 // make sure field is uncompressed and guard cells are zeroed
53 f.Uncompress();
54 FT zero = 0;
55 f.setGuardCells(zero);
56
57 const M& mesh = f.get_mesh();
58 // iterate through the particles and call scatter operation for those
59 // inside the half-open domain [min, max)
60 typename ParticleAttrib< Vektor<PT,Dim> >::const_iterator ppiter;
61 size_t i = 0;
62 for (ppiter = pp.cbegin(); i < pp.size(); ++i, ++ppiter) {
63 const Vektor<PT,Dim>& pos = *ppiter;
64 bool inside = true;
65 for (unsigned int d = 0; d < Dim; ++d)
66 inside = inside && (pos[d] >= domain[d].min() && pos[d] < domain[d].max());
67 if (inside)
68 IntOp::scatter(FT(1), f, pos, mesh);
69 }
70
71 // accumulate values in guard cells
73
74 INCIPPLSTAT(incParticleScatters);
75}
76
77
79// calculate a new RegionLayout for a given ParticleBase, and distribute the
80// new RegionLayout to all the nodes. This uses a Field BinaryBalancer.
81// With ParticleSpatialLayout::setOutsideToNearest(true), the particles
82// outside the domain of the RegionLayout do not count in the balance.
83template < class T, unsigned Dim, class Mesh, class CachingPolicy>
84bool
86
87
88
89 static IntNGP interp; // to scatter particle density
90
91 //Inform dbgmsg("Particle BinaryRepartition", INFORM_ALL_NODES);
92 //dbgmsg << "Performing particle load balancing, for ";
93 //dbgmsg << PB.getTotalNum() << " particles ..." << endl;
94
95 // get the internal FieldLayout from the Particle object's internal
96 // RegionLayout. From this, we make a new Field (we do not need a
97 RegionLayout<T,Dim,Mesh>& RL = PB.getLayout().getLayout();
98 if ( ! RL.initialized()) {
99 ERRORMSG("Cannot repartition particles: uninitialized layout." << endl);
100 return false;
101 }
103 Mesh& mesh = RL.getMesh();
104
105 // NDIndex which describes the entire domain ... if a particle is
106 // outside this region, we are in trouble!
107 const NDIndex<Dim>& TotalDomain = FL.getDomain();
108
109 // for all the particles, do the following:
110 // 1. get the position, and invert to the 'FieldLayout' index space
111 // 2. increment the field at the position near this index position
112 NDIndex<Dim> indx;
113
114 // By default, we do the number density computation and repartition of
115 // index space on a cell-centered Field. If FieldLayout is vertex-centered,
116 // we'll need to make some adjustments here.
117 bool CenterOffset[Dim];
118 int CenteringTotal = 0;
119 unsigned int d;
120 for (d=0; d<Dim; ++d) {
121 CenterOffset[d] = (TotalDomain[d].length() < mesh.gridSizes[d]);
122 CenteringTotal += CenterOffset[d];
123 }
124
125
126 // particles outside the domain are legal with this setting
127 const bool outsideToNearest = PB.getLayout().getOutsideToNearest();
128
129 if (CenteringTotal == Dim) { // allCell centering
131
132 // Now do a number density scatter on this Field
133 // Afterwards, the Field will be deleted, and will checkout of the
134 // FieldLayout. This is desired so that when we repartition the
135 // FieldLayout, we do not waste time redistributing the Field's data.
136 BF = offset;
137 if (outsideToNearest)
138 scatterInsideDomain(BF,PB.R,RL.getDomain(),interp);
139 else
140 scatter(BF,PB.R,interp);
141
142 // calculate a new repartitioning of the field, and use this to repartition
143 // the FieldLayout used inside the Particle object
144 try
145 {
146 indx = CalcBinaryRepartition(FL, BF);
147 }
149 {
150 return false;
151 }
152 }
153 else if (CenteringTotal == 0) { // allVert centering
155
156 // Now do a number density scatter on this Field
157 // Afterwards, the Field will be deleted, and will checkout of the
158 // FieldLayout. This is desired so that when we repartition the
159 // FieldLayout, we do not waste time redistributing the Field's data.
160 BF = offset;
161 if (outsideToNearest)
162 scatterInsideDomain(BF,PB.R,RL.getDomain(),interp);
163 else
164 scatter(BF,PB.R,interp);
165
166 // calculate a new repartitioning of the field, and use this to repartition
167 // the FieldLayout used inside the Particle object
168 try
169 {
170 indx = CalcBinaryRepartition(FL, BF);
171 }
173 {
174 return false;
175 }
176 }
177 else {
178 ERRORMSG("Not implemented for face- and edge-centered Fields!!" << endl);
179 Ippl::abort();
180 }
181
182 // now, we can repartition the FieldLayout within the RegionLayout
183 RL.RepartitionLayout(indx);
184 PB.update();
185 return true;
186}
187
188
189// the same, but taking a uniform layout (this will not actually do anything)
190template<class T, unsigned Dim>
191bool
193 // for a uniform layout, this repartition method does nothing, so just
194 // exit
195 return true;
196}
197
const unsigned Dim
NDIndex< Dim > CalcBinaryRepartition(FieldLayout< Dim > &, BareField< double, Dim > &)
void scatterInsideDomain(Field< FT, Dim, M, C > &f, const ParticleAttrib< Vektor< PT, Dim > > &pp, const NDRegion< PT, Dim > &domain, const IntOp &)
bool BinaryRepartition(IpplParticleBase< ParticleSpatialLayout< T, Dim, Mesh, CachingPolicy > > &PB, double offset)
T::PETE_Expr_t::PETE_Return_t min(const PETE_Expr< T > &expr, NDIndex< D > &loc)
bool scatter(Communicate &, InputIterator, InputIterator, RandomIterator, int *, int *, const ScatterOp &)
#define ERRORMSG(msg)
Definition IpplInfo.h:350
Inform & endl(Inform &inf)
Definition Inform.cpp:42
#define INCIPPLSTAT(stat)
Definition IpplStats.h:236
const_iterator cbegin() const
Definition Field.h:33
Mesh_t & get_mesh() const
Definition Field.h:110
void accumGuardCells()
void Uncompress() const
void setGuardCells(const T &) const
const NDIndex< Dim > & getDomain() const
Definition Mesh.h:35
bool initialized() const
void RepartitionLayout(NDIndex< Dim > *, NDIndex< Dim > *)
MeshType & getMesh()
const NDRegion< T, Dim > & getDomain() const
FieldLayout< Dim > & getFieldLayout()
static void abort(const char *=0)
Definition IpplInfo.cpp:617