OPAL (Object Oriented Parallel Accelerator Library) 2024.2
OPAL
ParticleSpatialLayout.hpp
Go to the documentation of this file.
1// -*- C++ -*-
2/***************************************************************************
3 *
4 * The IPPL Framework
5 *
6 ***************************************************************************/
7
8// include files
11#include "Index/NDIndex.h"
12#include "Region/RegionLayout.h"
13#include "Message/Communicate.h"
14#include "Message/Message.h"
15#include "Utility/IpplInfo.h"
16#include "Utility/IpplStats.h"
18
19
20// forward declarations
21template <unsigned Dim> class FieldLayout;
22class UserList;
23
25// constructor, from a FieldLayout
26template < class T, unsigned Dim, class Mesh, class CachingPolicy >
28 : RLayout(fl)
29{
30 setup(); // perform necessary setup
31}
32
33
35// constructor, from a FieldLayout
36template < class T, unsigned Dim, class Mesh, class CachingPolicy >
38 Mesh& mesh)
39 : RLayout(fl,mesh)
40{
41 setup(); // perform necessary setup
42}
43
44
46// constructor, from a RegionLayout
47template < class T, unsigned Dim, class Mesh, class CachingPolicy >
49 RegionLayout<T,Dim,Mesh>& rl) : RLayout(rl)
50{
51 setup(); // perform necessary setup
52}
53
54
56// default constructor ... this does not initialize the RegionLayout,
57// it will be instead initialized during the first update.
58template < class T, unsigned Dim, class Mesh, class CachingPolicy >
60{
61 setup(); // perform necessary setup
62}
63
64
66// perform common constructor tasks
67template < class T, unsigned Dim, class Mesh, class CachingPolicy >
69{
70
71 unsigned i; // loop variable
72
73 caching = false;
74
75 outsideToNearest_m = false;
76 outsideCount_m = 0;
77
78 // check ourselves in as a user of the RegionLayout
79 RLayout.checkin(*this);
80
81 // create storage for message pointers used in swapping particles
82 unsigned N = Ippl::getNodes();
83 SwapMsgList = new Message*[N];
84 for (i = 0; i < Dim; ++i)
85 SwapNodeList[i] = new bool[N];
86 PutList = new std::vector<size_t>[N];
88 // create storage for the number of particles on each node
89 // and flag for empty node domain
90 NodeCount = new size_t[N];
91 EmptyNode = new bool[N];
92 for (i = 0; i < N; ++i)
93 {
94 NodeCount[i] = 0;
95 EmptyNode[i] = false;
96 }
97}
99
101// destructor
102template < class T, unsigned Dim, class Mesh, class CachingPolicy >
104{
105
106 delete [] NodeCount;
107 delete [] EmptyNode;
108 delete [] SwapMsgList;
109 for (unsigned int i=0; i < Dim; i++)
110 delete [] (SwapNodeList[i]);
111 delete [] PutList;
112
113 // check ourselves out as a user of the RegionLayout
114 RLayout.checkout(*this);
115}
116
117
119// Update the location and indices of all atoms in the given IpplParticleBase
120// object. This handles swapping particles among processors if
121// needed, and handles create and destroy requests. When complete,
122// all nodes have correct layout information.
123template <class T, unsigned Dim, class Mesh, class CachingPolicy>
126 const ParticleAttrib<char>* canSwap)
127{
128
129 unsigned N = Ippl::getNodes();
130 unsigned myN = Ippl::myNode();
131 size_t LocalNum = PData.getLocalNum();
132 size_t DestroyNum = PData.getDestroyNum();
133 size_t TotalNum;
134 int node;
135
136 //Inform dbgmsg("SpatialLayout::update", INFORM_ALL_NODES);
137 //dbgmsg << "At start of update:" << endl;
138 //PData.printDebug(dbgmsg);
139
140 // delete particles in destroy list, update local num
141 PData.performDestroy();
142 LocalNum -= DestroyNum;
143
144 // set up our layout, if not already done ... we could also do this if
145 // we needed to expand our spatial region.
146 if ( ! RLayout.initialized())
147 rebuild_layout(LocalNum,PData);
148
149 // apply boundary conditions to the particle positions
150 if (this->getUpdateFlag(ParticleLayout<T,Dim>::BCONDS))
151 this->apply_bconds(LocalNum, PData.R, this->getBConds(), RLayout.getDomain());
152
153 if(caching)
154 this->updateCacheInformation(*this);
155
156 // swap particles, if necessary
157 if (N > 1 && this->getUpdateFlag(ParticleLayout<T,Dim>::SWAP))
158 {
159 // Now we can swap particles that have moved outside the region of
160 // local field space. This is done in several passes, one for each
161 // spatial dimension. The NodeCount values are updated by this routine.
163 static IpplMessageCounterRegion swapCounter("Swap Particles");
164 swapCounter.begin();
165 //MPI_Pcontrol( 1,"swap_particles");
166
167 if (canSwap==0)
168 LocalNum = new_swap_particles(LocalNum, PData);
169 else
170 LocalNum = new_swap_particles(LocalNum, PData, *canSwap);
171
172 swapCounter.end();
173 }
174
175 // Save how many local particles we have.
176 TotalNum = NodeCount[myN] = LocalNum;
177
178 // there is extra work to do if there are multipple nodes, to distribute
179 // the particle layout data to all nodes
180 if (N > 1)
181 {
182 // At this point, we can send our particle count updates to node 0, and
183 // receive back the particle layout.
186 if (myN != 0)
187 {
188 Message *msg = new Message;
189
190 // put local particle count in the message
191 msg->put(LocalNum);
192 // send this info to node 0
193 Ippl::Comm->send(msg, 0, tag1);
194
195 // receive back the number of particles on each node
196 node = 0;
197 Message* recmsg = Ippl::Comm->receive_block(node, tag2);
198 recmsg->get(NodeCount);
199 recmsg->get(TotalNum);
200 delete recmsg;
201 }
202 else // do update tasks particular to node 0
203 {
204 // receive messages from other nodes describing what they have
205 int notrecvd = N - 1; // do not need to receive from node 0
206 TotalNum = LocalNum;
207 while (notrecvd > 0)
208 {
209 // receive a message from another node. After recv, node == sender.
211 Message *recmsg = Ippl::Comm->receive_block(node, tag1);
212 size_t remNodeCount = 0;
213 recmsg->get(remNodeCount);
214 delete recmsg;
215 notrecvd--;
216
217 // update values based on data from remote node
218 TotalNum += remNodeCount;
219 NodeCount[node] = remNodeCount;
220 }
221
222 // send info back to all the client nodes
223 Message *msg = new Message;
224 msg->put(NodeCount, NodeCount + N);
225 msg->put(TotalNum);
226 Ippl::Comm->broadcast_others(msg, tag2);
227 }
228 }
229
230 // update our particle number counts
231 PData.setTotalNum(TotalNum); // set the total atom count
232 PData.setLocalNum(LocalNum); // set the number of local atoms
233
234 if(caching)
235 this->updateGhostParticles(PData, *this);
236
237 //dbgmsg << endl << "At end of update:" << endl;
238 //PData.printDebug(dbgmsg);
239}
240
241
242
243
244
246// print it out
247template < class T, unsigned Dim, class Mesh, class CachingPolicy >
248std::ostream& operator<<(std::ostream& out, const ParticleSpatialLayout<T,Dim,Mesh,CachingPolicy>& L)
249{
250
251 out << "ParticleSpatialLayout, with particle distribution:\n ";
252 for (unsigned int i=0; i < (unsigned int) Ippl::getNodes(); ++i)
253 out << "Number of particles " << L.getNodeCount(i) << " ";
254 out << "\nSpatialLayout decomposition = " << L.getLayout();
255 return out;
256}
257
258
260// Print out information for debugging purposes.
261template < class T, unsigned Dim, class Mesh, class CachingPolicy >
263{
264
265 o << "PSpatial: distrib = ";
266 for (int i=0; i < Ippl::getNodes(); ++i)
267 o << NodeCount[i] << " ";
268}
269
270
272// Repartition onto a new layout, if the layout changes ... this is a
273// virtual function called by a UserList, as opposed to the RepartitionLayout
274// function used by the particle load balancing mechanism.
275template < class T, unsigned Dim, class Mesh, class CachingPolicy >
279
280
282// Tell the subclass that the FieldLayoutBase is being deleted, so
283// don't use it anymore
284template < class T, unsigned Dim, class Mesh, class CachingPolicy >
286{
287
288 // really, nothing to do, since the RegionLayout we use only gets
289 // deleted when we are deleted ourselves.
290 return;
291}
const unsigned Dim
std::ostream & operator<<(std::ostream &out, const ParticleSpatialLayout< T, Dim, Mesh, CachingPolicy > &L)
#define P_LAYOUT_CYCLE
Definition Tags.h:86
#define P_SPATIAL_RETURN_TAG
Definition Tags.h:81
#define P_SPATIAL_LAYOUT_TAG
Definition Tags.h:80
Definition Mesh.h:35
bool send(Message *, int node, int tag, bool delmsg=true)
virtual int broadcast_others(Message *, int, bool delmsg=true)
Message * receive_block(int &node, int &tag)
Message & put(const T &val)
Definition Message.h:406
Message & get(const T &cval)
Definition Message.h:476
int next_tag(int t, int s=1000)
Definition TagMaker.h:39
int getNodeCount(unsigned i) const
virtual void Repartition(UserList *)
RegionLayout< T, Dim, Mesh > & getLayout()
virtual void notifyUserOfDelete(UserList *)
void update(IpplParticleBase< ParticleSpatialLayout< T, Dim, Mesh, CachingPolicy > > &p, const ParticleAttrib< char > *canSwap=0)
static int getNodes()
Definition IpplInfo.cpp:689
static int myNode()
Definition IpplInfo.cpp:710
static Communicate * Comm
Definition IpplInfo.h:84