OPAL (Object Oriented Parallel Accelerator Library) 2024.2
OPAL
DataSink.cpp
Go to the documentation of this file.
1//
2// Class DataSink
3// This class acts as an observer during the calculation. It generates diagnostic
4// output of the accelerated beam such as statistical beam descriptors of particle
5// positions, momenta, beam phase space (emittance) etc. These are written to file
6// at periodic time steps during the calculation.
7//
8// This class also writes the full beam phase space to an H5 file at periodic time
9// steps in the calculation (this period is different from that of the statistical
10// numbers).
11
12// Class also writes processor load balancing data to file to track parallel
13// calculation efficiency.
14//
15// Copyright (c) 2008 - 2020, Paul Scherrer Institut, Villigen PSI, Switzerland
16// All rights reserved
17//
18// This file is part of OPAL.
19//
20// OPAL is free software: you can redistribute it and/or modify
21// it under the terms of the GNU General Public License as published by
22// the Free Software Foundation, either version 3 of the License, or
23// (at your option) any later version.
24//
25// You should have received a copy of the GNU General Public License
26// along with OPAL. If not, see <https://www.gnu.org/licenses/>.
27//
28#include "Structure/DataSink.h"
29
30#include "OPALconfig.h"
31
33#include "Fields/Fieldmap.h"
34#include "Physics/Units.h"
38#include "Utilities/Options.h"
39#include "Utilities/Timer.h"
40#include "Utilities/Util.h"
42
43#ifdef __linux__
45#else
47#endif
48
49#ifdef ENABLE_AMR
51#endif
52
53#ifdef ENABLE_AMR
55#endif
56
57#include <sstream>
58
59
61 : isMultiBunch_m(false)
62{
63 this->init();
64}
65
66
67DataSink::DataSink(H5PartWrapper* h5wrapper, bool restart, short numBunch)
68 : isMultiBunch_m(numBunch > 1)
69{
70 if (restart && !Options::enableHDF5) {
71 throw OpalException("DataSink::DataSink()",
72 "Can not restart when HDF5 is disabled");
73 }
74
75 this->init(restart, h5wrapper, numBunch);
76
77 if ( restart )
79}
80
81
82DataSink::DataSink(H5PartWrapper* h5wrapper, short numBunch)
83 : DataSink(h5wrapper, false, numBunch)
84{ }
85
86
88 if (!Options::enableHDF5) return;
89
90 h5Writer_m->writePhaseSpace(beam, FDext);
91}
92
93
94int DataSink::dumpH5(PartBunchBase<double, 3>* beam, Vector_t FDext[], double meanEnergy,
95 int refSource,
96 double refPr, double refPt, double refPz,
97 double refR, double refTheta, double refZ,
98 double azimuth, double elevation, bool local) const
99{
100 if (!Options::enableHDF5) return -1;
101
102 return h5Writer_m->writePhaseSpace(beam, FDext, meanEnergy, refSource, refPr, refPt, refPz,
103 refR, refTheta, refZ, azimuth, elevation, local);
104}
105
106
108 const double& azimuth, const double& timeStep) const
109{
110 this->dumpSDDS_m(beam, FDext, losses_t(), azimuth, timeStep);
111}
112
113
115 const losses_t& losses, const double& azimuth) const
116{
117 this->dumpSDDS_m(beam, FDext, losses, azimuth, beam->getdT());
118}
119
120
122 const losses_t& losses, const double& azimuth,
123 const double& timeStep) const
124{
125 beam->calcBeamParameters();
126
127 size_t npOutside = 0;
129 npOutside = beam->calcNumPartsOutside(Options::beamHaloBoundary*beam->get_rrms());
130
132
133 statWriter_m->write(beam, FDext, timeStep, losses, azimuth, npOutside);
134
136
137 for (size_t i = 0; i < sddsWriter_m.size(); ++i) {
138 sddsWriter_m[i]->write(beam);
139 }
140
142}
143
144
146 if (!Options::enableHDF5) return;
147
148 h5Writer_m->storeCavityInformation();
149}
150
151
153 if (!Options::enableHDF5) return;
154
155 h5Writer_m->changeH5Wrapper(h5wrapper);
156}
157
158
159void DataSink::writeGeomToVtk(BoundaryGeometry& bg, const std::string& fn) {
160 if (Ippl::myNode() == 0 && Options::enableVTK) {
161 bg.writeGeomToVtk(fn);
162 }
163}
164
165
166void DataSink::writeImpactStatistics(const PartBunchBase<double, 3>* beam, long long& step, size_t& impact, double& sey_num,
167 size_t numberOfFieldEmittedParticles, bool nEmissionMode, std::string fn) {
168
169 double charge = 0.0;
170 size_t Npart = 0;
171 double Npart_d = 0.0;
172 if (!nEmissionMode) {
173 charge = -1.0 * beam->getCharge();
174 //reduce(charge, charge, OpAddAssign());
175 Npart_d = -1.0 * charge / beam->getChargePerParticle();
176 } else {
177 Npart = beam->getTotalNum();
178 }
179 if (Ippl::myNode() == 0) {
180 std::string ffn = fn + std::string(".dat");
181
182 std::unique_ptr<Inform> ofp(new Inform(nullptr, ffn.c_str(), Inform::APPEND, 0));
183 Inform &fid = *ofp;
184 fid.precision(6);
185 fid << std::setiosflags(std::ios::scientific);
186 double t = beam->getT() * Units::s2ns;
187 if (!nEmissionMode) {
188
189 if (step == 0) {
190 fid << "#Time/ns" << std::setw(18) << "#Geometry impacts" << std::setw(18) << "tot_sey" << std::setw(18)
191 << "TotalCharge" << std::setw(18) << "PartNum" << " numberOfFieldEmittedParticles " << endl;
192 }
193 fid << t << std::setw(18) << impact << std::setw(18) << sey_num << std::setw(18) << charge
194 << std::setw(18) << Npart_d << std::setw(18) << numberOfFieldEmittedParticles << endl;
195 } else {
196
197 if (step == 0) {
198 fid << "#Time/ns" << std::setw(18) << "#Geometry impacts" << std::setw(18) << "tot_sey" << std::setw(18)
199 << "ParticleNumber" << " numberOfFieldEmittedParticles " << endl;
200 }
201 fid << t << std::setw(18) << impact << std::setw(18) << sey_num
202 << std::setw(18) << double(Npart) << std::setw(18) << numberOfFieldEmittedParticles << endl;
203 }
204 }
205}
206
207
209 MultiBunchHandler* mbhandler_p) {
212
213 for (short b = 0; b < mbhandler_p->getNumBunch(); ++b) {
214 bool isOk = mbhandler_p->calcBunchBeamParameters(beam, b);
215 const MultiBunchHandler::beaminfo_t& binfo = mbhandler_p->getBunchInfo(b);
216 if (isOk) {
217 mbWriter_m[b]->write(beam, binfo);
218 }
219 }
220
221 for (size_t i = 0; i < sddsWriter_m.size(); ++i)
222 sddsWriter_m[i]->write(beam);
223
226}
227
228
230 for (short b = 0; b < mbhandler_p->getNumBunch(); ++b) {
231 MultiBunchHandler::beaminfo_t& binfo = mbhandler_p->getBunchInfo(b);
232 if (mbWriter_m[b]->exists()) {
233 binfo.pathlength = mbWriter_m[b]->getLastValue("s");
234 } else if (statWriter_m->exists()) {
235 binfo.pathlength = statWriter_m->getLastValue("s");
236 } else {
237 binfo.pathlength = 0.0;
238 }
239 }
240}
241
242
244 unsigned int linesToRewind = 0;
245
246 double spos = h5Writer_m->getLastPosition();
247 if (isMultiBunch_m) {
248 /* first check if multi-bunch restart
249 *
250 * first element of vector belongs to first
251 * injected bunch in machine --> rewind lines
252 * according to that file --> thus rewind in
253 * reversed order
254 */
255 for (std::vector<mbWriter_t>::reverse_iterator rit = mbWriter_m.rbegin();
256 rit != mbWriter_m.rend(); ++rit)
257 {
258 if ((*rit)->exists()) {
259 linesToRewind = (*rit)->rewindToSpos(spos);
260 (*rit)->replaceVersionString();
261 }
262 }
263 } else if ( statWriter_m->exists() ) {
264 // use stat file to get position
265 linesToRewind = statWriter_m->rewindToSpos(spos);
266 statWriter_m->replaceVersionString();
267 }
268 h5Writer_m->close();
269
270 // rewind all others
271 if ( linesToRewind > 0 ) {
272 for (size_t i = 0; i < sddsWriter_m.size(); ++i) {
273 sddsWriter_m[i]->rewindLines(linesToRewind);
274 sddsWriter_m[i]->replaceVersionString();
275 }
276 }
277}
278
279
280void DataSink::init(bool restart, H5PartWrapper* h5wrapper, short numBunch) {
281 std::string fn = OpalData::getInstance()->getInputBasename();
282
283 lossWrCounter_m = 0;
285
286 statWriter_m = statWriter_t(new StatWriter(fn + std::string(".stat"), restart));
287
288 sddsWriter_m.push_back(
289 sddsWriter_t(new LBalWriter(fn + std::string(".lbal"), restart))
290 );
291
292#ifdef ENABLE_AMR
293 if ( Options::amr ) {
294 sddsWriter_m.push_back(
295 sddsWriter_t(new GridLBalWriter(fn + std::string(".grid"), restart))
296 );
297 }
298#endif
299
300 if ( Options::memoryDump ) {
301#ifdef __linux__
302 sddsWriter_m.push_back(
303 sddsWriter_t(new MemoryProfiler(fn + std::string(".mem"), restart))
304 );
305#else
306 sddsWriter_m.push_back(
307 sddsWriter_t(new MemoryWriter(fn + std::string(".mem"), restart))
308 );
309#endif
310 }
311
312 if ( isMultiBunch_m ) {
313 initMultiBunchDump(numBunch);
314 }
315
316 if ( Options::enableHDF5 ) {
317 h5Writer_m = h5Writer_t(new H5Writer(h5wrapper, restart));
318 }
319}
320
321
322void DataSink::initMultiBunchDump(short numBunch) {
323 bool restart = OpalData::getInstance()->inRestartRun();
324 std::string fn = OpalData::getInstance()->getInputBasename();
325 short bunch = mbWriter_m.size();
326 while (bunch < numBunch) {
327 std::string fname = fn + std::string("-bunch-") +
328 convertToString(bunch, 4) + std::string(".smb");
329 mbWriter_m.push_back(
330 mbWriter_t(new MultiBunchDump(fname, restart))
331 );
332 ++bunch;
333 }
334}
Inform & endl(Inform &inf)
Definition Inform.cpp:42
constexpr double s2ns
Definition Units.h:44
bool memoryDump
Definition Options.cpp:103
bool enableVTK
If true VTK files are written.
Definition Options.cpp:81
bool enableHDF5
If true HDF5 files are written.
Definition Options.cpp:79
bool amr
Enable AMR if true.
Definition Options.cpp:97
double beamHaloBoundary
Definition Options.cpp:87
double getChargePerParticle() const
get the macro particle charge
size_t getTotalNum() const
size_t calcNumPartsOutside(Vector_t x)
returns the number of particles outside of a box defined by x
double getdT() const
Vector_t get_rrms() const
double getCharge() const
get the total charge per simulation particle
void gatherLoadBalanceStatistics()
double getT() const
std::string getInputBasename()
get input file name without extension
Definition OpalData.cpp:674
static OpalData * getInstance()
Definition OpalData.cpp:196
bool inRestartRun()
true if we do a restart run
Definition OpalData.cpp:312
bool calcBunchBeamParameters(PartBunchBase< double, 3 > *beam, short bunchNr)
beaminfo_t & getBunchInfo(short bunchNr)
short getNumBunch() const
void writeGeomToVtk(std::string fn)
void setMultiBunchInitialPathLengh(MultiBunchHandler *mbhandler_p)
Definition DataSink.cpp:229
void dumpSDDS(PartBunchBase< double, 3 > *beam, Vector_t FDext[], const double &azimuth, const double &timeStep) const
Write the stat file (OPAL-cycl)
Definition DataSink.cpp:107
std::unique_ptr< MultiBunchDump > mbWriter_t
Definition DataSink.h:54
void writeImpactStatistics(const PartBunchBase< double, 3 > *beam, long long int &step, size_t &impact, double &sey_num, size_t numberOfFieldEmittedParticles, bool nEmissionMode, std::string fn)
Definition DataSink.cpp:166
void rewindLines()
Definition DataSink.cpp:243
std::vector< sddsWriter_t > sddsWriter_m
Definition DataSink.h:144
void writeGeomToVtk(BoundaryGeometry &bg, const std::string &fn)
Definition DataSink.cpp:159
void changeH5Wrapper(H5PartWrapper *h5wrapper)
Definition DataSink.cpp:152
const bool isMultiBunch_m
Definition DataSink.h:155
StatWriter::losses_t losses_t
Definition DataSink.h:50
void initMultiBunchDump(short numBunch)
Definition DataSink.cpp:322
void dumpSDDS_m(PartBunchBase< double, 3 > *beam, Vector_t FDext[], const losses_t &losses, const double &azimuth, const double &timeStep) const
Definition DataSink.cpp:121
void init(bool restart=false, H5PartWrapper *h5wrapper=nullptr, short numBunch=1)
Definition DataSink.cpp:280
std::unique_ptr< SDDSWriter > sddsWriter_t
Definition DataSink.h:52
static std::string convertToString(int number, int setw=5)
Definition DataSink.h:162
std::vector< mbWriter_t > mbWriter_m
Definition DataSink.h:145
void storeCavityInformation()
Write cavity information from H5 file.
Definition DataSink.cpp:145
void dumpH5(PartBunchBase< double, 3 > *beam, Vector_t FDext[]) const
Definition DataSink.cpp:87
IpplTimings::TimerRef StatMarkerTimer_m
Timer to track statistics write time.
Definition DataSink.h:153
std::unique_ptr< H5Writer > h5Writer_t
Definition DataSink.h:53
unsigned int lossWrCounter_m
needed to create index for vtk file
Definition DataSink.h:150
statWriter_t statWriter_m
Definition DataSink.h:143
void writeMultiBunchStatistics(PartBunchBase< double, 3 > *beam, MultiBunchHandler *mbhandler)
Definition DataSink.cpp:208
DataSink()
Default constructor.
Definition DataSink.cpp:60
h5Writer_t h5Writer_m
Definition DataSink.h:142
std::unique_ptr< StatWriter > statWriter_t
Definition DataSink.h:51
The base class for all OPAL exceptions.
@ APPEND
Definition Inform.h:46
int precision() const
Definition Inform.h:112
static int myNode()
Definition IpplInfo.cpp:710
static TimerRef getTimer(const char *nm)
static void stopTimer(TimerRef t)
static void startTimer(TimerRef t)