OPAL (Object Oriented Parallel Accelerator Library) 2024.2
OPAL
LossDataSink.cpp
Go to the documentation of this file.
1//
2// Class LossDataSink
3// This class writes file attributes to describe phase space of loss files
4//
5// Copyright (c) 200x - 2020, Paul Scherrer Institut, Villigen PSI, Switzerland
6// All rights reserved
7//
8// This file is part of OPAL.
9//
10// OPAL is free software: you can redistribute it and/or modify
11// it under the terms of the GNU General Public License as published by
12// the Free Software Foundation, either version 3 of the License, or
13// (at your option) any later version.
14//
15// You should have received a copy of the GNU General Public License
16// along with OPAL. If not, see <https://www.gnu.org/licenses/>.
17//
19
22#include "Message/GlobalComm.h"
23#include "OPALconfig.h"
25#include "Utilities/Options.h"
26#include "Utilities/Util.h"
27
28#include "Utility/Inform.h"
29#include "Utility/IpplInfo.h"
30
31#include "H5hut.h"
32
33#include <algorithm>
34#include <cmath>
35#include <cstddef>
36#include <filesystem>
37#include <functional>
38#include <memory>
39#include <optional>
40#include <sstream>
41#include <string>
42#include <utility>
43#include <vector>
44
45extern Inform* gmsg;
46
47#define WRITE_FILEATTRIB_STRING(attribute, value) \
48 { \
49 h5_int64_t h5err = H5WriteFileAttribString(H5file_m, attribute, value); \
50 if (h5err <= H5_ERR) { \
51 std::stringstream ss; \
52 ss << "failed to write string attribute " << attribute << " to file " << fileName_m; \
53 throw GeneralClassicException(std::string(__func__), ss.str()); \
54 } \
55 }
56#define WRITE_STEPATTRIB_FLOAT64(attribute, value, size) \
57 { \
58 h5_int64_t h5err = H5WriteStepAttribFloat64(H5file_m, attribute, value, size); \
59 if (h5err <= H5_ERR) { \
60 std::stringstream ss; \
61 ss << "failed to write float64 attribute " << attribute << " to file " << fileName_m; \
62 throw GeneralClassicException(std::string(__func__), ss.str()); \
63 } \
64 }
65#define WRITE_STEPATTRIB_INT64(attribute, value, size) \
66 { \
67 h5_int64_t h5err = H5WriteStepAttribInt64(H5file_m, attribute, value, size); \
68 if (h5err <= H5_ERR) { \
69 std::stringstream ss; \
70 ss << "failed to write int64 attribute " << attribute << " to file " << fileName_m; \
71 throw GeneralClassicException(std::string(__func__), ss.str()); \
72 } \
73 }
74#define WRITE_DATA_FLOAT64(name, value) \
75 { \
76 h5_int64_t h5err = H5PartWriteDataFloat64(H5file_m, name, value); \
77 if (h5err <= H5_ERR) { \
78 std::stringstream ss; \
79 ss << "failed to write float64 data " << name << " to file " << fileName_m; \
80 throw GeneralClassicException(std::string(__func__), ss.str()); \
81 } \
82 }
83#define WRITE_DATA_INT64(name, value) \
84 { \
85 h5_int64_t h5err = H5PartWriteDataInt64(H5file_m, name, value); \
86 if (h5err <= H5_ERR) { \
87 std::stringstream ss; \
88 ss << "failed to write int64 data " << name << " to file " << fileName_m; \
89 throw GeneralClassicException(std::string(__func__), ss.str()); \
90 } \
91 }
92#define SET_STEP() \
93 { \
94 h5_int64_t h5err = H5SetStep(H5file_m, H5call_m); \
95 if (h5err <= H5_ERR) { \
96 std::stringstream ss; \
97 ss << "failed to set step " << H5call_m << " in file " << fileName_m; \
98 throw GeneralClassicException(std::string(__func__), ss.str()); \
99 } \
100 }
101#define GET_NUM_STEPS() \
102 { \
103 H5call_m = H5GetNumSteps(H5file_m); \
104 if (H5call_m <= H5_ERR) { \
105 std::stringstream ss; \
106 ss << "failed to get number of steps of file " << fileName_m; \
107 throw GeneralClassicException(std::string(__func__), ss.str()); \
108 } \
109 }
110#define SET_NUM_PARTICLES(num) \
111 { \
112 h5_int64_t h5err = H5PartSetNumParticles(H5file_m, num); \
113 if (h5err <= H5_ERR) { \
114 std::stringstream ss; \
115 ss << "failed to set number of particles to " << num << " in step " << H5call_m \
116 << " in file " << fileName_m; \
117 throw GeneralClassicException(std::string(__func__), ss.str()); \
118 } \
119 }
120
121#define OPEN_FILE(fname, mode, props) \
122 { \
123 H5file_m = H5OpenFile(fname, mode, props); \
124 if (H5file_m == (h5_file_t)H5_ERR) { \
125 std::stringstream ss; \
126 ss << "failed to open file " << fileName_m; \
127 throw GeneralClassicException(std::string(__func__), ss.str()); \
128 } \
129 }
130#define CLOSE_FILE() \
131 { \
132 h5_int64_t h5err = H5CloseFile(H5file_m); \
133 if (h5err <= H5_ERR) { \
134 std::stringstream ss; \
135 ss << "failed to close file " << fileName_m; \
136 throw GeneralClassicException(std::string(__func__), ss.str()); \
137 } \
138 }
139
140namespace {
141 void f64transform(
142 const std::vector<OpalParticle>& particles, unsigned int startIdx,
143 unsigned int numParticles, h5_float64_t* buffer,
144 std::function<h5_float64_t(const OpalParticle&)> select) {
145 std::transform(
146 particles.begin() + startIdx, particles.begin() + startIdx + numParticles, buffer,
147 select);
148 }
149 void i64transform(
150 const std::vector<OpalParticle>& particles, unsigned int startIdx,
151 unsigned int numParticles, h5_int64_t* buffer,
152 std::function<h5_int64_t(const OpalParticle&)> select) {
153 std::transform(
154 particles.begin() + startIdx, particles.begin() + startIdx + numParticles, buffer,
155 select);
156 }
157
158 void cminmax(double& min, double& max, double val) {
159 if (-val > min) {
160 min = -val;
161 } else if (val > max) {
162 max = val;
163 }
164 }
165} // namespace
166
168 : outputName_m(""),
169 spos_m(0.0),
170 refTime_m(0.0),
171 tmean_m(0.0),
172 trms_m(0.0),
173 nTotal_m(0),
174 RefPartR_m(0.0),
175 RefPartP_m(0.0),
176 rmin_m(0.0),
177 rmax_m(0.0),
178 rmean_m(0.0),
179 pmean_m(0.0),
180 rrms_m(0.0),
181 prms_m(0.0),
182 rprms_m(0.0),
183 normEmit_m(0.0),
184 rsqsum_m(0.0),
185 psqsum_m(0.0),
186 rpsum_m(0.0),
187 eps2_m(0.0),
188 eps_norm_m(0.0),
189 fac_m(0.0) {
190}
191
192LossDataSink::LossDataSink(const std::string& outfn, bool hdf5Save, CollectionType collectionType)
193 : h5hut_mode_m(hdf5Save),
194 H5file_m(0),
195 outputName_m(outfn),
196 H5call_m(0),
197 collectionType_m(collectionType) {
198 particles_m.clear();
199 turnNumber_m.clear();
200 bunchNumber_m.clear();
201
204 "LossDataSink::LossDataSink",
205 "You must select an OPTION to save Loss data files\n"
206 "Please, choose 'ENABLEHDF5=TRUE' or 'ASCIIDUMP=TRUE'");
207 }
208
210
211 if (OpalData::getInstance()->hasBunchAllocated()) {
213 }
214}
215
217 : h5hut_mode_m(rhs.h5hut_mode_m),
218 H5file_m(rhs.H5file_m),
219 outputName_m(rhs.outputName_m),
220 H5call_m(rhs.H5call_m),
221 RefPartR_m(rhs.RefPartR_m),
222 RefPartP_m(rhs.RefPartP_m),
223 globalTrackStep_m(rhs.globalTrackStep_m),
224 refTime_m(rhs.refTime_m),
225 spos_m(rhs.spos_m),
226 collectionType_m(rhs.collectionType_m) {
227 particles_m.clear();
228 turnNumber_m.clear();
229 bunchNumber_m.clear();
230}
231
233 if (H5file_m) {
234 CLOSE_FILE();
235 H5file_m = 0;
236 }
237}
238
239void LossDataSink::openH5(h5_int32_t mode) {
240 h5_prop_t props = H5CreateFileProp();
241 MPI_Comm comm = Ippl::getComm();
242 H5SetPropFileMPIOCollective(props, &comm);
243 OPEN_FILE(fileName_m.c_str(), mode, props);
244 H5CloseProp(props);
245}
246
248 // Write file attributes to describe phase space to H5 file.
249 std::stringstream OPAL_version;
250 OPAL_version << OPAL_PROJECT_NAME << " " << OPAL_PROJECT_VERSION << " # git rev. "
252 WRITE_FILEATTRIB_STRING("OPAL_version", OPAL_version.str().c_str());
253
254 WRITE_FILEATTRIB_STRING("SPOSUnit", "m");
255 WRITE_FILEATTRIB_STRING("TIMEUnit", "s");
256 WRITE_FILEATTRIB_STRING("RefPartRUnit", "m");
257 WRITE_FILEATTRIB_STRING("RefPartPUnit", "#beta#gamma");
258 WRITE_FILEATTRIB_STRING("GlobalTrackStepUnit", "1");
259
260 WRITE_FILEATTRIB_STRING("centroidUnit", "m");
261 WRITE_FILEATTRIB_STRING("RMSXUnit", "m");
262 WRITE_FILEATTRIB_STRING("MEANPUnit", "#beta#gamma");
263 WRITE_FILEATTRIB_STRING("RMSPUnit", "#beta#gamma");
264 WRITE_FILEATTRIB_STRING("#varepsilonUnit", "m rad");
265 WRITE_FILEATTRIB_STRING("#varepsilon-geomUnit", "m rad");
266 WRITE_FILEATTRIB_STRING("ENERGYUnit", "MeV");
267 WRITE_FILEATTRIB_STRING("dEUnit", "MeV");
268 WRITE_FILEATTRIB_STRING("TotalChargeUnit", "C");
269 WRITE_FILEATTRIB_STRING("TotalMassUnit", "MeV");
270
271 WRITE_FILEATTRIB_STRING("idUnit", "1");
272 WRITE_FILEATTRIB_STRING("xUnit", "m");
273 WRITE_FILEATTRIB_STRING("yUnit", "m");
274 WRITE_FILEATTRIB_STRING("zUnit", "m");
275 WRITE_FILEATTRIB_STRING("pxUnit", "#beta#gamma");
276 WRITE_FILEATTRIB_STRING("pyUnit", "#beta#gamma");
277 WRITE_FILEATTRIB_STRING("pzUnit", "#beta#gamma");
278 WRITE_FILEATTRIB_STRING("qUnit", "C");
279 WRITE_FILEATTRIB_STRING("mUnit", "MeV");
280
281 WRITE_FILEATTRIB_STRING("turnUnit", "1");
282 WRITE_FILEATTRIB_STRING("bunchNumUnit", "1");
283
284 WRITE_FILEATTRIB_STRING("timeUnit", "s");
285 WRITE_FILEATTRIB_STRING("meanTimeUnit", "s");
286 WRITE_FILEATTRIB_STRING("rmsTimeUnit", "s");
287
289 WRITE_FILEATTRIB_STRING("68-percentileUnit", "m");
290 WRITE_FILEATTRIB_STRING("95-percentileUnit", "m");
291 WRITE_FILEATTRIB_STRING("99-percentileUnit", "m");
292 WRITE_FILEATTRIB_STRING("99_99-percentileUnit", "m");
293 WRITE_FILEATTRIB_STRING("normalizedEps68PercentileUnit", "m rad");
294 WRITE_FILEATTRIB_STRING("normalizedEps95PercentileUnit", "m rad");
295 WRITE_FILEATTRIB_STRING("normalizedEps99PercentileUnit", "m rad");
296 WRITE_FILEATTRIB_STRING("normalizedEps99_99PercentileUnit", "m rad");
297 }
298
300 WRITE_FILEATTRIB_STRING("type", "temporal");
301 } else {
302 WRITE_FILEATTRIB_STRING("type", "spatial");
303 }
304}
305
307 bool hasTurn = hasTurnInformations();
308 if (Ippl::myNode() == 0) {
309 os_m << "# x (m), y (m), z (m), px ( ), py ( ), pz ( ), id";
310 if (hasTurn) {
311 os_m << ", turn ( ), bunchNumber ( )";
312 }
313 os_m << ", time (s)" << std::endl;
314 }
315}
316
318 const Vector_t& x, const Vector_t& p, double time, double spos, long long globalTrackStep) {
319 RefPartR_m.push_back(x);
320 RefPartP_m.push_back(p);
321 globalTrackStep_m.push_back((h5_int64_t)globalTrackStep);
322 spos_m.push_back(spos);
323 refTime_m.push_back(time);
324}
325
327 const OpalParticle& particle, const std::optional<std::pair<int, short>>& turnBunchNumPair) {
328 if (turnBunchNumPair) {
329 if (!particles_m.empty() && turnNumber_m.empty()) {
331 "LossDataSink::addParticle",
332 "Either no particle or all have turn number and bunch number");
333 }
334 turnNumber_m.push_back(turnBunchNumPair.value().first);
335 bunchNumber_m.push_back(turnBunchNumPair.value().second);
336 }
337
338 particles_m.push_back(particle);
339}
340
341bool LossDataSink::save(unsigned int numSets, OpalData::OpenMode openMode) {
342 if (outputName_m.empty())
343 return false;
345 return false;
346
347 if (openMode == OpalData::OpenMode::UNDEFINED) {
348 openMode = OpalData::getInstance()->getOpenMode();
349 }
350
351 namespace fs = std::filesystem;
352 if (h5hut_mode_m) {
353 fileName_m = outputName_m + std::string(".h5");
354 if (openMode == OpalData::OpenMode::WRITE || !fs::exists(fileName_m)) {
355 openH5();
357 } else {
358 openH5(H5_O_APPENDONLY);
360 }
361
362 for (unsigned int i = 0; i < numSets; ++i) {
363 saveH5(i);
364 }
365 CLOSE_FILE();
366 H5file_m = 0;
367
368 } else {
369 fileName_m = outputName_m + std::string(".loss");
370 if (openMode == OpalData::OpenMode::WRITE || !fs::exists(fileName_m)) {
371 openASCII();
373 } else {
374 appendASCII();
375 }
376 saveASCII();
377 closeASCII();
378 }
379 *gmsg << level2 << "Save '" << fileName_m << "'" << endl;
380
382
384
385 particles_m = std::vector<OpalParticle>();
386 turnNumber_m = std::vector<std::size_t>();
387 bunchNumber_m = std::vector<std::size_t>();
388 spos_m = std::vector<double>();
389 refTime_m = std::vector<double>();
390 RefPartR_m = std::vector<Vector_t>();
391 RefPartP_m = std::vector<Vector_t>();
392 globalTrackStep_m = std::vector<h5_int64_t>();
393
394 return true;
395}
396
397// Note: This was changed to calculate the global number of dumped particles
398// because there are two cases to be considered:
399// 1. ALL nodes have 0 lost particles -> nothing to be done.
400// 2. Some nodes have 0 lost particles, some not -> H5 can handle that but all
401// nodes HAVE to participate, otherwise H5 waits endlessly for a response from
402// the nodes that didn't enter the saveH5 function. -DW
404 std::size_t nLoc = particles_m.size();
405
406 reduce(nLoc, nLoc, OpAddAssign());
407
408 return nLoc == 0;
409}
410
412 bool hasTurnInformation = !turnNumber_m.empty();
413
414 allreduce(hasTurnInformation, 1, std::logical_or<bool>());
415
416 return hasTurnInformation > 0;
417}
418
419void LossDataSink::saveH5(unsigned int setIdx) {
420 std::size_t nLoc = particles_m.size();
421 std::size_t startIdx = 0, endIdx = nLoc;
422 if (setIdx + 1 < startSet_m.size()) {
423 startIdx = startSet_m[setIdx];
424 endIdx = startSet_m[setIdx + 1];
425 nLoc = endIdx - startIdx;
426 }
427
428 std::unique_ptr<std::size_t[]> locN(new std::size_t[Ippl::getNodes()]);
429 std::unique_ptr<std::size_t[]> globN(new std::size_t[Ippl::getNodes()]);
430
431 for (int i = 0; i < Ippl::getNodes(); i++) {
432 globN[i] = locN[i] = 0;
433 }
434
435 locN[Ippl::myNode()] = nLoc;
436 reduce(locN.get(), locN.get() + Ippl::getNodes(), globN.get(), OpAddAssign());
437
438 DistributionMoments engine;
439 engine.compute(particles_m.begin() + startIdx, particles_m.begin() + endIdx);
440
442 SET_STEP();
443 SET_NUM_PARTICLES(nLoc);
444
445 if (setIdx < spos_m.size()) {
446 WRITE_STEPATTRIB_FLOAT64("SPOS", &(spos_m[setIdx]), 1);
447 WRITE_STEPATTRIB_FLOAT64("TIME", &(refTime_m[setIdx]), 1);
448 WRITE_STEPATTRIB_FLOAT64("RefPartR", (h5_float64_t*)&(RefPartR_m[setIdx]), 3);
449 WRITE_STEPATTRIB_FLOAT64("RefPartP", (h5_float64_t*)&(RefPartP_m[setIdx]), 3);
450 WRITE_STEPATTRIB_INT64("GlobalTrackStep", &(globalTrackStep_m[setIdx]), 1);
451 }
452
453 Vector_t tmpVector;
454 double tmpDouble;
455 WRITE_STEPATTRIB_FLOAT64("centroid", (tmpVector = engine.getMeanPosition(), &tmpVector[0]), 3);
457 "RMSX", (tmpVector = engine.getStandardDeviationPosition(), &tmpVector[0]), 3);
458 WRITE_STEPATTRIB_FLOAT64("MEANP", (tmpVector = engine.getMeanMomentum(), &tmpVector[0]), 3);
460 "RMSP", (tmpVector = engine.getStandardDeviationMomentum(), &tmpVector[0]), 3);
462 "#varepsilon", (tmpVector = engine.getNormalizedEmittance(), &tmpVector[0]), 3);
464 "#varepsilon-geom", (tmpVector = engine.getGeometricEmittance(), &tmpVector[0]), 3);
465 WRITE_STEPATTRIB_FLOAT64("ENERGY", (tmpDouble = engine.getMeanKineticEnergy(), &tmpDouble), 1);
466 WRITE_STEPATTRIB_FLOAT64("dE", (tmpDouble = engine.getStdKineticEnergy(), &tmpDouble), 1);
467 WRITE_STEPATTRIB_FLOAT64("TotalCharge", (tmpDouble = engine.getTotalCharge(), &tmpDouble), 1);
468 WRITE_STEPATTRIB_FLOAT64("TotalMass", (tmpDouble = engine.getTotalMass(), &tmpDouble), 1);
469 WRITE_STEPATTRIB_FLOAT64("meanTime", (tmpDouble = engine.getMeanTime(), &tmpDouble), 1);
470 WRITE_STEPATTRIB_FLOAT64("rmsTime", (tmpDouble = engine.getStdTime(), &tmpDouble), 1);
473 "68-percentile", (tmpVector = engine.get68Percentile(), &tmpVector[0]), 3);
475 "95-percentile", (tmpVector = engine.get95Percentile(), &tmpVector[0]), 3);
477 "99-percentile", (tmpVector = engine.get99Percentile(), &tmpVector[0]), 3);
479 "99_99-percentile", (tmpVector = engine.get99_99Percentile(), &tmpVector[0]), 3);
480
482 "normalizedEps68Percentile",
483 (tmpVector = engine.getNormalizedEmittance68Percentile(), &tmpVector[0]), 3);
485 "normalizedEps95Percentile",
486 (tmpVector = engine.getNormalizedEmittance95Percentile(), &tmpVector[0]), 3);
488 "normalizedEps99Percentile",
489 (tmpVector = engine.getNormalizedEmittance99Percentile(), &tmpVector[0]), 3);
491 "normalizedEps99_99Percentile",
492 (tmpVector = engine.getNormalizedEmittance99_99Percentile(), &tmpVector[0]), 3);
493 }
494
495 WRITE_STEPATTRIB_FLOAT64("maxR", (tmpVector = engine.getMaxR(), &tmpVector[0]), 3);
496
497 // Write all data
498 std::vector<char> buffer(nLoc * sizeof(h5_float64_t));
499 h5_float64_t* f64buffer = nLoc > 0 ? reinterpret_cast<h5_float64_t*>(&buffer[0]) : nullptr;
500 h5_int64_t* i64buffer = nLoc > 0 ? reinterpret_cast<h5_int64_t*>(&buffer[0]) : nullptr;
501
502 ::i64transform(particles_m, startIdx, nLoc, i64buffer, [](const OpalParticle& particle) {
503 return particle.getId();
504 });
505 WRITE_DATA_INT64("id", i64buffer);
506 ::f64transform(particles_m, startIdx, nLoc, f64buffer, [](const OpalParticle& particle) {
507 return particle.getX();
508 });
509 WRITE_DATA_FLOAT64("x", f64buffer);
510 ::f64transform(particles_m, startIdx, nLoc, f64buffer, [](const OpalParticle& particle) {
511 return particle.getY();
512 });
513 WRITE_DATA_FLOAT64("y", f64buffer);
514 ::f64transform(particles_m, startIdx, nLoc, f64buffer, [](const OpalParticle& particle) {
515 return particle.getZ();
516 });
517 WRITE_DATA_FLOAT64("z", f64buffer);
518 ::f64transform(particles_m, startIdx, nLoc, f64buffer, [](const OpalParticle& particle) {
519 return particle.getPx();
520 });
521 WRITE_DATA_FLOAT64("px", f64buffer);
522 ::f64transform(particles_m, startIdx, nLoc, f64buffer, [](const OpalParticle& particle) {
523 return particle.getPy();
524 });
525 WRITE_DATA_FLOAT64("py", f64buffer);
526 ::f64transform(particles_m, startIdx, nLoc, f64buffer, [](const OpalParticle& particle) {
527 return particle.getPz();
528 });
529 WRITE_DATA_FLOAT64("pz", f64buffer);
530 ::f64transform(particles_m, startIdx, nLoc, f64buffer, [](const OpalParticle& particle) {
531 return particle.getCharge();
532 });
533 WRITE_DATA_FLOAT64("q", f64buffer);
534 ::f64transform(particles_m, startIdx, nLoc, f64buffer, [](const OpalParticle& particle) {
535 return particle.getMass();
536 });
537 WRITE_DATA_FLOAT64("m", f64buffer);
538
539 if (hasTurnInformations()) {
540 std::copy(
541 turnNumber_m.begin() + startIdx, turnNumber_m.begin() + startIdx + nLoc, i64buffer);
542 WRITE_DATA_INT64("turn", i64buffer);
543
544 std::copy(
545 bunchNumber_m.begin() + startIdx, bunchNumber_m.begin() + startIdx + nLoc, i64buffer);
546 WRITE_DATA_INT64("bunchNumber", i64buffer);
547 }
548
549 ::f64transform(particles_m, startIdx, nLoc, f64buffer, [](const OpalParticle& particle) {
550 return particle.getTime();
551 });
552 WRITE_DATA_FLOAT64("time", f64buffer);
553
554 ++H5call_m;
555}
556
558 /*
559 ASCII output
560 */
562 bool hasTurn = hasTurnInformations();
563 if (Ippl::Comm->myNode() == 0) {
564 const unsigned partCount = particles_m.size();
565
566 for (unsigned i = 0; i < partCount; i++) {
567 const OpalParticle& particle = particles_m[i];
568 os_m << particle.getX() << " ";
569 os_m << particle.getY() << " ";
570 os_m << particle.getZ() << " ";
571 os_m << particle.getPx() << " ";
572 os_m << particle.getPy() << " ";
573 os_m << particle.getPz() << " ";
574 os_m << particle.getId() << " ";
575 if (hasTurn) {
576 os_m << turnNumber_m[i] << " ";
577 os_m << bunchNumber_m[i] << " ";
578 }
579 os_m << particle.getTime() << std::endl;
580 }
581
582 int notReceived = Ippl::getNodes() - 1;
583 while (notReceived > 0) {
584 unsigned dataBlocks = 0;
585 int node = COMM_ANY_NODE;
586 Message* rmsg = Ippl::Comm->receive_block(node, tag);
587 if (rmsg == 0) {
588 ERRORMSG("LossDataSink: Could not receive from client nodes output." << endl);
589 }
590 notReceived--;
591 rmsg->get(&dataBlocks);
592 rmsg->get(&hasTurn);
593 for (unsigned i = 0; i < dataBlocks; i++) {
594 long id;
595 std::size_t bunchNum, turn;
596 double rx, ry, rz, px, py, pz, time;
597
598 os_m << (rmsg->get(&rx), rx) << " ";
599 os_m << (rmsg->get(&ry), ry) << " ";
600 os_m << (rmsg->get(&rz), rz) << " ";
601 os_m << (rmsg->get(&px), px) << " ";
602 os_m << (rmsg->get(&py), py) << " ";
603 os_m << (rmsg->get(&pz), pz) << " ";
604 os_m << (rmsg->get(&id), id) << " ";
605 if (hasTurn) {
606 os_m << (rmsg->get(&turn), turn) << " ";
607 os_m << (rmsg->get(&bunchNum), bunchNum) << " ";
608 }
609 os_m << (rmsg->get(&time), time) << std::endl;
610 }
611 delete rmsg;
612 }
613 } else {
614 Message* smsg = new Message();
615 const unsigned msgsize = particles_m.size();
616 smsg->put(msgsize);
617 smsg->put(hasTurn);
618 for (unsigned i = 0; i < msgsize; i++) {
619 const OpalParticle& particle = particles_m[i];
620 smsg->put(particle.getX());
621 smsg->put(particle.getY());
622 smsg->put(particle.getZ());
623 smsg->put(particle.getPx());
624 smsg->put(particle.getPy());
625 smsg->put(particle.getPz());
626 smsg->put(particle.getId());
627 if (hasTurn) {
628 smsg->put(turnNumber_m[i]);
629 smsg->put(bunchNumber_m[i]);
630 }
631 smsg->put(particle.getTime());
632 }
633 bool res = Ippl::Comm->send(smsg, 0, tag);
634 if (!res) {
635 ERRORMSG("LossDataSink Ippl::Comm->send(smsg, 0, tag) failed " << endl);
636 }
637 }
638}
639
659void LossDataSink::splitSets(unsigned int numSets) {
660 if (numSets <= 1 || particles_m.size() == 0)
661 return;
662
663 const std::size_t nLoc = particles_m.size();
664 std::size_t avgNumPerSet = nLoc / numSets;
665 std::vector<std::size_t> numPartsInSet(numSets, avgNumPerSet);
666 for (unsigned int j = 0; j < (nLoc - numSets * avgNumPerSet); ++j) {
667 ++numPartsInSet[j];
668 }
669 startSet_m.resize(numSets + 1, 0);
670
671 std::vector<double> data(2 * numSets, 0.0);
672 double* meanT = &data[0];
673 double* numParticles = &data[numSets];
674 std::vector<double> timeRange(numSets, 0.0);
675 double maxT = particles_m[0].getTime();
676
677 for (unsigned int iteration = 0; iteration < 2; ++iteration) {
678 std::size_t partIdx = 0;
679 for (unsigned int j = 0; j < numSets; ++j) {
680 const std::size_t& numThisSet = numPartsInSet[j];
681 for (std::size_t k = 0; k < numThisSet; ++k, ++partIdx) {
682 meanT[j] += particles_m[partIdx].getTime();
683 maxT = std::max(maxT, particles_m[partIdx].getTime());
684 }
685 numParticles[j] = numThisSet;
686 }
687
688 allreduce(&(data[0]), 2 * numSets, std::plus<double>());
689
690 for (unsigned int j = 0; j < numSets; ++j) {
691 meanT[j] /= numParticles[j];
692 }
693
694 for (unsigned int j = 0; j < numSets - 1; ++j) {
695 timeRange[j] = 0.5 * (meanT[j] + meanT[j + 1]);
696 }
697 timeRange[numSets - 1] = maxT;
698
699 std::fill(numPartsInSet.begin(), numPartsInSet.end(), 0);
700 std::size_t setNum = 0;
701 std::size_t idxPrior = 0;
702 for (std::size_t idx = 0; idx < nLoc; ++idx) {
703 if (particles_m[idx].getTime() > timeRange[setNum]) {
704 numPartsInSet[setNum] = idx - idxPrior;
705 idxPrior = idx;
706 ++setNum;
707 }
708 }
709 numPartsInSet[numSets - 1] = nLoc - idxPrior;
710 }
711
712 for (unsigned int i = 0; i < numSets; ++i) {
713 startSet_m[i + 1] = startSet_m[i] + numPartsInSet[i];
714 }
715}
716
718 SetStatistics stat;
719 double part[6];
720
721 const unsigned int totalSize = 45;
722 double plainData[totalSize];
723 double rminmax[6] = {};
724
725 Util::KahanAccumulation data[totalSize];
726 Util::KahanAccumulation* localCentroid = data + 1;
727 Util::KahanAccumulation* localMoments = data + 7;
728 Util::KahanAccumulation* localOthers = data + 43;
729
730 std::size_t startIdx = 0;
731 std::size_t nLoc = particles_m.size();
732 if (setIdx + 1 < startSet_m.size()) {
733 startIdx = startSet_m[setIdx];
734 nLoc = startSet_m[setIdx + 1] - startSet_m[setIdx];
735 }
736
737 data[0].sum = nLoc;
738
739 unsigned int idx = startIdx;
740 for (unsigned long k = 0; k < nLoc; ++k, ++idx) {
741 const OpalParticle& particle = particles_m[idx];
742
743 part[0] = particle.getX();
744 part[1] = particle.getPx();
745 part[2] = particle.getY();
746 part[3] = particle.getPy();
747 part[4] = particle.getZ();
748 part[5] = particle.getPz();
749
750 for (int i = 0; i < 6; i++) {
751 localCentroid[i] += part[i];
752 for (int j = 0; j <= i; j++) {
753 localMoments[i * 6 + j] += part[i] * part[j];
754 }
755 }
756 localOthers[0] += particle.getTime();
757 localOthers[1] += std::pow(particle.getTime(), 2);
758
759 ::cminmax(rminmax[0], rminmax[1], particle.getX());
760 ::cminmax(rminmax[2], rminmax[3], particle.getY());
761 ::cminmax(rminmax[4], rminmax[5], particle.getZ());
762 }
763
764 for (int i = 0; i < 6; i++) {
765 for (int j = 0; j < i; j++) {
766 localMoments[j * 6 + i] = localMoments[i * 6 + j];
767 }
768 }
769
770 for (unsigned int i = 0; i < totalSize; ++i) {
771 plainData[i] = data[i].sum;
772 }
773
774 allreduce(plainData, totalSize, std::plus<double>());
775 allreduce(rminmax, 6, std::greater<double>());
776
777 if (plainData[0] == 0.0)
778 return stat;
779
780 double* centroid = plainData + 1;
781 double* moments = plainData + 7;
782 double* others = plainData + 43;
783
785 stat.spos_m = spos_m[setIdx];
786 stat.refTime_m = refTime_m[setIdx];
787 stat.RefPartR_m = RefPartR_m[setIdx];
788 stat.RefPartP_m = RefPartP_m[setIdx];
789 stat.nTotal_m = (unsigned long)std::round(plainData[0]);
790
791 for (unsigned int i = 0; i < 3u; i++) {
792 stat.rmean_m(i) = centroid[2 * i] / stat.nTotal_m;
793 stat.pmean_m(i) = centroid[(2 * i) + 1] / stat.nTotal_m;
794 stat.rsqsum_m(i) =
795 (moments[2 * i * 6 + 2 * i] - stat.nTotal_m * std::pow(stat.rmean_m(i), 2));
796 stat.psqsum_m(i) = std::max(
797 0.0,
798 moments[(2 * i + 1) * 6 + (2 * i) + 1] - stat.nTotal_m * std::pow(stat.pmean_m(i), 2));
799 stat.rpsum_m(i) =
800 (moments[(2 * i) * 6 + (2 * i) + 1]
801 - stat.nTotal_m * stat.rmean_m(i) * stat.pmean_m(i));
802 }
803 stat.tmean_m = others[0] / stat.nTotal_m;
804 stat.trms_m = std::sqrt(std::max(0.0, (others[1] / stat.nTotal_m - std::pow(stat.tmean_m, 2))));
805
806 stat.eps2_m =
807 ((stat.rsqsum_m * stat.psqsum_m - stat.rpsum_m * stat.rpsum_m)
808 / (1.0 * stat.nTotal_m * stat.nTotal_m));
809
810 stat.rpsum_m /= stat.nTotal_m;
811
812 for (unsigned int i = 0; i < 3u; i++) {
813 stat.rrms_m(i) = std::sqrt(std::max(0.0, stat.rsqsum_m(i)) / stat.nTotal_m);
814 stat.prms_m(i) = std::sqrt(std::max(0.0, stat.psqsum_m(i)) / stat.nTotal_m);
815 stat.eps_norm_m(i) = std::sqrt(std::max(0.0, stat.eps2_m(i)));
816 double tmp = stat.rrms_m(i) * stat.prms_m(i);
817 stat.fac_m(i) = (tmp == 0) ? 0.0 : 1.0 / tmp;
818 stat.rmin_m(i) = -rminmax[2 * i];
819 stat.rmax_m(i) = rminmax[2 * i + 1];
820 }
821
822 stat.rprms_m = stat.rpsum_m * stat.fac_m;
823
824 return stat;
825}
Inform * gmsg
Definition Main.cpp:70
#define GET_NUM_STEPS()
#define OPEN_FILE(fname, mode, props)
#define SET_STEP()
#define WRITE_DATA_FLOAT64(name, value)
#define WRITE_STEPATTRIB_INT64(attribute, value, size)
#define WRITE_FILEATTRIB_STRING(attribute, value)
#define SET_NUM_PARTICLES(num)
#define WRITE_STEPATTRIB_FLOAT64(attribute, value, size)
#define WRITE_DATA_INT64(name, value)
Inform * gmsg
Definition Main.cpp:70
#define CLOSE_FILE()
CollectionType
T::PETE_Expr_t::PETE_Return_t max(const PETE_Expr< T > &expr, NDIndex< D > &loc)
T::PETE_Expr_t::PETE_Return_t min(const PETE_Expr< T > &expr, NDIndex< D > &loc)
#define IPPL_APP_CYCLE
Definition Tags.h:103
#define IPPL_APP_TAG3
Definition Tags.h:96
const int COMM_ANY_NODE
Definition Communicate.h:40
void allreduce(const T *input, T *output, int count, Op op)
bool reduce(Communicate &, InputIterator, InputIterator, OutputIterator, const ReduceOp &, bool *IncludeVal=0)
#define ERRORMSG(msg)
Definition IpplInfo.h:350
Inform & level2(Inform &inf)
Definition Inform.cpp:46
Inform & endl(Inform &inf)
Definition Inform.cpp:42
bool computePercentiles
Definition Options.cpp:109
bool enableHDF5
If true HDF5 files are written.
Definition Options.cpp:79
std::string getGitRevision()
Definition Util.cpp:38
void checkAndAddOutputFileName(const std::string &outfn)
checks the output file names of all items to avoid duplicates
Definition OpalData.cpp:680
OpenMode getOpenMode() const
Definition OpalData.cpp:353
static OpalData * getInstance()
Definition OpalData.cpp:196
OpenMode
Enum for writing to files.
Definition OpalData.h:64
void setOpenMode(OpenMode openMode)
Definition OpalData.cpp:349
Vector_t getStandardDeviationMomentum() const
Vector_t get99Percentile() const
double getMeanKineticEnergy() const
Vector_t getStandardDeviationPosition() const
Vector_t getNormalizedEmittance95Percentile() const
Vector_t getNormalizedEmittance68Percentile() const
double getTotalCharge() const
Vector_t getNormalizedEmittance99_99Percentile() const
Vector_t getNormalizedEmittance99Percentile() const
Vector_t getMeanPosition() const
Vector_t get99_99Percentile() const
void compute(const std::vector< OpalParticle >::const_iterator &, const std::vector< OpalParticle >::const_iterator &)
double getStdKineticEnergy() const
Vector_t getNormalizedEmittance() const
Vector_t getMeanMomentum() const
Vector_t get95Percentile() const
Vector_t get68Percentile() const
Vector_t getGeometricEmittance() const
double getPy() const
Get vertical momentum (no dimension).
double getPz() const
Get relative momentum error (no dimension).
double getY() const
Get vertical displacement in m.
double getCharge() const
Get charge in Coulomb.
double getZ() const
Get longitudinal displacement c*t in m.
double getMass() const
Get mass in GeV/c^2.
double getTime() const
Get time.
int64_t getId() const
Get the id of the particle.
double getPx() const
Get horizontal momentum (no dimension).
double getX() const
Get horizontal position in m.
Vector_t eps_norm_m
Vector_t rmax_m
Vector_t RefPartP_m
std::string outputName_m
Vector_t prms_m
Vector_t pmean_m
Vector_t psqsum_m
Vector_t rmean_m
unsigned long nTotal_m
Vector_t rsqsum_m
Vector_t eps2_m
Vector_t rrms_m
Vector_t fac_m
Vector_t RefPartR_m
Vector_t rpsum_m
Vector_t rmin_m
Vector_t rprms_m
bool save(unsigned int numSets=1, OpalData::OpenMode openMode=OpalData::OpenMode::UNDEFINED)
Returns whether the file was written; false (on all ranks) if there was nothing to save.
std::vector< Vector_t > RefPartR_m
void addReferenceParticle(const Vector_t &x, const Vector_t &p, double time, double spos, long long globalTrackStep)
std::vector< std::size_t > bunchNumber_m
h5_file_t H5file_m
used to write out data in H5hut mode
~LossDataSink() noexcept(false)
std::vector< std::size_t > turnNumber_m
std::vector< double > refTime_m
CollectionType collectionType_m
std::string outputName_m
void writeHeaderASCII()
std::ofstream os_m
std::vector< Vector_t > RefPartP_m
h5_int64_t H5call_m
Current record, or time step, of H5 file.
void addParticle(const OpalParticle &, const std::optional< std::pair< int, short int > > &turnBunchNumPair=std::nullopt)
std::string fileName_m
std::vector< double > spos_m
void openH5(h5_int32_t mode=H5_O_WRONLY)
SetStatistics computeSetStatistics(unsigned int setIdx)
void splitSets(unsigned int numSets)
std::vector< OpalParticle > particles_m
bool hasTurnInformations() const
void appendASCII()
void saveH5(unsigned int setIdx)
std::vector< h5_int64_t > globalTrackStep_m
LossDataSink()=default
void closeASCII()
bool hasNoParticlesToDump() const
std::vector< unsigned long > startSet_m
long double sum
Definition Util.h:304
bool send(Message *, int node, int tag, bool delmsg=true)
Message * receive_block(int &node, int &tag)
void barrier(void)
Message & put(const T &val)
Definition Message.h:406
Message & get(const T &cval)
Definition Message.h:476
int next_tag(int t, int s=1000)
Definition TagMaker.h:39
static MPI_Comm getComm()
Definition IpplInfo.h:152
static int getNodes()
Definition IpplInfo.cpp:689
static int myNode()
Definition IpplInfo.cpp:710
static Communicate * Comm
Definition IpplInfo.h:84