OPAL (Object Oriented Parallel Accelerator Library) 2024.2
OPAL
Main.cpp
Go to the documentation of this file.
1//
2// Copyright (c) 2008 - 2020, Paul Scherrer Institut, Villigen PSI, Switzerland
3//
4// All rights reserved
5//
6// This file is part of OPAL.
7//
8// OPAL is free software: you can redistribute it and/or modify
9// it under the terms of the GNU General Public License as published by
10// the Free Software Foundation, either version 3 of the License, or
11// (at your option) any later version.
12//
13// You should have received a copy of the GNU General Public License
14// along with OPAL. If not, see <https://www.gnu.org/licenses/>.
15//
16#include "opal.h"
17
18#include "H5hut.h"
19
23#include "Parser/FileStream.h"
24#include "Utilities/Timer.h"
25#include "Fields/Fieldmap.h"
26#include "Track/TrackRun.h"
27#include "FixedAlgebra/FTps.h"
28
29#include "BasicActions/Option.h"
30#include "Utilities/Options.h"
33#include "Utilities/Util.h"
34
36
37#include "OPALconfig.h"
38
39#ifdef ENABLE_AMR
40#include <AMReX_ParallelDescriptor.H>
41#endif
42
43// IPPL
44#include "Message/Communicate.h"
45#include "Utility/Inform.h"
47#include "Utility/IpplInfo.h"
48#include "Utility/IpplTimings.h"
49#include "GSLErrorHandling.h"
50
51#include <gsl/gsl_errno.h>
52
53#include <cstring>
54#include <filesystem>
55#include <iomanip>
56#include <iostream>
57#include <set>
58#include <system_error>
59
60// ippl and gmsg need to be defined within the python module (these global
61// objects are shared across several python modules (dynamically loaded libs).
62// if we are using non-python OPAL, they are defined here, else they are defined
63// in PyOpal::Globals
64#ifdef DONT_DEFINE_IPPL_GMSG
65 extern Ippl *ippl;
66 extern Inform *gmsg;
67 extern Inform *gmsgALL;
68#else
72#endif
73
74namespace {
75 void printStdoutHeader() {
76 OPALTimer::Timer simtimer;
77 std::string dateStr(simtimer.date());
78 std::string timeStr(simtimer.time());
79 std::string mySpace(" ");
80
81 *gmsg << mySpace << " ____ _____ ___ " << endl;
82 *gmsg << mySpace << " / __ \\| __ \\ /\\ | | " << endl;
83 *gmsg << mySpace << " | | | | |__) / \\ | |" << endl;
84 *gmsg << mySpace << " | | | | ___/ /\\ \\ | |" << endl ;
85 *gmsg << mySpace << " | |__| | | / ____ \\| |____" << endl;
86 *gmsg << mySpace << " \\____/|_| /_/ \\_\\______|" << endl;
87
88 std::string gitRevision = "git rev. " + Util::getGitRevision();
89 std::string copyRight = "(c) PSI, http://amas.web.psi.ch";
90 *gmsg << endl
91 << "This is OPAL (Object Oriented Parallel Accelerator Library) Version " << OPAL_PROJECT_VERSION << "\n"
92 << std::setw(37 + gitRevision.length() / 2) << std::right << gitRevision << "\n\n" << endl
93 << std::setw(37 + copyRight.length() / 2) << std::right << copyRight << "\n\n" << endl
94 << "The optimiser (former opt-Pilot) is integrated " << endl
95 << endl;
96
97 *gmsg << "Please send cookies, goodies or other motivations (wine and beer ... ) \nto the OPAL developers " << PACKAGE_BUGREPORT << "\n" << endl;
98 *gmsg << "Time: " << timeStr << " date: " << dateStr << "\n" << endl;
99 }
100
101 void printHelp() {
102 ::printStdoutHeader();
103
104 INFOMSG("\n");
105 INFOMSG("Usage: opal [<option> <option> ...]\n");
106 INFOMSG(" The possible values for <option> are:\n");
107 INFOMSG(" --version : Print the version of opal.\n");
108 INFOMSG(" --version-full : Print the version of opal with additional informations.\n");
109 INFOMSG(" --git-revision : Print the revision hash of the repository.\n");
110 INFOMSG(" --input <fname> : Specifies the input file <fname>.\n");
111 INFOMSG(" --restart <n> : Performes a restart from step <n>.\n");
112 INFOMSG(" --restartfn <fname> : Uses the file <fname> to restart from.\n");
113#ifdef ENABLE_AMR
114 INFOMSG(" --noInitAMR : Disable initialization of AMR\n");
115#endif
117 INFOMSG(" --help-command <command> : Display the help for the command <command>\n");
118 INFOMSG(" --help : Display this command-line summary.\n");
119 INFOMSG(endl);
120 }
121}
122
123
124bool checkInitAmrFlag(int argc, char* argv[]) {
125 std::string noamr = "noInitAMR";
126 bool initAMR = true;
127 for (int i = 0; i < argc; ++i) {
128 std::string sargv = std::string(argv[i]);
129 if ( sargv.find(noamr) != std::string::npos ) {
130 initAMR = false;
131 break;
132 }
133 }
134 return initAMR;
135}
136
137int opalMain(int argc, char *argv[]);
138
139int main(int argc, char *argv[]) {
140 // python has its own main function that can interfere with opal main;
141 // so when calling from python we call opalMain instead
142 // Keep the pointer: 'delete ippl' at the end of opalMain() destroys the last
143 // IpplInfo, which deletes the communicator and so calls MPI_Finalize.
144 ippl = new Ippl(argc, argv);
145 gmsg = new Inform("OPAL");
146 gmsgALL = new Inform("OPAL", INFORM_ALL_NODES);
147 return opalMain(argc, argv);
148}
149
150int opalMain(int argc, char *argv[]) {
151
152 namespace fs = std::filesystem;
153
154#ifdef ENABLE_AMR
155 bool initAMR = checkInitAmrFlag(argc, argv);
156 if ( initAMR ) {
157 // false: build no parmparse, we use the OPAL parser instead.
158 amrex::Initialize(argc, argv, false, Ippl::getComm());
159 }
160#endif
161
162 H5SetVerbosityLevel(1); //65535);
163
164 gsl_set_error_handler(&handleGSLErrors);
165
166 static IpplTimings::TimerRef mainTimer = IpplTimings::getTimer("mainTimer");
167 IpplTimings::startTimer(mainTimer);
168
169
170 if(Ippl::myNode() == 0) remove("errormsg.txt");
171
172 const OpalParser parser;
173
174 std::cout.precision(16);
175 std::cout.setf(std::ios::scientific, std::ios::floatfield);
176 std::cerr.precision(16);
177 std::cerr.setf(std::ios::scientific, std::ios::floatfield);
178
179 // Set global truncation orders.
183
185
186 /*
187 Make a directory data for some of the output
188 */
189 if(Ippl::myNode() == 0) {
190 if (!fs::exists(opal->getAuxiliaryOutputDirectory())) {
191 std::error_code error_code;
192 if (!fs::create_directory(opal->getAuxiliaryOutputDirectory(), error_code)) {
193 std::cerr << error_code.message() << std::endl;
194 // use error code to prevent create_directory from throwing an exception
195 }
196 }
197 }
199 if (!fs::is_directory(opal->getAuxiliaryOutputDirectory())) {
200 std::cerr << "unable to create directory; aborting" << std::endl;
201 abort();
202 }
203
204 opal->storeArguments(argc, argv);
205 try {
207
208 // Read startup file.
210
211 char *startup = getenv("HOME");
212 std::filesystem::path p = strncat(startup, "/init.opal", 20);
213 if (startup != nullptr && is_regular_file(p)) {
214
215 FileStream::setEcho(false);
216 FileStream *is;
217
218 try {
219 is = new FileStream(startup);
220 } catch(...) {
221 is = 0;
222 ERRORMSG("Could not open startup file '" << startup << "'\n"
223 << "Note: this is not mandatory for an OPAL simulation!\n");
224 }
225
226 if(is) {
227 *gmsg << "Reading startup file '" << startup << "'" << endl;
228 parser.run(is);
229 *gmsg << "Finished reading startup file." << endl;
230 }
232 } else {
233 *gmsg << level5
234 << "Couldn't find startup file '" << startup << "'\n"
235 << "Note: this is not mandatory for an OPAL simulation!\n" << endl;
236 }
237
238 if(argc <= 1) {
239 ::printHelp();
240 exit(1);
241 }
242 int inputFileArgument = -1;
243 std::string fname;
244 std::string restartFileName;
245
246 for(int ii = 1; ii < argc; ++ ii) {
247 std::string argStr = std::string(argv[ii]);
248 if (argStr == std::string("-h") ||
249 argStr == std::string("-help") ||
250 argStr == std::string("--help")) {
251 ::printHelp();
252 exit(0);
253 } else if (argStr == std::string("--help-command")) {
254 if (argc < ii + 2) {
255 ::printHelp();
256 exit(1);
257 }
258 ::printStdoutHeader();
259 const std::string cmdName = Util::toUpper(argv[ii + 1]);
260 Object *object = OpalData::getInstance()->find(cmdName);
261
262 if(object == 0) {
263 *gmsg << "\nOpalParser::printHelp(): Unknown object \""
264 << cmdName << "\".\n" << endl;
265 exit(1);
266 }
267
268 object->printHelp(std::cout);
269 exit(0);
270 } else if (argStr == std::string("--version")) {
271 if (Ippl::myNode() == 0) {
272 std::cout << OPAL_PROJECT_VERSION << std::endl;
273 }
274 exit(0);
275 } else if (argStr == std::string("--version-full")) {
276 ::printStdoutHeader();
277 INFOMSG("OPAL Version " << OPAL_PROJECT_VERSION << ", git rev. " << Util::getGitRevision() << endl);
279 std::string options = (IpplInfo::compileOptions() +
280 std::string(" ") +
281 std::string(OPAL_COMPILE_OPTIONS) +
282 std::string(" "));
283 std::set<std::string> uniqOptions;
284 while (options.length() > 0) {
285 size_t n = options.find_first_of(' ');
286 while (n == 0) {
287 options = options.substr(n + 1);
288 n = options.find_first_of(' ');
289 }
290
291 uniqOptions.insert(options.substr(0, n));
292 options = options.substr(n + 1);
293 }
294 for (auto it: uniqOptions) {
295 options += it + " ";
296 }
297
298 std::string header("Compile-time options: ");
299 while (options.length() > 58) {
300 std::string line = options.substr(0, 58);
301 size_t n = line.find_last_of(' ');
302 INFOMSG(header << line.substr(0, n) << "\n");
303
304 header = std::string(22, ' ');
305 options = options.substr(n + 1);
306 }
307 INFOMSG(header << options << endl);
308 exit(0);
309 } else if (argStr == std::string("--git-revision")) {
310 if (Ippl::myNode() == 0) {
311 std::cout << Util::getGitRevision() << std::endl;
312 }
313 exit(0);
314 } else if (argStr == std::string("--input")) {
315 ++ ii;
316 inputFileArgument = ii;
317 continue;
318 } else if (argStr == std::string("-restart") ||
319 argStr == std::string("--restart")) {
320 opal->setRestartRun();
321 opal->setRestartStep(atoi(argv[++ ii]));
322 continue;
323 } else if (argStr == std::string("-restartfn") ||
324 argStr == std::string("--restartfn")) {
325 restartFileName = std::string(argv[++ ii]);
326 continue;
327 } else if ( argStr.find("noInitAMR") != std::string::npos) {
328 // do nothing here
329 } else {
330 if (inputFileArgument == -1 &&
331 (ii == 1 || ii + 1 == argc) &&
332 argv[ii][0] != '-') {
333 inputFileArgument = ii;
334 continue;
335 } else {
336 INFOMSG("Unknown argument \"" << argStr << "\"" << endl);
337 ::printHelp();
338 exit(1);
339 }
340 }
341 }
342
343 ::printStdoutHeader();
344 if (inputFileArgument == -1) {
345 INFOMSG("No input file provided!" << endl);
346 exit(1);
347 }
348
349 fname = std::string(argv[inputFileArgument]);
350 if (!fs::exists(fname)) {
351 INFOMSG("Input file '" << fname << "' doesn't exist!" << endl);
352 exit(1);
353 }
354
355 opal->storeInputFn(fname);
356
357 if (opal->inRestartRun()) {
358 if (restartFileName.empty()) {
359 restartFileName = opal->getInputBasename() + std::string(".h5");
360 }
361 if (!fs::exists(restartFileName)) {
362 INFOMSG("Restart file '" << restartFileName << "' doesn't exist!" << endl);
363 exit(1);
364 }
365 opal->setRestartFileName(restartFileName);
366 }
367
368 FileStream *is;
369
370 try {
371 is = new FileStream(fname);
372 } catch(...) {
373 is = 0;
374 *gmsg << "Input file '" << fname << "' not found." << endl;
375 }
376
377 if(is) {
378 *gmsg << "* Reading input stream '" << fname << "'" << endl;
379 parser.run(is);
380 *gmsg << "* End of input stream '" << fname << "'" << endl;
381 }
382
383 if(Ippl::myNode() == 0) {
384 std::ifstream errormsg("errormsg.txt");
385 if(errormsg.good()) {
386 char buffer[256];
387 std::string closure(" *\n");
388 ERRORMSG("\n"
389 << "* **********************************************************************************\n"
390 << "* ************** W A R N I N G / E R R O R * * M E S S A G E S *********************\n"
391 << "* **********************************************************************************"
392 << endl);
393 errormsg.getline(buffer, 256);
394 while(errormsg.good()) {
395 ERRORMSG("* ");
396 if(errormsg.gcount() == 1) {
397 ERRORMSG(closure);
398 } else if ((size_t)errormsg.gcount() <= closure.size()) {
399 ERRORMSG(buffer << closure.substr(errormsg.gcount() - 1));
400 } else {
401 ERRORMSG(buffer << endl);
402 }
403 errormsg.getline(buffer, 256);
404 }
405 ERRORMSG("* " << closure
406 << "* **********************************************************************************\n"
407 << "* **********************************************************************************"
408 << endl);
409 }
410 errormsg.close();
411 }
412
413 } catch(EarlyLeaveException& ex) {
414 // do nothing here
415 } catch(OpalException &ex) {
416 Inform errorMsg("Error", std::cerr, INFORM_ALL_NODES);
417 errorMsg << "\n*** User error detected by function \""
418 << ex.where() << "\"\n";
419 // stat->printWhere(errorMsg, true);
420 std::string what = ex.what();
421 size_t pos = what.find_first_of('\n');
422 do {
423 errorMsg << " " << what.substr(0, pos) << endl;
424 what = what.substr(pos + 1, std::string::npos);
425 pos = what.find_first_of('\n');
426 } while (pos != std::string::npos);
427 errorMsg << " " << what << endl;
428
429 MPI_Abort(MPI_COMM_WORLD, -100);
430 } catch(ClassicException &ex) {
431 Inform errorMsg("Error", std::cerr, INFORM_ALL_NODES);
432 errorMsg << "\n*** User error detected by function \""
433 << ex.where() << "\"\n";
434 // stat->printWhere(errorMsg, true);
435 std::string what = ex.what();
436 size_t pos = what.find_first_of('\n');
437 do {
438 errorMsg << " " << what.substr(0, pos) << endl;
439 what = what.substr(pos + 1, std::string::npos);
440 pos = what.find_first_of('\n');
441 } while (pos != std::string::npos);
442 errorMsg << " " << what << endl;
443
444 MPI_Abort(MPI_COMM_WORLD, -100);
445 } catch(SDDSParserException &ex) {
446 Inform errorMsg("Error", std::cerr, INFORM_ALL_NODES);
447
448 errorMsg << "\n*** Error detected by function \""
449 << ex.where() << "\"\n";
450 std::string what = ex.what();
451 size_t pos = what.find_first_of('\n');
452 do {
453 errorMsg << " " << what.substr(0, pos) << endl;
454 what = what.substr(pos + 1, std::string::npos);
455 pos = what.find_first_of('\n');
456 } while (pos != std::string::npos);
457 errorMsg << " " << what << endl;
458
459 MPI_Abort(MPI_COMM_WORLD, -100);
460 } catch(IpplException &ex) {
461 Inform errorMsg("Error", std::cerr, INFORM_ALL_NODES);
462
463 errorMsg << "\n*** Error detected by function \""
464 << ex.where() << "\"\n";
465 std::string what = ex.what();
466 size_t pos = what.find_first_of('\n');
467 do {
468 errorMsg << " " << what.substr(0, pos) << endl;
469 what = what.substr(pos + 1, std::string::npos);
470 pos = what.find_first_of('\n');
471 } while (pos != std::string::npos);
472 errorMsg << " " << what << endl;
473
474 MPI_Abort(MPI_COMM_WORLD, -100);
475 } catch(std::bad_alloc &ex) {
476 Inform errorMsg("Error", std::cerr, INFORM_ALL_NODES);
477 errorMsg << "\n*** Error:\n";
478 errorMsg << " Sorry, virtual memory exhausted.\n"
479 << ex.what()
480 << endl;
481
482 MPI_Abort(MPI_COMM_WORLD, -100);
483 } catch(assertion &ex) {
484 Inform errorMsg("Error", std::cerr, INFORM_ALL_NODES);
485 errorMsg << "\n*** Runtime-error ******************\n";
486 std::string what = ex.what();
487 size_t pos = what.find_first_of('\n');
488 do {
489 errorMsg << " " << what.substr(0, pos) << endl;
490 what = what.substr(pos + 1, std::string::npos);
491 pos = what.find_first_of('\n');
492 } while (pos != std::string::npos);
493 errorMsg << " " << what << endl;
494
495 errorMsg << "\n************************************\n" << endl;
496 throw std::runtime_error("in Parser");
497 } catch(std::exception &ex) {
498 Inform errorMsg("Error", std::cerr, INFORM_ALL_NODES);
499 errorMsg << "\n"
500 << "*** Error:\n"
501 << " Internal OPAL error: \n";
502 std::string what = ex.what();
503 size_t pos = what.find_first_of('\n');
504 do {
505 errorMsg << " " << what.substr(0, pos) << endl;
506 what = what.substr(pos + 1, std::string::npos);
507 pos = what.find_first_of('\n');
508 } while (pos != std::string::npos);
509 errorMsg << " " << what << endl;
510
511 MPI_Abort(MPI_COMM_WORLD, -100);
512 } catch(...) {
513 Inform errorMsg("Error", std::cerr, INFORM_ALL_NODES);
514 errorMsg << "\n*** Error:\n"
515 << " Unexpected exception caught.\n" << endl;
516
517 MPI_Abort(MPI_COMM_WORLD, -100);
518 }
519
520
521 IpplTimings::stopTimer(mainTimer);
522
524
525 IpplTimings::print(std::string("timing.dat"),
526 OpalData::getInstance()->getProblemCharacteristicValues());
527
529 // Before clearDictionary() and before Ippl is torn down: the tracker is held in a
530 // static shared_ptr and would otherwise be destroyed after MPI_Finalize.
531#ifndef DONT_DEFINE_IPPL_GMSG
533#endif
536 delete gmsg;
537
538#ifdef ENABLE_AMR
539 if ( initAMR ) {
540 amrex::Finalize(true);
541 }
542#endif
543
544 // Calls MPI_Finalize, collectively like the barrier above. Nothing after this,
545 // static destructors included, may use MPI, parallel HDF5 or Ippl::Comm.
546#ifndef DONT_DEFINE_IPPL_GMSG
547 delete ippl;
548#endif
549 delete Ippl::Info;
550 delete Ippl::Warn;
551 delete Ippl::Error;
552 delete Ippl::Debug;
553
554 return 0;
555}
int opalMain(int argc, char *argv[])
Definition Main.cpp:150
int main(int argc, char *argv[])
Definition Main.cpp:139
Inform * gmsgALL
Definition Main.cpp:71
Ippl * ippl
Definition Main.cpp:69
bool checkInitAmrFlag(int argc, char *argv[])
Definition Main.cpp:124
Inform * gmsg
Definition Main.cpp:70
void handleGSLErrors(const char *reason, const char *file, int, int)
#define ERRORMSG(msg)
Definition IpplInfo.h:350
IpplInfo Ippl
Definition IpplInfo.h:353
#define INFOMSG(msg)
Definition IpplInfo.h:348
Inform & endl(Inform &inf)
Definition Inform.cpp:42
Inform & level5(Inform &inf)
Definition Inform.cpp:49
#define INFORM_ALL_NODES
Definition Inform.h:39
bool echo
Echo flag.
Definition Options.cpp:24
std::string toUpper(const std::string &str)
Definition Util.cpp:152
std::string getGitRevision()
Definition Util.cpp:38
void configure()
Configure all commands.
The base class for all OPAL objects.
Definition Object.h:48
The global OPAL structure.
Definition OpalData.h:49
void storeInputFn(const std::string &fn)
store opals input filename
Definition OpalData.cpp:654
std::string getInputBasename()
get input file name without extension
Definition OpalData.cpp:674
void setRestartRun(const bool &value=true)
set OPAL in restart mode
Definition OpalData.cpp:316
Object * find(const std::string &name)
Find entry.
Definition OpalData.cpp:571
static void deleteInstance()
Definition OpalData.cpp:206
void setRestartFileName(std::string s)
store opals restart h5 format filename
Definition OpalData.cpp:332
static OpalData * getInstance()
Definition OpalData.cpp:196
void storeArguments(int argc, char *argv[])
Definition OpalData.cpp:768
void setRestartStep(int s)
store the location where to restart
Definition OpalData.cpp:320
std::string getAuxiliaryOutputDirectory() const
get the name of the the additional data directory
Definition OpalData.cpp:666
bool inRestartRun()
true if we do a restart run
Definition OpalData.cpp:312
static void clearDictionary()
Definition Fieldmap.cpp:274
static void setGlobalTruncOrder(int order)
Set the global truncation order.
Definition FTps.hpp:419
A stream of input tokens.
Definition FileStream.h:32
static void setEcho(bool flag)
Set echo flag.
The abstract base class for all exceptions in CLASSIC.
The default parser for OPAL-9.
Definition OpalParser.h:44
virtual void run() const
Read current stream.
static void releaseTracker()
Definition TrackRun.cpp:63
virtual const std::string & what() const
Return the message string for the exception.
virtual const std::string & where() const
Return the name of the method or function which detected the exception.
The base class for all OPAL exceptions.
std::string date() const
Return date.
Definition Timer.cpp:35
std::string time() const
Return time.
Definition Timer.cpp:42
void barrier(void)
static void printVersion(void)
Definition IpplInfo.cpp:755
static Inform * Warn
Definition IpplInfo.h:79
static void printHelp()
Definition IpplInfo.cpp:763
static Inform * Info
Definition IpplInfo.h:78
static Inform * Error
Definition IpplInfo.h:80
static MPI_Comm getComm()
Definition IpplInfo.h:152
static Inform * Debug
Definition IpplInfo.h:81
static int myNode()
Definition IpplInfo.cpp:710
static Communicate * Comm
Definition IpplInfo.h:84
static const char * compileOptions()
Definition IpplInfo.cpp:855
Timing::TimerRef TimerRef
static TimerRef getTimer(const char *nm)
static void stopTimer(TimerRef t)
static void print()
static void startTimer(TimerRef t)