Main.cpp

Go to the documentation of this file.

 
//
// Copyright (c) 2008 - 2020, Paul Scherrer Institut, Villigen PSI, Switzerland
//
// All rights reserved
//
// This file is part of OPAL.
//
// OPAL is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// You should have received a copy of the GNU General Public License
// along with OPAL. If not, see <https://www.gnu.org/licenses/>.
//
 
#include <cstring>
 
// Ensure real MPI types are defined before including H5hut.
// This avoids H5hut's serial stubs redefining MPI_Comm/MPI_Datatype.
#include <mpi.h>
 
extern "C" {
#include "H5hut.h"
}
 
#include "AbstractObjects/OpalData.h"
#include "Fields/Fieldmap.h"
#include "OpalConfigure/Configure.h"
#include "OpalParser/FileStream.h"
#include "OpalParser/OpalParser.h"
#include "Utilities/Timer.h"
 
#include "BasicActions/Option.h"
#include "Utilities/OpalException.h"
#include "Utilities/Options.h"
#include "Utilities/Util.h"
 
#include "BuildInfo.h"
 
// IPPL
#include "GSLErrorHandling.h"
#include "Utility/Inform.h"
#include "Utility/IpplException.h"
#include "Utility/IpplInfo.h"
#include "Utility/IpplTimings.h"
 
#include "Utilities/GSLCompat.h"
 
#include <filesystem>
#include <system_error>
 
#include <cstring>
#include <iomanip>
#include <iostream>
#include <set>
 
extern Inform* gmsg;
 
namespace OPALXMAIN {
    void printStdoutHeader() {
        OPALTimer::Timer simtimer;
        std::string dateStr(simtimer.date());
        std::string timeStr(simtimer.time());
        std::string mySpace("            ");
 
        *gmsg << mySpace << "  ____  _____        _       __   __" << endl;
        *gmsg << "\033[1;32m";
        *gmsg << R"(
 
  ██████  ██████   █████  ██      ██   ██
 ██    ██ ██   ██ ██   ██ ██       ██ ██
 ██    ██ ██████  ███████ ██        ███
 ██    ██ ██      ██   ██ ██       ██ ██
  ██████  ██      ██   ██ ███████ ██   ██
 
         This is OPALX (Object Oriented Parallel Accelerator Library for Exascale)
 
)" << "\033[0m"
              << endl;
 
        std::string gitRevision = "git rev. " + Util::getGitRevision();
        std::string copyRight   = "(c) PSI, https://github.com/OPALX-project/OPALX";
        *gmsg << endl
              << "OPALX is the performance portable version of OPAL :: Version "
              << buildinfo::project_version << "\n"
              << std::setw(37 + gitRevision.length() / 2) << std::right << gitRevision << "\n\n"
              << endl
              << std::setw(37 + copyRight.length() / 2) << std::right << copyRight << "\n\n"
              << endl;
 
        *gmsg << "Please send cookies, goodies or other motivations (wine and beer ... ) \nto the "
                 "OPAL developers "
              << buildinfo::bugreport << "\n"
              << endl;
        *gmsg << "Time: " << timeStr << " date: " << dateStr << "\n" << endl;
 
        // Check which host device is being used
        *gmsg << "* Host:   " << Kokkos::HostSpace::execution_space::name() << endl;
 
        // Check which device is being used (this works for CUDA, HIP, or any device-enabled
        // execution space)
#ifdef KOKKOS_ENABLE_CUDA
        *gmsg << "* Device: " << Kokkos::Cuda::name() << endl << endl;
#elif defined(KOKKOS_ENABLE_HIP)
        *gmsg << "* Device: " << Kokkos::Experimental::HIP::name() << endl << endl;
#elif defined(KOKKOS_ENABLE_OPENMP)
        *gmsg << "* Device: " << Kokkos::OpenMP::name() << endl << endl;
#else
        *gmsg << "* Device: Serial execution" << endl << endl;
#endif
    }
 
    void printHelp() {
        OPALXMAIN::printStdoutHeader();
 
        *ippl::Info << "\n";
        *ippl::Info << "Usage: opal [<option> <option> ...]\n";
        *ippl::Info << "   The possible values for <option> are:\n";
        *ippl::Info << "   --version                : Print the version of opal.\n";
        *ippl::Info << "   --version-full           : Print the version of opal with additional "
                       "informations.\n";
        *ippl::Info << "   --git-revision           : Print the revision hash of the repository.\n";
        *ippl::Info << "   --input <fname>          : Specifies the input file <fname>.\n";
        *ippl::Info << "   --restart <n>            : Performes a restart from step <n>.\n";
        *ippl::Info << "   --restartfn <fname>      : Uses the file <fname> to restart from.\n";
        //*ippl::printHelp();
        *ippl::Info << "   --help-command <command> : Display the help for the command <command>\n";
        *ippl::Info << "   --help                   : Display this command-line summary.\n";
        *ippl::Info << endl;
    }
}  // namespace OPALXMAIN
 
int main(int argc, char* argv[]) {
    ippl::initialize(argc, argv);
    {
        gmsg         = new Inform("OPAL-X");
        namespace fs = std::filesystem;
 
        H5SetVerbosityLevel(1);  // 65535);
 
        gsl_set_error_handler(&handleGSLErrors);
 
        static IpplTimings::TimerRef mainTimer = IpplTimings::getTimer("mainTimer");
        IpplTimings::startTimer(mainTimer);
 
        if (ippl::Comm->rank() == 0) remove("errormsg.txt");
 
        const OpalParser parser;
 
        std::cout.precision(16);
        std::cout.setf(std::ios::scientific, std::ios::floatfield);
        std::cerr.precision(16);
        std::cerr.setf(std::ios::scientific, std::ios::floatfield);
 
        OpalData* opal = OpalData::getInstance();
 
        if (ippl::Comm->rank() == 0) {
            if (!fs::exists(opal->getAuxiliaryOutputDirectory())) {
                std::error_code error_code;
                if (!fs::create_directory(opal->getAuxiliaryOutputDirectory(), error_code)) {
                    std::cerr << error_code.message() << std::endl;
                    //  use error code to prevent create_directory from throwing an exception
                }
            }
        }
        ippl::Comm->barrier();
        if (!fs::is_directory(opal->getAuxiliaryOutputDirectory())) {
            std::cerr << "unable to create directory; aborting" << std::endl;
            abort();
        }
 
        opal->storeArguments(argc, argv);
 
        try {
            Configure::configure();
 
            // Read startup file.
            FileStream::setEcho(Options::echo);
 
            char* startup           = getenv("HOME");
            std::filesystem::path p = strncat(startup, "/init.opal", 20);
            if (startup != nullptr && fs::is_regular_file(p)) {
                FileStream::setEcho(false);
                FileStream* is;
 
                try {
                    is = new FileStream(startup);
                } catch (...) {
                    is = 0;
                    *ippl::Error << "Could not open startup file '" << startup << "'\n"
                                 << "Note: this is not mandatory for an OPAL simulation!\n";
                }
 
                if (is) {
                    *gmsg << "Reading startup file '" << startup << "'" << endl;
                    parser.run(is);
                    *gmsg << "Finished reading startup file." << endl;
                }
                FileStream::setEcho(Options::echo);
            } else {
                *gmsg << level5 << "Couldn't find startup file '" << startup << "'\n"
                      << "Note: this is not mandatory for an OPAL simulation!\n"
                      << endl;
            }
 
            if (argc <= 1) {
                OPALXMAIN::printHelp();
                exit(1);
            }
            int inputFileArgument = -1;
            std::string fname;
            std::string restartFileName;
            for (int ii = 1; ii < argc; ++ii) {
                std::string argStr = std::string(argv[ii]);
                if (argStr == std::string("-h") || argStr == std::string("-help")
                    || argStr == std::string("--help")) {
                    OPALXMAIN::printHelp();
                    exit(0);
                } else if (argStr == std::string("--help-command")) {
                    if (argc < ii + 2) {
                        OPALXMAIN::printHelp();
                        exit(1);
                    }
                    OPALXMAIN::printStdoutHeader();
                    const std::string cmdName = Util::toUpper(argv[ii + 1]);
                    Object* object            = OpalData::getInstance()->find(cmdName);
 
                    if (object == 0) {
                        *gmsg << "\nOpalParser::printHelp(): Unknown object \"" << cmdName
                              << "\".\n"
                              << endl;
                        exit(1);
                    }
 
                    object->printHelp(std::cout);
                    exit(0);
                } else if (argStr == std::string("--version")) {
                    if (ippl::Comm->rank() == 0) {
                        std::cout << buildinfo::project_version << std::endl;
                    }
                    exit(0);
                } else if (argStr == std::string("--version-full")) {
                    OPALXMAIN::printStdoutHeader();
                    *ippl::Info << "OPAL Version " << buildinfo::project_version << ", git rev. "
                                << Util::getGitRevision() << endl;
                    IpplInfo::printVersion();
                    std::string options =
                            (IpplInfo::compileOptions() + std::string(" ")
                             + std::string(buildinfo::compile_options) + std::string(" "));
                    std::set<std::string> uniqOptions;
                    while (options.length() > 0) {
                        size_t n = options.find_first_of(' ');
                        while (n == 0) {
                            options = options.substr(n + 1);
                            n       = options.find_first_of(' ');
                        }
 
                        uniqOptions.insert(options.substr(0, n));
                        options = options.substr(n + 1);
                    }
                    for (auto it : uniqOptions) {
                        options += it + " ";
                    }
 
                    std::string header("Compile-time options: ");
                    while (options.length() > 58) {
                        std::string line = options.substr(0, 58);
                        size_t n         = line.find_last_of(' ');
                        *ippl::Info << header << line.substr(0, n) << "\n";
 
                        header  = std::string(22, ' ');
                        options = options.substr(n + 1);
                    }
                    *ippl::Info << header << options << endl;
                    exit(0);
                } else if (argStr == std::string("--git-revision")) {
                    if (ippl::Comm->rank() == 0) {
                        std::cout << Util::getGitRevision() << std::endl;
                    }
                    exit(0);
                } else if (argStr == std::string("--input")) {
                    ++ii;
                    inputFileArgument = ii;
                    continue;
                } else if (
                        argStr == std::string("-restart") || argStr == std::string("--restart")) {
                    opal->setRestartRun();
                    opal->setRestartStep(atoi(argv[++ii]));
                    continue;
                } else if (
                        argStr == std::string("-restartfn")
                        || argStr == std::string("--restartfn")) {
                    restartFileName = std::string(argv[++ii]);
                    continue;
                } else if (argStr == std::string("--info")) {
                    ++ii;
                    continue;
                } else if (argStr == std::string("--overallocate")) {
                    ++ii;
                    continue;
                } else {
                    if (inputFileArgument == -1 && (ii == 1 || ii + 1 == argc)
                        && argv[ii][0] != '-') {
                        inputFileArgument = ii;
                        continue;
                    } else {
                        *gmsg << "Unknown argument \"" << argStr << "\"" << endl;
                        OPALXMAIN::printHelp();
                        exit(1);
                    }
                }
            }
 
            OPALXMAIN::printStdoutHeader();
            if (inputFileArgument == -1) {
                *ippl::Info << "No input file provided!" << endl;
                exit(1);
            }
 
            fname = std::string(argv[inputFileArgument]);
            if (!fs::exists(fname)) {
                *ippl::Info << "Input file '" << fname << "' doesn't exist!" << endl;
                exit(1);
            }
 
            opal->storeInputFn(fname);
 
            if (opal->inRestartRun()) {
                if (restartFileName.empty()) {
                    restartFileName = opal->getInputBasename() + std::string(".h5");
                }
                if (!fs::exists(restartFileName)) {
                    *ippl::Info << "Restart file '" << restartFileName << "' doesn't exist!"
                                << endl;
                    exit(1);
                }
                opal->setRestartFileName(restartFileName);
            }
 
            FileStream* is;
 
            try {
                is = new FileStream(fname);
            } catch (...) {
                is = 0;
                *gmsg << "Input file '" << fname << "' not found." << endl;
            }
 
            if (is) {
                *gmsg << "* Reading input stream '" << fname << "'" << endl;
                parser.run(is);
                *gmsg << "* End of input stream '" << fname << "'" << endl;
            }
 
            if (ippl::Comm->rank() == 0) {
                std::ifstream errormsg("errormsg.txt");
                if (errormsg.good()) {
                    char buffer[256];
                    std::string closure(
                            "                                                                      "
                            "    "
                            "    "
                            "   *\n");
                    *ippl::Error
                            << "\n"
                            << "* "
                               "*******************************************************************"
                               "****"
                               "****"
                               "*******\n"
                            << "* ************** W A R N I N G / E R R O R * * M E S S A G E S "
                               "*********************\n"
                            << "* "
                               "*******************************************************************"
                               "****"
                               "****"
                               "*******"
                            << endl;
                    errormsg.getline(buffer, 256);
                    while (errormsg.good()) {
                        *ippl::Error << "* ";
                        if (errormsg.gcount() == 1) {
                            *ippl::Error << closure;
                        } else if ((size_t)errormsg.gcount() <= closure.size()) {
                            *ippl::Error << buffer << closure.substr(errormsg.gcount() - 1);
                        } else {
                            *ippl::Error << buffer << endl;
                        }
                        errormsg.getline(buffer, 256);
                    }
                    *ippl::Error << "* " << closure
                                 << "* "
                                    "**************************************************************"
                                    "********"
                                    "************\n"
                                 << "* "
                                    "**************************************************************"
                                    "********"
                                    "************"
                                 << endl;
                }
                errormsg.close();
            }
 
        } catch (OpalException& ex) {
            Inform errorMsg("Error", std::cerr, INFORM_ALL_NODES);
            errorMsg << "\n*** User error detected by function \"" << ex.where() << "\"\n";
            // stat->printWhere(errorMsg, true);
            std::string what = ex.what();
            size_t pos       = what.find_first_of('\n');
            do {
                errorMsg << "    " << what.substr(0, pos) << endl;
                what = what.substr(pos + 1, std::string::npos);
                pos  = what.find_first_of('\n');
            } while (pos != std::string::npos);
            errorMsg << "    " << what << endl;
 
            MPI_Abort(MPI_COMM_WORLD, -100);
        } catch (IpplException& ex) {
            Inform errorMsg("Error", std::cerr, INFORM_ALL_NODES);
 
            errorMsg << "\n*** Error detected by function \"" << ex.where() << "\"\n";
            std::string what = ex.what();
            size_t pos       = what.find_first_of('\n');
            do {
                errorMsg << "    " << what.substr(0, pos) << endl;
                what = what.substr(pos + 1, std::string::npos);
                pos  = what.find_first_of('\n');
            } while (pos != std::string::npos);
            errorMsg << "    " << what << endl;
 
            MPI_Abort(MPI_COMM_WORLD, -100);
        } catch (std::bad_alloc& ex) {
            Inform errorMsg("Error", std::cerr, INFORM_ALL_NODES);
            errorMsg << "\n*** Error:\n";
            errorMsg << "    Sorry, virtual memory exhausted.\n" << ex.what() << endl;
 
            MPI_Abort(MPI_COMM_WORLD, -100);
        } catch (assertion& ex) {
            Inform errorMsg("Error", std::cerr, INFORM_ALL_NODES);
            errorMsg << "\n*** Runtime-error ******************\n";
            std::string what = ex.what();
            size_t pos       = what.find_first_of('\n');
            do {
                errorMsg << "    " << what.substr(0, pos) << endl;
                what = what.substr(pos + 1, std::string::npos);
                pos  = what.find_first_of('\n');
            } while (pos != std::string::npos);
            errorMsg << "    " << what << endl;
 
            errorMsg << "\n************************************\n" << endl;
            throw std::runtime_error("in Parser");
        } catch (std::exception& ex) {
            Inform errorMsg("Error", std::cerr, INFORM_ALL_NODES);
            errorMsg << "\n"
                     << "*** Error:\n"
                     << "    Internal OPAL error: \n";
            std::string what = ex.what();
            size_t pos       = what.find_first_of('\n');
            do {
                errorMsg << "    " << what.substr(0, pos) << endl;
                what = what.substr(pos + 1, std::string::npos);
                pos  = what.find_first_of('\n');
            } while (pos != std::string::npos);
            errorMsg << "    " << what << endl;
 
            MPI_Abort(MPI_COMM_WORLD, -100);
        } catch (...) {
            Inform errorMsg("Error", std::cerr, INFORM_ALL_NODES);
            errorMsg << "\n*** Error:\n"
                     << "    Unexpected exception caught.\n"
                     << endl;
 
            MPI_Abort(MPI_COMM_WORLD, -100);
        }
 
        // print dictionary
        //        opal->printAllNames(std::cout);
 
        IpplTimings::stopTimer(mainTimer);
 
        IpplTimings::print();
 
        IpplTimings::print(
                std::string("timing.dat"),
                OpalData::getInstance()->getProblemCharacteristicValues());
 
        ippl::Comm->barrier();
        Fieldmap::clearDictionary();
 
        // \todo we should not need this OpalData::deleteInstance();
 
        delete gmsg;
        ippl::finalize();
        return 0;
    }
}