Build OPAL on Piz Daint at CSCS

1. Account

1.1. Creation

First you need a proper CSCS account. In order to do that click on this link.

1.2. Login

One has first to access Ela before being able to login to Piz Daint.

$ ssh username@ela.cscs.ch
> ssh daint

2. Modules

The hybrid architecture of Piz Daint allows to run on GPUs and CPUs. Therefore, it provides two modules in order to set up the environment properly:

> module load daint-gpu
> module load daint-mc

This instruction only shows the installation of OPAL for the multicore architecture (i.e. the module daint-mc).

2.1. Load environment

> module load daint-mc
> module switch PrgEnv-cray/6.0.8 PrgEnv-gnu/6.0.8
> module load CMake/3.14.5

2.2. Export environment variables for builds

> export TARGET_DIR=$HOME/gcc/8.3.0
> export DOWNLOADS_DIR=$SCRATCH
> export SRC_DIR=$SCRATCH
> export NJOBS=4

2.3. Building OpenBLAS

2.3.1. Installation script (openblas_install.sh)

#!/bin/bash

# recipe for:
P=OpenBLAS
V=0.2.20

# download
curl -L \
    --output "${DOWNLOADS_DIR}/$P-$V.tar.gz" \
    "http://github.com/xianyi/$P/archive/v$V.tar.gz"

# unpack
mkdir -p "${SRC_DIR}/$P" && cd "$_"
tar xvf "${DOWNLOADS_DIR}/$P-$V.tar.gz"
cd "$P-$V"

# configure
# nothing to configure

# compile & install
make TARGET=CORE2
make TARGET=CORE2 PREFIX="${PREFIX}" install

2.3.2. Installation procedure

> export PREFIX=$TARGET_DIR/OpenBLAS/0.2.20
> ./openblas_install.sh
> export OPENBLAS_DIR=$PREFIX
> export OPENBLAS_HOME=$OPENBLAS_DIR
> export OPENBLAS_INCLUDE_DIR=$OPENBLAS_HOME/include
> export OPENBLAS_LIBRARY_DIR=$OPENBLAS_HOME/lib
> export OPENBLAS_PREFIX=$OPENBLAS_HOME
> export OPENBLAS_VERSION=0.2.20
> export BLASLIB=OPENBLAS_LIBRARY_DIR/libopenblas.a

2.4. Building HDF5

2.4.1. Installation script (hdf5_install.sh)

#!/bin/bash

# recipe for:
P=hdf5
V_MAJOR=1
V_MINOR=10
V=${V_MAJOR}.${V_MINOR}.6

# download
curl -L \
     --output "${DOWNLOADS_DIR}/$P-$V.tar.gz" \
     "https://support.hdfgroup.org/ftp/HDF5/releases/$P-$V_MAJOR.$V_MINOR/$P-$V/src/$P-$V.tar.gz"

# unpack
mkdir -p "${SRC_DIR}/$P" && cd "$_"
tar xvf "${DOWNLOADS_DIR}/$P-$V.tar.gz"

#configure
mkdir -p "${SRC_DIR}/$P/build" && cd "$_"
CXX=CC CC=cc Ftn=ftn \
    "${SRC_DIR}/$P/$P-$V/configure" \
    --prefix="${PREFIX}" \
    --enable-shared \
    --enable-parallel \
    --enable-cxx \
    --enable-unsupported \
    --with-pic

# compile & install
make -j ${NJOBS}
make install

2.4.2. Installation procedure

> export PREFIX=$TARGET_DIR/hdf5/1.10.6
> ./hdf5_install.sh
> export HDF5_DIR=$PREFIX
> export HDF5_HOME=$HDF5_DIR
> export HDF5_ROOT=$HDF5_DIR
> export HDF5_PREFIX=$HDF5_DIR
> export HDF5_INCLUDE_DIR=$HDF5_HOME/include
> export HDF5_LIBRARY_DIR=$HDF5_HOME/lib
> export HDF5_VERSION=1.10.6

2.5. Building H5hut

2.5.1. Installation script (h5hut_install.sh)

#!/bin/bash

# recipe for:
P=H5hut
V=2.0.0rc6
SHA=7e7a85f14fc3ee123c122859a641db624de41b2a

# download
curl -L \
    --output "${DOWNLOADS_DIR}/$P-$V.tar.gz" \
    "https://gitlab.psi.ch/$P/src/repository/archive.tar.gz?ref=$P-$V"

# unpack
mkdir -p "${SRC_DIR}/$P" && cd "$_"
tar xvf "${DOWNLOADS_DIR}/$P-$V.tar.gz"

# pre-configure
cd "src-$P-$V-$SHA"
./autogen.sh
cd ..

# configure
mkdir -p "${SRC_DIR}/$P/build" && cd "$_"
CXX=CC CC=cc FC=ftn \
    "${SRC_DIR}/$P/src-$P-$V-$SHA/configure" \
    --prefix="${PREFIX}" \
    --enable-parallel \
    --with-hdf5="${HDF5_PREFIX}" \
    --with-pic

# compile & install
make -j ${NJOBS}
make install

2.5.2. Installation procedure

> export PREFIX=$TARGET_DIR/H5hut/2.0.0rc6
> ./h5hut_install.sh
> export H5HUT_DIR=$TARGET_DIR/H5hut/2.0.0rc6
> export H5HUT_HOME=$H5HUT_DIR
> export H5HUT_PREFIX=$H5HUT_HOME
> export H5HUT_INCLUDE_DIR=$H5HUT_HOME/include
> export H5HUT_LIBRARY_DIR=$H5HUT_HOME/lib
> export H5HUT_VERSION=2.0.0rc6

2.6. Building Boost

2.6.1. Installation script (boost_install.sh)

#!/bin/bash

# recipe for:
P=boost
V=1_70_0

# download
curl -L \
    --output "${DOWNLOADS_DIR}/${P}_$V.tar.gz" \
    "https://netcologne.dl.sourceforge.net/project/$P/$P/${V//_/.}/${P}_$V.tar.gz"

# unpack
mkdir -p "${SRC_DIR}/$P" && cd "$_"
tar xvf "${DOWNLOADS_DIR}/${P}_$V.tar.gz"

# configure
mkdir -p "${SRC_DIR}/$P/build"
cd "${SRC_DIR}/$P/${P}_$V"
./bootstrap.sh \
    --prefix="${PREFIX}" \
    --with-libraries=mpi,chrono,filesystem,iostreams,regex,serialization,system,timer

echo "using mpi : : " >> project-config.jam
echo "<include>\"${MPICH_DIR}/include\"" >> project-config.jam
echo "<library-path>\"${MPICH_DIR}/lib\"" >> project-config.jam
echo "<find-shared-library>mpich" >> project-config.jam
echo "<find-shared-library>mpichcxx" >> project-config.jam
echo "<find-shared-library>fmpich ;" >> project-config.jam

# compile & install
./b2 \
    --build-dir=""${SRC_DIR}/$P/build"" \
    --layout=system \
    variant=release \
    link=shared,static \
    threading=multi \
    stage
./b2 \
    --build-dir=""${SRC_DIR}/$P/build"" \
    --layout=system \
    variant=release \
    link=shared,static \
    threading=multi \
    install

2.6.2. Installation procedure

> export PREFIX=$TARGET_DIR/boost/1.70.0
> ./boost_install.sh
> export BOOST_DIR=$PREFIX
> export BOOST_HOME=$BOOST_DIR
> export BOOST_INCLUDE_DIR=$BOOST_DIR/include
> export BOOST_LIBRARY_DIR=$BOOST_DIR/lib
> export BOOST_PREFIX=$BOOST_DIR
> export BOOST_VERSION=1.70.0

2.7. Building GSL

2.7.1. Installation script (gsl_install.sh)

#!/bin/bash

# recipe for:
P=gsl
V=${GSL_VERSION:-2.6}

FNAME="$P-$V.tar.gz"
DOWNLOAD_URL="ftp://ftp.gnu.org/gnu/$P/${FNAME}"
SRC_FILE="${DOWNLOADS_DIR}/${FNAME}"

# download
test -r "${SRC_FILE}" || curl -L --output "$_" "${DOWNLOAD_URL}"

# unpack
mkdir -p "${SRC_DIR}/$P" && cd "$_"
tar xvf "${SRC_FILE}"

# configure
mkdir -p "${SRC_DIR}/$P/build" && cd "$_"
"${SRC_DIR}/$P/$P-$V/configure" \
                --prefix="${PREFIX}" \
                --disable-shared

# compile & install
make -j ${NJOBS}
make install

2.7.2. Installation procedure

> export PREFIX=$TARGET_DIR/gsl/2.6
> ./gsl_install.sh
> export GSL_DIR=$PREFIX
> export GSL_HOME=$GSL_DIR
> export GSL_INCLUDE_DIR=$GSL_HOME/include
> export GSL_LIBRARY_DIR=$GSL_HOME/lib
> export GSL_PREFIX=$GSL_DIR
> export GSL_VERSION=2.6

2.8. Building OPAL

2.8.1. Installation script (opal_install.sh)

#!/bin/bash

# recipe for:
P=OPAL
V=2.4.0
SHA=ffded40c6dd317d29776f58f33080e2919319d89

# download
mkdir -p "${DOWNLOADS_DIR}/$P" && cd "$_"
curl -L \
    --output "${DOWNLOADS_DIR}/${P}-$V.tar.gz" \
    "https://gitlab.psi.ch/$P/src/repository/archive.tar.gz?ref=$P-$V"

# unpack
mkdir -p "${SRC_DIR}/$P" && cd "$_"
tar xvf "${DOWNLOADS_DIR}/$P-$V.tar.gz"
mv "${SRC_DIR}/$P/src-$P-$V-$SHA" "${SRC_DIR}/$P/$P-$V"

# configure
mkdir -p "${SRC_DIR}/$P/build" && cd "$_"
CXX=CC CC=cc FC=ftn cmake \
    -DCMAKE_INSTALL_PREFIX:STRING="${PREFIX}" \
    -DCMAKE_BUILD_TYPE:STRING=Release \
    -DCMAKE_C_COMPILER=cc \
    -DCMAKE_CXX_COMPILER=CC \
    "${SRC_DIR}/$P/$P-$V"

# compile & install
make -j ${NJOBS}
make install

2.8.2. Installation procedure

> export PREFIX=$TARGET_DIR/opal/2.4.0
> ./opal_install.sh

You might add the executable path to the environment variable:

> export OPAL_EXE_PATH=$PREFIX/bin
> export PATH=$OPAL_EXE_PATH:$PATH

Make sure you use the right directory. You can check if it worked with

> which opal

> whereis opal

3. Modification of ~/.bashrc

It’s convenient to add a helper function to your ~/.bashrc in order to make life easier:

load_opal_gcc830() {
    module load daint-mc
    module switch PrgEnv-cray/6.0.8 PrgEnv-gnu/6.0.8
    module load CMake/3.14.5

    TARGET_DIR=$HOME/gcc/8.3.0

    export MPI_HOME=$MPICH_DIR
    export MPI_PREFIX=$MPI_HOME
    export MPI_INCLUDE_DIR=$MPI_HOME/include
    export MPI_LIBRARY_DIR=$MPI_HOME/lib

    export OPENBLAS_DIR=$TARGET_DIR/OpenBLAS/0.2.20
    export OPENBLAS_HOME=$OPENBLAS_DIR
    export OPENBLAS_INCLUDE_DIR=$OPENBLAS_HOME/include
    export OPENBLAS_LIBRARY_DIR=$OPENBLAS_HOME/lib
    export OPENBLAS_PREFIX=$OPENBLAS_HOME
    export OPENBLAS_VERSION=0.2.20
    export BLASLIB=$OPENBLAS_LIBRARY_DIR/libopenblas.a

    export LD_LIBRARY_PATH=$OPENBLAS_LIBRARY_DIR/:$LD_LIBRARY_PATH

    export BOOST_DIR=$TARGET_DIR/boost/1.70.0
    export BOOST_HOME=$BOOST_DIR
    export BOOST_INCLUDE_DIR=$BOOST_DIR/include
    export BOOST_LIBRARY_DIR=$BOOST_DIR/lib
    export BOOST_PREFIX=$BOOST_DIR
    export BOOST_VERSION=1.70.0

    export GSL_DIR=$TARGET_DIR/gsl/2.6
    export GSL_HOME=$GSL_DIR
    export GSL_INCLUDE_DIR=$GSL_HOME/include
    export GSL_LIBRARY_DIR=$GSL_HOME/lib
    export GSL_PREFIX=$GSL_DIR
    export GSL_VERSION=2.6

    export HDF5_DIR=$TARGET_DIR/hdf5/1.10.6
    export HDF5_HOME=$HDF5_DIR
    export HDF5_PREFIX=$HDF5_DIR
    export HDF5_ROOT=$HDF5_DIR
    export HDF5_INCLUDE_DIR=$HDF5_HOME/include
    export HDF5_LIBRARY_DIR=$HDF5_HOME/lib
    export HDF5_VERSION=1.10.6

    export H5HUT_DIR=$TARGET_DIR/H5hut/2.0.0rc6
    export H5HUT_HOME=$H5HUT_DIR
    export H5HUT_PREFIX=$H5HUT_HOME
    export H5HUT_INCLUDE_DIR=$H5HUT_HOME/include
    export H5HUT_LIBRARY_DIR=$H5HUT_HOME/lib
    export H5HUT_VERSION=2.0.0rc6

    export OPAL_EXE_PATH=$TARGET_DIR/opal/2.4.0/bin
    export PATH=$OPAL_EXE_PATH:$PATH

    export CRAYPE_LINK_TYPE=dynamic
}

At the next login (or by just doing source ~/.bashrc) you can load everything by executing

> load_opal_gcc830