''' !!! WORK IN PROGRESS    !!! '''\\
Last Update 02.2017, Authors: Jens Henrik Göbbert, ...

== OpenFOAM Settings for use with Score-P and Intel compiler

* This includes additional information on how to build OpenFOAM for use with Score-P and the Intel compiler.
* Please check first
  * [https://trac.version.fz-juelich.de/vis/wiki/Software/OpenFOAM]
  * [https://trac.version.fz-juelich.de/vis/wiki/Software/OpenFOAM/config_ScoreP]

=== Set Environment ===

The environment must be set up according to where the source code directories are located.
{{{ #!sh
INST_DIR=/homeb/zam/${USER}/my_OpenFOAM # must NOT (!!) end with a '/', or build will fail with "touch [...] permission denied"
cd $INST_DIR

# load modules (Intel)
module --force purge
module use /usr/local/software/jureca/OtherStages
module load Stages/2016a
module load Intel/2016.2.181-GCC-4.9.3-2.25
module load IntelMPI/5.1.3.181
module load flex/.2.5.39 # flex 2.6.0 causes errors
module load Boost/1.60.0-Python-2.7.11
module load MPFR/3.1.3
module load GMP/6.1.0  
module load CGAL/4.7-Python-2.7.11
module load METIS/5.1.0-RTW64-IDX32

# configure Score-P
module load Score-P/3.0-p1
export SCOREP_WRAPPER_INSTRUMENTER_FLAGS="--nomemory --verbose"
#export SCOREP_WRAPPER_COMPILER_FLAGS=

# configure environment
export FOAM_INST_DIR=$INST_DIR
export WM_COMPILER=Icc
export WM_MPLIB=EASYBUILDMPI
export MPICC=scorep-mpicc
export MPICXX=scorep-mpiicpc
export MPICC_ALT=mpicc
export MPICXX_ALT=mpiicpc
export CC=scorep-icc
export CXX=scorep-icpc
export CC_ALT=icc
export CXX_ALT=icpc
export CFLAGS="-no-prec-div -O2 -fp-model precise -diag-disable 654,1125"
export CXXFLAGS="-O2 -fp-model precise -diag-disable 654,1125 -DMPICH_SKIP_MPICXX"

export WM_NCOMPPROCS=8  # parallel build
export WM_LABEL_SIZE=32 # 32bit for labels/indices - relevant for METIS (if 64 needed patch METIS first)

# load environment
VERSION=3.0.1
source OpenFOAM-${VERSION}/etc/bashrc
foam
foamSystemCheck
}}}

=== Scotch ===
Building of Scotch library builds and then runs an executable as part of the compilation which is called dummysizes. This must not be compiled with Score-P instrumentation. Hence, we compile it with ${CCD_ALT} and define this in the config files.

Patch the Icc-configfile linux64'''Icc'''Int32:
 * $WM_THIRD_PARTY_DIR/platforms/linux64'''Icc'''Int32/scotch_6.0.3/src/libscotch/Makefile
{{{ #!sh
# libscotch/Makefile
+ dummysizes$(EXE) : dummysizes.c                            \
[...]
                     $(CCD_ALT) $(CFLAGS) -DSCOTCH_VERSION=$(VERSION) -DSCOTCH_RELEASE=$(RELEASE) -DSCOTCH_PATCHLEVEL=$(PATCHLEVEL) $(<) -o $(@) $(LDFLAGS)

+ ptdummysizes$(EXE) : dummysizes.c                            \
[...]
+                     $(CCD_ALT) $(CFLAGS) -DSCOTCH_VERSION=$(VERSION) -DSCOTCH_RELEASE=$(RELEASE) -DSCOTCH_PATCHLEVEL=$(PATCHLEVEL) $(<) -o $(@) $(LDFLAGS)
}}}