!!! WORK IN PROGRESS !!!
Last Update 02.2017, Authors: Jens Henrik Göbbert, ...

OpenFOAM Settings for Xeon Phi

• This includes additional information on how to build OpenFOAM for Xeon Phi.
  1. follow https://trac.version.fz-juelich.de/vis/wiki/Software/OpenFOAM, but do NOT build OpenFOAM, yet.
  2. follow the additional instructions from this page
  3. follow the build instructions from https://trac.version.fz-juelich.de/vis/wiki/Software/OpenFOAM#a5.BuildTestOpenFOAM

Set Environment

The environment must be set up according to where the source code directories are located.

INST_DIR=/homeb/zam/${USER}/my_OpenFOAM_knl # must not end with a '/', or build will fail with "touch [...] permission denied"
cd $INST_DIR

# load modules
#module --force purge
#module load Stages/Devel
#module load Intel/2017.0.098-GCC-5.4.0
#module load IntelMPI/2017.0.098

# use thirdparty libs compiled for KNL
export EBVERSIONBOOST=1.61.0
export EBROOTBOOST=${BASE}/deps/Boost/install/knl
export EBROOTMPFR=${BASE}/deps/MPFR/install/knl  
export  EBROOTGMP=${BASE}/deps/GMP/install/knl 
export EBROOTCGAL=${BASE}/deps/CGAL/install/knl 
#export EBROOTMETIS=${BASE}/deps/METIS/install/knl 

# configure environment
export FOAM_INST_DIR=$INST_DIR
export WM_COMPILER=Icc
export WM_MPLIB=EASYBUILDMPI
export MPICC=mpicc
export MPICXX=mpiicpc
export CC=icc
export CXX=icpc
export OPTFLAGS="-qopt-streaming-stores-always -qopt-prefetch-distance=72"
export CFLAGS="-no-prec-div -O3 -xMIC-AVX512 -diag-disable 654,1125 ${OPTCFLAGS}"
export CXXFLAGS="-O3 -xMIC-AVX512 -diag-disable 654,1125 ${OPTFLAGS} -DMPICH_SKIP_MPICXX"

export WM_NCOMPPROCS=8  # parallel build
export WM_LABEL_SIZE=32 # 32bit for labels/indices - relevant for METIS (if 64 needed patch METIS first)

# load environment
VERSION=3.0.1
source OpenFOAM-${VERSION}/etc/bashrc
foam
foamSystemCheck

wmake/rules/linux64Icc/c++

CC = ${MPICXX} -O3 -xMIC-avx512 -DvectorMachine -std=c++11 -fp-trap=common -fp-model precise -fp-speculation=safe -opt-streaming-stores-always -qopt-prefetch-distance=72

• ​https://github.com/OpenFOAM/OpenFOAM-dev/blob/master/wmake/rules/linux64IccKNL/c++
• more possible optimizations for Intel compiler
  • ​-qopt-streaming-stores-always
  • ​-qopt-prefetch-distance=72
  • ​-fimf-precision=low
  • ​-fimf-domain-exclusion=31
  • ​-restrict
  • ​-ansi-alias
  • ​-no-ansi-alias-check

Scotch

Patch $WM_THIRD_PARTY_DIR/etc/wmakeFiles/scotch/Makefile.inc.i686_pc_linux2.shlib-OpenFOAM-64Int32

# scotch/Makefile.inc.i686_pc_linux2.shlib-OpenFOAM-64Int32
+ AR             = ${CC}
+ CCS            = ${CC}
+ CCP            = ${MPICC}
+ CCD            = ${MPICC}
+ CFLAGS         = -O3 -xMIC-AVX512 -DCOMMON_FILE_COMPRESS_GZ -DCOMMON_RANDOM_FIXED_SEED -DSCOTCH_RENAME -Drestrict=__restrict

CGAL

Boost

CFLAGS="-xMIC-AVX512"
CXXFLAGS="-xMIC-AVX512"
LDFLAGS="-xMIC-AVX512"

./bootstrap.sh --with-toolset=intel-linux --prefix=${BASE}/deps/Boost/install/knl
./bjam toolset=intel -j 8 --disable-icu \
  cflags="$CFLAGS" cxxflags="$CXXFLAGS" linkflags="$LDFLAGS" \
  --without-iostreams --without-python
./b2 install --prefix=${BASE}/deps/Boost/install/knl

• more on Boost ...

GMP

./configure CC=icc CXX=icpc \
       CFLAGS="-fPIC -O2 -xMIC-AVX512" \
     CXXFLAGS="-fPIC -O2 -xMIC-AVX512" \
     --host=x86_64-k1om-linux \
     --enable-cxx \
     --prefix=${BASE}/deps/GMP/install/knl

• more on GMP ...

MPFR

./configure CC=icc CXX=icpc \
     CFLAGS="-fPIC -O2 -xMIC-AVX512" \
   CXXFLAGS="-fPIC -O2 -xMIC-AVX512" \
   --host=x86_64-k1om-linux \
   --prefix=${BASE}/deps/MPFR/install/knl \
   --with-gmp=${BASE}/deps/GMP/install/knl \
   --prefix=${BASE}/deps/MPFR/install/knl

• more on MPFR ...

CGAL

export CC=icc
export CXX=icpc
export   CFLAGS="-O2 -xMIC-AVX512 "
export CXXFLAGS="-O2 -xMIC-AVX512 "
cmake -DCMAKE_VERBOSE_MAKEFILE=ON \
      -DBOOST_ROOT=${BASE}/deps/Boost/install/knl \
      -DMPFR_INCLUDE_DIR=${BASE}/deps/MPFR/install/knl/include/ -DMPFR_LIBRARIES=${BASE}/deps/MPFR/install/knl/lib/ \
      -DGMP_INCLUDE_DIR=${BASE}/deps/GMP/install/knl/include/   -DGMP_LIBRARIES=${BASE}/deps/GMP/install/knl/lib/  \
      -DCMAKE_INSTALL_PREFIX=${BASE}/deps/CGAL/install/knl \
      ../cgal-releases-CGAL-4.8.2

• more on CGAL ...

Reference

• optimizations for OpenFOAM for Xeon Phi
  • ​https://www.ixpug.org/docs/tcs_ixpug14.pdf
• general information on Xeon Phi
  • ​https://software.intel.com/en-us/articles/compiling-for-the-intel-xeon-phi-processor-and-the-intel-avx-512-isa
• fp-model precise/strict
  • Such code cannot be vectorized. This may have a substantial impact on performance.
  • ​https://software.intel.com/en-us/articles/differences-in-floating-point-arithmetic-between-intel-xeon-processors-and-the-intel-xeon
• -axCOMMON_AVX512
  • subset of AVX512 function set (valid for skylake, too)