''' !!! WORK IN PROGRESS    !!! '''\\
Last Update 02.2017, Authors: Jens Henrik Göbbert, ...

== OpenFOAM Settings for Xeon Phi

* This includes additional information on how to build OpenFOAM for Xeon Phi.
  1. follow [https://trac.version.fz-juelich.de/vis/wiki/Software/OpenFOAM], but do NOT build OpenFOAM, yet.
  2. follow the additional instructions from this page
  3. follow the build instructions from [https://trac.version.fz-juelich.de/vis/wiki/Software/OpenFOAM#a5.BuildTestOpenFOAM]


=== Set Environment ===

The environment must be set up according to where the source code directories are located.
{{{ #!sh
INST_DIR=/homeb/zam/${USER}/my_OpenFOAM_knl # must not end with a '/', or build will fail with "touch [...] permission denied"
cd $INST_DIR

# load modules
#module --force purge
#module load Stages/Devel
#module load Intel/2017.0.098-GCC-5.4.0
#module load IntelMPI/2017.0.098

# use thirdparty libs compiled for KNL
export EBVERSIONBOOST=1.61.0
export EBROOTBOOST=${BASE}/deps/Boost/install/knl
export EBROOTMPFR=${BASE}/deps/MPFR/install/knl  
export  EBROOTGMP=${BASE}/deps/GMP/install/knl 
export EBROOTCGAL=${BASE}/deps/CGAL/install/knl 
#export EBROOTMETIS=${BASE}/deps/METIS/install/knl 

# configure environment
export FOAM_INST_DIR=$INST_DIR
export WM_COMPILER=Icc
export WM_MPLIB=EASYBUILDMPI
export MPICC=mpicc
export MPICXX=mpiicpc
export CC=icc
export CXX=icpc
export OPTFLAGS="-qopt-streaming-stores-always -qopt-prefetch-distance=72"
export CFLAGS="-no-prec-div -O3 -xMIC-AVX512 -diag-disable 654,1125 ${OPTCFLAGS}"
export CXXFLAGS="-O3 -xMIC-AVX512 -diag-disable 654,1125 ${OPTFLAGS} -DMPICH_SKIP_MPICXX"

export WM_NCOMPPROCS=8  # parallel build
export WM_LABEL_SIZE=32 # 32bit for labels/indices - relevant for METIS (if 64 needed patch METIS first)

# load environment
VERSION=3.0.1
source OpenFOAM-${VERSION}/etc/bashrc
foam
foamSystemCheck
}}}

==== wmake/rules/linux64Icc/c++ ====
{{{ #!sh
CC = ${MPICXX} -O3 -xMIC-avx512 -DvectorMachine -std=c++11 -fp-trap=common -fp-model precise -fp-speculation=safe -opt-streaming-stores-always -qopt-prefetch-distance=72
}}}
* [https://github.com/OpenFOAM/OpenFOAM-dev/blob/master/wmake/rules/linux64IccKNL/c++]
* more possible optimizations for Intel compiler
  * [https://software.intel.com/en-us/node/522896 -qopt-streaming-stores-always]
  * [https://software.intel.com/en-us/node/693294 -qopt-prefetch-distance=72]
  * [https://software.intel.com/en-us/node/522977 -fimf-precision=low]
  * [https://software.intel.com/en-us/node/522975 -fimf-domain-exclusion=31]
  * [https://software.intel.com/en-us/node/682747 -restrict]
  * [https://software.intel.com/en-us/node/522859 -ansi-alias]
  * [https://software.intel.com/en-us/node/693226 -no-ansi-alias-check]

==== Scotch ====
Patch $WM_THIRD_PARTY_DIR/etc/wmakeFiles/scotch/Makefile.inc.i686_pc_linux2.shlib-OpenFOAM-64Int32
{{{ #!sh
# scotch/Makefile.inc.i686_pc_linux2.shlib-OpenFOAM-64Int32
+ AR             = ${CC}
+ CCS            = ${CC}
+ CCP            = ${MPICC}
+ CCD            = ${MPICC}
+ CFLAGS         = -O3 -xMIC-AVX512 -DCOMMON_FILE_COMPRESS_GZ -DCOMMON_RANDOM_FIXED_SEED -DSCOTCH_RENAME -Drestrict=__restrict
}}}

=== CGAL ===

==== Boost ====
{{{#!sh
CFLAGS="-xMIC-AVX512"
CXXFLAGS="-xMIC-AVX512"
LDFLAGS="-xMIC-AVX512"

./bootstrap.sh --with-toolset=intel-linux --prefix=${BASE}/deps/Boost/install/knl
./bjam toolset=intel -j 8 --disable-icu \
  cflags="$CFLAGS" cxxflags="$CXXFLAGS" linkflags="$LDFLAGS" \
  --without-iostreams --without-python
./b2 install --prefix=${BASE}/deps/Boost/install/knl
}}}
* [wiki:/Software/Boost more on Boost ...]

==== GMP ====
{{{#!sh
./configure CC=icc CXX=icpc \
       CFLAGS="-fPIC -O2 -xMIC-AVX512" \
     CXXFLAGS="-fPIC -O2 -xMIC-AVX512" \
     --host=x86_64-k1om-linux \
     --enable-cxx \
     --prefix=${BASE}/deps/GMP/install/knl
}}}
* [wiki:/Software/GMP more on GMP ...]

==== MPFR ====
{{{#!sh
./configure CC=icc CXX=icpc \
     CFLAGS="-fPIC -O2 -xMIC-AVX512" \
   CXXFLAGS="-fPIC -O2 -xMIC-AVX512" \
   --host=x86_64-k1om-linux \
   --prefix=${BASE}/deps/MPFR/install/knl \
   --with-gmp=${BASE}/deps/GMP/install/knl \
   --prefix=${BASE}/deps/MPFR/install/knl
}}}
* [wiki:/Software/MPFR more on MPFR ...]

==== CGAL ====
{{{#!sh
export CC=icc
export CXX=icpc
export   CFLAGS="-O2 -xMIC-AVX512 "
export CXXFLAGS="-O2 -xMIC-AVX512 "
cmake -DCMAKE_VERBOSE_MAKEFILE=ON \
      -DBOOST_ROOT=${BASE}/deps/Boost/install/knl \
      -DMPFR_INCLUDE_DIR=${BASE}/deps/MPFR/install/knl/include/ -DMPFR_LIBRARIES=${BASE}/deps/MPFR/install/knl/lib/ \
      -DGMP_INCLUDE_DIR=${BASE}/deps/GMP/install/knl/include/   -DGMP_LIBRARIES=${BASE}/deps/GMP/install/knl/lib/  \
      -DCMAKE_INSTALL_PREFIX=${BASE}/deps/CGAL/install/knl \
      ../cgal-releases-CGAL-4.8.2
}}}
* [wiki:/Software/CGAL more on CGAL ...]

=== Reference ===
* optimizations for OpenFOAM for Xeon Phi
  * https://www.ixpug.org/docs/tcs_ixpug14.pdf
* general information on Xeon Phi
  * https://software.intel.com/en-us/articles/compiling-for-the-intel-xeon-phi-processor-and-the-intel-avx-512-isa
* fp-model precise/strict
  * Such code cannot be vectorized. This may have a substantial impact on performance.
  * https://software.intel.com/en-us/articles/differences-in-floating-point-arithmetic-between-intel-xeon-processors-and-the-intel-xeon
* -axCOMMON_AVX512
  * subset of AVX512 function set (valid for skylake, too)