!----------------------------------------------------------------------------
! This file is part of psOpen
!
! Version 2.2
!
! Copyright (C) 2011-2012 Jens Henrik Goebbert <jens.henrik.goebbert()rwth-aachen.de>
!
!    psOpen is free software; you can redistribute it and/or modify
!    it under the terms of the GNU General Public License as published by
!    the Free Software Foundation; either version 2 of the License, or
!    (at your option) any later version.

!    This program is distributed in the hope that it will be useful,
!    but WITHOUT ANY WARRANTY; without even the implied warranty of
!    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
!    GNU General Public License for more details.

!    You should have received a copy of the GNU General Public License
!    along with this program; if not, write to the Free Software
!    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
!
!    Please consider mentioning the copyright owners in any publication
!    if results of psOpen are used.
!----------------------------------------------------------------------------

!========================================
!> @addtogroup _visit_
!! @{
!!
!> @file psopen_visit.F90
!! @brief file of the visit module
!! @details
!! @author Jens Henrik Goebbert
!!
!! @}
!========================================

!!!! TODO: check https://software.intel.com/en-us/forums/intel-visual-fortran-compiler-for-windows/topic/514614 !!!

#include "psopen_defines.inc"

#define VISIT_SUCCESS_OR_FAILURE(err_val) \
            if (err_val /= VISIT_OKAY) then; \
              if (vis_mpi_proc_id == 0) write (*,*) ' FAILED with err=', err_val; \
              return; \
            else; \
              if (vis_mpi_proc_id == 0) write (*,*) ' SUCCESS'; \
            end if
#define WRITE_RANK0_NOADVANCE(fmt_val,val,verbose_val)        if(vis_mpi_proc_id == 0 .and. verbose_val <= vis_verbose_level) write(*,fmt=fmt_val,advance='no') val
#define WRITE_RANK0(fmt_val,val,verbose_val)                  if(vis_mpi_proc_id == 0 .and. verbose_val <= vis_verbose_level) write(*,fmt=fmt_val) val
#define WRITE_RANK0_ARGS2(fmt_val,val1,val2,verbose_val)      if(vis_mpi_proc_id == 0 .and. verbose_val <= vis_verbose_level) write(*,fmt=fmt_val) val1,val2
#define WRITE_RANK0_ARGS3(fmt_val,val1,val2,val3,verbose_val) if(vis_mpi_proc_id == 0 .and. verbose_val <= vis_verbose_level) write(*,fmt=fmt_val) val1,val2,val3

module psopen_visit
#ifdef USE_MPI_MODULE
    use mpi
    implicit none
#else
    implicit none
    include 'mpif.h'
#endif
    include "visitfortransimV2interface.inc"
    save
    private

    ! verbose level <= 1: show debug messages in init_visit()
    ! verbose level <= 2: show debug messages in visit_checkstatus()
    ! verbose level <= 3: -- not used --
    ! verbose level <= 4: show debug messages in callback functions
    ! verbose level <= 5: show all debug messages
    integer, parameter, public :: vis_verbose_level = 2

    integer, public :: do_visit = 0
    integer, public :: visit_port = -1

    integer, public :: vis_mpi_proc_id = -1
    integer, public :: vis_mpi_nproc   = -1
    integer, public :: vis_mpi_comm    = -1

    integer, parameter, public :: SIMMODE_RUN      = 1
    integer, parameter, public :: SIMMODE_PAUSE    = 2
    integer, parameter, public :: SIMMODE_STEP     = 3
    integer, public :: sim_mode = SIMMODE_PAUSE

    integer, parameter, public :: VISMODE_DISCONNECTED  = 0
    integer, parameter, public :: VISMODE_UPDATE_FAST   = 1
    integer, parameter, public :: VISMODE_UPDATE_NEVER  = 2
    integer, parameter, public :: VISMODE_UPDATE_ONCE   = 3
    integer, parameter, public :: VISMODE_UPDATE_SYNC   = 4
    integer, public :: vis_mode = VISMODE_DISCONNECTED

    integer, public :: conn_freq = 1
    integer, public :: sync_mode = -1
    integer, public :: sync_count = -1

    integer, parameter, public :: SIMCMD_NONE = 0
    integer, parameter, public :: SIMCMD_DUMP = 1
    integer, parameter, public :: SIMCMD_STAT = 2
    integer, parameter, public :: SIMCMD_SYNC = 3
    integer, public :: sim_cmd = SIMCMD_NONE

    integer, parameter, public :: VISIT_COMMAND_PROCESS = 0
    integer, parameter, public :: VISIT_COMMAND_SUCCESS = 1
    integer, parameter, public :: VISIT_COMMAND_FAILURE = 2

    logical :: visit_tracefile_onlyrank0 = .true.
    character (len=1024) :: visit_tracefile = '' ! trace file off by default - else set 'visit_trace.out'
    character (len=1024) :: visit_path = VISIT_F77NULLSTRING ! empty = take visit from $PATH
    character (len=1024) :: visit_args = VISIT_F77NULLSTRING
    !
    ! "visit_args" will be complemented by additional args passed via the host profile (Launch Profiles -> Additional Arguments) from the VisIt GUI.
    ! ---------------------------
    ! -pid
    !    this option adds process ids to debug log files - logs are not overwritten by different processes writing to a single file
    ! -debug [level][d]
    !    level 1-5
    !    d "d" for additional __FILE__ and __LINE__ information in log files
    ! -reverse_launch
    ! -fixed-buffer-sockets
    !    this option enables VisIt to connect to machines which expect fixed size send/recv buffers (like default settings of BlueGene)
    !

    character (len=1024) :: visit_simbase = 'psOpen'
    character (len=1024) :: visit_simcomment = 'psOpen Simulation'
    character (len=1024) :: visit_simpath = VISIT_F77NULLSTRING ! directory path to where simulation was started
    character (len=1024) :: visit_siminput = VISIT_F77NULLSTRING ! path+name to simulation input file
    character (len=1024) :: visit_simgui = VISIT_F77NULLSTRING ! xml user interface file, which VisIt can use to create a custom user interface for controlling the simulation

    integer, public :: minRealIndex (3), maxRealIndex (3)
    integer, public :: minGhosts (3), maxGhosts (3)

    real, dimension (:, :, :, :), allocatable, target :: visit_vardata_vec3df_mem
    real, dimension (:, :, :, :), pointer, public :: visit_vardata_vec3df

    real, dimension (:, :, :), allocatable, target :: visit_vardata_sca3df_mem
    real, dimension (:, :, :), pointer, public :: visit_vardata_sca3df

    integer, public :: sim_iloop = -1
    double precision, public :: sim_time = -1.d0

    integer :: sim_iloop_last = -1
    integer :: visit_cmd_serie = 0

    public :: init_visit, visit_checkstatus
    public :: visit_varupdate_sca3df, visit_varupdate_vec3df
    public :: visit_getdata_sca3df, visit_getdata_vec3df, visit_metadata_vec3d

contains

  ! --------------------------------------
  !
  !  initialize VisIt plugin
  !
  ! --------------------------------------
    subroutine init_visit (ierr)
      use psopen_data
      implicit none

    ! function args
      integer :: ierr

    ! other vars
      integer :: env_len_in, env_len = 1024
      character (len=1), dimension(:), allocatable :: env
      character (len=1024) :: visit_tracefile_wid

      ierr = 0

      vis_mpi_proc_id = mpi_proc_id
      vis_mpi_nproc   = mpi_nproc
      vis_mpi_comm    = mpi_mycomm

      ! check if memory arrays are enabled in psopen_data.F90
      if(.not. associated(cwork10)) then; PRTERR1('work10 not enabled in psopen_data_F90'); end if
      if(.not. associated(cwork11)) then; PRTERR1('work11 not enabled in psopen_data_F90'); end if
      if(.not. associated(cwork12)) then; PRTERR1('work12 not enabled in psopen_data_F90'); end if

      if(.not. associated(cwork20)) then; PRTERR1('work20 not enabled in psopen_data_F90'); end if
      if(.not. associated(cwork21)) then; PRTERR1('work21 not enabled in psopen_data_F90'); end if
      if(.not. associated(cwork22)) then; PRTERR1('work22 not enabled in psopen_data_F90'); end if

      if(.not. associated(cwork30)) then; PRTERR1('work30 not enabled in psopen_data_F90'); end if
      if(.not. associated(cwork31)) then; PRTERR1('work31 not enabled in psopen_data_F90'); end if
      if(.not. associated(cwork32)) then; PRTERR1('work32 not enabled in psopen_data_F90'); end if

      if(.not. associated(cwork100)) then; PRTERR1('work100 not enabled in psopen_data_F90'); end if

    ! tell libsim whether the simulation is parallel
      if (vis_mpi_nproc > 1) then

        ! By default, the Libsim runtime library will create a duplicate of the MPI_COMM_WORLD communicator unless you specify an alternate communicator.
        ! visitsetmpicommunicator() was introduced in VisIt 2.9.2, but does not work before VisIt 2.10.0
        !WRITE_RANK0_NOADVANCE('(a)', 'VisIt: set MPI communicator ...', 1)
        !ierr = visitsetmpicommunicator(vis_mpi_comm); CHKWARN1(ierr, VISIT_OKAY)
        !VISIT_SUCCESS_OR_FAILURE(ierr)

        WRITE_RANK0_NOADVANCE('(a)', 'VisIt: set parallel mode ...', 1 )
        ierr = visItSetParallel(1); CHKWARN1(ierr, VISIT_OKAY)
             VISIT_SUCCESS_OR_FAILURE(ierr)
      end if

      WRITE_RANK0_NOADVANCE('(a)', 'VisIt: set parallel rank ...', 1 )
      ierr = visItSetParallelRank(vis_mpi_proc_id); CHKWARN1(ierr, VISIT_OKAY)
           VISIT_SUCCESS_OR_FAILURE(ierr)

    ! dump trace of visit function calls for better debugging
      if (len_trim(visit_tracefile) > 0) then
        WRITE_RANK0_NOADVANCE('(a)', 'VisIt: open trace file ...', 1 )
        if(visit_tracefile_onlyrank0) then
          if (vis_mpi_proc_id == 0) then
            ierr = visItOpenTraceFile(visit_tracefile,len_trim(visit_tracefile)); CHKWARN1(ierr, VISIT_OKAY)
                 VISIT_SUCCESS_OR_FAILURE(ierr)
          end if
        else
          write(visit_tracefile_wid, fmt='(a,a,i9.9)') trim(visit_tracefile), '_', vis_mpi_proc_id
          ierr = visItOpenTraceFile(visit_tracefile_wid,len_trim(visit_tracefile_wid)); CHKWARN1(ierr, VISIT_OKAY)
               VISIT_SUCCESS_OR_FAILURE(ierr)
        end if
      end if

    ! set visits binary directory
      if (len_trim(visit_path) > 0) then
        WRITE_RANK0_NOADVANCE('(a)', 'VisIt: set directory ...', 1 )
        ierr = visItSetDirectory(visit_path,len_trim(visit_path)); CHKWARN1(ierr, VISIT_OKAY)
             VISIT_SUCCESS_OR_FAILURE(ierr)
      end if

!    ! set port on which visit is listening
!      if (visit_port > 0) then
!        visit_args
!      end if

    ! parse command line arguments to visit
      if (len_trim(visit_args) > 0) then
        WRITE_RANK0_NOADVANCE('(a)', 'VisIt: set options ...', 1 )
        ierr = visItSetOptions(visit_args,len_trim(visit_args)); CHKWARN1(ierr, VISIT_OKAY)
             VISIT_SUCCESS_OR_FAILURE(ierr)
      end if

    ! prepare visit environment for loading plugins
      WRITE_RANK0_NOADVANCE('(a)', 'VisIt: setup enviroment ...', 1 )

    !---- best way to setup environment if VisIt >2.10.0 installed (or an older,but patched version)
      ! get VisIt environment from rank 0 ...
      ! (read the output of command "visit -env -engine")
      ! ATTENTION: this needs VisIt 2.10.1 at minimum (or visitgetenv() in visitfortransimV2interface.c to be patched)
      if (vis_mpi_proc_id == 0) then
        env_len_in = env_len+1
        ! loop until string is large enough to hold all characters
        do while (env_len+1 == env_len_in) ! see comment at visItGetEnv(..) for env_len+1
          env_len = env_len *2
          env_len_in = env_len
          !WRITE_RANK0_ARGS2('(a,i6)', 'env_len_in: ', env_len_in, 1)
          if(allocated(env)) deallocate(env)
          allocate(character(len=1) :: env(env_len), stat=ierr); CHKERRQ0 (ierr)
          ! if max. characters of env are too small to hold full string, visItGetEnv() returns env_len, which is max. characters of env minus 1 (for "\0")
          ierr = visItGetEnv(env,env_len); CHKWARN1(ierr, VISIT_OKAY)
          !WRITE_RANK0_ARGS2('(a,i6)', 'env_len_out: ', env_len, 1)
        end do
        VISIT_SUCCESS_OR_FAILURE(ierr)
      end if
      call MPI_BCAST(env_len, 1, MPI_INTEGER, 0, vis_mpi_comm, ierr); CHKERRQ1(ierr, MPI_SUCCESS)
      if(vis_mpi_proc_id /= 0) then
        allocate(character(len=1) :: env(env_len), stat=ierr); CHKERRQ0 (ierr)
      end if
      ! ... and pass the VisIt environment to all other processors collectively
      ierr = visItSetupEnv2(env,env_len); CHKWARN1(ierr, VISIT_OKAY)
           VISIT_SUCCESS_OR_FAILURE(ierr)
      deallocate(env)

!    !---- else if VisIt <=2.10.0 installed
!      ierr = visItSetupEnv(); CHKWARN1(ierr, VISIT_OKAY)
!           VISIT_SUCCESS_OR_FAILURE(ierr)

    ! initialize libsim library and write .sim2 file to visit_simpath
      if (vis_mpi_proc_id == 0) then
        WRITE_RANK0_NOADVANCE('(a)', 'VisIt: initialization ...', 1 )
        ierr = visItInitializeSim(visit_simbase,    len_trim(visit_simbase),    &
                                  visit_simcomment, len_trim(visit_simcomment), &
                                  visit_simpath,    len_trim(visit_simpath),    &
                                  visit_siminput,   len_trim(visit_siminput),   &
                                  visit_simgui,     len_trim(visit_simgui),     &
                                  VISIT_F77NULLSTRING, VISIT_F77NULLSTRINGLEN); CHKWARN1(ierr, VISIT_OKAY)
             VISIT_SUCCESS_OR_FAILURE(ierr)
      end if

    ! calc subdomain size of this mpi process
    ! (values stored at x)
    !
    !  0   1   2   3   4   5   6   7   8   9   10
    !  |~x~:~x~|-x-:-x-:-x-:-x-:-x-:-x-|-x-:-x-|
    !          ^                       ^
    !      minRealIndex            maxRealIndex

      ! default for no ghostcell
      minGhosts(:) = 0
      maxGhosts(:) = 0
      minRealIndex(:) = 0
      maxRealIndex(1) = isize(1)
      maxRealIndex(2) = isize(2)
      maxRealIndex(3) = isize(3)

      ! set ghostcells (except on mesh-borders)
      if(istart(1) /= 1) then
        minGhosts(1) = gsize
        minRealIndex(1) = minGhosts(1)
        maxRealIndex(1) = minGhosts(1) +isize(1)
      end if
      if(istart(2) /= 1) then
        minGhosts(2) = gsize
        minRealIndex(2) = minGhosts(2)
        maxRealIndex(2) = minGhosts(2) +isize(2)
      end if
      if(istart(3) /= 1) then
        minGhosts(3) = gsize
        minRealIndex(3) = minGhosts(3)
        maxRealIndex(3) = minGhosts(3) +isize(3)
      end if
      if(iend(1)  /= n1) maxGhosts(1) = gsize
      if(iend(2)  /= n2) maxGhosts(2) = gsize
      if(iend(3)  /= n3) maxGhosts(3) = gsize

    ! allocate memory for variables
      allocate(visit_vardata_vec3df_mem( 3, 0:(maxRealIndex(1)+maxGhosts(1)), &
                                            0:(maxRealIndex(2)+maxGhosts(2)), &
                                            0:(maxRealIndex(3)+maxGhosts(3))), stat=ierr); CHKERRQ0(ierr)
      visit_vardata_vec3df => visit_vardata_vec3df_mem

      allocate(visit_vardata_sca3df_mem(  0:(maxRealIndex(1)+maxGhosts(1)), &
                                          0:(maxRealIndex(2)+maxGhosts(2)), &
                                          0:(maxRealIndex(3)+maxGhosts(3))), stat=ierr); CHKERRQ0(ierr)
      visit_vardata_sca3df => visit_vardata_sca3df_mem

    end subroutine init_visit

  ! --------------------------------------
  !
  !  check connection to VisIT server
  !
  ! --------------------------------------
    recursive integer function visit_checkstatus(sim_iloop_in, sim_time_in)
      implicit none
    ! function args
      integer, intent(in) :: sim_iloop_in
      double precision, intent(in) :: sim_time_in

    ! other vars
      integer :: ierr
      integer :: blocking_call
      integer :: visitState
      integer :: visitResult

      ierr = 0
      visit_checkstatus = 0

  ! -- QUICK RETURN, IF NOTHING TO DO --------------------------------------

    ! do we want to check for new status now?
    ! - if simulation is doing nothing, we always check for new status
    ! - if visit_checkstatus() is called recursively, we always check for new status
    ! - if simulation is doing something, we check with the frequency of 'conn_freq'
      if(sim_iloop_in > sim_iloop_last .and. &
         sim_iloop_in < sim_iloop_last +conn_freq) then
          return
      end if
      sim_iloop = sim_iloop_in
      sim_time  = sim_time_in

    ! if single step than switch to pause
      if (sim_mode == SIMMODE_STEP) then
        sim_mode = SIMMODE_PAUSE
        return ! go back to solver once
      end if

  ! -- UPDATE PLOTS, IF REQUIRED --------------------------------------------

    ! update time and plots
      if( (vis_mode /= VISMODE_DISCONNECTED) .and. &  ! if we are connected
          (vis_mode /= VISMODE_UPDATE_NEVER) .and. &  ! if update enabled
          (sim_iloop /= sim_iloop_last) ) then        ! if new iteration reached
        if( (vis_mode == VISMODE_UPDATE_FAST) .or. &  ! if always update
            (vis_mode == VISMODE_UPDATE_ONCE) .or. &  ! if one single update requested
            (vis_mode == VISMODE_UPDATE_SYNC) ) then  ! if sync mode
          ierr = visItTimestepChanged(); CHKERRQ1(ierr,VISIT_OKAY)
          ierr = visItUpdatePlots(); CHKERRQ1(ierr,VISIT_OKAY) ! get meta data ...
        end if
        if(vis_mode == VISMODE_UPDATE_ONCE) then
          vis_mode = VISMODE_UPDATE_NEVER ! go back to 'update never'
        end if
      end if
      sim_iloop_last = sim_iloop

  ! -- CHECK FOR NEW COMMANDS, IF REQUIRED ------------------------------------

    ! detect input from proc 0 and broadcast that input to all others
      if (vis_mpi_proc_id == 0) then
        blocking_call = 0

      ! sync simulation and visualization update ?
        if (vis_mode == VISMODE_UPDATE_SYNC) then
        ! wait for new commands only if called from simulation loop directly
          if(visit_cmd_serie == 0) then
          ! only pause after a certain number of command series
            if(sync_count >= 0) then
              WRITE_RANK0('(a)', 'VisIt: wait for input/sync ...', 2)
              blocking_call = 1
            end if
          end if
      ! pause simulation ?
        else if (sim_mode == SIMMODE_PAUSE) then
          if(visit_cmd_serie == 0) then
            WRITE_RANK0('(a)', 'VisIt: wait for input ...', 2)
          end if
          blocking_call = 1
        end if

      ! check for new commands in queue
        visitState = VisItDetectInputWithTimeout(blocking_call, 0, -1); ! blocking?, no console input
      end if
      call MPI_BCAST(visitState, 1, MPI_INTEGER, 0, vis_mpi_comm, ierr); CHKERRQ1(ierr, MPI_SUCCESS)
      visit_cmd_serie = visit_cmd_serie +1

  ! -- PROCESS RETURNED VISIT STATE ------------------------------------------

    ! VisItDetectInput() returns with "no new input" = cmd queue empty
    ! (visitState == 0 is never reached in blocking mode!)
      if (visitState == 0) then
        WRITE_RANK0('(a)', 'VisIt: nothing to do', 5)
        visit_checkstatus = 0
        visit_cmd_serie = 0    ! command series has ended

        ! return to simulation, if number of command series has been reached
        if(sync_count <= 0) then
          sync_count = sync_mode ! reset sync_count
          WRITE_RANK0('(a)', 'VisIt: return to simulation', 5)
          return
        else
          sync_count = sync_count-1
        end if

    ! VisItDetectInput() returns with "VisIt is trying to connect" -> open connection/stop tstep
      else if(visitState == 1) then

        WRITE_RANK0_NOADVANCE('(a)', 'VisIt: try to connect to client ... ', 2)
        visit_checkstatus = 1

        visitResult = visitAttemptConnection()
        if (visitResult == VISIT_OKAY) then
          WRITE_RANK0('(a)', 'connected and listening', 2)
          vis_mode = VISMODE_UPDATE_FAST
          conn_freq= 1
        else
          WRITE_RANK0('(a)', 'connection attempt FAILED', 2)
          vis_mode = VISMODE_DISCONNECTED
        end if

    ! VisItDetectInput() returns with "VisIt wants to tell the engine something"
      else if (visitState == 2) then

        if(visit_cmd_serie == 1) then
          WRITE_RANK0('(a)', 'VisIt: process command', 2)
        else
          WRITE_RANK0_ARGS2('(a,i6)', 'VisIt process command ', visit_cmd_serie, 5)
        end if
        visit_checkstatus = 2

        ! process VisIt command
        visitResult = visit_processCommand ()

        ! disconnect on an error or closed connection
        if (visitResult /= VISIT_OKAY) then

           visitResult = VisItDisconnect ()
           if (visitResult == VISIT_OKAY) then
             WRITE_RANK0('(a)', 'disconnected.', 2)
           else
             WRITE_RANK0('(a)', 'disconnect FAILED!', 2)
           end if

          sim_mode = SIMMODE_RUN
          vis_mode = VISMODE_DISCONNECTED
          conn_freq= 1
        end if

    ! VisItDetectInput() returns with "error" -> cleanup/disconnect
      else if (visitState < 0) then

        WRITE_RANK0_NOADVANCE('(a)', 'VisIt: cannot recover from error ... ', 2)
        visit_checkstatus = -1

        visitResult = VisItDisconnect()
        if (visitResult == VISIT_OKAY) then
          WRITE_RANK0('(a)', 'disconnected.', 2)
        else
          WRITE_RANK0('(a)', 'disconnect FAILED!', 2)
        end if

        vis_mode = VISMODE_DISCONNECTED
        sim_mode = SIMMODE_RUN

      end if

      ! check for other waiting cmds until all processed
      visit_checkstatus = visit_checkstatus(sim_iloop, sim_time)
      return

    end function visit_checkstatus

 ! --------------------------------------
 !
 !  check connection to VisIT server
 !
 ! --------------------------------------
    function visit_processCommand ()
      implicit none

   ! function args
      integer :: visit_processCommand

   ! other vars
      integer :: ierr, ret, success, command

      visit_processCommand = VISIT_ERROR

      if (vis_mpi_proc_id == 0) then

     ! get commands from visit client and broadcast them
     ! to other mpi_procs using visitSlaveProcessCallback()
        success = visitProcessEngineCommand ()
        if (success == VISIT_OKAY) then
          command = VISIT_COMMAND_SUCCESS
          ret = VISIT_OKAY
        else
          command = VISIT_COMMAND_FAILURE
          ret = VISIT_ERROR
        end if

     ! broadcast success or failure
     ! info: the corresponding broadcast for "process"
     !       is deep inside of visitProcessEngineCommand()
        call MPI_BCAST (command, 1, MPI_INTEGER, 0, vis_mpi_comm, ierr); CHKERRQ1(ierr, MPI_SUCCESS)

      else

        command = VISIT_COMMAND_PROCESS
        do while (command == VISIT_COMMAND_PROCESS)
          call MPI_BCAST (command, 1, MPI_INTEGER, 0, vis_mpi_comm, ierr); CHKERRQ1(ierr, MPI_SUCCESS)

          if (command == VISIT_COMMAND_PROCESS) then
            success = visitProcessEngineCommand ()
          else if (command == VISIT_COMMAND_SUCCESS) then
            ret = VISIT_OKAY
          else ! command == VISIT_COMMAND_FAILURE
            ret = VISIT_ERROR
          end if
        end do

      end if

      visit_processCommand = ret
    end function visit_processCommand

 ! --------------------------------------
 !
 !  update variables as float/real(4) (write from sim-memory to visit-memory)
 !
 ! --------------------------------------
    subroutine visit_varupdate_sca3df (simmem, visitmem, ghostmem)
      use psopen_data
#ifdef USE_P3DFFT
        use p3dfft_ghosts
#endif
#ifdef USE_NB3DFFT
        use nb3dfft_ghosts
#endif
      implicit none

   ! function vars
      real(kind=MP), dimension (:, :, :), pointer, intent (in) :: simmem
      real, dimension (:, :, :), pointer, intent (out) :: visitmem
      real(kind=MP), dimension (:, :, :), pointer :: ghostmem

   ! other vars
      integer :: i, j, k
      double precision :: ghost_time
      integer (kind=8) :: memsize1d

      memsize1d = int(msize(1)+2*gsize,8) *int(msize(2)+2*gsize,8) *int(msize(3)+2*gsize,8)

      if(gsize>0) then
        WRITE_RANK0('(a)', 'VisIt: update scalar data (incl. ghostcells)', 5)
        ! TODO: do a direct copy from simmem first and then
        !       copy only ghostcell related data to ghostmem and
        !       loop only over these ghostcells afterwards
        ghostmem (:, :, :) = simmem (:, :, :)
        call ghosts_update (ghostmem, 3, ghost_time)
        do k = 0, maxRealIndex (3) + maxGhosts (3)
          do j = 0, maxRealIndex (2) + maxGhosts (2)
            do i = 0, maxRealIndex (1) + maxGhosts (1)
!             visitmem(i,j,k) = ghosts_ijk_debug(ghostmem, istart(1)-minGhosts(1)+i, &
!                                                         istart(2)-minGhosts(2)+j, &
!                                                         istart(3)-minGhosts(3)+k, memsize1d)
              visitmem(i,j,k) = ghosts_ijk(ghostmem, istart(1)-minGhosts(1)+i, &
                                                   istart(2)-minGhosts(2)+j, &
                                                   istart(3)-minGhosts(3)+k)
            end do
          end do
        end do
      else
        WRITE_RANK0('(a)', 'VisIt: update scalar data', 5)
        do k = 0, maxRealIndex(3)
          do j = 0, maxRealIndex(2)
            do i = 0, maxRealIndex(1)
              visitmem(i,j,k) = simmem( istart(1) +i, &
                                        istart(2) +j, &
                                        istart(3) +k)
            end do
          end do
        end do
      end if

   end subroutine visit_varupdate_sca3df

 ! --------------------------------------
 !
 !  update variables as float/real(4) (write from sim-memory to visit-memory)
 !
 ! --------------------------------------
    subroutine visit_varupdate_vec3df (simmem1, simmem2, simmem3, visitmem, ghostmem)
      use psopen_data
#ifdef USE_P3DFFT
        use p3dfft_ghosts
#endif
#ifdef USE_NB3DFFT
        use nb3dfft_ghosts
#endif
      implicit none

   ! function vars
      real(kind=MP), dimension (:, :, :), pointer, intent (in) :: simmem1, simmem2, simmem3
      real, dimension (:, :, :, :), pointer, intent (out) :: visitmem
      real(kind=MP), dimension (:, :, :), pointer :: ghostmem

   ! other vars
      integer :: i, j, k
      double precision :: ghost_time
      integer (kind=8) :: memsize1d

      memsize1d = int(msize(1)+2*gsize,8) *int(msize(2)+2*gsize,8) *int(msize(3)+2*gsize,8)

      WRITE_RANK0('(a)', 'VisIt: update vector data', 5)

      if(gsize > 0) then
        ! TODO: do a direct copy from simmem first and then
        !       copy only ghostcell related data to ghostmem and
        !       loop only over these ghostcells afterwards
        ghostmem (:, :, :) = simmem1 (:, :, :)
        call ghosts_update (ghostmem, 3, ghost_time)
        do k = 0, maxRealIndex (3) + maxGhosts (3)
          do j = 0, maxRealIndex (2) + maxGhosts (2)
            do i = 0, maxRealIndex (1) + maxGhosts (1)
              visitmem(1,i,j,k) = ghosts_ijk(ghostmem, istart(1)-minGhosts(1)+i, &
                                                       istart(2)-minGhosts(2)+j, &
                                                       istart(3)-minGhosts(3)+k)

!              visitmem(1,i,j,k) = ghosts_ijk_debug(ghostmem, istart(1)-minGhosts(1)+i, &
!                                                             istart(2)-minGhosts(2)+j, &
!                                                             istart(3)-minGhosts(3)+k, memsize1d)
!              visitmem(1,i,j,k) = istart(3)-minGhosts(3)+k
!              visitmem(1,i,j,k) = 0.0
!              if( (istart(1)-minGhosts(1)+i) == 3 .and. &
!                  (istart(2)-minGhosts(2)+j) == 1 .and. &
!                  (istart(3)-minGhosts(3)+k) == 1) then
!                visitmem(1,i,j,k) = 1.0
!              end if
!              if(visitmem(1,i,j,k) < -1.0 .or. visitmem(1,i,j,k) > 1.0.or. visitmem(1,i,j,k) /= visitmem(1,i,j,k)) then
!                WRITE_RANK0_ARGS6('(a,i6,f,3(i6))', WRITE_ARGS9('VALUE 1-ERROR: ',vis_mpi_proc_id, visitmem(2,i,j,k), i,j,k), 5)
!              endif
            end do
          end do
        end do
      else
        do k = 0, maxRealIndex(3)
          do j = 0, maxRealIndex(2)
            do i = 0, maxRealIndex(1)
              visitmem(1,i,j,k) = simmem1( istart(1) +i, &
                                           istart(2) +j, &
                                           istart(3) +k)
            end do
          end do
        end do
      end if

      if(gsize > 0) then
        ! TODO: do a direct copy from simmem first and then
        !       copy only ghostcell related data to ghostmem and
        !       loop only over these ghostcells afterwards
        ghostmem (:, :, :) = simmem2 (:, :, :)
        call ghosts_update (ghostmem, 3, ghost_time)
        do k = 0, maxRealIndex (3) + maxGhosts (3)
          do j = 0, maxRealIndex (2) + maxGhosts (2)
            do i = 0, maxRealIndex (1) + maxGhosts (1)
              visitmem(2,i,j,k) = ghosts_ijk(ghostmem, istart(1)-minGhosts(1)+i, &
                                                       istart(2)-minGhosts(2)+j, &
                                                       istart(3)-minGhosts(3)+k)

!              visitmem(2,i,j,k) = ghosts_ijk_debug(ghostmem, istart(1)-minGhosts(1)+i, &
!                                                             istart(2)-minGhosts(2)+j, &
!                                                             istart(3)-minGhosts(3)+k, memsize1d)
!              visitmem(2,i,j,k) = istart(3)-minGhosts(3)+k
!              visitmem(2,i,j,k) = 0.0
!              if( (istart(1)-minGhosts(1)+i) == 1 .and. &
!                  (istart(2)-minGhosts(2)+j) == 5 .and. &
!                  (istart(3)-minGhosts(3)+k) == 1) then
!                visitmem(2,i,j,k) = 1.0
!              end if
!              if(visitmem(2,i,j,k) < -1.0 .or. visitmem(2,i,j,k) > 1.0 .or. visitmem(2,i,j,k) /= visitmem(2,i,j,k)) then
!                WRITE_RANK0_ARGS6('(a,i6,f,3(i6))', WRITE_ARGS9('VALUE 2-ERROR: ',vis_mpi_proc_id, visitmem(2,i,j,k), i,j,k), 5)
!              end if
            end do
          end do
        end do
      else
        do k = 0, maxRealIndex(3)
          do j = 0, maxRealIndex(2)
            do i = 0, maxRealIndex(1)
              visitmem(2,i,j,k) = simmem2( istart(1) +i, &
                                           istart(2) +j, &
                                           istart(3) +k)
            end do
          end do
        end do
      end if

      if(gsize > 0) then
        ! TODO: do a direct copy from simmem first and then
        !       copy only ghostcell related data to ghostmem and
        !       loop only over these ghostcells afterwards
        ghostmem (:, :, :) = simmem3 (:, :, :)
        call ghosts_update (ghostmem, 3, ghost_time)
        do k = 0, maxRealIndex (3) + maxGhosts (3)
          do j = 0, maxRealIndex (2) + maxGhosts (2)
            do i = 0, maxRealIndex (1) + maxGhosts (1)
              visitmem(3,i,j,k) = ghosts_ijk(ghostmem, istart(1)-minGhosts(1)+i, &
                                                       istart(2)-minGhosts(2)+j, &
                                                       istart(3)-minGhosts(3)+k)
!              visitmem(3,i,j,k) = ghosts_ijk_debug(ghostmem, istart(1)-minGhosts(1)+i, &
!                                                             istart(2)-minGhosts(2)+j, &
!                                                             istart(3)-minGhosts(3)+k, memsize1d)
!              visitmem(3,i,j,k) = istart(3)-minGhosts(3)+k
!              visitmem(3,i,j,k) = 0.0
!              if( (istart(1)-minGhosts(1)+i) == 1 .and. &
!                  (istart(2)-minGhosts(2)+j) == 1 .and. &
!                  (istart(3)-minGhosts(3)+k) == 8) then
!                visitmem(3,i,j,k) = 1.0
!              end if
!              if(visitmem(3,i,j,k) < -1.0 .or. visitmem(3,i,j,k) > 1.0.or. visitmem(3,i,j,k) /= visitmem(3,i,j,k)) then
!                WRITE_RANK0_ARGS6('(a,i6,f,3(i6))', WRITE_ARGS9('VALUE 3-ERROR: ',vis_mpi_proc_id, visitmem(3,i,j,k), i,j,k), 5)
!              end if
            end do
          end do
        end do
      else
        do k = 0, maxRealIndex(3)
          do j = 0, maxRealIndex(2)
            do i = 0, maxRealIndex(1)
              visitmem(3,i,j,k) = simmem3( istart(1) +i, &
                                           istart(2) +j, &
                                           istart(3) +k)
            end do
          end do
        end do
      end if

    end subroutine visit_varupdate_vec3df

   ! --------------------------------------
   !
   !  add mesh variables to meta data
   !
   ! --------------------------------------
   subroutine visit_metadata_vec3d(md, mname, vname, ierr)
      implicit none

    ! function vars
      integer, intent(inout) :: md
      character(len=*) :: mname, vname
      integer, intent(out) :: ierr

      ! other vars
        integer :: vmd
        character(len=100) :: meshname, vecname

      ! to pass it to C/VisIt, we have to copy character to stack, that somehow fixes some problems
        meshname = repeat(' ',len(meshname))
        vecname = repeat(' ',len(vecname))
        meshname = trim(mname)
        vecname = trim(vname)

      ierr = 1

        !meshname = 'mesh3d'
        !vecname = 'velocity'
        !WRITE_RANK0_ARGS3('(a,a,i6)', 'meshn : .',trim(meshname),'.', len_trim(meshname), 5)
        !WRITE_RANK0_ARGS3('(a,a,i6)', 'vecna : .', trim(vecname),'.', len_trim(vecname), 5)
        if (visitmdvaralloc(vmd) == VISIT_OKAY) then
          ierr = visitmdVarSetName(vmd, vecname, len_trim(vecname)); CHKERRQ1(ierr,VISIT_OKAY)
          ierr = visitmdVarSetMeshName(vmd, meshname, len_trim(meshname)); CHKERRQ1(ierr,VISIT_OKAY)
          ierr = visitmdVarSetCentering(vmd, VISIT_VARCENTERING_NODE); CHKERRQ1(ierr,VISIT_OKAY)
          ierr = visitmdVarSetType(vmd, VISIT_VARTYPE_VECTOR); CHKERRQ1(ierr,VISIT_OKAY)

          ierr = visitmdSimAddVariable(md, vmd); CHKERRQ1(ierr,VISIT_OKAY)

          ierr = 0
        end if

  end subroutine visit_metadata_vec3d

     ! --------------------------------------
     !
     !  add mesh scalar-variable to meta data
     !
     ! --------------------------------------
     integer function visit_getdata_sca3df(meshname, vecname, simmem, ghostmem, ierr)
        use psopen_data ! using MP
        implicit none

      ! function vars
        character(len=*) :: meshname, vecname
        real(kind=MP), dimension (:, :, :), pointer, intent (in) :: simmem
        real(kind=MP), dimension (:, :, :), pointer :: ghostmem
        integer, intent(out) :: ierr

      ! function
        integer :: nTuples

        ierr = 1
        visit_getdata_sca3df = VISIT_INVALID_HANDLE

        if (visitvardataalloc(visit_getdata_sca3df) == VISIT_OKAY) then
            WRITE_RANK0('(a)', 'VisIt: get scalar data', 5)

            call visit_varupdate_sca3df (simmem, visit_vardata_sca3df, ghostmem)

          ! set variable velocity with 3 doubles for each node
            nTuples = size (visit_vardata_sca3df, 1) * size (visit_vardata_sca3df, 2) * size (visit_vardata_sca3df, 3)
            ierr = visitvardatasetf (visit_getdata_sca3df, VISIT_OWNER_COPY, 1, nTuples, visit_vardata_sca3df); CHKERRQ1(ierr,VISIT_OKAY)

            ierr = 0

        else
          visit_getdata_sca3df = VISIT_INVALID_HANDLE
          WRITE_RANK0('(a)', 'VisIt: get scalar data FAILED - visitVarDataAlloc() failed', 5)
        end if

      end function visit_getdata_sca3df

     ! --------------------------------------
     !
     !  add mesh vector-variables to meta data
     !
     ! --------------------------------------
     integer function visit_getdata_vec3df(meshname, vecname, simmem1, simmem2, simmem3, ghostmem, ierr)
        use psopen_data ! using MP
        implicit none

      ! function vars
        character(len=*) :: meshname, vecname
        real(kind=MP), dimension (:, :, :), pointer, intent (in) :: simmem1, simmem2, simmem3
         real(kind=MP), dimension (:, :, :), pointer :: ghostmem
        integer, intent(out) :: ierr

      ! function
        integer :: nTuples, nComponents

        ierr = 1
        visit_getdata_vec3df = VISIT_INVALID_HANDLE

      if (visitvardataalloc(visit_getdata_vec3df) == VISIT_OKAY) then
          WRITE_RANK0('(a)', 'VisIt: get vector data', 5)

          call visit_varupdate_vec3df (simmem1, simmem2, simmem3, visit_vardata_vec3df, ghostmem)

          ! set variable velocity with 3 doubles for each node
          nTuples = size (visit_vardata_vec3df, 2) * size (visit_vardata_vec3df, 3) * size (visit_vardata_vec3df, 4)
          nComponents = size (visit_vardata_vec3df, 1)
          ierr = visitvardatasetf (visit_getdata_vec3df, VISIT_OWNER_COPY, nComponents, nTuples, visit_vardata_vec3df); CHKERRQ1(ierr,VISIT_OKAY)

          ierr = 0

      else
        visit_getdata_vec3df = VISIT_INVALID_HANDLE
          WRITE_RANK0('(a)', 'VisIt: get vector data FAILED - visitVarDataAlloc() failed', 5)
      end if

      end function visit_getdata_vec3df

end module psopen_visit

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
!
! These functions must be defined to satisfy the visitfortransimV2interface lib.
!
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

!--------------------------------------------------------------------
! visitgetmetadata()
!--------------------------------------------------------------------
integer function visitgetmetadata ()
    use psopen_visit
    implicit none
    include "visitfortransimV2interface.inc"

  ! other vars
    integer :: md, m1, emd, cmd, ierr

    character(len=100) :: meshname, vecname, scaname
    integer :: vmd

    visitgetmetadata = VISIT_INVALID_HANDLE

    WRITE_RANK0('(a)', 'VisIt: get metadata ', 4)

  ! set meta data for simulation
    if (visitmdsimalloc(md) == VISIT_OKAY) then

      ierr = visitmdSimSetCycleTime (md, sim_iloop, sim_time)
      if (sim_mode == SIMMODE_RUN) then
        ierr = visitmdsimsetmode(md, VISIT_SIMMODE_RUNNING); CHKERRQ1(ierr,VISIT_OKAY)
      else
        ierr = visitmdsimsetmode(md, VISIT_SIMMODE_STOPPED); CHKERRQ1(ierr,VISIT_OKAY)
      end if

    ! Add meshes to meta data
      if (visitmdmeshalloc(m1) == VISIT_OKAY) then
        ierr = visitmdMeshSetName(m1, "mesh3d", 6); CHKERRQ1(ierr,VISIT_OKAY)
        ierr = visitmdMeshSetMeshType(m1,VISIT_MESHTYPE_RECTILINEAR); CHKERRQ1(ierr,VISIT_OKAY)
        ierr = visitmdMeshSetTopologicalDim(m1, 3); CHKERRQ1(ierr,VISIT_OKAY)
        ierr = visitmdMeshSetSpatialDim(m1, 3); CHKERRQ1(ierr,VISIT_OKAY)
        ierr = visitmdMeshSetNumDomains(m1, vis_mpi_nproc); CHKERRQ1(ierr,VISIT_OKAY)

        !ierr = visitmdMeshSetXUnits(m1, 'pt.', 3); CHKERRQ1(ierr,VISIT_OKAY)
        !ierr = visitmdMeshSetYUnits(m1, 'pt.', 3); CHKERRQ1(ierr,VISIT_OKAY)
        !ierr = visitmdMeshSetZUnits(m1, 'pt.', 3); CHKERRQ1(ierr,VISIT_OKAY)

        ierr = visitmdMeshSetXLabel(m1, 'X', 1); CHKERRQ1(ierr,VISIT_OKAY)
        ierr = visitmdMeshSetYLabel(m1, 'Y', 1); CHKERRQ1(ierr,VISIT_OKAY)
        ierr = visitmdMeshSetZLabel(m1, 'Z', 1); CHKERRQ1(ierr,VISIT_OKAY)

        ierr = visitmdsimaddmesh (md, m1)
      end if

    ! Add mesh variables to meta data
!      call visit_metadata_vec3d(md, 'velocity', 'mesh3d', ierr)
      meshname = 'mesh3d'
      vecname = 'velocity'
        if (visitmdvaralloc(vmd) == VISIT_OKAY) then
          ierr = visitmdVarSetName(vmd, trim(vecname), len_trim(vecname)); CHKERRQ1(ierr,VISIT_OKAY)
          ierr = visitmdVarSetMeshName(vmd, trim(meshname), len_trim(meshname)); CHKERRQ1(ierr,VISIT_OKAY)
          ierr = visitmdVarSetCentering(vmd, VISIT_VARCENTERING_NODE); CHKERRQ1(ierr,VISIT_OKAY)
          ierr = visitmdVarSetType(vmd, VISIT_VARTYPE_VECTOR); CHKERRQ1(ierr,VISIT_OKAY)

          ierr = visitmdSimAddVariable(md, vmd); CHKERRQ1(ierr,VISIT_OKAY)

          ierr = 0
        end if
    ! Add mesh variables to meta data
!      call visit_metadata_vec3d(md, 'velocity_fluc', 'mesh3d', ierr)
      meshname = 'mesh3d'
      vecname = 'velocity_fluc'
        if (visitmdvaralloc(vmd) == VISIT_OKAY) then
          ierr = visitmdVarSetName(vmd, trim(vecname), len_trim(vecname)); CHKERRQ1(ierr,VISIT_OKAY)
          ierr = visitmdVarSetMeshName(vmd, trim(meshname), len_trim(meshname)); CHKERRQ1(ierr,VISIT_OKAY)
          ierr = visitmdVarSetCentering(vmd, VISIT_VARCENTERING_NODE); CHKERRQ1(ierr,VISIT_OKAY)
          ierr = visitmdVarSetType(vmd, VISIT_VARTYPE_VECTOR); CHKERRQ1(ierr,VISIT_OKAY)

          ierr = visitmdSimAddVariable(md, vmd); CHKERRQ1(ierr,VISIT_OKAY)

          ierr = 0
        end if

    ! Add mesh variables to meta data
!      call visit_metadata_vec3d(md, 'vorticity','mesh3d', ierr)
      meshname = 'mesh3d'
      vecname = 'vorticity'
        if (visitmdvaralloc(vmd) == VISIT_OKAY) then
          ierr = visitmdVarSetName(vmd, trim(vecname), len_trim(vecname)); CHKERRQ1(ierr,VISIT_OKAY)
          ierr = visitmdVarSetMeshName(vmd, trim(meshname), len_trim(meshname)); CHKERRQ1(ierr,VISIT_OKAY)
          ierr = visitmdVarSetCentering(vmd, VISIT_VARCENTERING_NODE); CHKERRQ1(ierr,VISIT_OKAY)
          ierr = visitmdVarSetType(vmd, VISIT_VARTYPE_VECTOR); CHKERRQ1(ierr,VISIT_OKAY)

          ierr = visitmdSimAddVariable(md, vmd); CHKERRQ1(ierr,VISIT_OKAY)

          ierr = 0
        end if

    ! Add mesh variables to meta data
      !call visit_metadata_sca3d(md, 'kturb','mesh3d', ierr)
      meshname = 'mesh3d'
      scaname = 'k'
        if (visitmdvaralloc(vmd) == VISIT_OKAY) then
          ierr = visitmdVarSetName(vmd, trim(scaname), len_trim(scaname)); CHKERRQ1(ierr,VISIT_OKAY)
          ierr = visitmdVarSetMeshName(vmd, trim(meshname), len_trim(meshname)); CHKERRQ1(ierr,VISIT_OKAY)
          ierr = visitmdVarSetCentering(vmd, VISIT_VARCENTERING_NODE); CHKERRQ1(ierr,VISIT_OKAY)
          ierr = visitmdVarSetType(vmd, VISIT_VARTYPE_SCALAR); CHKERRQ1(ierr,VISIT_OKAY)

          ierr = visitmdSimAddVariable(md, vmd); CHKERRQ1(ierr,VISIT_OKAY)

          ierr = 0
        end if

    ! Add mesh variables to meta data
      !call visit_metadata_sca3d(md, 'kturb','mesh3d', ierr)
      meshname = 'mesh3d'
      scaname = 'k_fluc'
        if (visitmdvaralloc(vmd) == VISIT_OKAY) then
          ierr = visitmdVarSetName(vmd, trim(scaname), len_trim(scaname)); CHKERRQ1(ierr,VISIT_OKAY)
          ierr = visitmdVarSetMeshName(vmd, trim(meshname), len_trim(meshname)); CHKERRQ1(ierr,VISIT_OKAY)
          ierr = visitmdVarSetCentering(vmd, VISIT_VARCENTERING_NODE); CHKERRQ1(ierr,VISIT_OKAY)
          ierr = visitmdVarSetType(vmd, VISIT_VARTYPE_SCALAR); CHKERRQ1(ierr,VISIT_OKAY)

          ierr = visitmdSimAddVariable(md, vmd); CHKERRQ1(ierr,VISIT_OKAY)

          ierr = 0
        end if

    ! Add mesh variables to meta data
      !call visit_metadata_sca3d(md, 'duds','mesh3d', ierr)
      meshname = 'mesh3d'
      scaname = 'duds'
        if (visitmdvaralloc(vmd) == VISIT_OKAY) then
          ierr = visitmdVarSetName(vmd, trim(scaname), len_trim(scaname)); CHKERRQ1(ierr,VISIT_OKAY)
          ierr = visitmdVarSetMeshName(vmd, trim(meshname), len_trim(meshname)); CHKERRQ1(ierr,VISIT_OKAY)
          ierr = visitmdVarSetCentering(vmd, VISIT_VARCENTERING_NODE); CHKERRQ1(ierr,VISIT_OKAY)
          ierr = visitmdVarSetType(vmd, VISIT_VARTYPE_SCALAR); CHKERRQ1(ierr,VISIT_OKAY)

          ierr = visitmdSimAddVariable(md, vmd); CHKERRQ1(ierr,VISIT_OKAY)

          ierr = 0
        end if

    ! Add mesh variables to meta data
      !call visit_metadata_sca3d(md, 'duds','mesh3d', ierr)
      meshname = 'mesh3d'
      scaname = 'duds_fluc'
        if (visitmdvaralloc(vmd) == VISIT_OKAY) then
          ierr = visitmdVarSetName(vmd, trim(scaname), len_trim(scaname)); CHKERRQ1(ierr,VISIT_OKAY)
          ierr = visitmdVarSetMeshName(vmd, trim(meshname), len_trim(meshname)); CHKERRQ1(ierr,VISIT_OKAY)
          ierr = visitmdVarSetCentering(vmd, VISIT_VARCENTERING_NODE); CHKERRQ1(ierr,VISIT_OKAY)
          ierr = visitmdVarSetType(vmd, VISIT_VARTYPE_SCALAR); CHKERRQ1(ierr,VISIT_OKAY)

          ierr = visitmdSimAddVariable(md, vmd); CHKERRQ1(ierr,VISIT_OKAY)

          ierr = 0
        end if

    ! Add mesh variables to meta data
      !call visit_metadata_sca3d(md, 'lambda2','mesh3d', ierr)
      meshname = 'mesh3d'
      scaname = 'lambda2'
        if (visitmdvaralloc(vmd) == VISIT_OKAY) then
          ierr = visitmdVarSetName(vmd, trim(scaname), len_trim(scaname)); CHKERRQ1(ierr,VISIT_OKAY)
          ierr = visitmdVarSetMeshName(vmd, trim(meshname), len_trim(meshname)); CHKERRQ1(ierr,VISIT_OKAY)
          ierr = visitmdVarSetCentering(vmd, VISIT_VARCENTERING_NODE); CHKERRQ1(ierr,VISIT_OKAY)
          ierr = visitmdVarSetType(vmd, VISIT_VARTYPE_SCALAR); CHKERRQ1(ierr,VISIT_OKAY)

          ierr = visitmdSimAddVariable(md, vmd); CHKERRQ1(ierr,VISIT_OKAY)

          ierr = 0
        end if

    ! Add expressions to meta data
      if (visitmdexpralloc(emd) == VISIT_OKAY) then
        ierr = visitmdExprSetName(emd, "nid", 3); CHKERRQ1(ierr,VISIT_OKAY)
        ierr = visitmdExprSetDefinition(emd, "nodeid(mesh3d)", 14); CHKERRQ1(ierr,VISIT_OKAY)
        ierr = visitmdExprSetType(emd, VISIT_VARTYPE_SCALAR); CHKERRQ1(ierr,VISIT_OKAY)

        ierr = visitmdSimAddExpression(md, emd); CHKERRQ1(ierr,VISIT_OKAY)
      end if

    ! Add simulation commands to meta data

      ! set simulation mode
      if (visitmdcmdalloc(cmd) == VISIT_OKAY) then
        ierr = visitmdcmdsetname(cmd, "sim. run", 8); CHKERRQ1(ierr,VISIT_OKAY)
        ierr = visitmdsimaddgenericcommand(md, cmd); CHKERRQ1(ierr,VISIT_OKAY)
      end if
      if (visitmdcmdalloc(cmd) == VISIT_OKAY) then
        ierr = visitmdcmdsetname(cmd, "sim. pause", 10); CHKERRQ1(ierr,VISIT_OKAY)
        ierr = visitmdsimaddgenericcommand(md, cmd); CHKERRQ1(ierr,VISIT_OKAY)
      end if
      if (visitmdcmdalloc(cmd) == VISIT_OKAY) then
        ierr = visitmdcmdsetname(cmd, "sim. step", 9); CHKERRQ1(ierr,VISIT_OKAY)
        ierr = visitmdsimaddgenericcommand(md, cmd); CHKERRQ1(ierr,VISIT_OKAY)
      end if

      ! set connection mode
      if (visitmdcmdalloc(cmd) == VISIT_OKAY) then
        ierr = visitmdcmdsetname(cmd, "conn. freq. 1", 13); CHKERRQ1(ierr,VISIT_OKAY)
        ierr = visitmdsimaddgenericcommand(md, cmd); CHKERRQ1(ierr,VISIT_OKAY)
      end if
      if (visitmdcmdalloc(cmd) == VISIT_OKAY) then
        ierr = visitmdcmdsetname(cmd, "conn. freq. 5", 13); CHKERRQ1(ierr,VISIT_OKAY)
        ierr = visitmdsimaddgenericcommand(md, cmd); CHKERRQ1(ierr,VISIT_OKAY)
      end if
      if (visitmdcmdalloc(cmd) == VISIT_OKAY) then
        ierr = visitmdcmdsetname(cmd, "conn. freq. 10", 14); CHKERRQ1(ierr,VISIT_OKAY)
        ierr = visitmdsimaddgenericcommand(md, cmd); CHKERRQ1(ierr,VISIT_OKAY)
      end if

      ! set visualization mode
      if (visitmdcmdalloc(cmd) == VISIT_OKAY) then
        ierr = visitmdcmdsetname(cmd, "update fast", 11); CHKERRQ1(ierr,VISIT_OKAY)
        ierr = visitmdsimaddgenericcommand(md, cmd); CHKERRQ1(ierr,VISIT_OKAY)
      end if
      if (visitmdcmdalloc(cmd) == VISIT_OKAY) then
        ierr = visitmdcmdsetname(cmd, "update never", 12); CHKERRQ1(ierr,VISIT_OKAY)
        ierr = visitmdsimaddgenericcommand(md, cmd); CHKERRQ1(ierr,VISIT_OKAY)
      end if
      if (visitmdcmdalloc(cmd) == VISIT_OKAY) then
        ierr = visitmdcmdsetname(cmd, "update once", 11); CHKERRQ1(ierr,VISIT_OKAY)
        ierr = visitmdsimaddgenericcommand(md, cmd); CHKERRQ1(ierr,VISIT_OKAY)
      end if

      if (visitmdcmdalloc(cmd) == VISIT_OKAY) then
        ierr = visitmdcmdsetname(cmd, "update sync 1", 13); CHKERRQ1(ierr,VISIT_OKAY)
        ierr = visitmdsimaddgenericcommand(md, cmd); CHKERRQ1(ierr,VISIT_OKAY)
      end if
      if (visitmdcmdalloc(cmd) == VISIT_OKAY) then
        ierr = visitmdcmdsetname(cmd, "update sync (movie)", 19); CHKERRQ1(ierr,VISIT_OKAY)
        ierr = visitmdsimaddgenericcommand(md, cmd); CHKERRQ1(ierr,VISIT_OKAY)
      end if
      if (visitmdcmdalloc(cmd) == VISIT_OKAY) then
        ierr = visitmdcmdsetname(cmd, "---", 3); CHKERRQ1(ierr,VISIT_OKAY)
        ierr = visitmdsimaddgenericcommand(md, cmd); CHKERRQ1(ierr,VISIT_OKAY)
      end if

      ! set special commands
      if (visitmdcmdalloc(cmd) == VISIT_OKAY) then
        ierr = visitmdcmdsetname(cmd, "dump", 4); CHKERRQ1(ierr,VISIT_OKAY)
        ierr = visitmdsimaddgenericcommand(md, cmd); CHKERRQ1(ierr,VISIT_OKAY)
      end if
      if (visitmdcmdalloc(cmd) == VISIT_OKAY) then
        ierr = visitmdcmdsetname(cmd, "stat", 4); CHKERRQ1(ierr,VISIT_OKAY)
        ierr = visitmdsimaddgenericcommand(md, cmd); CHKERRQ1(ierr,VISIT_OKAY)
      end if
    end if

    visitgetmetadata = md

end function visitgetmetadata

!-----------------------------------------------------------------
! visitcommandcallback
!-----------------------------------------------------------------
subroutine visitcommandcallback (cmd, lcmd, args, largs)
    use psopen_visit
    implicit none
    include "visitfortransimV2interface.inc"

  ! function args
    integer :: lcmd, largs
    character (len=1) :: cmd (lcmd)
    character (len=1) :: args (largs)

  ! other vars
    integer :: result

    WRITE_RANK0_NOADVANCE('(a)', 'VisIt: callback command ... ', 4)

  ! do not get confused if disconnection and command call overlaps
    if(vis_mode == VISMODE_DISCONNECTED) return

  ! Handle the commands that we define in visitgetmetadata.

    ! set simulation mode
    if (visitstrcmp(cmd, lcmd, "sim. run", 8) == 0) then
      WRITE_RANK0('(a)', 'sim run', 4)
      sim_mode = SIMMODE_RUN
      vis_mode = VISMODE_UPDATE_FAST
      !conn_freq= do not change
    else if (visitstrcmp(cmd, lcmd, "sim. pause", 10) == 0) then
      WRITE_RANK0('(a)', 'sim pause', 4)
      sim_mode = SIMMODE_PAUSE
      vis_mode = VISMODE_UPDATE_FAST
      !conn_freq= do not change
    else if (visitstrcmp(cmd, lcmd, "sim. step", 9) == 0) then
      WRITE_RANK0('(a)', 'sim step', 4)
      sim_mode = SIMMODE_STEP
      vis_mode = VISMODE_UPDATE_FAST
      !conn_freq= do not change

    ! set connection frequency
    else if (visitstrcmp(cmd, lcmd, "conn. freq. 1", 13) == 0) then
      WRITE_RANK0('(a)', 'conn. freq. 1', 4)
      !sim_mode = do not change
      !vis_mode = do not change
      conn_freq = 1
    else if (visitstrcmp(cmd, lcmd, "conn. freq. 5", 13) == 0) then
      WRITE_RANK0('(a)', 'conn. freq. 5', 4)
      !sim_mode = do not change
      !vis_mode = do not change
      conn_freq = 5
    else if (visitstrcmp(cmd, lcmd, "conn. freq. 10", 14) == 0) then
      WRITE_RANK0('(a)', 'conn. freq. 10', 4)
      !sim_mode = do not change
      !vis_mode = do not change
      conn_freq = 10

    ! set visualization mode
    else if (visitstrcmp(cmd, lcmd, "update fast", 11) == 0) then
      WRITE_RANK0('(a)', 'update fast', 4)
      !sim_mode = do not change
      vis_mode = VISMODE_UPDATE_FAST
      !conn_freq = do not change
    else if (visitstrcmp(cmd, lcmd, "update never", 12) == 0) then
      WRITE_RANK0('(a)', 'update never', 4)
      !sim_mode = do not change
      vis_mode = VISMODE_UPDATE_NEVER
      !conn_freq = do not change
    else if (visitstrcmp(cmd, lcmd, "update once", 11) == 0) then
      WRITE_RANK0('(a)', 'update once', 4)
      !sim_mode = do not change
      vis_mode = VISMODE_UPDATE_ONCE
      !conn_freq = do not change

    ! for sync we have to make sure, to exit recursive call of visit_checkstatus()
    !   - disable blocking call to detect input
    !   - enable blocking call after visit_checkstatus() is called from simulation loop again
    else if (visitstrcmp(cmd, lcmd, "update sync 1", 13) == 0) then
      WRITE_RANK0('(a)', 'update sync 1', 4)
      vis_mode = VISMODE_UPDATE_SYNC
      sync_mode = 1
      sync_count = sync_mode
    else if (visitstrcmp(cmd, lcmd, "update sync (movie)", 19) == 0) then
      WRITE_RANK0('(a)', 'update sync (movie)', 4)
      vis_mode = VISMODE_UPDATE_SYNC
      sync_mode = 4
      sync_count = sync_mode
    else if (visitstrcmp(cmd, lcmd, "---", 3) == 0) then
      WRITE_RANK0('(a)', '---', 4)
      sync_mode = 0
      sync_count = sync_mode

    else if (visitstrcmp(cmd, lcmd, "dump", 4) == 0) then
      WRITE_RANK0('(a)', 'dump', 4)
      sim_cmd = SIMCMD_DUMP
    else if (visitstrcmp(cmd, lcmd, "stat", 4) == 0) then
      WRITE_RANK0('(a)', 'stat', 4)
      sim_cmd = SIMCMD_STAT
    else
      WRITE_RANK0_ARGS2(*, 'unknown = ',cmd, 4)
    end if

end subroutine visitcommandcallback

!---------------------------------------------------------------------------
! visitgetmesh
!---------------------------------------------------------------------------
integer function visitgetmesh (domain, name, lname)
    use psopen_data
    use psopen_visit
    implicit none
    include "visitfortransimV2interface.inc"

  ! function args
    integer domain, lname
    character (len=1) name (lname)

  ! other vars
    integer :: h, hx, hy, hz, ierr
    real :: cx (0:isize(1)+2*gsize)
    real :: cy (0:isize(2)+2*gsize)
    real :: cz (0:isize(3)+2*gsize)
    integer :: i, j, k
    integer :: baseIndex (3)
    integer :: xfac, yfac, zfac
    real(kind=MP) :: xshift, yshift, zshift

    WRITE_RANK0_ARGS2(*, 'VisIt: get mesh ',name, 4)

    !------------------------------------------------------!
    ! domain = sub-domain(1) + sub-domain(2)               !
    ! (values stored at x = cell-centered)                 !
    !     |-x-:-x-:-x-:-x-:-x-:-x-:-x-:-x-|                !
    !                                                      !
    !       0   1   2   3   4   5               (cell ids) !
    !     |-x-:-x-:-x-:-x-|~x~:~x~|           sub-domain 1 !
    !     0   1   2   3   4   5   6          (coordinates) !
    !     ^               ^                                !
    ! minRealIndex     maxRealIndex                        !
    !                                                      !
    !               0   1   2   3   4   5       (cell ids) !
    !             |~x~:~x~|-x-:-x-:-x-:-x-|   sub-domain 2 !
    !             0   1   2   3   4   5   6  (coordinates) !
    !                     ^               ^                !
    !                 minRealIndex     maxRealIndex        !
    !                                                      !
    ! :-x-: grid cell                                      !
    ! :~x~: ghost cell                                     !
    !------------------------------------------------------!

    visitgetmesh = VISIT_INVALID_HANDLE
    h  = VISIT_INVALID_HANDLE
    hx = VISIT_INVALID_HANDLE
    hy = VISIT_INVALID_HANDLE
    hz = VISIT_INVALID_HANDLE

    ! set base index
    !   baseIndex is an offset in X,Y,Z that will be added to your mesh’s zone numbers and node numbers (=cell number) when VisIt displays information about your mesh.
    !   You can leave these values set at zero. However, when you want to create a multi-domain mesh that has global zone and node numbers (=cell number),
    !   you should set the values for baseIndex
    baseIndex (1) = istart (1) - 1
    baseIndex (2) = istart (2) - 1
    baseIndex (3) = istart (3) - 1

    !WRITE_RANK0('(a)', 'CX corr---------------------', 4)
    ! set to ascending coords and start with 0.0
    xfac = 1
    xshift = 0._MP
    if(coordX(1) > coordX(2)) xfac = -1
    if(coordX(1) /= 0._MP)     xshift = -coordX(1)*xfac
    ! calc coords shown in VisIt
    do i = 0, maxRealIndex(1) + maxGhosts(1)
      cx (i) = (coordX (istart(1)-minGhosts(1)+i) *xfac) +xshift
      !if(vis_mpi_proc_id == 0) write(*,*) i, cx(i)
    end do

    !WRITE_RANK0('(a)', 'CY corr---------------------', 4)
    ! set to ascending coords and start with 0.0
    yfac = 1
    yshift = 0._MP
    if(coordY(1) > coordY(2)) yfac = -1
    if(coordY(1) /= 0._MP)     yshift = -coordY(1)*yfac
    ! calc coords shown in VisIt
    do j = 0, maxRealIndex(2) + maxGhosts(2)
      cy (j) = (coordY (istart(2)-minGhosts(2)+j) *yfac) +yshift
      ! if(vis_mpi_proc_id == 0) write(*,*) j, cy(j)
    end do

    !WRITE_RANK0('(a)', 'CZ corr---------------------', 4)
    ! set to ascending coords and start with 0.0
    zfac = 1
    zshift = 0._MP
    if(coordZ(1) > coordZ(2)) zfac = -1
    if(coordZ(1) /= 0._MP)     zshift = -coordZ(1)*zfac
    ! calc coords shown in VisIt
    do k = 0, maxRealIndex(3) + maxGhosts(3)
      cz (k) = (coordZ (istart(3)-minGhosts(3)+k) *zfac) +zshift
      ! if(vis_mpi_proc_id == 0) write(*,*) k, cz(k)
    end do

    if (visitstrcmp(name, lname, "mesh3d", 6) == 0) then

    ! allocate a rectilinear mesh
      if (visitrectmeshalloc(h) == VISIT_OKAY) then
        WRITE_RANK0('(a)', 'VisIt: mesh3d set coords', 4)

      ! set the coordinates
        ierr = visitvardataalloc(hx); CHKERRQ1(ierr,VISIT_OKAY)
        ierr = visitvardataalloc(hy); CHKERRQ1(ierr,VISIT_OKAY)
        ierr = visitvardataalloc(hz); CHKERRQ1(ierr,VISIT_OKAY)
        ! attention: visitvardatasetf(<handle>, <flag>, <no.components per tuple>, <no.tuples>, <memory>)
        !!!                                         NOT <start in array>,<end in array> !!!
        ierr = visitvardatasetf(hx,VISIT_OWNER_COPY,1,maxRealIndex(1) +maxGhosts(1) +1,cx); CHKERRQ1(ierr,VISIT_OKAY)
        ierr = visitvardatasetf(hy,VISIT_OWNER_COPY,1,maxRealIndex(2) +maxGhosts(2) +1,cy); CHKERRQ1(ierr,VISIT_OKAY)
        ierr = visitvardatasetf(hz,VISIT_OWNER_COPY,1,maxRealIndex(3) +maxGhosts(3) +1,cz); CHKERRQ1(ierr,VISIT_OKAY)

        ierr = visitrectmeshsetcoordsxyz(h, hx, hy, hz); CHKERRQ1(ierr,VISIT_OKAY)

      ! set the real indices of the mesh of this mpi process (which tells VisIt about ghostcells)
        ierr = visitrectmeshsetrealindices(h, minRealIndex, maxRealIndex); CHKERRQ1(ierr,VISIT_OKAY)

      ! set the base index of the mesh of this mpi process
        ierr = visitrectmeshsetbaseindex(h, baseIndex); CHKERRQ1(ierr,VISIT_OKAY)

      else
        WRITE_RANK0('(a)', 'VisIt: mesh3d set coords ... FAILED', 4)
      end if

    end if

    visitgetmesh = h

end function visitgetmesh

!---------------------------------------------------------------------------
! visitgetvariable
!---------------------------------------------------------------------------
integer function visitgetvariable (domain, name, lname)
    use psopen_data
    use psopen_visit
    use psopen_fft
    use psopen_utils_flow
    use psopen_utils_stream
    use psopen_utils_vortex
    implicit none
    include "visitfortransimV2interface.inc"

  ! function args
    integer domain, lname
    character (len=1) name (lname)

  ! other vars
    integer :: h
    integer :: k, ierr

    WRITE_RANK0_ARGS2(*, 'VisIt: get variable ',name, 4)

    h = VISIT_INVALID_HANDLE
    visitgetvariable = VISIT_INVALID_HANDLE

    ! velocity
    if (visitstrcmp(name, lname, "velocity", 8) == 0) then
      call fft_c2r(cu_arch, rwork10)
      call fft_c2r(cv_arch, rwork11)
      call fft_c2r(cw_arch, rwork12)
      visitgetvariable = visit_getdata_vec3df('mesh3d', 'velocity', rwork10, rwork11, rwork12, rwork100, ierr)

    ! velocity fuctuations
    else &
    if (visitstrcmp(name, lname, "velocity_fluc", 13) == 0) then
      call fft_c2r(cu_arch, rwork10)
      call fft_c2r(cv_arch, rwork11)
      call fft_c2r(cw_arch, rwork12)
      if(.not. read_u_zprofile) then
        call calc_zlayer_mean(rwork10, u_zprofile, ierr)
      endif
      do k=istart(3), iend(3)
        rwork10(:,:,k) = rwork10(:,:,k) -u_zprofile(k)
      end do
      visitgetvariable = visit_getdata_vec3df('mesh3d', 'velocity_fluc', rwork10, rwork11, rwork12, rwork100, ierr)

    ! vorticity
    else &
    if (visitstrcmp(name, lname, "vorticity", 9) == 0) then
      call calc_vorticity( cu_arch, cv_arch, cw_arch, cwork10, cwork11, cwork12, cwork100)
      call fft_c2r(cwork10, rwork10)
      call fft_c2r(cwork11, rwork11)
      call fft_c2r(cwork12, rwork12)
      visitgetvariable = visit_getdata_vec3df('mesh3d', 'vorticity', rwork10, rwork11, rwork12, rwork100, ierr)

    ! vorticity fluctuations
    else &
    if (visitstrcmp(name, lname, "vorticity_fluc", 14) == 0) then
      call calc_vorticity( cu_arch, cv_arch, cw_arch, cwork10, cwork11, cwork12, cwork100)
      call fft_c2r(cwork10, rwork10)
      call fft_c2r(cwork11, rwork11)
      call fft_c2r(cwork12, rwork12)
      visitgetvariable = visit_getdata_vec3df('mesh3d', 'vorticity_fluc', rwork10, rwork11, rwork12, rwork100, ierr)

    ! kinetic energy
    else &
    if (visitstrcmp(name, lname, "k", 1) == 0) then
      call fft_c2r(cu_arch, rwork10)
      call fft_c2r(cv_arch, rwork11)
      call fft_c2r(cw_arch, rwork12)
      call calc_kturb( rwork10, rwork11, rwork12, rwork20, ierr)
      visitgetvariable = visit_getdata_sca3df('mesh3d', 'k', rwork20, rwork100, ierr)

    ! kinetic energy fluctuations
    else &
    if (visitstrcmp(name, lname, "k_fluc", 6) == 0) then
      call fft_c2r(cu_arch, rwork10)
      call fft_c2r(cv_arch, rwork11)
      call fft_c2r(cw_arch, rwork12)
      if(.not. read_u_zprofile) then
        call calc_zlayer_mean(rwork10, u_zprofile, ierr)
      endif
      do k=istart(3), iend(3)
        rwork10(:,:,k) = rwork10(:,:,k) -u_zprofile(k)
      end do
      call calc_kturb( rwork10, rwork11, rwork12, rwork20, ierr)
      visitgetvariable = visit_getdata_sca3df('mesh3d', 'k_fluc', rwork20, rwork100, ierr)

    ! du/ds
    else &
    if (visitstrcmp(name, lname, "duds", 4) == 0) then
      call fft_c2r(cu_arch, rwork10)
      call fft_c2r(cv_arch, rwork11)
      call fft_c2r(cw_arch, rwork12)
      call calc_streamline_duds(  rwork10,  rwork11, rwork12,  rwork32,  &
                                  rwork20,  rwork21, rwork22,            &
                                  cwork30,  cwork31, rwork100, ierr)
      visitgetvariable = visit_getdata_sca3df('mesh3d', 'duds', rwork32, rwork100, ierr)

    ! du/ds fluctuations
    else &
    if (visitstrcmp(name, lname, "duds_fluc", 9) == 0) then
      call fft_c2r(cu_arch, rwork10)
      call fft_c2r(cv_arch, rwork11)
      call fft_c2r(cw_arch, rwork12)
      if(.not. read_u_zprofile) then
        call calc_zlayer_mean(rwork10, u_zprofile, ierr)
      endif
      do k=istart(3), iend(3)
        rwork10(:,:,k) = rwork10(:,:,k) -u_zprofile(k)
      end do
      call calc_streamline_duds(  rwork10,  rwork11, rwork12,  rwork32,  &
                                  rwork20,  rwork21, rwork22,            &
                                  cwork30,  cwork31, rwork100, ierr)
      visitgetvariable = visit_getdata_sca3df('mesh3d', 'duds_fluc', rwork32, rwork100, ierr)

    ! lambda2
    else &
    if (visitstrcmp(name, lname, "lambda2", 7) == 0) then
      call calc_vortex_lambda123(cu_arch, cv_arch, cw_arch,                      &
                           cwork10, cwork11, cwork12, cwork20, cwork21, cwork22, &
                           rwork10, rwork11, rwork12, rwork20, rwork21, rwork22, &
                           rwork100, rwork101, rwork102, rwork110, rwork111, rwork112, &
                           cwork120, cwork121, cwork122, &
                           rwork120, rwork121, rwork122, & ! eigenvalues 1,2,3
                           ierr)
      visitgetvariable = visit_getdata_sca3df('mesh3d', 'lambda2', rwork121, rwork100, ierr)

    else
      WRITE_RANK0_ARGS2(*, 'VisIt: variable UNKNOWN - ',name, 4)
    endif

    WRITE_RANK0('(a)', 'VisIt: get variable ... FINISHED', 4)

end function visitgetvariable

!---------------------------------------------------------------------------
! visitgetmixedvariable
!---------------------------------------------------------------------------
integer function visitgetmixedvariable (domain, name, lname)
    use psopen_visit
    implicit none
    include "visitfortransimV2interface.inc"

  ! function args
    integer domain, lname
    character (len=1) name (lname)

  ! other vars
    integer :: h
    integer :: k, ierr

    WRITE_RANK0_ARGS2(*, 'VisIt: get mixed variable ',name, 4)

    h = VISIT_INVALID_HANDLE
    visitgetmixedvariable = VISIT_INVALID_HANDLE

    WRITE_RANK0_ARGS2(*, 'VisIt: mixed variable UNKNOWN - ',name, 4)

    WRITE_RANK0('(a)', 'VisIt: get mixed variable ... FINISHED', 4)

end function visitgetmixedvariable

!---------------------------------------------------------------------------
! visitbroadcastintfunction
!---------------------------------------------------------------------------
integer function visitbroadcastintfunction (value, sender)
    use psopen_visit
#ifdef USE_MPI_MODULE
    use mpi
    implicit none
#else
    implicit none
    include 'mpif.h'
#endif
    include "visitfortransimV2interface.inc"

  ! function vars
    integer :: value, sender

  ! other vars
    integer :: ierr

    call MPI_BCAST(value, 1, MPI_INTEGER, sender, vis_mpi_comm, ierr); CHKERRQ1(ierr, MPI_SUCCESS)
    visitbroadcastintfunction = ierr

end function visitbroadcastintfunction

!---------------------------------------------------------------------------
! visitbroadcaststringfunction
!---------------------------------------------------------------------------
integer function visitbroadcaststringfunction (str, lstr, sender)
    use psopen_visit
#ifdef USE_MPI_MODULE
    use mpi
    implicit none
#else
    implicit none
    include 'mpif.h'
#endif
    include "visitfortransimV2interface.inc"

  ! function vars
    integer :: lstr, sender
    character (len=1) :: str (lstr)

  ! other vars
    integer :: ierr

    call MPI_BCAST(str, lstr, MPI_CHARACTER, sender, vis_mpi_comm, ierr); CHKERRQ1(ierr, MPI_SUCCESS)
    visitbroadcaststringfunction = ierr

end function visitbroadcaststringfunction

!---------------------------------------------------------------------------
! visitslaveprocesscallback
!---------------------------------------------------------------------------
subroutine visitslaveprocesscallback ()
    use psopen_visit
#ifdef USE_MPI_MODULE
    use mpi
    implicit none
#else
    implicit none
    include 'mpif.h'
#endif

  ! other vars
    integer :: c, ierr

    c = VISIT_COMMAND_PROCESS
    call MPI_BCAST(c, 1, MPI_INTEGER, 0, vis_mpi_comm, ierr); CHKERRQ1(ierr, MPI_SUCCESS)
end subroutine visitslaveprocesscallback

!---------------------------------------------------------------------------
! visitactivatetimestep
!---------------------------------------------------------------------------
integer function visitactivatetimestep ()
    implicit none
    include "visitfortransimV2interface.inc"
    visitactivatetimestep = VISIT_OKAY
end function visitactivatetimestep

!---------------------------------------------------------------------------
! visitgetcurve
!---------------------------------------------------------------------------
integer function visitgetcurve (name, lname)
    implicit none
    character * 8 name
    integer lname
    include "visitfortransimV2interface.inc"
    visitgetcurve = VISIT_INVALID_HANDLE
end function visitgetcurve

!---------------------------------------------------------------------------
! visitgetdomainlist
!---------------------------------------------------------------------------
integer function visitgetdomainlist (name, lname)
    use psopen_visit
    implicit none
    include "visitfortransimV2interface.inc"

  ! function vars
    integer :: lname
    character (len=1) :: name (lname)

  ! other vars
    integer :: h, dl, ierr

    WRITE_RANK0_ARGS2(*, 'VisIt: get domain list ',name, 4)

    visitgetdomainlist = VISIT_INVALID_HANDLE
    h = VISIT_INVALID_HANDLE
    dl = VISIT_INVALID_HANDLE

  ! Tell VisIt that there are as many domains as processors and this
  ! processor just has one of them.
    h = VISIT_INVALID_HANDLE
    if (visitdomainlistalloc(h) == VISIT_OKAY) then
      if (visitvardataalloc(dl) == VISIT_OKAY) then
        ierr = visitvardataseti(dl, VISIT_OWNER_SIM, 1, 1, vis_mpi_proc_id); CHKERRQ1(ierr,VISIT_OKAY)
        ierr = visitdomainlistsetdomains(h, vis_mpi_nproc, dl); CHKERRQ1(ierr,VISIT_OKAY)
      end if
    end if
    visitgetdomainlist = h

end function visitgetdomainlist

!---------------------------------------------------------------------------
! visitgetdomainbounds
!---------------------------------------------------------------------------
integer function visitgetdomainbounds (name, lname)
    implicit none
    character * 8 name
    integer lname
    include "visitfortransimV2interface.inc"
    visitgetdomainbounds = VISIT_INVALID_HANDLE
end function visitgetdomainbounds

!---------------------------------------------------------------------------
! visitgetdomainnesting
!---------------------------------------------------------------------------
integer function visitgetdomainnesting (name, lname)
    implicit none
    character * 8 name
    integer lname
    include "visitfortransimV2interface.inc"
    visitgetdomainnesting = VISIT_INVALID_HANDLE
end function visitgetdomainnesting

!---------------------------------------------------------------------------
! visitgetmaterial
!---------------------------------------------------------------------------
integer function visitgetmaterial (domain, name, lname)
    implicit none
    character * 8 name
    integer domain, lname
    include "visitfortransimV2interface.inc"
    visitgetmaterial = VISIT_INVALID_HANDLE
end function visitgetmaterial
