#!/bin/bash
# set up paths
module load boost/1.73
export LD_LIBRARY_PATH=/beegfs/users/mtree/precice/precice-2.1.1/install/lib64:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=/beegfs/users/mtree/precice/yaml-cpp/build:$LD_LIBRARY_PATH

cd ${0%/*} || exit 1    # Run from this directory
. $WM_PROJECT_DIR/bin/tools/RunFunctions    # Tutorial run functions

# This script prepares and runs all the participants in one terminal,
# forwarding the solvers' output to log files.

# The script "Allclean" cleans-up the result and log files.
# Set up the run parameters:
#./Allclean

# 1 for true, 0 for false
parallel=0
if [ "$1" = "-parallel" ]; then
    parallel=1
fi

# =============== Participant 1: Fluid ===========================
Participant1="fluid"
Solver1="pimpleDyMFoam"

    # Prepare
    echo "Preparing the ${Participant1} participant..."

    cd $Participant1 
        Solver1=$(getApplication)    # solver
        echo "  ${Participant1} Solver: ${Solver1}."
    cd ..

    if [ $parallel -eq 1 ]; then
        echo "  Decomposing the case..."
        decomposePar -force -case ${Participant1} > ${Participant1}_decomposePar.log 2>&1
        cd ${Participant1}
            nproc=$(getNumberOfProcessors)
        cd ..
        echo "  Starting the ${Participant1} participant in parallel..."
        mpirun -np ${nproc} ${Solver1} -parallel -case ${Participant1} > ${Participant1}.log 2>&1 &
    else
        echo "  Starting the ${Participant1} participant in serial..."
        ${Solver1} -case ${Participant1} > ${Participant1}.log 2>&1 &
    fi
    PIDParticipant1=$!

# =============== Participant 2: Solid ===========================
Participant2="solid"
Solver2="/beegfs/users/mtree/CalculiX/calculix-adapter-master/bin/ccx_preCICE"

    # Run and get the process id
    echo "Starting the ${Participant2} participant..."
    export OMP_NUM_THREADS=1
    export CCX_NPROC_EQUATION_SOLVER=1
    ${Solver2} -i ${Participant2}/aorta -precice-participant solid > ${Participant2}.log 2>&1 &
    PIDParticipant2=$!

# Wait for all the participants to finish
 # =============== Wait for all the participants to finish =======
echo "Waiting for the participants to exit..., PIDs: ${PIDParticipant1}, ${PIDParticipant2}"
echo "(you may run 'tail -f ${Participant1}.log' in another terminal to check the progress)"
echo "To kill midrun, press 'c'. Ctrl+C will cause the processes to run in the background."
# loop="ps -p ${PIDParticipant1}"
while [ -e /proc/${PIDParticipant1} ]; do
    # loop="ps -p ${PIDParticipant1}"
    read -r -t1 -n1 input
    if [ "$input" = "c" ]; then
        kill ${PIDParticipant1}
        kill ${PIDParticipant2}
        false
    fi
done
 if [ $? -ne 0 ] || [ "$(grep -c -E "error:" ${Participant1}.log)" -ne 0 ] || [ "$(grep -c -E "error:" ${Participant2}.log)" -ne 0 ]; then
    echo ""
    echo "Something went wrong... See the log files for more."
    # precaution
    kill ${PIDParticipant1}
    kill ${PIDParticipant2}
else
    echo ""
    echo "The simulation completed! Reconstructing fields (if parallel)"
    ./removeObsoleteFolders.sh      # For some versions of openfoam, the functionobject is written in an empty timeolder. remove these.
    if [ $parallel -eq 1 ]; then
        reconstructPar -case ${Participant1} > ${Participant1}/reconstructPar.log 2>&1 & 
    fi
    echo "You may now open '${Participant1}/${Participant1}.foam' in ParaView."
    # Note: ".foam" triggers the native OpenFOAM reader of ParaView.
    # Change to ".OpenFOAM" to use the OpenFOAM reader provided with OpenFOAM.
fi

echo ""
echo "### NOTE ### Make sure to use the correct solver for your OpenFOAM version! (pimpleFoam for OpenFOAM v1806, OpenFOAM 6, or newer, vs pimpleDyMFoam for older) You may change this in your Fluid/system/controlDict file, if needed."