#!/bin/bash
#SBATCH --job-name=preCICE
#SBATCH --account=cesg
#SBATCH --partition=cesg
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=28 # each cesg group node has 40 total "cores" (i.e. 2 threads per 20 slots), but the UW Hyak Wiki recommends using 28 to run jobs most efficiently
#SBATCH --mem=128G
#SBATCH --time=24:00:00
#SBATCH --mail-type=ALL
#SBATCH --mail-user=nsaoirse@uw.edu

# This script prepares and runs all the participants in one terminal,
# forwarding the solvers' output to log files.
# Alternatively, you may execute the scripts "runSolid" and "runFluid"
# in separate terminals.

# The script "Allclean" cleans-up the result and log files.

module load test/gcc_8.2.1-ompi_4.0.2;
module load cmake/3.11.2;
source /gscratch/sw/contrib/OpenFOAM7-CESG/OpenFOAM-7/etc/bashrc;
source /gscratch/sw/contrib/OpenFOAM7-CESG/ThirdParty-7/Anaconda3/etc/profile.d/conda.sh;
source /gscratch/sw/contrib/preCICE/spack/share/spack/setup-env.sh;
spack load /tpl3o4x;

solverroot="./"


Participant1="Solid1"
configfile="precice-config.xml"

echo "Starting ${participant2} participant..."
python3 ${solverroot}${Participant1}Solver.py ${solverroot}${configfile} > ${Participant1}.log 2>&1 &
PIDParticipant1=$!



# Participant 2: Fluid;
Participant2="Fluid1"
Solver2="interFoam"
    # Prepare;
    echo "Preparing the ${Participant2} participant..."
	cp Fluid1/0/alpha.water.org Fluid1/0/alpha.water
	blockMesh -case ${Participant2}
    setFields -case ${Participant2}
    # Run and get the process id;
    echo "Starting the ${Participant2} participant..."
	
    ${Solver2} -case ${Participant2}> ${Participant2}.log 2>&1 &
    PIDParticipant2=$!

# Wait for all the participants to finish
echo "Waiting for the participants to exit..."
echo "(you may run 'tail -f ${Participant1}.log' in another terminal to check the progress)"
wait ${PIDParticipant2}
if [ $? -ne 0 ] || [ "$(grep -c -E "error:" ${Participant1}.log)" -ne 0 ];  || [ "$(grep -c -E "error:" ${Participant2}.log)" -ne 0 ]; then
    echo ""
    echo "Something went wrong... See the log files for more."
else
    echo ""
    echo "The simulation completed!"
    echo "You may now open '${Participant1}/${Participant1}.foam' and '${Participant2}/${Participant2}.foam' in ParaView."
    # Note: ".foam" triggers the native OpenFOAM reader of ParaView.
    # Change to ".OpenFOAM" to use the OpenFOAM reader provided with OpenFOAM.
fi