#!/usr/bin/env bash
#
#SBATCH --job-name=fenics-native
###SBATCH --output=output-bothsolvers.txt
#
#SBATCH --ntasks=16
###SBATCH --ntasks-per-node=4
#SBATCH --time=02:00:00


source /home/jaustar/software/sgscl/spack/var/spack/environments/fractures-fenics-openblas/loads
module load gcc-9.2.0
source /home/jaustar/software/sgscl/fenics/share/dolfin/dolfin.conf
 
echo "Loaded modules"
module list
echo "Operating from path:"
echo $PWD

#simulation_dir=/import/sgs.scratch/jaustar/precice-fractures/nonlinearproblem3d-precice
simulation_dir=${SLURM_SUBMIT_DIR}
cd "${simulation_dir}"



#list_of_biotsolver_ids=$(ps -ef | grep biot_solver.py | grep -v grep | awk '{print $2}')
#for id in $list_of_biotsolver_ids; do
#  kill -9 $id
#done
#
#list_of_hdflow_solver_ids=$(ps -ef | grep hdflow_solver.py | grep -v grep | awk '{print $2}')
#for id in $list_of_hdflow_solver_ids; do
#  kill -9 $id
#done

time srun --mpi=pmi2 pkill -9 -f python3
time srun --mpi=pmi2 pkill -9 -f python

rm -rf precice-run
sleep 5

#mpirun -n 6 python3 biot_solver.py biot.input > biotsolver.log 2>&1  &
cd biot_solver
echo "Starting biot solver from ${PWD}"
#time srun --mpi=pmi2 -n 6 python3 biot_solver.py biot.input > biotsolver.log 2>&1  &
#time srun --mpi=pmi2 -n 6 python3 biot_solver.py biot.input
time mpirun.mpich -n 12 python3 biot_solver.py biot.input > biotsolver-${SLURM_JOBID}.log 2>&1  &
pid_biot=$!
#time mpirun.mpich -n 6 python3 biot_solver.py biot.input
cd ../hdflow_solver
echo "Starting hdflow solver from ${PWD}"
#mpirun -n 2 python3 hdflow_solver.py flow.input > hdflowsolver.log 2>&1 &
#time srun --mpi=pmi2 -n 2 python3 hdflow_solver.py flow.input > hdflowsolver.log 2>&1 &
time mpirun.mpich -n 4 python3 hdflow_solver.py flow.input > hdflowsolver-${SLURM_JOBID}.log 2>&1 &
pid_flow=$!
cd "${simulation_dir}"

echo "Waiting for processes ${pid_biot} and ${pid_flow}"
wait ${pid_biot} ${pid_flow}