#!/bin/bash
#PBS -N bw3d
#PBS -j oe
#PBS -q normal
#PBS -l walltime=2:00:00
#PBS -l select=1:mpiprocs=8:model=bro

# Sample PBS job script to run the MPI gamera code on the bw3d test case.
# This script was designed to run on pleiades.

# Example usage
# qsub -v bw3d.pbs

echo "Job $PBS_JOBID started at `date` on `hostname`."

# Specify the ID string for the run.
export RUNID=bw3d

# Load the required modules for MPI kaiju.
module purge
module load pkgsrc/2021Q2
module load comp-intel/2020.4.304
module load mpi-hpe/mpt.2.23
module load hdf5/1.8.18_mpt
echo "The following modules are loaded:"
module list

# Define the kaiju installation location.
export KAIJU_INSTALL_DIR=$HOME/kaiju

# Set kaiju-related environment variables.
source $KAIJU_INSTALL_DIR/scripts/setupEnvironment.sh

# Add the kaiju binary directory to the command path.
export PATH=$KAIJUHOME/build_mpi/bin:$PATH

# Add the kaiju quickstart code for this model to the PATH.
export KAIJU_QUICKSTART_DIR=$KAIJUHOME/quickstart
export PATH=$KAIJU_QUICKSTART_DIR/bw3d:$PATH

# Set the MPI_TYPE_DEPTH to 32.
# If this is not done, gamera_mpi.x will crash with a stack traceback that
# includes an error messge like this:
# MPT ERROR: The program attempted to construct a derived datatype with
# depth 15, but the maximum allowed depth is 14. You can increase
export MPI_TYPE_DEPTH=32

echo "The active environment variables are:"
printenv

# Move to the job directory (this was not required before about 2022-03-01).
# If this is not done, the current working directory is $HOME.
cd $PBS_O_WORKDIR
echo "The current directory is `pwd`."

# Run the model.
EXE=gamera_mpi.x
echo "Running $EXE on model $RUNID."
mpiexec $EXE $RUNID.xml >& ${EXE}.${RUNID}.out

echo "Job $PBS_JOBID ended at `date` on `hostname`."