#!/bin/bash
# The line above this is the "shebang" line.  It must be first line in script
#-----------------------------------------------------
#	Example Submission Script: Hello-UMD, hybrid (MPI + multithreaded)
#	Runs a hello-umd command with mpi and multiple threads per task
#-----------------------------------------------------
#
# Slurm sbatch parameters section:
# 	These SBATCH lines should immediately follow the "shebang" line
# 	They must occur before any non-empty/non-comment lines in the script
#
#	Request 3 MPI tasks with 15 CPU cores each
#SBATCH -n 3
#SBATCH -c 15
#	Request 5 minutes of walltime
#SBATCH -t 5
#	Request 1 GB of memory per CPU core
#SBATCH --mem-per-cpu=1024
#	Allow other jobs to run on same node
#SBATCH --oversubscribe
#	Run on debug partition for rapid turnaround.  You will need
#	to change this (remove the line) if walltime > 15 minutes
#SBATCH --partition=debug

# This job will run the hello-umd with multiple MPI tasks each using 
# multiple threads.  We create a directory on parallel filesystem from 
# where we actually will run the job.

# Process our dot files (and in particular define the "module" command)
source /etc/profile

# Module load section
# First clear our module list 
module purge
# and reload the standard modules
module load hpcc/zaratan
# Load the desired compiler,  MPI, and package modules
# NOTE: You need to use the same compiler and MPI module used
# when compiling the MPI-enabled code you wish to run (in this
# case hello-umd). 
# NOTE: You need to module load the MPI library (openmpi) before hello-umd;
# otherwise you will get a non-MPI version of hello-umd
module load gcc
module load openmpi
module load hello-umd

# Section to make a scratch directory for this job
# Because different MPI tasks, which might be on different nodes, and will need
# access to it, we put it in a parallel file system.  
# We include the SLURM jobid in the directory name to avoid interference 
# if multiple jobs running at same time.
TMPWORKDIR="${HOME}/scratch/job.${SLURM_JOB_ID}"
mkdir $TMPWORKDIR
cd $TMPWORKDIR

# Section to output information identifying the job, etc.
echo "Slurm job submit-hello-umd-hybrid.sh ${SLURM_JOB_ID} running on:"
hostname
echo "Ntasks=${SLURM_NTASKS}; CPU cores per task=${SLURM_CPUS_PER_TASK}"
echo "On ${SLURM_JOB_NUM_NODES} nodes: ${SLURM_JOB_NODELIST}"
date
pwd
echo "Loaded modules are:"
module list
echo "Job will be run in $TMPWORKDIR"

# Setting this variable will suppress the warnings
# about lack of CUDA support on non-GPU enabled nodes.  We
# are not using CUDA, so warning is harmless.
export OMPI_MCA_mpi_cuda_support=0

# Get the full path to our hello-umd executable.  It is best
# to provide the full path of our executable to mpirun, etc.
MYEXE=`which hello-umd`
echo "Using executable $MYEXE"

# Run our code using mpirun to launch the different tasks
# We do not specify the number of tasks here, and instead rely on
# it defaulting to the number of tasks requested of Slurm
# We add -t $SLURM_CPUS_PER_TASK to have hello-umd run threads on all cores 
# allocated to the job
mpirun  ${MYEXE} -t $SLURM_CPUS_PER_TASK  > hello.out 2>&1
# Save the exit code from the previous command
ECODE=$?

echo "Job finished with exit code $ECODE.  Work dir is $TMPWORKDIR"
date

# Exit with the cached exit code
exit $ECODE