#!/bin/bash
# The line above this is the "shebang" line.  It must be first line in script
#-----------------------------------------------------
#	Example Submission Script: Hello-UMD, multithreaded
#	Runs a multithreaded hello-umd command
#-----------------------------------------------------
#
# Slurm sbatch parameters section:
#	Request a single task using 10 CPU cores
#SBATCH --ntasks=1
#SBATCH --cpus-per-task=10
#	Request 5 minute of walltime
#SBATCH -t 5
#	Request 10 GiB of total memory for the job
#SBATCH --mem=10240
#	Allow other jobs to run on same node
#SBATCH --oversubscribe
#	Run on debug partition for rapid turnaround.  You will need
#	to change this (remove the line) if walltime > 15 minutes
#SBATCH --partition=debug

# This job will run our hello-umd demo binary in multithreaded mode
# Output will go to local /tmp scratch space on the node we are running
# on, and then will be copied back to our work directory.

# Process our dot files (and in particular define the "module" command)
source /etc/profile

# Module load section
# First clear our module list 
module purge
# and reload the standard modules
module load hpcc/zaratan
# Set up environment for hello-umd
module load hello-umd

# Section to make a scratch directory for this job
# Since this is a single node job, the local /tmp filesystem is a good choice
# We include the SLURM jobid in the #directory name to avoid interference if 
# multiple jobs running at same time.
TMPWORKDIR="/tmp/job.${SLURM_JOB_ID}"
mkdir $TMPWORKDIR
cd $TMPWORKDIR

# Section to output information identifying the job, etc.
echo "Slurm job submit-hello-umd-multithreaded.sh ${SLURM_JOB_ID} running on:"
hostname
echo "Ntasks=${SLURM_NTASKS}; CPU cores per task=${SLURM_CPUS_PER_TASK}"
echo "On ${SLURM_JOB_NUM_NODES} nodes: ${SLURM_JOB_NODELIST}"
date
pwd
echo "Loaded modules are:"
module list

# Run our code, giving -t 0 to use all available CPUs (10 in this case)
hello-umd -t 0 > hello.out 2>&1
# Save the exit code from the previous command
ECODE=$?

# Copy results back to submit dir
cp hello.out ${SLURM_SUBMIT_DIR}/hello-${SLURM_JOB_ID}.out

echo "Job finished with exit code $ECODE"
date

# Exit with the cached exit code
exit $ECODE