#!/bin/bash -l

##############################
#       Job blueprint        #
##############################

#SBATCH --account=nn9405k

# Give your job a name, so you can recognize it in the queue overview
#SBATCH --job-name=hel256

# Define, how many nodes you need. On Fram, each node has 32 cores(?)
#SBATCH --nodes=8 --ntasks-per-node=32

# You can further define the number of tasks with --ntasks-per-*
# See "man sbatch" for details. e.g. --ntasks=4 will ask for 4 cpus.

# Define, how long the job will run in real time. This is a hard cap meaning
# that if the job runs longer than what is written here, it will be
# force-stopped by the server. If you make the expected time too long, it will
# take longer for the job to start. Here, we say the job will take 5 minutes.
#              d-hh:mm:ss
#SBATCH --time=0-00:05:00

# Define the partition on which the job shall run. May be omitted.
#SBATCH --qos=short

# How much memory you need.
# --mem will define memory per node and
# --mem-per-cpu will define memory per CPU/core. Choose one of those.
##SBATCH --mem-per-cpu=1500MB
##SBATCH --mem=5GB    # this one is not in effect, due to the double hash

# Turn on mail notification. There are many possible self-explaining values:
# NONE, BEGIN, END, FAIL, ALL (including all aforementioned)
# For more values, check "man sbatch"
#SBATCH --mail-type=END,FAIL

# You may not place any commands before the last SBATCH directive

# load necessary compilers
#module load intel/13.0
#module load OpenMPI/1.6.2-intel-13.0
module load GCCcore/9.3.0
module load OpenMPI/4.0.3-GCC-9.3.0


## Define and create a unique scratch directory for this job
#SCRATCH_DIRECTORY=/global/work/${USER}/${SLURM_JOBID}.stallo-adm.uit.no
#mkdir -p ${SCRATCH_DIRECTORY}
#echo Working directory is ${SCRATCH_DIRECTORY}
#echo Running on host `hostname`
#echo Time is `date`
#cd ${SCRATCH_DIRECTORY}
#echo Running in `pwd`

## You can copy everything you need to the scratch directory
## ${SLURM_SUBMIT_DIR} points to the path where this script was submitted from
#mkdir src
#mkdir data
#cp -r ${SLURM_SUBMIT_DIR}/data/* ${SCRATCH_DIRECTORY}/data/.
#cp ${SLURM_SUBMIT_DIR}/src/start.x ${SCRATCH_DIRECTORY}/src/.
#cp ${SLURM_SUBMIT_DIR}/src/run.x ${SCRATCH_DIRECTORY}/src/.
#cp ${SLURM_SUBMIT_DIR}/src/cparam.local ${SCRATCH_DIRECTORY}/src/.
#cp ${SLURM_SUBMIT_DIR}/*.in ${SCRATCH_DIRECTORY}/.

# This is where the actual work is done. In this case, the script only waits.
# The time command is optional, but it may give you a hint on how long the
# command worked
pc_start
pc_run

## After the job is done we copy our output back to $SLURM_SUBMIT_DIR
#cp -r ${SCRATCH_DIRECTORY}/data/. ${SLURM_SUBMIT_DIR}/data/.

# In addition to the copied files, you will also find a file called
# slurm-1234.out in the submit directory. This file will contain all output that
# was produced during runtime, i.e. stdout and stderr.

## After everything is saved to the home directory, delete the work directory to
## save space on /global/work
#cd ${SLURM_SUBMIT_DIR}
#rm -rf ${SCRATCH_DIRECTORY}

# Finish the script
exit 0
