#! /bin/ksh
###############################################################################
#                              test0086b.run
#
# Generated by $Id: mkexp 421 2016-06-01 12:50:59Z m221078 $
#
# $Id: DEFAULT.run.tmpl 9187 2017-08-09 14:54:45Z m211003 $
#
# $Id: DEFAULT.config 8788 2016-09-20 14:14:42Z m211003 $
# $Id: SETUP.config.in 8426 2016-02-18 16:04:00Z m221078 $
# $Id: piControl-CR.config 8788 2016-09-20 14:14:42Z m211003 $
# $Id: no_hamocc.config -1 2015-08-28 12:30:51Z m221078 $
# $Id: no_hamocc.config -1 2015-08-28 12:30:51Z m221078 $
# $Id: hdoutput m211003 $
# $Id: ebm_output_hrly.config 1 2016-07-22 14:32:06Z m300467 $
# $Id: mistral.config 8788 2016-09-20 14:14:42Z m211003 $
#
###############################################################################
#
# Setup for mistral (SLURM)
#
# $Id: mistral.tmpl 9187 2017-08-09 14:54:45Z m211003 $
#
#SBATCH --job-name=test_run
#SBATCH --partition=compute
#SBATCH --ntasks-per-node=24
#SBATCH --ntasks=96
#SBATCH --cpus-per-task=1
#SBATCH --time=00:01:00
#SBATCH --output=test_run_%j.log
#SBATCH --error=test_run_%j.log
#SBATCH --mail-type=FAIL
#SBATCH --mail-user=pgierz@awi.de
#SBATCH --account=ba0989
ulimit -s 102400
ulimit -c 0

# OpenMPI optimised
export OMPI_MCA_pml=ob1         # sets the point-to-point management layer
export OMPI_MCA_mtl=^mxm        # sets the matching transport layer (MPI-2 one-sided comm.)

SLURM_JOB_NAME=test_run

AWI_ND=3
PISM_ND=1
hostname="m"

# Job nodes extraction
echo "nodelist = $SLURM_JOB_NODELIST"
nodeslurm=$SLURM_JOB_NODELIST
tmp=${nodeslurm#"m["}
nodes=${tmp%]*}
myarray=(`echo $nodes | sed 's/,/\n/g'`)
idx=0
for element in "${myarray[@]}"
do
if [[ "$element" == *"-"* ]]; then
array=(`echo $element | sed 's/-/\n/g'`)
for node in $(seq ${array[0]} ${array[1]})
do
   nodelist[$idx]=$node
   idx=$idx+1
done
else
nodelist[$idx]=$element
idx=$idx+1
fi
done

for element in "${nodelist[@]}"
do
echo "$element"
done

# AWI-CM nodes list
awi=""
for idx in $(seq 0 $((AWI_ND-1)))
do
  if [[ $idx == $((AWI_ND-1)) ]]; then
    awi="$awi${nodelist[$idx]}"
  else
    awi="$awi${nodelist[$idx]},"
  fi
done
echo "AWI-CM nodes: $awi"

# PISM nodes list
length=${#nodelist[@]}
pism=""
for idx in $(seq $AWI_ND $((length-1)))
do
  if [[ $idx == $((length-1)) ]]; then
    pism="$pism${nodelist[$idx]}"
  else
    pism="$pism${nodelist[$idx]},"
  fi
done
echo "PISM nodes: $pism"

STARTTIME=`date +%s`
# Run PISM
#echo "srun --propagate=STACK,CORE --nodes=$PISM_ND --ntasks=$((SLURM_NTASKS_PER_NODE*PISM_ND)) --ntasks-per-node=$SLURM_NTASKS_PER_NODE --nodelist="$hostname[$pism]" --cpu_bind=cores ./pism &"
srun --nodelist="$hostname[$pism]" ./pism > pism.log &

# Run AWI-CM
#echo "srun --propagate=STACK,CORE --nodes=$AWI_ND --ntasks=$((SLURM_NTASKS_PER_NODE*AWI_ND)) --ntasks-per-node=$SLURM_NTASKS_PER_NODE --nodelist="$hostname[$awi]" --cpu_bind=cores ./awi_cm &"
srun --nodelist="$hostname[$awi]" ./awi_cm > awi.log &

wait
ENDTIME=`date +%s`
echo "Total time: $((ENDTIME - STARTTIME)) seconds"
echo "Done"