#!/bin/bash
# This is the script for running the HPCG benchmark
#
###SHELLPACK preamble hpcg-bench 3.0

MPIRUN=
MPIOPT=

###SHELLPACK parseargBegin
###SHELLPACK parseargParam	--duration		HPCG_DURATION
###SHELLPACK parseargParam	--iterations		HPCG_ITERATIONS
###SHELLPACK parseargParam	--parallelise-type	HPCG_PARALLELISE
###SHELLPACK parseargEnd
###SHELLPACK monitor_hooks

###SHELLPACK check_install_required hpcg-${VERSION}
###SHELLPACK init_complete

TEMPLATE=
template_set_param() {
	KEY=$1
	VALUE=$2
	VALUE_ESCAPED=`echo "$VALUE" | sed -e 's/\//\\\\\//g'`
	sed -i -e "s/$KEY\s*=.*/$KEY = $VALUE_ESCAPED/" setup/Make.$TEMPLATE
}

round_down_power_2_cpus() {
	POWER=1

	while [ $((1<<$POWER)) -le $NUMCPUS ]; do
		POWER=$((POWER+1))
	done

	MPICPUS=$((1<<(POWER-1)))
}
round_down_power_2_cpus

# Configure build template
case $HPCG_PARALLELISE in
gcc-omp)
	TEMPLATE=GCC_OMP
	export OMP_NUM_THREADS=$MPICPUS
	unset MPIRUN
	unset MPIOPT
	;;
gcc-mpi)
	TEMPLATE=Linux_MPI
	unset OMP_NUM_THREADS
	MPIRUN=$HPCG_MPI_PATH/mpirun
	MPIOPT="--allow-run-as-root -mca btl ^openib,udapl -np $MPICPUS"
	template_set_param CXX   $HPCG_MPI_PATH/mpicxx
	template_set_param MPdir $HPCG_MPI_PATH
	template_set_param MPlib ""
	;;
gcc-omp-mpi)
	TEMPLATE=MPI_GCC_OMP
	export OMP_NUM_THREADS=$MPICPUS
	MPIRUN=$HPCG_MPI_PATH/mpirun
	MPIOPT="--allow-run-as-root -mca btl ^openib,udapl -np $MPICPUS"
	template_set_param CXX   $HPCG_MPI_PATH/mpicxx
	template_set_param MPdir $HPCG_MPI_PATH
	template_set_param MPlib ""
	;;
serial)
	TEMPLATE=Linux_Serial
	MPICPUS=1
	unset OMP_NUM_THREADS
	unset MPIRUN
	unset MPIOPT
	;;
*)
	die  Unsupported parallelise option $HPCG_PARALLELISE
	;;
esac
make clean || die Failed to clean hpcg
make arch=$TEMPLATE
if [ $? -ne 0 ]; then
	echo Attempting build with g++ 4.8
	install-depends gcc48-c++
	make CXX=g++-4.8 arch=$TEMPLATE || die Failed to build hpcg
fi

# Configure benchmark
cat > hpcg.dat <<EOF
HPCG benchmark input file
Sandia National Laboratories; University of Tennessee, Knoxville
104 104 104
$HPCG_DURATION
EOF

monitor_pre_hook $LOGDIR_RESULTS $P
###SHELLPACK iteration_begin $HPCG_ITERATIONS
	mmtests_activity hpcg-$ITERATION
	echo Running hpcg iteration $ITERATION/$HPCG_ITERATIONS cpus $MPICPUS
	eval $MPIRUN $MPIOPT ./bin/xhpcg 2>&1 | tee $LOGDIR_RESULTS/hpcg-${ITERATION}.raw
	mv HPCG-Benchmark-*.yaml $LOGDIR_RESULTS/hpcg-${ITERATION}.yaml
	mv hpcg_log*.txt $LOGDIR_RESULTS/hpcg-${ITERATION}.log
###SHELLPACK iteration_end
monitor_post_hook $LOGDIR_RESULTS $P

exit $SHELLPACK_SUCCESS
