#!/bin/bash
# This is the script for running the graph500 benchmark
#
P=graph500-bench
DEFAULT_VERSION=2.1.4
. $SHELLPACK_INCLUDE/common.sh
TIME_CMD=`which time`
if [ "$TIME_CMD" = "" ]; then
        TIMEFORMAT="%2Uuser %2Ssystem %Relapsed %P%%CPU"
        TIME_CMD="time"
fi

MPIRUN=
MPIOPT=

# Basic argument parser
TASKSET_SERVER=
TASKSET_CLIENT=
SERVERSIDE_COMMAND=none
SERVERSIDE_NAME=`date +%Y%m%d-%H%M-%S`

while [ "$1" != "" ]; do
	case "$1" in
	-v)
		VERSION=$2
		shift 2
		;;
	--serverside-command)
		SERVERSIDE_COMMAND=$2
		shift 2
		;;
	--serverside-name)
		SERVERSIDE_NAME=$2
		shift 2
		;;
	--workset)
		GRAPH500_WORKSET=$2
		shift 2
		;;
	--parallelize)
		GRAPH500_PARALLELIZE=$2
		shift 2
		;;
	--use-hugetlbfs)
		GRAPH500_HUGETLBFS=yes
		shift
		;;
	*)
		echo Unrecognised option: $1
		shift
	esac
done
if [ "$TASKSET_SERVER" != "" ]; then
	echo TASKSET_SERVER: $TASKSET_SERVER
	echo TASKSET_CLIENT: $TASKSET_CLIENT
fi
if [ -z "$VERSION" ]; then
	VERSION=$DEFAULT_VERSION
fi

# Approximate equation for memory usage in gigabytes is
# ((2**SCALE) * (2 * EDGE + 1))*8/1048576/1024
#
# Multipliers apply depending on how it is parallelised.
#
# toy and mini are defined by the graph500 table of classes. The smaller
# classes are defined by mmtests in the interest in testing graph500 on
# single nodes instead of clusters.
SCALE=
EDGE=
case $GRAPH500_WORKSET in
infant)
	# 1G
	SCALE=22
	EDGE=14
	;;
kinder)
	# 8G
	SCALE=25
	EDGE=16
	;;
toy)
	# 17GB
	SCALE=26
	EDGE=16
	;;
mini)
	# 136 GB
	SCALE=29
	EDGE=16
	;;
esac

# Apply scaling multiplier
APPROXIMATE_USAGE=$((((2**SCALE)*(2*EDGE+1))*8))
case $GRAPH500_PARALLELIZE in
omp)
	GRAPH500_BINARY="./omp-csr/omp-csr"
	APPROXIMATE_USAGE=`perl -e "print int ($APPROXIMATE_USAGE*2.2)"`
	;;
mpi-simple)
	GRAPH500_BINARY="./mpi/graph500_mpi_simple"
	APPROXIMATE_USAGE=$((APPROXIMATE_USAGE*7/4))
	;;
mpi-one-sided)
	GRAPH500_BINARY="./mpi/graph500_mpi_one_sided"
	APPROXIMATE_USAGE=$((APPROXIMATE_USAGE*7/4))
	;;
mpi-replicated)
	GRAPH500_BINARY="./mpi/graph500_mpi_replicated"
	APPROXIMATE_USAGE=$((APPROXIMATE_USAGE*7/4))
	;;
mpi-replicated)
	GRAPH500_BINARY="./mpi/graph500_mpi_replicated_csc"
	APPROXIMATE_USAGE=$((APPROXIMATE_USAGE*7/4))
	;;
mpi-custom)
	GRAPH500_BINARY="./mpi/graph500_mpi_custom"
	APPROXIMATE_USAGE=$((APPROXIMATE_USAGE*7/4))
	;;
*)
	die Unrecognised parallelize method $GRAPH500_PARALLELIZE
	;;
esac

# Check available memory
if [ $APPROXIMATE_USAGE -gt $((MEMTOTAL_BYTES*4/5)) ]; then
	die Estimated memory usage $APPROXIMATE_USAGE bytes exceeds estimated available $((MEMTOTAL_BYTES*4/5))
fi

if [ "$INSTALL_FORCE" = "yes" ]; then
	rm -rf $SHELLPACK_SOURCES/graph500-${VERSION}
fi
if [ ! -d $SHELLPACK_SOURCES/graph500-${VERSION}-installed ]; then
	mmtests_activity source-install
	$SHELLPACK_INCLUDE/shellpack-install-graph500 -v ${VERSION}  || die graph500 install script returned error
	mmtests_activity source-installed
fi
cd $SHELLPACK_SOURCES/graph500-${VERSION}-installed || die Failed to cd to graph500 install directory
if [ "$INSTALL_ONLY" = "yes" ]; then
	echo graph500 installed only as requested.
	exit $SHELLPACK_SUCCESS
fi
# Include monitor hooks
. $SHELLPACK_INCLUDE/include-monitor.sh

round_down_power_2_cpus() {
	POWER=1

	while [ $((1<<$POWER)) -le $NUMCPUS ]; do
		POWER=$((POWER+1))
	done

	MPICPUS=$((1<<(POWER-1)))
}
round_down_power_2_cpus

# Generate make.inc
echo "CFLAGS = -g -std=c99 -O3 -march=native -fgcse-sm -fgcse-las -fgcse-after-reload -floop-strip-mine -ftree-loop-im -fivopts -funswitch-loops" > make.inc
echo "LDLIBS = -lm -lrt" >> make.inc
if [ "$GRAPH500_HUGETLBFS" = "yes" ]; then
	echo "CPPFLAGS = -DUSE_MMAP_LARGE -DUSE_MMAP_LARGE_EXT"	>> make.inc
else
	echo "CPPFLAGS = "					>> make.inc
fi

# Set parallelize options
case $GRAPH500_PARALLELIZE in
omp)
	GRAPH500_COMMAND="$GRAPH500_BINARY -s $SCALE -e $EDGE"
	export OMP_NUM_THREADS=$MPICPUS
	unset MPIRUN
	unset MPIOPT
	echo BUILD_OPENMP = Yes		>> make.inc
	echo CFLAGS_OPENMP = -fopenmp	>> make.inc
	;;
mpi-simple)
	GRAPH500_COMMAND="$GRAPH500_BINARY $SCALE $EDGE"
	echo BUILD_MPI=Yes		>> make.inc
	echo MPICC=mpicc		>> make.inc
	export PATH=$GRAPH500_MPI_PATH:$PATH
	MPIRUN=$GRAPH500_MPI_PATH/mpirun
	MPIOPT="--allow-run-as-root -mca btl ^openib,udapl -np $MPICPUS"
	unset OMP_NUM_THREADS
esac

make clean || die Failed to clean
make
if [ $? -ne 0 ]; then
	echo Stripping loop-strip-mine
	sed -i -e 's/-floop-strip-mine//' make.inc
	make || die Failed to build
fi

monitor_pre_hook $LOGDIR_RESULTS graph500
mmtests_activity hpcg-$ITERATION
echo Running graph500 scale $SCALE edge $EDGE
eval $MPIRUN $MPIOPT $GRAPH500_COMMAND 2>&1 | tee $LOGDIR_RESULTS/graph500.log
monitor_post_hook $LOGDIR_RESULTS graph500

exit $SHELLPACK_SUCCESS
#### Description graph500
#### Details graph500-bench 11
