#!/bin/bash
P=thpfioscale-bench
DEFAULT_VERSION=2.99
. $SHELLPACK_INCLUDE/common.sh
TIME_CMD=`which time`
if [ "$TIME_CMD" = "" ]; then
        TIMEFORMAT="%2Uuser %2Ssystem %Relapsed %P%%CPU"
        TIME_CMD="time"
fi

THPFIOSCALE_FIO_FILESIZE=${THPFIOSCALE_FIO_FILESIZE:-65536}
MADV_HUGEPAGE=0

# Basic argument parser
TASKSET_SERVER=
TASKSET_CLIENT=
TASKSET_ALL=
SERVERSIDE_COMMAND=none
SERVERSIDE_NAME=`date +%Y%m%d-%H%M-%S`

while [ "$1" != "" ]; do
	case "$1" in
	-v)
		VERSION=$2
		shift 2
		;;
	--serverside-command)
		SERVERSIDE_COMMAND=$2
		shift 2
		;;
	--serverside-name)
		SERVERSIDE_NAME=$2
		shift 2
		;;
	--install-only)
		INSTALL_ONLY=yes
		shift
		;;
	--install-force)
		INSTALL_FORCE=yes
		shift
		;;
	--min-threads)
		THPFIOSCALE_MIN_THREADS=$2
		shift 2
		;;
	--max-threads)
		THPFIOSCALE_MAX_THREADS=$2
		shift 2
		;;
	--fio-threads)
		THPFIOSCALE_FIO_THREADS=$2
		shift 2
		;;
	--thp-size)
		THPFIOSCALE_THP_SIZE=$2
		shift 2
		;;
	--fio-size)
		THPFIOSCALE_FIO_SIZE=$2
		shift 2
		;;
	--madv-huge)
		THPFIOSCALE_MADV_HUGEPAGE=$2
		shift 2
		;;
	*)
		echo Unrecognised option: $1
		shift
	esac
done
if [ "$TASKSET_SERVER" != "" ]; then
	echo TASKSET_SERVER: $TASKSET_SERVER
	echo TASKSET_CLIENT: $TASKSET_CLIENT
fi
if [ -z "$VERSION" ]; then
	VERSION=$DEFAULT_VERSION
fi

if [ "$INSTALL_FORCE" = "yes" ]; then
	rm -rf $SHELLPACK_SOURCES/fio-${VERSION}
fi
if [ ! -d $SHELLPACK_SOURCES/fio-${VERSION}-installed ]; then
	mmtests_activity source-install
	$SHELLPACK_INCLUDE/shellpack-install-fio -v ${VERSION}  || die fio install script returned error
	mmtests_activity source-installed
fi
cd $SHELLPACK_SOURCES/fio-${VERSION}-installed || die Failed to cd to fio install directory
if [ "$INSTALL_ONLY" = "yes" ]; then
	echo fio installed only as requested.
	exit $SHELLPACK_SUCCESS
fi
if [ "$INSTALL_FORCE" = "yes" ]; then
	rm -rf $SHELLPACK_SOURCES/thpfioscale-${VERSION}
fi
if [ ! -d $SHELLPACK_SOURCES/thpfioscale-${VERSION}-installed ]; then
	mmtests_activity source-install
	$SHELLPACK_INCLUDE/shellpack-install-thpfioscale -v ${VERSION}  || die thpfioscale install script returned error
	mmtests_activity source-installed
fi
cd $SHELLPACK_SOURCES/thpfioscale-${VERSION}-installed || die Failed to cd to thpfioscale install directory
if [ "$INSTALL_ONLY" = "yes" ]; then
	echo thpfioscale installed only as requested.
	exit $SHELLPACK_SUCCESS
fi
# Include monitor hooks
. $SHELLPACK_INCLUDE/include-monitor.sh

if [ "$THPFIOSCALE_MADV_HUGEPAGE" = "yes" ]; then
	MADV_HUGEPAGE=1
fi

# Initialise directories
for i in `seq 0 $((THPFIOSCALE_FIO_THREADS-1))`; do
	mkdir -p $SHELLPACK_DATA/thpfilescale/$i
done

# Initialise files
echo Initialising fio files
FIO_NR_FILES=$((THPFIOSCALE_FIO_SIZE/THPFIOSCALE_FIO_FILESIZE/THPFIOSCALE_FIO_THREADS))
FIO_NR_BLOCKS=$((THPFIOSCALE_FIO_SIZE/4096))
FIO_JOBFILE="/tmp/fio-jobfile-$$"
FIO_ALLOCSIZE=$((FIO_NR_BLOCKS*4/1024))
ulimit -n $((FIO_NR_FILES+1024))
cat >$FIO_JOBFILE <<END
[global]
direct=0
ioengine=sync
blocksize=4096
invalidate=0
fallocate=none
create_on_open=1

[writer]
nrfiles=$FIO_NR_FILES
filesize=$THPFIOSCALE_FIO_FILESIZE
readwrite=write
numjobs=$THPFIOSCALE_FIO_THREADS
filename_format=thpfilescale/\$jobnum/workfile.\$filenum
END
echo Parameters
echo "o Directory   $SHELLPACK_DATA"
echo "o Total sz    $((THPFIOSCALE_FIO_SIZE/1048576/1024))gb"
echo "o Thread sz   $((THPFIOSCALE_FIO_SIZE/THPFIOSCALE_FIO_THREADS/1048576/1024))gb"
echo "o Alloc size  ${FIO_ALLOCSIZE}kb"
echo "o nr blocks   ${FIO_NR_BLOCKS}"
echo "o nr files    ${FIO_NR_FILES}"
echo "o total files $((FIO_NR_FILES*THPFIOSCALE_FIO_THREADS))"
echo "o nr jobs     ${THPFIOSCALE_FIO_THREADS}"
echo "o thp min     ${THPFIOSCALE_MIN_THREADS}"
echo "o thp max     ${THPFIOSCALE_MAX_THREADS}"
ulimit -a
START=`date +%s`
$SHELLPACK_SOURCES/fio-${VERSION}-installed/fio \
	--directory="$SHELLPACK_DATA" 		\
	--alloc-size=$FIO_ALLOCSIZE		\
	--minimal				\
	$FIO_JOBFILE				\
	&> $LOGDIR_RESULTS/fio-init.log
if [ $? -ne 0 ]; then
	cat $LOGDIR_RESULTS/fio-init.log
	die "Failed to initialse files"
fi
END=`date +%s`
INIT_TIME=$((END-START))
sync

# Initialise random reader
echo Initialising random reader
cat >$FIO_JOBFILE <<END
[global]
direct=0
runtime=3600
time_based
ioengine=sync
blocksize=4096
invalidate=0

[reader]
nrfiles=$FIO_NR_FILES
filesize=$THPFIOSCALE_FIO_FILESIZE
file_service_type=random:16
readwrite=randread
filename_format=thpfilescale/\$jobnum/workfile.\$filenum

numjobs=$THPFIOSCALE_FIO_THREADS
END
$SHELLPACK_SOURCES/fio-${VERSION}-installed/fio \
	--directory="$SHELLPACK_DATA" 		\
	--alloc-size=$FIO_ALLOCSIZE		\
	--minimal				\
	$FIO_JOBFILE				\
	&> $LOGDIR_RESULTS/fio-read.log &
FIO_PID=$!

THPFIOSCALE_FIO_WARMUP=${THPFIOSCALE_FIO_WARMUP:-$INIT_TIME}
date
echo Warming up fio pid $FIO_PID for $THPFIOSCALE_FIO_WARMUP seconds
sleep $THPFIOSCALE_FIO_WARMUP

echo Warmup complete, starting THP
THREADS=
START_THREAD=$THPFIOSCALE_MIN_THREADS
END_THREAD=$THPFIOSCALE_MAX_THREADS
if [ $END_THREAD -gt 32 ]; then
	THREADS=`seq $START_THREAD 3 8`
	THREADS="$THREADS `seq 12 9 32`"
	THREADS="$THREADS `seq 48 31 $END_THREAD`"
elif [ $END_THREAD -gt 8 ]; then
	THREADS=`seq $START_THREAD 2 8`
	THREADS="$THREADS `seq 12 6 $END_THREAD`"
else
	THREADS=`seq $START_THREAD 2 $END_THREAD`
fi
if [ `echo $THREADS | awk '{print $NF}'` -ne $END_THREAD ]; then
	THREADS="$THREADS $END_THREAD"
fi

for NR_THREADS in $THREADS; do
	if [ $NR_THREADS -lt $THPFIOSCALE_MIN_THREADS ]; then
		continue
	fi
	mmtests_activity process $NR_THREADS/$END_THREAD
	monitor_pre_hook $LOGDIR_RESULTS $NR_THREADS
	echo Starting $NR_THREADS/$THPFIOSCALE_MAX_THREADS
	$TIME_CMD -o $LOGDIR_RESULTS/threads-${NR_THREADS}.time \
		./thpfioscale $NR_THREADS $THPFIOSCALE_THP_SIZE $MADV_HUGEPAGE \
			> $LOGDIR_RESULTS/threads-${NR_THREADS}.log 2>&1
	monitor_post_hook $LOGDIR_RESULTS $NR_THREADS
done

SIGNAL="INT"
ATTEMPT=0
echo -n Shutting down fio pid $FIO_PID
kill -$SIGNAL $FIO_PID
while [ $? -eq 0 ]; do
	echo -n .
	kill -$SIGNAL $FIO_PID
	ATTEMPT=$((ATTEMPT+1))
	if [ $ATTEMPT -gt 20 ]; then
		SIGNAL="KILL"
	fi
	sleep 5
	ps -p $FIO_PID &> /dev/null
done

echo
echo Checking remaining fio processes
ps auxw | grep "fio " | grep -v grep
NR=`ps auxw | grep "fio " | grep -v grep | wc -l`
if [ $NR -gt 0 ]; then
	echo Warning: Killing all fio processes
	killall -KILL fio
fi

echo Cleaning up fio
rm $FIO_JOBFILE
rm -rf $SHELLPACK_DATA/thpfilescale

exit $SHELLPACK_SUCCESS
#### Description thpfioscale
#### Details thpfioscale-bench 15
