#!/bin/bash
P=graphdb-install
DEFAULT_VERSION=0
. $SHELLPACK_INCLUDE/common.sh
TIME_CMD=`which time`
if [ "$TIME_CMD" = "" ]; then
        TIMEFORMAT="%2Uuser %2Ssystem %Relapsed %P%%CPU"
        TIME_CMD="time"
fi
WEB_LOCATION="https://gist.githubusercontent.com/apurvam/6803958/raw/979550355453c9058865e63b1ad571fb2a3ffe75/graphdb-simulator.cpp"
MIRROR_LOCATION="$WEBROOT/graphdb/graphdb-simulator.cpp"

# Basic argument parser
TASKSET_SERVER=
TASKSET_CLIENT=
SERVERSIDE_COMMAND=none
SERVERSIDE_NAME=`date +%Y%m%d-%H%M-%S`

while [ "$1" != "" ]; do
	case "$1" in
	-v)
		VERSION=$2
		shift 2
		;;
	--serverside-command)
		SERVERSIDE_COMMAND=$2
		shift 2
		;;
	--serverside-name)
		SERVERSIDE_NAME=$2
		shift 2
		;;
	*)
		echo Unrecognised option: $1
		shift
	esac
done
if [ "$TASKSET_SERVER" != "" ]; then
	echo TASKSET_SERVER: $TASKSET_SERVER
	echo TASKSET_CLIENT: $TASKSET_CLIENT
fi
if [ -z "$VERSION" ]; then
	VERSION=$DEFAULT_VERSION
fi

cd $SHELLPACK_SOURCES || die Sources directory does not exist
rm -rf graphdb-${VERSION}-installed
mkdir -p graphdb-${VERSION}-installed
sources_fetch $WEB_LOCATION $MIRROR_LOCATION $SHELLPACK_SOURCES/graphdb-${VERSION}-installed/graphdb.cpp

LINESTART=`grep -n "==== BEGIN log-latencies.patch" $0 | tail -1 | awk -F : '{print $1}'`
LINEEND=`grep -n "==== END log-latencies.patch" $0 | tail -1 | awk -F : '{print $1}'`
if [ "$LINEEND" = "" ]; then
	LINECOUNT=`wc -l $0 | awk '{print $1}'`
fi
if [ "$LINESTART" = "" ]; then
	die Failed to find start of file log-latencies.patch
fi
echo Extracting $SHELLPACK_TEMP/log-latencies.patch
sed -n $((LINESTART+1)),$((LINEEND-1))p $0 > $SHELLPACK_TEMP/log-latencies.patch
cd $SHELLPACK_SOURCES/graphdb-${VERSION}-installed/
cat $SHELLPACK_TEMP/log-latencies.patch | patch -p1 || exit $SHELLPACK_FAILURE

exit $SHELLPACK_SUCCESS

==== BEGIN log-latencies.patch ====
diff --git a/graphdb.cpp b/graphdb.cpp
index 8e712ad51241..d1119e6e80f0 100644
--- a/graphdb.cpp
+++ b/graphdb.cpp
@@ -17,8 +17,6 @@
  *
  * - Apurva Mehta <amehta@linkedin.com>
  */
- 
-
 #include <iostream>
 #include <string>
 #include <sys/mman.h>
@@ -35,40 +33,47 @@
 #include <sys/resource.h>
 #include <pthread.h>
 
-using namespace std; 
+using namespace std;
 
 static const unsigned int FILE_SIZE = 1048576 * 10; // 10MB FILE_SIZE
 static const string SET_0 = "./mmap_set_0";
 static const string LIVE_DATA = "./mmap_live_data";
 static const string SET_1 = "./mmap_set_1";
-static const unsigned int NUM_FILES = 2500;
+static const unsigned int NUM_FILES = 10;
 static const unsigned int NUM_READ_THREADS = 25;
 static const unsigned int NUM_WRITE_THREADS = 10;
+static const unsigned int DURATION = 300;
+static const unsigned long long IO_READ_LATENCY_THRESHOLD = 200;
+static const unsigned long long IO_WRITE_LATENCY_THRESHOLD = 400;
+static const unsigned long long VM_LATENCY_THRESHOLD = 10;
+static const useconds_t read_delay = 20000;
+static const useconds_t max_write_delay = 100000;
 static pthread_rwlock_t addr_lock = PTHREAD_RWLOCK_INITIALIZER;
 
+unsigned long long finish_time = 0;
 
 vector<unsigned long* > open_and_mmap_files_from_dir(string dname)
 {
-        vector<unsigned long* > addresses(NUM_FILES);
-	
+	vector<unsigned long* > addresses(NUM_FILES);
+
 	for (unsigned int i = 0; i < NUM_FILES; ++i){
-		ostringstream os; 
+		ostringstream os;
+		string ss;
 		os << dname << "/" << i << ".dat";
-		const char *filename = os.str().c_str();
+		ss = os.str();
+		const char *filename = ss.c_str();
 
 		int fd = open(filename, O_RDONLY);
-
 		if (fd == -1){
-			cerr << "Unable to open " << filename << ". Error code: " << errno << endl; 
+			cerr << "Unable to open " << dname << "/" << filename << ". Error code: " << errno << endl;
 			exit(1);
 		}
 
 		unsigned long *data = (unsigned long *)mmap(0, FILE_SIZE, PROT_READ, MAP_PRIVATE, fd, 0);
-
 		if ((void *)data == MAP_FAILED) {
-			cerr << "Could not mmap " << filename << ". Error code: " << errno << endl; 
+			cerr << "Could not mmap " << filename << ". Error code: " << errno << endl;
 			exit(1);
-		}		
+		}
 
 		close(fd);
 
@@ -89,7 +94,6 @@ unsigned long get_user_time_ms(struct rusage start_time, struct rusage end_time)
 	return (timeval_to_us(end_time.ru_utime) - timeval_to_us(start_time.ru_utime)) / 1000;
 }
 
-
 unsigned long get_system_time_ms(struct rusage start_time, struct rusage end_time)
 {
 	return (timeval_to_us(end_time.ru_stime) - timeval_to_us(start_time.ru_stime)) / 1000;
@@ -103,60 +107,97 @@ string get_time_string()
 
 	time(&raw_time);
 	timeinfo = localtime(&raw_time);
-	strftime(buffer, 80, "%D %T %Z", timeinfo);
+	strftime(buffer, 80, "info %D %T %Z", timeinfo);
 
 	return string(buffer);
+}
 
+void start_timer(void)
+{
+	struct timeval current;
+	gettimeofday(&current, NULL);
+	finish_time = timeval_to_us(current) + DURATION * 1000000;
+	cerr << get_time_string() << " | started timer" << endl;
 }
 
+bool finished(void)
+{
+	struct timeval current;
+
+	if (!finish_time)
+		return false;
 
+	gettimeofday(&current, NULL);
+	return timeval_to_us(current) > finish_time;
+}
 
 void open_mmap_read_close(const char *filename)
 {
 	int fd = open(filename, O_RDONLY);
 
 	if (fd == -1){
-		cerr << "Unable to open " << filename << ". Error code: " << errno << endl; 
+		cerr << "Unable to open " << filename << ". Error code: " << errno << endl;
 		exit(1);
 	}
 
 	unsigned long *data = (unsigned long *)mmap(0, FILE_SIZE, PROT_READ, MAP_PRIVATE, fd, 0);
-
 	if (data == MAP_FAILED) {
-		cerr << "Could not mmap " << filename << ". Error code: " << errno << endl; 
+		cerr << "Could not mmap " << filename << ". Error code: " << errno << endl;
 		exit(1);
 	}
+
 	unsigned int num_ints = FILE_SIZE / sizeof(unsigned long);
-	int read_bytes = 0; 
-	unsigned long tmp; 
+	int read_bytes = 0;
+	unsigned long tmp, sum = 0;
 	for (unsigned int i = 0; i < num_ints; ++i ){
-	      tmp = data[i];
-	      read_bytes += sizeof(unsigned long);
+		tmp = data[i];
+		sum += tmp;
+		read_bytes += sizeof(unsigned long);
 	}
 
-	if (munmap(data, FILE_SIZE) == -1) cerr << "Cound not unmap file" << endl; 
+	if (munmap(data, FILE_SIZE) == -1) cerr << "Could not unmap file with junk value " << sum << endl;
 
-	cout << get_time_string() << " | Opened, mmapped, and closed file: " << string(filename) << ". Read: " << read_bytes << " bytes." << endl; 
+	cerr << get_time_string() << " | Opened, mmapped, and closed file: " << string(filename) << ". Read: " << read_bytes << " bytes" << endl;
 	close(fd);
 }
 
+ofstream *open_latency(void)
+{
+	ostringstream fname;
+	fname << (unsigned int) pthread_self() << "-latency.dat";
+
+	return new ofstream(fname.str().c_str(), ios::trunc | ios::binary);
+}
+
+void report_latency(ofstream *fout, unsigned long long current_time, const char *op, unsigned long long latency)
+{
+	char buf[128];
+
+	sprintf(buf, "%llu %s %llu", current_time, op, latency);
+	*fout << buf << endl;
+}
+
+void close_latency(ofstream *fout)
+{
+	fout->close();
+	delete fout;
+}
+
 void *read_randomly_from_addresses(void *addrs)
 {
-	
-	struct rusage r_start_time;
-	struct rusage r_end_time; 
 	struct timeval start_time;
 	struct timeval end_time;
+	ofstream *fout;
 
-	unsigned long long start; 
-	unsigned long long end; 
+	unsigned long long start;
+	unsigned long long end;
 
-	unsigned long tmp; 
+	unsigned long tmp = 0, sum = 0;
 
 	vector<unsigned long*> *mapped_addrs =  (vector<unsigned long*> *) addrs;
+	fout = open_latency();
 
-	while(true){
-
+	while(!finished()) {
 		int lock_rc = 0;
 		while ((lock_rc = pthread_rwlock_rdlock(&addr_lock)) == EBUSY);
 
@@ -167,43 +208,42 @@ void *read_randomly_from_addresses(void *addrs)
 
 		unsigned int file_no = rand() % mapped_addrs->size();
 		unsigned int file_sz = FILE_SIZE/sizeof(unsigned long);
-		unsigned int offset = 0 ; 
+		unsigned int offset = 0 ;
 		unsigned int rd_size = rand() % (file_sz - offset);
 
-		getrusage(RUSAGE_THREAD, &r_start_time);
 		gettimeofday(&start_time, NULL);
-
-		for (unsigned int i = 0; i < rd_size; i++)
+		for (unsigned int i = 0; i < rd_size; i++) {
 			tmp = (*mapped_addrs)[file_no][offset+i];
+		}
+		sum += tmp;
+		gettimeofday(&end_time, NULL);
 
 		lock_rc = pthread_rwlock_unlock(&addr_lock);
 
 		if (lock_rc != 0)
-			cerr << get_time_string() << " | WARN : Could not release read lock in thread " << (unsigned int) pthread_self() << ". Error " << lock_rc << endl;
-	
-		getrusage(RUSAGE_THREAD, &r_end_time);
-		gettimeofday(&end_time, NULL);
+			cerr << get_time_string() << " | WARN : Could not release read lock in thread " << (unsigned int) pthread_self() << ". Error " << lock_rc << " junk " << sum << endl;
 
-		start = timeval_to_us(start_time); 
-		end = timeval_to_us(end_time);
 
-		if ((end - start) > 100000){
-			// print diagnostics if memory access took more that 500ms.
-			cout << get_time_string() << " | usr: " << get_user_time_ms(r_start_time, r_end_time) << " ms | sys: " << get_system_time_ms(r_start_time, r_end_time) << " ms | elapsed time : " << (end - start) / 1000 << " ms." << endl; 
+		start = timeval_to_us(start_time);
+		end = timeval_to_us(end_time);
 
-		}
-		usleep(20000);
+		if ((end - start) > IO_READ_LATENCY_THRESHOLD)
+			report_latency(fout, end, "read", (end - start));
+		if (read_delay)
+			usleep(read_delay);
 	}
+	close_latency(fout);
 
 	return 0;
-
 }
 
 void *create_data_and_update_pool(void *pool)
 {
 	vector<unsigned long *> *addresses = (vector<unsigned long *> *) pool;
 
-	while (true) {
+	ofstream *flatency = open_latency();
+
+	while (!finished()) {
 		// Create a new .dat file. Name will be Epoch time in microseconds.
 
 		ostringstream fname;
@@ -212,29 +252,35 @@ void *create_data_and_update_pool(void *pool)
 		unsigned long long time = timeval_to_us(tv);
 		fname << LIVE_DATA << "/" << (unsigned int) pthread_self() << "-" <<  time << ".dat";
 		ofstream fout;
-		fout.open(fname.str().c_str(), ios::trunc | ios::binary);
 
 		// Populate the file in random sized increments, with random waits (upto 10 ms) in between updates. This simulates our write traffic
 
+		struct timeval start_time, end_time;
 		unsigned int num_written = 0;
-		const unsigned int num_ints = FILE_SIZE/sizeof(unsigned long); 
+		unsigned long long latency = 0;
+		const unsigned int num_ints = FILE_SIZE/sizeof(unsigned long);
+		fout.open(fname.str().c_str(), ios::trunc | ios::binary);
+
 		while (num_written < (num_ints-1)){
 			unsigned int wr_size = rand() % (num_ints - num_written);
 
+			gettimeofday(&start_time, NULL);
 			for (unsigned int i = 0; i < wr_size; i++){
 				unsigned long data = (unsigned long)rand();
 				fout.write(reinterpret_cast<const char *>(&data), sizeof(data));
 				++num_written;
 			}
-		       
-			usleep(rand() % 100000);
-		}
-		
+			gettimeofday(&end_time, NULL);
+			latency += timeval_to_us(end_time) - timeval_to_us(start_time);
 
+			if (max_write_delay)
+				usleep(rand() % max_write_delay);
+		}
 		fout.close();
-		cout << get_time_string() << " | Finished writing " << fname.str() << endl;
+		if (latency > IO_WRITE_LATENCY_THRESHOLD)
+			report_latency(flatency, timeval_to_us(end_time), "write", latency);
 
-		// Reopen the file in readonly mode. 
+		// Reopen the file in readonly mode.
 
 		int fd = open(fname.str().c_str(), O_RDONLY);
 
@@ -242,15 +288,20 @@ void *create_data_and_update_pool(void *pool)
 			cerr << get_time_string() << " | Could not open " << fname.str() << " for reading. Error code: " << errno << endl;
 			exit(1);
 		}
+		gettimeofday(&start_time, NULL);
 		unsigned long *base = (unsigned long *) mmap(NULL, FILE_SIZE, PROT_READ, MAP_PRIVATE, fd, 0);
 		if ((void *)base == MAP_FAILED) {
 			cerr << get_time_string() << " | Could not mmap " << fname.str() << " for reading. Error code : " << errno << endl;
 			exit(1);
 		}
+		gettimeofday(&end_time, NULL);
+		latency = timeval_to_us(end_time) - timeval_to_us(start_time);
+		if (latency > VM_LATENCY_THRESHOLD)
+			report_latency(flatency, timeval_to_us(end_time), "mmap", latency);
 		close(fd);
 
-		// Lock the table and update it. 
-		int lock_rc = 0; 
+		// Lock the table and update it.
+		int lock_rc = 0;
 		while ((lock_rc = pthread_rwlock_wrlock(&addr_lock)) == EBUSY);
 
 		if (lock_rc != 0) {
@@ -264,63 +315,61 @@ void *create_data_and_update_pool(void *pool)
 		// Add the new file
 		(*addresses)[file_no] = base ;
 
-		// Unmap the old file. 
+		// Unmap the old file.
+		gettimeofday(&start_time, NULL);
 		if (munmap(old_addr, FILE_SIZE) != 0)
 			cerr << get_time_string() << " | Warn: could not mmap file at offset " << file_no << ". Error code : " << errno << endl;
+		gettimeofday(&end_time, NULL);
+		latency = timeval_to_us(end_time) - timeval_to_us(start_time);
+		if (latency > VM_LATENCY_THRESHOLD)
+			report_latency(flatency, timeval_to_us(end_time), "munmap", latency);
+
 
-		
 		lock_rc = pthread_rwlock_unlock(&addr_lock);
 		if (lock_rc != 0){
 			cerr << get_time_string() << " | Could not unlock write lock for file : " << fname.str() << ". Aborting" <<endl;
 			exit(1);
 		}
-
-		cout << get_time_string() << " | Successfully added " << fname.str() << " to working set at index " << file_no << endl;
-		
-	}		
+	}
+	close_latency(flatency);
 	return pool;
 }
-	
 
 int main()
 {
 	srand(time(0));
- 	for (unsigned int i = 0; i < NUM_FILES; ++i){
-		ostringstream os; 
+	for (unsigned int i = 0; i < NUM_FILES; ++i){
+		const char *filename;
+		stringstream os;
+		string ss;
+
 		os << SET_0 << "/" << i << ".dat";
-		const char *filename = os.str().c_str();
-		open_mmap_read_close(filename);	
+		ss = os.str();
+		filename = ss.c_str();
+		open_mmap_read_close(filename);
 	}
-	
+
 	vector<unsigned long*> mapped_addrs = open_and_mmap_files_from_dir(SET_1);
 	vector<pthread_t> read_threads(NUM_READ_THREADS);
-	vector<pthread_t> write_threads(NUM_WRITE_THREADS); 
-	void *res; 
+	vector<pthread_t> write_threads(NUM_WRITE_THREADS);
+	void *res;
+
+	pthread_rwlock_init(&addr_lock, NULL);
 
-	pthread_rwlock_init(&addr_lock, NULL); 
-	
-	cout << get_time_string() << " | Starting read threads.. " << endl; 
+	cerr << get_time_string() << " | Starting read threads.. " << endl;
 	for (unsigned int i = 0; i < NUM_READ_THREADS; ++i)
 		pthread_create(&read_threads[i], 0, &read_randomly_from_addresses, (void *) &mapped_addrs);
- 
-
-
-	cout << get_time_string() << " | Starting write threads.. " << endl; 
-
 
+	cerr << get_time_string() << " | Starting write threads.. " << endl;
 	for(unsigned int i = 0; i < NUM_WRITE_THREADS; ++i)
 		pthread_create(&write_threads[i], 0, &create_data_and_update_pool, (void *) &mapped_addrs);
 
-	
-
-	pthread_join(write_threads[0], &res); 
+	start_timer();
+	pthread_join(write_threads[0], &res);
 	pthread_join(read_threads[0], &res);
 
-	       
-		
 	for(vector<unsigned long*>::iterator i = mapped_addrs.begin(); i != mapped_addrs.end(); ++i)
-	  	munmap(*i, FILE_SIZE);
-	
+		munmap(*i, FILE_SIZE);
 
 	return 0;
 
==== END log-latencies.patch ====
#### Description graphdb
#### Details graphdb 29
