#!/usr/bin/ksh
#
# (c) 2009, lists@nerdbynature.de
#
# Yet another filesystem benchmark script
#
# - bonnie++
# - dbench
# - iozone3
# - tiobench
# - generic operations, like tar/touch/rm
#
# TODO:
# - bonnie++ 1.96 is out!
# - different mount options for each filesystems
# - different benchmark options
# - integrate fio? (http://git.kernel.dk/?p=fio.git)
#
# v0.1 - initial version
# v0.2 - disabled 2 filesystems
#        ufs - http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=526586
#        nilfs2 - filesystem fills up until ENOSPC
# v0.3 - run tiobench with only 1 thread, otherwise we get:
#        Illegal division by zero at /usr/bin/tiobench line 163
# v0.4 - rewrite for ksh: this way we have fancy features to replace seq(1)
#        but still don't have to use a full blown bash.
# v0.5 - replace "date +%s"
#        enable NILFS2, UFS and ZFS again
#        rework the generic tests
# 
CONF="/usr/local/etc/fs-bench.conf"

log() {
echo "`date +'%F %H:%M:%S'`: $1"
[ -n "$2" ] && exit "$2"
}

# little helper function to warn if we might run out of diskspace during the benchmark.
# Takes three arguments (count in 1024, size in bytes and a multiplier) seperated by a colon,
# result will be given in KB:
#    $ echo "1:1024:2" | chkfree
#   -> 2048
chkfree() {
eval `awk -F: '{p=sprintf ("%.0f", ($1 * $2 * $3)); print "ESTIM="p }'`
eval `df -k $MPT | awk '!/Filesystem/ {print "AVAIL="$4}'`
if [ $ESTIM -ge $AVAIL ]; then
	log "WARNING: $ESTIM KB estimated but only $AVAIL KB available - $b could fail!"
else
	log "DEBUG:  $ESTIM KB estimated, $AVAIL KB available" >> $LOG/raw/bonnie-$fs.log
fi
}

# sanity checks
if [ ! -b "$1" -o ! -d "$2" -o ! -f $CONF ]; then
	log "Usage: `basename $0` [dev] [mpt]"
	log "Make sure $CONF exists!" 1
else
	DEV="$1"
	MPT="`echo $2 | sed 's/\/$//'`"
	. "$CONF"
fi

# overwrite results dir?
if [ -d "$LOG" ]; then
	printf "Directory $LOG already exists, overwrite? (y/n) " && read c
	if [ $c = "y" ]; then
		$DEBUG rm -rf "$LOG"
	else
		log "Aborted." 1
	fi
fi
$DEBUG mkdir -p "$LOG"/raw "$LOG"/env
$DEBUG cp "$0"  "$LOG"/env/`basename $0`.txt
$DEBUG cp /proc/config* /boot/config-$(uname -r) "$LOG"/env 2>/dev/null
$DEBUG dmesg > "$LOG"/env/dmesg.txt
$DEBUG lspci > "$LOG"/env/lspci.txt
$DEBUG hdparm -tT "$DEV" > "$LOG"/env/hdparm.txt

########################################################
# MKFS
########################################################
mkfs_btrfs() {
$DEBUG mkfs.btrfs $DEV 1>/dev/null
$DEBUG mount -t btrfs -o noatime $DEV $MPT
ERR=$?
}

mkfs_ext2() {
$DEBUG mkfs.ext2 -Fq $DEV
$DEBUG mount -t ext2 -o noatime,user_xattr $DEV $MPT
ERR=$?
}

mkfs_ext3() {
$DEBUG mkfs.ext3 -Fq $DEV
$DEBUG mount -t ext3 -o noatime,user_xattr $DEV $MPT
ERR=$?
}

mkfs_ext4() {
$DEBUG mkfs.ext4 -Fq $DEV
$DEBUG mount -t ext4 -o noatime,user_xattr $DEV $MPT
ERR=$?
}

mkfs_jfs() {
$DEBUG mkfs.jfs -q $DEV 1>/dev/null
$DEBUG mount -t jfs -o noatime $DEV $MPT
ERR=$?
}

mkfs_nilfs2() {
$DEBUG mkfs.nilfs2 -q $DEV
$DEBUG mount -t nilfs2 -o noatime $DEV $MPT 2>/dev/null
ERR=$?
}

mkfs_reiserfs() {
$DEBUG mkfs.reiserfs -qf $DEV > /dev/null 2>&1
$DEBUG mount -t reiserfs -o noatime,user_xattr $DEV $MPT
ERR=$?
}

mkfs_ufs() {
$DEBUG mkfs.ufs -J -O2 -U $DEV > /dev/null
$DEBUG mount -t ufs -o ufstype=ufs2,noatime $DEV $MPT
ERR=$?
}

mkfs_xfs() {
$DEBUG mkfs.xfs -qf $DEV
$DEBUG mount -t xfs -o noatime $DEV $MPT
ERR=$?
}

mkfs_zfs() {
$DEBUG zpool create -f -O atime=off -m $MPT ztest $DEV
ERR=$?
}

########################################################
# BENCHMARKS
########################################################

run_bonnie() {
eval conf_bonnie
log "Running $b on $fs..."
echo $NUMFILES | awk -F: '{printf $1 ":" $2 ":1.1"}' | chkfree
$DEBUG bonnie++ -d $MPT -s $SIZE -n $NUMFILES -m $fs -r $RAM -x $NUMTESTS -u root 1>$LOG/raw/bonnie-$fs.csv 2>$LOG/raw/bonnie-$fs.err
$DEBUG cat $LOG/raw/bonnie-*.csv | bon_csv2html > $LOG/bonnie.html
}

########################################################
run_stress() {
#
# Based on http://oss.oracle.com/~mason/stress.sh
#      Copyright (C) 1999 Bibliotech Ltd., 631-633 Fulham Rd., London SW6 5UQ.
#      $Id: stress.sh,v 1.2 1999/02/10 10:58:04 rich Exp $
#
eval conf_stress
log "Running $b on $fs (size: `du -sk "$CONTENT" | awk '{print $1 / 1024 " MB"}'`)..."
$DEBUG mkdir $MPT/stress || log "cannot create $MPT/stress" 1
$DEBUG    cd $MPT/stress || log "cannot cd into $MPT/stress" 1

# computing MD5 sums over content directory
find $CONTENT -type f -exec md5sum '{}' + | sort > $MPT/content.sums

# starting stress test processes
p=1
while [ $p -le $CONCURRENT ]; do
	(
	# wait for all processes to start up.
	if [ "$STAGGER" = "yes" ]; then
		sleep `expr 2 \* $p`
	else
		sleep 1
	fi

	r=1
	while [ $r -le $RUNS ]; do
		log "Running stresstest in $MPT/stress/$p (r: $r)..."
		# Remove old directories.
		$DEBUG rm -rf $MPT/stress/$p

		# copy content
		$DEBUG mkdir $MPT/stress/$p || log "cannot create $MPT/stress/$p"
		$DEBUG cp -dRx $CONTENT/* $MPT/stress/$p || log "cannot copy $CONTENT to $MPT/stress/$p"

		# compare the content and the copy.
		$DEBUG cd $MPT/stress/$p
		$DEBUG find . -type f -exec md5sum '{}' + | sort > $MPT/stress.$p
		$DEBUG diff -q $MPT/content.sums $MPT/stress.$p
		if [ $? != 0 ]; then
			log "corruption found in $MPT/stress/$p (r: $r)"
			continue
		fi
		$DEBUG cd $MPT/stress
		$DEBUG rm -f $MPT/stress.$p
	r=`expr $r + 1`
	done
	) &
p=`expr $p + 1`
done
}

########################################################
run_dbench() {
eval conf_dbench
log "Running $b on $fs..."
$DEBUG dbench -x -t $TIME -D $MPT $NPROC > $LOG/raw/dbench-$fs.log
echo "$fs:	`egrep '^Throughput' $LOG/raw/dbench-$fs.log`" >> $LOG/dbench.txt
}

run_iozone() {
eval conf_iozone
log "Running $b on $fs..."
$DEBUG cd $MPT || log "cannot cd into $MPT" 1
$DEBUG iozone -a -c -S $CACHESIZE -s $FILESIZE > $LOG/raw/iozone-$fs.log
}

run_tiobench() {
eval conf_tiobench
log "Running $b on $fs..."
$DEBUG tiobench --identifier fs_"$fs" --size $SIZE --numruns $NUMRUNS --dir $MPT \
	--block 4096 --block 8192 --threads 1 \
	1>$LOG/raw/tiobench-$fs.log 2>$LOG/raw/tiobench-$fs.err

# results are hard to summarize
echo "                               File  Blk   Num                   Avg      Maximum      Lat%     Lat%    CPU" >  $LOG/tiobench.txt
echo "                               Size  Size  Thr   Rate  (CPU%)  Latency    Latency      >2s      >10s    Eff" >> $LOG/tiobench.txt
for t in "Sequential Reads" "Sequential Writes" "Random Reads" "Random Writes"; do
	echo $t
	# adjust -An for more/less than 2 different blocksizes!
	grep -h -A5 "$t" $LOG/raw/tiobench-*.log | egrep '^fs_'
	echo
done >> $LOG/tiobench.txt
}

run_generic() {
eval conf_generic
CONTENT_SIZE=`du -sk "$CONTENT" | awk '{print $1}'`
ERR=""
log "Running $b on $fs..."
# - copy content to fs
# - tar up local content
# - copy content within same fs
# - create NUMFILES in NUMDIRS

# tar to
GEN_BEGIN=$(printf '%(%s)T')
$DEBUG mkdir "$MPT"/tar
$DEBUG cd "$CONTENT"
$DEBUG tar -cf - . 2>>$LOG/raw/generic-$fs.err | tar -C "$MPT"/tar -xf - 2>>$LOG/raw/generic-$fs.err
DIFF="`diff -r "$CONTENT" "$MPT"/tar 2>&1 | grep -v 'No such file or directory'`"
[ -z "$DIFF" ] || ERR="- FAILED"
$DEBUG sync
$DEBUG cd /tmp
GEN_END=$(printf '%(%s)T')
GEN_DUR=`echo "scale=2; $GEN_END - $GEN_BEGIN" | bc -l`
  SPEED=`echo "scale=2; ( $CONTENT_SIZE / 1024 ) / $GEN_DUR" | bc -l 2>/dev/null`
log "$fs: 1-tar content to $MPT and running diff took $GEN_DUR seconds. ($SPEED MB/s) $ERR"  > $LOG/raw/generic-$fs.log
ERR=""

# tar from
GEN_BEGIN=$(printf '%(%s)T')
$DEBUG cd "$MPT"/tar
$DEBUG tar -cf - . 2>>$LOG/raw/generic-$fs.err | dd of=/dev/null 2>>$LOG/raw/generic-$fs.err
[ $? = 0 ] || ERR="- FAILED"
$DEBUG sync
$DEBUG cd /tmp
GEN_END=$(printf '%(%s)T')
GEN_DUR=`echo "scale=2; $GEN_END - $GEN_BEGIN" | bc -l`
  SPEED=`echo "scale=2; ( $CONTENT_SIZE / 1024 ) / $GEN_DUR" | bc -l 2>/dev/null`
log "$fs: 2-tar content on $MPT took $GEN_DUR seconds. ($SPEED MB/s) $ERR" >> $LOG/raw/generic-$fs.log
ERR=""

# copy
GEN_BEGIN=$(printf '%(%s)T')
$DEBUG mkdir "$MPT"/copy
$DEBUG cp -fpR "$CONTENT" "$MPT"/copy 2>>$LOG/raw/generic-$fs.err
[ $? = 0 ] || ERR="- FAILED"
$DEBUG sync
GEN_END=$(printf '%(%s)T')
GEN_DUR=`echo "scale=2; $GEN_END - $GEN_BEGIN" | bc -l`
  SPEED=`echo "scale=2; ( $CONTENT_SIZE / 1024 ) / $GEN_DUR" | bc -l 2>/dev/null`
log "$fs: 3-cp content took $GEN_DUR seconds. ($SPEED MB/s) $ERR" >> $LOG/raw/generic-$fs.log
ERR=""

# cleanup before starting the next test
CONTENT_SIZE_2=`du -sk "$MPT" | awk '{print $1}'`
GEN_BEGIN=$(printf '%(%s)T')
$DEBUG echo "DEBUG-1: `df -i "$MPT" | grep -v Filesystem`" >> $LOG/raw/generic-$fs.err
$DEBUG rm -rf "$MPT"/tar "$MPT"/copy 2>>$LOG/raw/generic-$fs.err
[ $? = 0 ] || ERR="- FAILED"
$DEBUG echo "DEBUG-2: `df -i "$MPT" | grep -v Filesystem`" >> $LOG/raw/generic-$fs.err
GEN_END=$(printf '%(%s)T')
GEN_DUR=`echo "scale=2; $GEN_END - $GEN_BEGIN" | bc -l`
  SPEED=`echo "scale=2; ( $CONTENT_SIZE_2 / 1024 ) / $GEN_DUR" | bc -l 2>/dev/null`
log "$fs: 4-rm content took $GEN_DUR seconds. ($SPEED MB/s) $ERR" >> $LOG/raw/generic-$fs.log
ERR=""

# create many files
GEN_BEGIN=$(printf '%(%s)T')
$DEBUG mkdir "$MPT"/manyfiles
for d in `seq 1 $NUMDIRS`; do
	$DEBUG mkdir -p $MPT/manyfiles/dir."$d" 2>>$LOG/raw/generic-$fs.err
	$DEBUG cd $MPT/manyfiles/dir."$d"       2>>$LOG/raw/generic-$fs.err
	seq 1 $NUMFILES | xargs touch           2>>$LOG/raw/generic-$fs.err
done
sync
GEN_END=$(printf '%(%s)T')
GEN_DUR=`echo "scale=2; $GEN_END - $GEN_BEGIN" | bc -l`
 NUMDIRS_C=`find $MPT/manyfiles -type d | wc -l`
NUMFILES_C=`find $MPT/manyfiles -type f | wc -l`
[ `echo "$NUMDIRS_C - 1" | bc` = $NUMDIRS -a $NUMFILES_C = `echo "$NUMDIRS * $NUMFILES" | bc` ] || ERR="- FAILED"
log "$fs: 5-creating $NUMFILES files in each of the $NUMDIRS directories took $GEN_DUR seconds. $ERR" >> $LOG/raw/generic-$fs.log
ERR=""

# cleanup before starting the next test
GEN_BEGIN=$(printf '%(%s)T')
$DEBUG echo "DEBUG-3: `df -i "$MPT" | grep -v Filesystem`" >> $LOG/raw/generic-$fs.err
$DEBUG rm -rf "$MPT"/manyfiles 2>>$LOG/raw/generic-$fs.err
[ $? = 0 ] || ERR="- FAILED"
$DEBUG echo "DEBUG-4: `df -i "$MPT" | grep -v Filesystem`" >> $LOG/raw/generic-$fs.err
GEN_END=$(printf '%(%s)T')
GEN_DUR=`echo "scale=2; $GEN_END - $GEN_BEGIN" | bc -l`
 INODES=`echo "$NUMFILES * $NUMDIRS" | bc`
  SPEED=`echo "scale=2; $INODES / $GEN_DUR" | bc -l 2>/dev/null`
log "$fs: 6-rm $INODES inodes took $GEN_DUR seconds. ($SPEED i/s) $ERR" >> $LOG/raw/generic-$fs.log
ERR=""

# create many dirs (we just replace NUMDIRS with NUMFILES and vice versa)
GEN_BEGIN=$(printf '%(%s)T')
$DEBUG mkdir -p "$MPT"/manydirs
for d in `seq 1 $NUMFILES`; do
	$DEBUG mkdir -p $MPT/manydirs/dir."$d" 2>>$LOG/raw/generic-$fs.err
	$DEBUG cd $MPT/manydirs/dir."$d"       2>>$LOG/raw/generic-$fs.err
	seq 1 $NUMDIRS | xargs touch           2>>$LOG/raw/generic-$fs.err
done
sync
GEN_END=$(printf '%(%s)T')
GEN_DUR=`echo "scale=2; $GEN_END - $GEN_BEGIN" | bc -l`
 NUMDIRS_C=`find $MPT/manydirs -type d | wc -l`
NUMFILES_C=`find $MPT/manydirs -type f | wc -l`
[ `echo "$NUMDIRS_C - 1" | bc -l` = $NUMFILES -a $NUMFILES_C = `echo "$NUMDIRS * $NUMFILES" | bc -l` ] || ERR="- FAILED"
log "$fs: 7-creating $NUMDIRS files in each of the $NUMFILES directories took $GEN_DUR seconds. $ERR" >> $LOG/raw/generic-$fs.log
ERR=""

# cleanup 
GEN_BEGIN=$(printf '%(%s)T')
$DEBUG echo "DEBUG-5: `df -i "$MPT" | grep -v Filesystem`" >> $LOG/raw/generic-$fs.err
$DEBUG rm -rf "$MPT"/manydirs 2>>$LOG/raw/generic-$fs.err
[ $? = 0 ] || ERR="- FAILED"
$DEBUG echo "DEBUG-6: `df -i "$MPT" | grep -v Filesystem`" >> $LOG/raw/generic-$fs.err
GEN_END=$(printf '%(%s)T')
GEN_DUR=`echo "scale=2; $GEN_END - $GEN_BEGIN" | bc -l`
 INODES=`echo "$NUMFILES * $NUMDIRS" | bc`
  SPEED=`echo "scale=2; $INODES / $GEN_DUR" | bc -l 2>/dev/null`
log "$fs: 8-rm $INODES inodes took $GEN_DUR seconds. ($SPEED i/s) $ERR" >> $LOG/raw/generic-$fs.log
ERR=""

# results across all tests
CONTENT_SIZE_MB=`echo "scale=2; $CONTENT_SIZE / 1024" | bc -l`
log "Content size is $CONTENT_SIZE_MB" > $LOG/generic.txt
egrep -h -- ": [0-9]-" $LOG/raw/generic-*.log | cut -d\  -f3-20 | sed 's/ /	/' >> $LOG/generic.txt
}

########################################################
# MAIN
########################################################
mount | grep -q "$MPT" && log "$MPT is already mounted!" 1

BEGIN=$(printf '%(%s)T')
# iterating through filesystems
for fs in $FILESYSTEMS; do
	echo "========================================================"
	FS_BEG=$(printf '%(%s)T')

	# mkfs, mount, umount for every benchmark
	for b in $BENCHMARKS; do
		mkfs_"$fs"
		if [ ! $ERR = 0 ]; then
			log "mkfs failed ($fs, $b)"
			continue
		fi
		BM_BEG=$(printf '%(%s)T')
		run_"$b"
		$DEBUG cd /
		$DEBUG sync
		$DEBUG sleep 5
		$DEBUG sync
		# we have to special case ZFS here :(
		case $fs in
			zfs)
			$DEBUG zpool destroy -f ztest 2>/dev/null
			# sometimes, this just fails (why?)
			if [ ! $? = 0 ]; then
				$DEBUG sync
				$DEBUG sleep 5
				$DEBUG zpool destroy -f ztest || log "Unmounting $fs failed!" 1
			fi
			;;

			*)
			$DEBUG umount $MPT || log "Unmounting $fs failed!" 1
			;;
		esac
		BM_END=$(printf '%(%s)T')
		BM_DUR=`echo "scale=2; ( $BM_END - $BM_BEG ) / 60" | bc -l`
		log "Running $b on $fs took $BM_DUR minutes."
	done
	FS_END=$(printf '%(%s)T')
	FS_DUR=`echo "scale=2; ( $FS_END - $FS_BEG ) / 60" | bc -l`
	echo
	log "Running all benchmarks on $fs took $FS_DUR minutes."
	echo
done 2>&1 | tee $LOG/fs-bench.log

END=$(printf '%(%s)T')
DUR=`echo "scale=2; ( $END - $BEGIN ) / 60" | bc -l`
log "Finished after $DUR minutes."