From 5ab676d0f3c49f152649fa6ad366e65b040dc932 Mon Sep 17 00:00:00 2001 From: "Victor Grishchenko (mughal)" Date: Thu, 18 Feb 2010 10:24:18 +0100 Subject: [PATCH] Manifold to run the swift executable. Channel graphs are prettified. --- mfold/build.default.sh | 8 ++++++++ mfold/clean.default.sh | 13 ++++++++----- mfold/compile.default.sh | 22 ++++++++-------------- mfold/docmd | 4 ++++ mfold/dohrv | 12 +++++++++--- mfold/dotop | 9 +++++++++ mfold/loggraphs | 30 +++++++++++++++++++++++------- mfold/logparse | 21 +++++++++++++++++++++ mfold/logreport | 2 -- mfold/run.default.sh | 6 +++--- mfold/run.seeder.sh | 5 ++--- 11 files changed, 95 insertions(+), 37 deletions(-) create mode 100755 mfold/dotop diff --git a/mfold/build.default.sh b/mfold/build.default.sh index 33e6319..647633c 100644 --- a/mfold/build.default.sh +++ b/mfold/build.default.sh @@ -45,4 +45,12 @@ echo building CPPPATH=~/include LIBPATH=~/lib scons -j4 || exit 7 echo testing tests/connecttest || exit 8 + +# TODO: one method +if [ ! -e bin ]; then mkdir bin; fi +g++ -I. *.cpp compat/*.cpp ext/seq_picker.cpp -pg -o bin/swift-pg & +g++ -I. *.cpp compat/*.cpp ext/seq_picker.cpp -g -o bin/swift-dbg & +g++ -I. *.cpp compat/*.cpp ext/seq_picker.cpp -O3 -o bin/swift-o3 & +wait + echo done diff --git a/mfold/clean.default.sh b/mfold/clean.default.sh index 5ea4fa8..c2035fd 100644 --- a/mfold/clean.default.sh +++ b/mfold/clean.default.sh @@ -1,9 +1,12 @@ +if [ -e .netem-on ]; then + sudo tc qdisc del dev `cat .netem-on` root + rm .netem-on +fi +sudo iptables -F & cd swift -rm -f chunk core lout lerr +rm -f chunk core killall leecher killall seeder -if [ -e .netem-iface ]; then - sudo tc qdisc del dev `cat .netem-iface` root - rm .netem-iface -fi +killall swift-o3 +killall swift-dbg echo DONE diff --git a/mfold/compile.default.sh b/mfold/compile.default.sh index 40db746..1d277d6 100644 --- a/mfold/compile.default.sh +++ b/mfold/compile.default.sh @@ -1,18 +1,12 @@ cd swift || exit 1 +if [ ! -d bin ]; then mkdir bin; fi git pull || exit 2 -rm exec/*-pg exec/*-o3 - -g++ -I. exec/leecher.cpp *.cpp compat/*.cpp ext/seq_picker.cpp -pg -o exec/leecher-pg & -g++ -I. exec/seeder.cpp *.cpp compat/*.cpp ext/seq_picker.cpp -pg -o exec/seeder-pg & -g++ -I. exec/leecher.cpp *.cpp compat/*.cpp ext/seq_picker.cpp -O3 -o exec/leecher-o3 & -g++ -I. exec/seeder.cpp *.cpp compat/*.cpp ext/seq_picker.cpp -O3 -o exec/seeder-o3 & +rm bin/swift-pg bin/swift-o3 bin/swift-dbg +g++ -I. *.cpp compat/*.cpp ext/seq_picker.cpp -pg -o bin/swift-pg & +g++ -I. *.cpp compat/*.cpp ext/seq_picker.cpp -g -o bin/swift-dbg & +g++ -I. *.cpp compat/*.cpp ext/seq_picker.cpp -O3 -o bin/swift-o3 & wait - -cd exec - -if [ ! -e leecher-pg ]; then exit 3; fi -if [ ! -e leecher-o3 ]; then exit 4; fi -if [ ! -e seeder-pg ]; then exit 5; fi -if [ ! -e seeder-o3 ]; then exit 6; fi - +if [ ! -e bin/swift-pg ]; then exit 4; fi +if [ ! -e bin/swift-dbg ]; then exit 5; fi +if [ ! -e bin/swift-o3 ]; then exit 6; fi diff --git a/mfold/docmd b/mfold/docmd index 4b9e102..87de81e 100755 --- a/mfold/docmd +++ b/mfold/docmd @@ -11,6 +11,10 @@ else fi if [ ! -d logs ]; then mkdir logs; fi +if [ ! -e $SHSC ]; then + echo $HOST $CMD EMPTY + exit 0 +fi if ( cat $ENV $SHSC | ssh $HOST ) > logs/$HOST.$CMD.out 2> logs/$HOST.$CMD.err; then echo $HOST $CMD OK diff --git a/mfold/dohrv b/mfold/dohrv index ec170fb..2e50856 100755 --- a/mfold/dohrv +++ b/mfold/dohrv @@ -16,10 +16,15 @@ for s in `grep -v '#' $SERVERS`; do done for s in `grep -v '#' $SERVERS`; do - (if ssh $s "cat ./swift/lout.gz" | gunzip | \ - ./logparse $s \ - > harvest/$s.fifo ; then + (if ssh $s \ + "cd swift/ && \ + rm -rf harvest && mkdir harvest && \ + ( zcat lout.gz | ./mfold/logparse $s | gzip )" \ + | gunzip > harvest/$s.fifo ; then + + ssh $s "cd swift/; tar cz harvest" | tar xz echo $s harvest OK + else echo $s harvest FAIL fi) & @@ -29,6 +34,7 @@ done # batch-size is critical for performance LC_ALL=C sort -m -s --batch-size=64 harvest/*.fifo | gzip > harvest/swarm.log.gz & wait +./loggraphs ./logreport > harvest/index.html rm harvest/*.fifo diff --git a/mfold/dotop b/mfold/dotop new file mode 100755 index 0000000..33a6c36 --- /dev/null +++ b/mfold/dotop @@ -0,0 +1,9 @@ +#!/bin/bash + +while true; do + rm logs/*status.out + ( ./doall status > /dev/null ) & + wait + clear + cat logs/*status.out | sort +done diff --git a/mfold/loggraphs b/mfold/loggraphs index 222beea..579bef9 100755 --- a/mfold/loggraphs +++ b/mfold/loggraphs @@ -6,6 +6,9 @@ fi cd harvest +HEAD=`head -1 *.log | grep -v '^$' | cut -f1 | sort | head -1` +TAIL=`tail -n1 -q *.log | grep -v '^$' | cut -f1 | sort | tail -n1 -q` + for from in `grep -v '^#' ../$SERVERS`; do for to in `grep -v '^#' ../$SERVERS`; do CWNDLOG="$from-$to-cwnd.log" @@ -21,30 +24,43 @@ for from in `grep -v '^#' ../$SERVERS`; do echo "set y2label 'packets'" >> $GP echo "set ylabel 'microseconds'" >> $GP echo "set xlabel 'run time millis'" >> $GP + #echo "set xrange [$HEAD:$TAIL]" >> $GP CWNDLOG="$from-$to-cwnd.log" - echo -ne "plot '$CWNDLOG' using 1:2 with lines title 'cwnd'"\ + echo -ne "plot '$CWNDLOG' using 1:2 with lines lt rgb '#00aa00' title 'cwnd'"\ " axis x1y2, "\ - " '$CWNDLOG' using 1:3 with lines title 'data out'"\ + " '$CWNDLOG' using 1:3 with lines lt rgb '#99ff99' title 'data out'"\ " axis x1y2 "\ >> $GP RTTLOG="$from-$to-rtt.log" if [ -e $RTTLOG ]; then - echo -ne ", '$RTTLOG' using 1:2 with lines title 'rtt' "\ + echo -ne ", '$RTTLOG' using 1:2 with lines lt rgb '#2833ff' title 'rtt' "\ "axis x1y1, "\ - "'$RTTLOG' using 1:3 with lines title 'dev' "\ + "'$RTTLOG' using 1:3 with lines lt rgb '#8844ff' title 'dev' "\ "axis x1y1"\ >> $GP fi OWDLOG="$from-$to-owd.log" if [ -e $OWDLOG ]; then - echo -ne ", '$OWDLOG' using 1:2 with lines title 'owd' "\ + echo -ne ", '$OWDLOG' using 1:2 with lines lt rgb '#8800ee' title 'owd' "\ "axis x1y1, "\ - "'$OWDLOG' using 1:3 with lines title 'min owd'"\ + "'$OWDLOG' using 1:3 with lines lw 2 lt rgb '#0044cc' title 'min owd'"\ "axis x1y1, "\ - "'$OWDLOG' using 1:($2+25000) with lines title 'target'"\ + "'$OWDLOG' using 1:(\$3+25000) with lines lw 2 lt rgb '#0000ff' title 'target'"\ "axis x1y1 "\ >> $GP fi + RDATALOG="$from-$to-rdata.log" + if [ -e $RDATALOG ]; then + echo -ne ", '$RDATALOG' using 1:(1) with points "\ + "lt rgb '#0f0000' title 'r-losses'"\ + >> $GP + fi + TDATALOG="$from-$to-tdata.log" + if [ -e $TDATALOG ]; then + echo -ne ", '$TDATALOG' using 1:(1) with points "\ + "lt rgb '#ff0000' title 't-losses'"\ + >> $GP + fi echo >> $GP ( cat $GP | gnuplot ) & done diff --git a/mfold/logparse b/mfold/logparse index e1fbcb7..4a0229c 100755 --- a/mfold/logparse +++ b/mfold/logparse @@ -11,6 +11,8 @@ $SERVER=shift; %CWNDLOG = (); %RTTLOG = (); %OWDLOG = (); +%TDATALOG = (); +%RDATALOG = (); $SENTB = 0; $RCVDB = 0; @@ -66,6 +68,18 @@ while (<>) { } print {$RTTLOG{$host}} "$ms\t$1\t$2\n"; } + } elsif ($event eq "Tdata") { + if (not exists $TDATALOG{$host}) { + open(my $handle, '>', "harvest/$SERVER-$host-tdata.log") or die; + $TDATALOG{$host} = $handle; + } + print {$TDATALOG{$host}} "$ms\n"; + } elsif ($event eq "Rdata") { + if (not exists $RDATALOG{$host}) { + open(my $handle, '>', "harvest/$SERVER-$host-rdata.log") or die; + $RDATALOG{$host} = $handle; + } + print {$RDATALOG{$host}} "$ms\n"; } $EVENTS{"$host"} = { "+hs"=>0 } if not exists $EVENTS{"$host"}; @@ -86,6 +100,13 @@ for $host (keys %OWDLOG) { for $host (keys %RTTLOG) { close ($RTTLOG{$host}); } +for $host (keys %TDATALOG) { + close ($TDATALOG{$host}); +} +for $host (keys %RDATALOG) { + close ($RDATALOG{$host}); +} + open(LEGEND,"> harvest/$SERVER-legend.txt") or die; for $channel (keys %CHANN) { diff --git a/mfold/logreport b/mfold/logreport index bafe4c3..346e4b4 100755 --- a/mfold/logreport +++ b/mfold/logreport @@ -1,7 +1,5 @@ #!/bin/bash -./loggraphs - cd harvest echo '' diff --git a/mfold/run.default.sh b/mfold/run.default.sh index 2afdf4c..0ad6ddc 100644 --- a/mfold/run.default.sh +++ b/mfold/run.default.sh @@ -3,10 +3,10 @@ # env variables are set in env.default.sh HASH=66b9644bb01eaad09269354df00172c8a924773b -sleep 1 ulimit -c 1024000 cd swift || exit 1 rm -f core rm -f chunk -#valgrind --leak-check=yes \ -./exec/leecher $HASH chunk $SEEDER:$SEEDPORT 0.0.0.0:$RUNPORT 2>lerr | gzip > lout.gz || exit 2 +sleep $(( $RANDOM % 5 )) +bin/swift-dbg -w -h $HASH -f chunk -t $SEEDER:$SEEDPORT \ + -l 0.0.0.0:$RUNPORT -p -D 2>lerr | gzip > lout.gz || exit 2 diff --git a/mfold/run.seeder.sh b/mfold/run.seeder.sh index afacc46..3479952 100644 --- a/mfold/run.seeder.sh +++ b/mfold/run.seeder.sh @@ -6,7 +6,6 @@ if [ ! -e ScottKim_2008P.mp4 ]; then wget -c http://video.ted.com/talks/podcast/ScottKim_2008P.mp4 || exit 1 fi -(./exec/seeder ScottKim_2008P.mp4 0.0.0.0:$SEEDPORT \ - 2> lerr | gzip > lout.gz ) >/dev/null 2> /dev/null & - +bin/swift-dbg -w -f ScottKim_2008P.mp4 -p -D \ + -l 0.0.0.0:$SEEDPORT 2>lerr | gzip > lout.gz || exit 2 exit -- 2.20.1