From 59f328257e0b84e7601bffc5c0171ef437c8d7d6 Mon Sep 17 00:00:00 2001 From: "Victor Grishchenko (mughal)" Date: Thu, 11 Feb 2010 13:00:26 +0100 Subject: [PATCH] Finally, Manifold is more or less usable. --- mfold/doall | 15 +++++++------- mfold/docmd | 10 ++++++++- mfold/dohrv | 20 +++++++++++++----- mfold/env.default.sh | 5 +++++ mfold/logparse | 29 +++++++++++++++++++++++++++ mfold/net.aussie.sh | 2 +- mfold/net.lossy.sh | 2 +- mfold/net.messy.sh | 2 +- mfold/run.default.sh | 8 ++++---- mfold/{run.media.sh => run.seeder.sh} | 5 ++++- mfold/unnet.default.sh | 3 --- 11 files changed, 77 insertions(+), 24 deletions(-) create mode 100644 mfold/env.default.sh create mode 100755 mfold/logparse rename mfold/{run.media.sh => run.seeder.sh} (58%) delete mode 100644 mfold/unnet.default.sh diff --git a/mfold/doall b/mfold/doall index b61d969..1f5b82c 100755 --- a/mfold/doall +++ b/mfold/doall @@ -5,20 +5,21 @@ # docmd.sh); all failed executions are # put to the FAILURES file rm -f FAILURES +rm -rf logs +mkdir logs if [ -z "$SERVERS" ]; then - SERVERS="das2.txt" + SERVERS="servers.txt" fi -HOSTS=`cat $SERVERS | awk '{print $2}'` -for srv in $HOSTS; do - ( for cmd in $@; do - if ! ./docmd.sh $srv $cmd; then +for srv in `grep -v '^#' $SERVERS`; do + ( + if ! ./docmd $srv $1; then echo $srv >> FAILURES - echo $src FAILED + echo $srv FAILED break fi - done ) & + ) & done wait diff --git a/mfold/docmd b/mfold/docmd index e0a2963..ee6fcdb 100755 --- a/mfold/docmd +++ b/mfold/docmd @@ -2,6 +2,11 @@ HOST=$1 CMD=$2 +ENV=env.default.sh + +if [ ! $SERVERS ]; then + SERVERS="servers.txt" +fi if [ -e $CMD.$HOST.sh ] ; then SHSC=$CMD.$HOST.sh ; @@ -9,11 +14,14 @@ else SHSC=$CMD.default.sh ; fi -if ( cat $SHSC | ssh $HOST ) > .$HOST.$CMD.out 2> .$HOST.$CMD.err; then +if [ ! -d logs ]; then mkdir logs; fi + +if ( cat $ENV $SHSC | ssh $HOST ) > logs/$HOST.$CMD.out 2> logs/$HOST.$CMD.err; then echo $HOST $CMD OK exit 0 else echo $HOST $CMD FAIL + cat $SHSC cat .$HOST.$CMD.out .$HOST.$CMD.err exit 1 fi diff --git a/mfold/dohrv b/mfold/dohrv index a61c878..698dad4 100755 --- a/mfold/dohrv +++ b/mfold/dohrv @@ -1,5 +1,9 @@ #!/bin/bash - +# The script downloads logs in parallel, +# feeds them into fifos; sort takes logs +# from fifos, merges and gzips them; +# the result is put into harvest/ +# if [ ! $SERVERS ]; then SERVERS="servers.txt" fi @@ -7,18 +11,24 @@ fi rm -rf harvest mkdir harvest +for s in `cat $SERVERS`; do + mkfifo harvest/$s.log +done + for s in `cat $SERVERS`; do (if ssh $s "cat ./swift/lout.gz" | gunzip | \ - perl -ne 's/(#\d+)/'$s' $1/ && print' \ - > harvest/$s.log ; then + ./logparse $s \ + > harvest/$s.log-fifo ; then echo $s harvest OK else echo $s harvest FAIL fi) & done +# Ensure your version of sort is recent enough +# batch-size is critical for performance +LC_ALL=C sort -m -s --batch-size=64 harvest/*.log-fifo | gzip > harvest/swarm.log.gz & wait -echo 'harvested all; sorting' -sort -m harvest/*.log | ./swlognm | gzip > harvest/swarm.log.gz +rm harvest/*.log-fifo echo DONE diff --git a/mfold/env.default.sh b/mfold/env.default.sh new file mode 100644 index 0000000..2fb63a4 --- /dev/null +++ b/mfold/env.default.sh @@ -0,0 +1,5 @@ +# This script sets up shared environment variables +# at the servers +export SEEDER=130.161.211.198 +export SEEDPORT=10001 +export RUNPORT=10002 diff --git a/mfold/logparse b/mfold/logparse new file mode 100755 index 0000000..948448d --- /dev/null +++ b/mfold/logparse @@ -0,0 +1,29 @@ +#!/usr/bin/perl -w + +$SERVER=shift; +%HOSTS = (); +%CHANN = (); +$CHANN{"#0"} = "none"; + +open(SRV,$ENV{"HOME"}."/.ssh/config") or die; +while () { + $srvname=$1 if /Host (\S+)/; + $HOSTS{$1}=$srvname if /HostName (\S+)/; +} +close SRV; + +while (<>) { + /([\d\_]+) (#\d+) (\S+) (.*)/ or next; + my $time = $1; + my $channel = $2; + my $event = $3; + my $rest = $4; + if ($event eq "sent") { + $rest =~ /\d+b ([\d\.]+):/; + $ip = $1; + $CHANN{"$channel"} = $HOSTS{$ip}; + } + my $host = $CHANN{"$channel"}; + $host = "unknown" if not $host; + print "$time $SERVER $host$channel $event $rest\n"; +} diff --git a/mfold/net.aussie.sh b/mfold/net.aussie.sh index 893a0be..7d0bb29 100644 --- a/mfold/net.aussie.sh +++ b/mfold/net.aussie.sh @@ -1,3 +1,3 @@ -touch .netem-on +echo eth0 > .netem-on sudo tc qdisc add dev eth0 root netem delay 400ms diff --git a/mfold/net.lossy.sh b/mfold/net.lossy.sh index 1d588a4..8fb9ed8 100644 --- a/mfold/net.lossy.sh +++ b/mfold/net.lossy.sh @@ -1,3 +1,3 @@ -touch .netem-on +echo eth0 > .netem-on sudo tc qdisc add dev eth0 root netem delay 100ms loss 5.0% diff --git a/mfold/net.messy.sh b/mfold/net.messy.sh index f376abe..5cfd8aa 100644 --- a/mfold/net.messy.sh +++ b/mfold/net.messy.sh @@ -1,3 +1,3 @@ -touch .netem-on +echo eth0 > .netem-on sudo tc qdisc add dev eth0 root netem delay 100ms 20ms 25% diff --git a/mfold/run.default.sh b/mfold/run.default.sh index 0d891d4..2afdf4c 100644 --- a/mfold/run.default.sh +++ b/mfold/run.default.sh @@ -1,12 +1,12 @@ #!/bin/bash - +# This script runs a leecher at some server; +# env variables are set in env.default.sh HASH=66b9644bb01eaad09269354df00172c8a924773b -HEAD=83.96.143.114 sleep 1 -rm -f core ulimit -c 1024000 cd swift || exit 1 +rm -f core rm -f chunk #valgrind --leak-check=yes \ -./exec/leecher $HASH chunk $HEAD:10001 0.0.0.0:10002 2>lerr | gzip > lout.gz || exit 2 +./exec/leecher $HASH chunk $SEEDER:$SEEDPORT 0.0.0.0:$RUNPORT 2>lerr | gzip > lout.gz || exit 2 diff --git a/mfold/run.media.sh b/mfold/run.seeder.sh similarity index 58% rename from mfold/run.media.sh rename to mfold/run.seeder.sh index 1c1d422..afacc46 100644 --- a/mfold/run.media.sh +++ b/mfold/run.seeder.sh @@ -6,4 +6,7 @@ if [ ! -e ScottKim_2008P.mp4 ]; then wget -c http://video.ted.com/talks/podcast/ScottKim_2008P.mp4 || exit 1 fi -./exec/seeder ScottKim_2008P.mp4 0.0.0.0:10001 2> lerr | gzip > lout.gz +(./exec/seeder ScottKim_2008P.mp4 0.0.0.0:$SEEDPORT \ + 2> lerr | gzip > lout.gz ) >/dev/null 2> /dev/null & + +exit diff --git a/mfold/unnet.default.sh b/mfold/unnet.default.sh deleted file mode 100644 index e4a52f5..0000000 --- a/mfold/unnet.default.sh +++ /dev/null @@ -1,3 +0,0 @@ -if [ -e .netem-on ]; then - sudo tc qdisc del dev eth0 root -fi -- 2.20.1