Finally, Manifold is more or less usable.

diff --git a/mfold/doall b/mfold/doall
index b61d969..1f5b82c 100755 (executable)
--- a/mfold/doall
+++ b/mfold/doall
@@ -5,20 +5,21 @@
 # docmd.sh); all failed executions are
 # put to the FAILURES file
 rm -f FAILURES
+rm -rf logs
+mkdir logs
 
 if [ -z "$SERVERS" ]; then
-    SERVERS="das2.txt"
+    SERVERS="servers.txt"
 fi
-HOSTS=`cat $SERVERS | awk '{print $2}'`
 
-for srv in $HOSTS; do
-    ( for cmd in $@; do
-        if ! ./docmd.sh $srv $cmd; then
+for srv in `grep -v '^#' $SERVERS`; do
+    (
+        if ! ./docmd $srv $1; then
             echo $srv >> FAILURES
-            echo $src FAILED
+            echo $srv FAILED
             break
         fi
-    done ) &
+    ) &
 done
 
 wait

diff --git a/mfold/docmd b/mfold/docmd
index e0a2963..ee6fcdb 100755 (executable)
--- a/mfold/docmd
+++ b/mfold/docmd
@@ -2,6 +2,11 @@
 
 HOST=$1
 CMD=$2
+ENV=env.default.sh
+
+if [ ! $SERVERS ]; then
+    SERVERS="servers.txt"
+fi
 
 if [ -e $CMD.$HOST.sh ] ; then 
     SHSC=$CMD.$HOST.sh ;
@@ -9,11 +14,14 @@ else
     SHSC=$CMD.default.sh ;
 fi
 
-if ( cat $SHSC | ssh $HOST ) > .$HOST.$CMD.out 2> .$HOST.$CMD.err; then
+if [ ! -d logs ]; then mkdir logs; fi
+
+if ( cat $ENV $SHSC | ssh $HOST ) > logs/$HOST.$CMD.out 2> logs/$HOST.$CMD.err; then
     echo $HOST  $CMD    OK
     exit 0
 else
     echo $HOST  $CMD    FAIL
+    cat $SHSC
     cat .$HOST.$CMD.out .$HOST.$CMD.err
     exit 1
 fi

diff --git a/mfold/dohrv b/mfold/dohrv
index a61c878..698dad4 100755 (executable)
--- a/mfold/dohrv
+++ b/mfold/dohrv
@@ -1,5 +1,9 @@
 #!/bin/bash
-
+#   The script downloads logs in parallel,
+#   feeds them into fifos; sort takes logs
+#   from fifos, merges and gzips them;
+#   the result is put into harvest/
+#
 if [ ! $SERVERS ]; then
     SERVERS="servers.txt"
 fi
@@ -7,18 +11,24 @@ fi
 rm -rf harvest
 mkdir harvest
 
+for s in `cat $SERVERS`; do
+    mkfifo harvest/$s.log
+done
+
 for s in `cat $SERVERS`; do
     (if ssh $s "cat ./swift/lout.gz" | gunzip | \
-        perl -ne 's/(#\d+)/'$s' $1/ && print' \
-        > harvest/$s.log ; then
+        ./logparse $s \
+        > harvest/$s.log-fifo ; then
         echo $s harvest OK
     else
         echo $s harvest FAIL
     fi) &
 done  
 
+# Ensure your version of sort is recent enough
+# batch-size is critical for performance
+LC_ALL=C sort -m -s --batch-size=64 harvest/*.log-fifo | gzip > harvest/swarm.log.gz &
 wait
-echo 'harvested all; sorting'
-sort -m harvest/*.log | ./swlognm | gzip > harvest/swarm.log.gz
+rm harvest/*.log-fifo
 
 echo DONE

diff --git a/mfold/env.default.sh b/mfold/env.default.sh
new file mode 100644 (file)
index 0000000..2fb63a4
--- /dev/null
+++ b/mfold/env.default.sh
@@ -0,0 +1,5 @@
+# This script sets up shared environment variables
+# at the servers
+export SEEDER=130.161.211.198
+export SEEDPORT=10001
+export RUNPORT=10002

diff --git a/mfold/logparse b/mfold/logparse
new file mode 100755 (executable)
index 0000000..948448d
--- /dev/null
+++ b/mfold/logparse
@@ -0,0 +1,29 @@
+#!/usr/bin/perl -w
+
+$SERVER=shift;
+%HOSTS = ();
+%CHANN = ();
+$CHANN{"#0"} = "none";
+
+open(SRV,$ENV{"HOME"}."/.ssh/config") or die;
+while (<SRV>) {
+    $srvname=$1 if /Host (\S+)/;
+    $HOSTS{$1}=$srvname if /HostName (\S+)/;
+}
+close SRV;
+
+while (<>) {
+    /([\d\_]+) (#\d+) (\S+) (.*)/ or next;
+    my $time = $1;
+    my $channel = $2;
+    my $event = $3;
+    my $rest = $4;
+    if ($event eq "sent") {
+        $rest =~ /\d+b ([\d\.]+):/;
+        $ip = $1;
+        $CHANN{"$channel"} = $HOSTS{$ip};
+    }
+    my $host = $CHANN{"$channel"};
+    $host = "unknown" if not $host;
+    print "$time $SERVER $host$channel $event $rest\n";
+}

diff --git a/mfold/net.aussie.sh b/mfold/net.aussie.sh
index 893a0be..7d0bb29 100644 (file)
--- a/mfold/net.aussie.sh
+++ b/mfold/net.aussie.sh
@@ -1,3 +1,3 @@
-touch .netem-on
+echo eth0 > .netem-on
 
 sudo tc qdisc add dev eth0 root netem delay 400ms

diff --git a/mfold/net.lossy.sh b/mfold/net.lossy.sh
index 1d588a4..8fb9ed8 100644 (file)
--- a/mfold/net.lossy.sh
+++ b/mfold/net.lossy.sh
@@ -1,3 +1,3 @@
-touch .netem-on
+echo eth0 > .netem-on
 
 sudo tc qdisc add dev eth0 root netem delay 100ms loss 5.0%

diff --git a/mfold/net.messy.sh b/mfold/net.messy.sh
index f376abe..5cfd8aa 100644 (file)
--- a/mfold/net.messy.sh
+++ b/mfold/net.messy.sh
@@ -1,3 +1,3 @@
-touch .netem-on
+echo eth0 > .netem-on
 
 sudo tc qdisc add dev eth0 root netem delay 100ms 20ms 25%

diff --git a/mfold/run.default.sh b/mfold/run.default.sh
index 0d891d4..2afdf4c 100644 (file)
--- a/mfold/run.default.sh
+++ b/mfold/run.default.sh
@@ -1,12 +1,12 @@
 #!/bin/bash
-
+# This script runs a leecher at some server;
+# env variables are set in env.default.sh
 HASH=66b9644bb01eaad09269354df00172c8a924773b
-HEAD=83.96.143.114
 
 sleep 1 
-rm -f core
 ulimit -c 1024000
 cd swift || exit 1
+rm -f core
 rm -f chunk
 #valgrind --leak-check=yes \
-./exec/leecher $HASH chunk $HEAD:10001 0.0.0.0:10002 2>lerr | gzip > lout.gz || exit 2
+./exec/leecher $HASH chunk $SEEDER:$SEEDPORT 0.0.0.0:$RUNPORT 2>lerr | gzip > lout.gz || exit 2

diff --git a/mfold/run.media.sh b/mfold/run.seeder.sh
similarity index 58%
rename from mfold/run.media.sh
rename to mfold/run.seeder.sh
index 1c1d422..afacc46 100644 (file)
--- a/mfold/run.media.sh
+++ b/mfold/run.seeder.sh
@@ -6,4 +6,7 @@ if [ ! -e ScottKim_2008P.mp4 ]; then
     wget -c http://video.ted.com/talks/podcast/ScottKim_2008P.mp4 || exit 1
 fi
 
-./exec/seeder ScottKim_2008P.mp4 0.0.0.0:10001 2> lerr | gzip > lout.gz
+(./exec/seeder ScottKim_2008P.mp4 0.0.0.0:$SEEDPORT \
+    2> lerr | gzip > lout.gz ) >/dev/null 2> /dev/null &
+
+exit

diff --git a/mfold/unnet.default.sh b/mfold/unnet.default.sh
deleted file mode 100644 (file)
index e4a52f5..0000000
--- a/mfold/unnet.default.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-if [ -e .netem-on ]; then
-    sudo tc qdisc del dev eth0 root
-fi