X-Git-Url: http://p2p-next.cs.pub.ro/gitweb/?a=blobdiff_plain;f=src%2Flibswift%2Fmfold%2Fdohrv;fp=src%2Flibswift%2Fmfold%2Fdohrv;h=c0b37c208fba47697eb9c92da6fa4dc34eaf777b;hb=45963a7511531cd1656ad5d3847d2dafd015c54d;hp=0000000000000000000000000000000000000000;hpb=d069796805ad79542fd7e4406d1e9c6d2d8c2ef7;p=swifty.git

diff --git a/src/libswift/mfold/dohrv b/src/libswift/mfold/dohrv
new file mode 100755
index 0000000..c0b37c2
--- /dev/null
+++ b/src/libswift/mfold/dohrv
@@ -0,0 +1,117 @@
+#!/bin/bash
+#   The script downloads logs in parallel,
+#   feeds them into fifos; sort takes logs
+#   from fifos, merges and gzips them;
+#   the result is put into harvest/
+#
+if [ ! $SERVERS ]; then
+    export SERVERS="servers.txt"
+fi
+
+. env.default.sh
+
+if [ ! $TMPDIR ]; then
+    TMPDIR=/tmp
+fi
+
+if [ ! $MAXHARVEST ]; then
+    MAXHARVEST=100
+fi
+
+mv harvest .hrv-old
+rm -rf .hrv-old &
+mkdir harvest
+
+i=0
+j=1
+for sstr in `grep -v '#' $SERVERS`; do
+    s=${sstr%:*}
+    mkfifo harvest/$s-$j.fifo
+    # yas, yes, yes
+    (
+	if ssh $s \
+	    "if [ -e ~/.dohrv_copying ]; then exit 1; \
+            else touch ~/.dohrv_copying; fi" ; then
+	    scp ~/.ssh/config $s:.ssh/config > /dev/null 
+	    scp $SERVERS $s:swift/mfold/servers.txt > /dev/null
+	    ssh $s "rm -f ~/.dohrv_copying"
+	fi
+    ) &
+    let i++
+    if [ $i == $MAXHARVEST ]; then
+	wait
+	i=0
+	mkfifo harvest/swpart$j.fifo
+	let j++
+    fi
+done
+if [[ $i>0 ]]; then
+    wait
+    mkfifo harvest/swpart$j.fifo
+fi
+
+echo 'Done making fifos and copying configs.'
+
+i=0
+j=1
+for sstr in `grep -v '#' $SERVERS`; do
+    s=${sstr%:*}
+    (
+	if ssh $s \
+            "cd swift/  && \
+        rm -rf $s-harvest && mkdir $s-harvest && \
+        ( zcat $s-lout.gz | ./mfold/logparse $s | gzip )" \
+            | gunzip > harvest/$s-$j.fifo ; then
+
+            ssh $s "cd swift/; tar cz $s-harvest" | tar xz
+	    mv $s-harvest/* harvest/
+	    rmdir $s-harvest
+            echo $s harvest OK
+	    
+	else
+            echo $s harvest FAIL
+	fi
+    ) &
+    let i++
+    if [ $i == $MAXHARVEST ]; then
+	# Ensure your version of sort is recent enough
+	# batch-size is critical for performance
+	LC_ALL=C sort -m -s -T $TMPDIR --batch-size=64 --compress-program=gzip \
+	    harvest/*-$j.fifo | gzip > harvest/swpart$j.log.gz &
+	wait
+	i=0
+	let j++
+    fi
+done  
+if [[ $i>0 ]]; then
+    LC_ALL=C sort -m -s -T $TMPDIR --batch-size=64 --compress-program=gzip \
+	harvest/*-$j.fifo | gzip > harvest/swpart$j.log.gz &
+    wait
+    let j++
+fi
+
+echo 'Done sorting of swarm parts.'
+
+if [[ $j>2 ]]; then
+    for (( i=1; i<j; i++ )); do
+	(zcat harvest/swpart$i.log.gz > harvest/swpart$i.fifo) &
+    done
+    LC_ALL=C sort -m -s -T $TMPDIR --batch-size=64 --compress-program=gzip \
+	harvest/swpart*.fifo | gzip > harvest/swarm.log.gz &
+    wait
+else
+    mv harvest/swpart1.log.gz harvest/swarm.log.gz
+fi
+
+echo 'Done sorting of whole swarm.'
+
+rm harvest/*.fifo
+rm harvest/swpart*.log.gz
+./loggraphs
+./logreport > harvest/index.html
+#./logdistr
+
+cp report.css harvest
+# scp -rq harvest mfold.libswift.org:/storage/mfold-granary/`date +%d%b_%H:%M`_`whoami` &
+
+echo DONE