2 # The script downloads logs in parallel,
3 # feeds them into fifos; sort takes logs
4 # from fifos, merges and gzips them;
5 # the result is put into harvest/
7 if [ ! $SERVERS ]; then
8 export SERVERS="servers.txt"
13 if [ ! $TMPDIR ]; then
17 if [ ! $MAXHARVEST ]; then
27 for sstr in `grep -v '#' $SERVERS`; do
29 mkfifo harvest/$s-$j.fifo
33 "if [ -e ~/.dohrv_copying ]; then exit 1; \
34 else touch ~/.dohrv_copying; fi" ; then
35 scp ~/.ssh/config $s:.ssh/config > /dev/null
36 scp $SERVERS $s:swift/mfold/servers.txt > /dev/null
37 ssh $s "rm -f ~/.dohrv_copying"
41 if [ $i == $MAXHARVEST ]; then
44 mkfifo harvest/swpart$j.fifo
50 mkfifo harvest/swpart$j.fifo
53 echo 'Done making fifos and copying configs.'
57 for sstr in `grep -v '#' $SERVERS`; do
62 rm -rf $s-harvest && mkdir $s-harvest && \
63 ( zcat $s-lout.gz | ./mfold/logparse $s | gzip )" \
64 | gunzip > harvest/$s-$j.fifo ; then
66 ssh $s "cd swift/; tar cz $s-harvest" | tar xz
67 mv $s-harvest/* harvest/
76 if [ $i == $MAXHARVEST ]; then
77 # Ensure your version of sort is recent enough
78 # batch-size is critical for performance
79 LC_ALL=C sort -m -s -T $TMPDIR --batch-size=64 --compress-program=gzip \
80 harvest/*-$j.fifo | gzip > harvest/swpart$j.log.gz &
87 LC_ALL=C sort -m -s -T $TMPDIR --batch-size=64 --compress-program=gzip \
88 harvest/*-$j.fifo | gzip > harvest/swpart$j.log.gz &
93 echo 'Done sorting of swarm parts.'
96 for (( i=1; i<j; i++ )); do
97 (zcat harvest/swpart$i.log.gz > harvest/swpart$i.fifo) &
99 LC_ALL=C sort -m -s -T $TMPDIR --batch-size=64 --compress-program=gzip \
100 harvest/swpart*.fifo | gzip > harvest/swarm.log.gz &
103 mv harvest/swpart1.log.gz harvest/swarm.log.gz
106 echo 'Done sorting of whole swarm.'
109 rm harvest/swpart*.log.gz
111 ./logreport > harvest/index.html
114 cp report.css harvest
115 # scp -rq harvest mfold.libswift.org:/storage/mfold-granary/`date +%d%b_%H:%M`_`whoami` &