From 200166602fa517c3668ea61def0ca4e677454c76 Mon Sep 17 00:00:00 2001 From: Adriana Draghici Date: Fri, 6 Nov 2009 13:00:58 +0000 Subject: [PATCH] Tribler: StatusParser parses log files without interacting with the database. Changed the date format used for logging. Signed-off-by: Adriana Draghici --- log-parser/tribler/StatusParser.py | 256 ++++++++++++++++++++++++++++- log-samples/tribler/log/test2.txt | 104 ++++++------ tribler-mod/scrips/modify.sh | 2 +- tribler-mod/scrips/unmodify.sh | 2 +- 4 files changed, 308 insertions(+), 56 deletions(-) diff --git a/log-parser/tribler/StatusParser.py b/log-parser/tribler/StatusParser.py index 96ba9b9..05757c7 100644 --- a/log-parser/tribler/StatusParser.py +++ b/log-parser/tribler/StatusParser.py @@ -1,6 +1,258 @@ #!/usr/bin/env python import sys -from DatabaseWriter import DatabaseWriter +import getopt +import re +#from DatabaseWriter import DatabaseWriter +#from DatabaseCommander import DatabaseCommander +#import julian +import datetime -# TODO +DEBUG = True + +files_sizes = {} # dictionary: key - filename, value - filesize + +def usage(): + print "Usage: python StatusParser.py -i|--id id status_file" + print "id:" + print "\t--id" + print "\t-i\t\tclient_session_id" + print "\tstatus_file:" + print "\t--file" + print "\t-f\t\tstatus_file for tribler" + print "\tdatabase\t\tSQLite database file" + print "\t--help" + print "\t-h\t\t\tprint this help screen" + + +""" All status messages contain a status string (e.g. DLSTATUS_DOWNLOADING). """ +def tribler_is_status_line(line): + if line.find("DLSTATUS_DOWNLOADING") > -1 or line.find("DLSTATUS_SEEDING") > -1: + return True + return False + +def tribler_is_single_download_line(line): + if line.find("SingleDownload") == -1: + return False + return True + +""" Parse a line with this format: SingleDownload: save_as( u'' '' ) + Saves the file name and size in the dictionary file_sizes. + If the line does not correspond to this format, it does nothing. +""" +def tribler_get_file_size(line): + index = -1 + parts = [] + if line.find("save_as") != -1: + parts = line.split("'") + files_sizes[parts[1]] = int(parts[2]) # saves the filename and its size in bytes + +def tribler_canon_num_peers(non_canon_value): + return int(non_canon_value) + +def tribler_canon_dht(non_canon_value): + return int(non_canon_value) + +# 119.51kb/s -> 119 +def tribler_canon_download_speed(non_canon_value): + return int(float(non_canon_value.strip("KB/s"))) + +# 12119.51kb/s -> 12119 +def tribler_canon_upload_speed(non_canon_value): + return int(float(non_canon_value.strip("KB/s"))) + +# 25% -> 25*file_size/100 +def tribler_canon_download_size(non_canon_value, filename): + return float(non_canon_value.strip("%")) * files_sizes[filename] / 100 + +# 492mb -> 492*1024*1024 +def tribler_canon_upload_size(non_canon_value): + pass + +def tribler_canon_eta(non_canon_value): + return int(float(non_canon_value)) + +# +# sample tribler status line +# 03 Nov 2009 12:18:55 aqua.mpeg DLSTATUS_DOWNLOADING 29.84% None up 0.00KB/s down 4414.39KB/s eta 12 peers 2 +# +def tribler_parse_status_line(line): + num_peers = 0 + dht = 0 + download_speed = 0 + upload_speed = 0 + download_size = 0 + upload_size = 0 + eta = 0 + filename = "" + timestamp = None + string_array = re.split("\ *", line) + + if DEBUG == True: + print "string_array is: ", string_array + if len(string_array) != 16: + print "Error: Wrong line format!" + return None + + # get timestamp and transform it in datetime format + time_array = string_array[3].split(":"); + if len(time_array) != 3: + timestamp = None + else: + timestamp = datetime.datetime(int(string_array[2]), int(string_array[1]), int(string_array[0]), #year, month, day + int(time_array[0]), int(time_array[1]), int(time_array[2])) #hour, min, sec + + filename = string_array[4] + + i = 5 + while i < len(string_array): #string_array: + if string_array[i] == "peers": + num_peers = tribler_canon_num_peers(string_array[i+1]) + print "num_peers = %d" %(num_peers) + i = i + 2 + continue + if string_array[i] == "down": + download_speed = tribler_canon_download_speed(string_array[i+1]) + print "download_speed = %d" %(download_speed) + i = i + 2 + continue + if string_array[i] == "up": + upload_speed = tribler_canon_upload_speed(string_array[i+1]) + print "upload_speed= %d" %(upload_speed) + i = i + 2 + continue + if string_array[i] == "DLSTATUS_DOWNLOADING" or string_array[i] == "DLSTATUS_SEEDING": + download_size = tribler_canon_download_size(string_array[i+1], filename) + print "download_size = %d" %(download_size) + i = i + 2 + continue + if string_array[i] == "eta": + eta = tribler_canon_eta(string_array[i+1]) + print "eta = %d" %(eta) + i = i + 2 + continue + i = i + 1 + + print "-----------------------------gata o linie----------------" + return (timestamp, num_peers, dht, download_speed, upload_speed, download_size, upload_size, eta) + +def tribler_parse_status_file(dbw, client_session_id, filename): + + try: + fin = open(filename, "r") + while 1: + line = fin.readline() + if not line: + break + + line = line.strip() + if tribler_is_single_download_line(line) == True: + tribler_get_file_size(line) + + if tribler_is_status_line(line) == False: + continue + + (time, num_peers, dht, download_speed, upload_speed, download_size, upload_size, eta_time) = tribler_parse_status_line(line) + + + # if DEBUG == True: + print "(%d, %s, %d, %d kb/s, %d kb/s, %d bytes, %d bytes)" % (num_peers, time, eta_time, download_speed, upload_speed, download_size, upload_size) + + # dbw.add_status_message_datetime(client_session_id, time, num_peers, dht, download_speed, upload_speed, download_size, upload_size, eta_time) + + except IOError: + print "Error processing file %s." %filename + +def main_just_parse(): + filename = sys.argv[1] + client_session_id = 1 + tribler_parse_status_file(None, client_session_id, filename) + + +def main(): + try: + opts, args = getopt.getopt(sys.argv[1:], "hi:f:", ["help", + "id=", "file="]) + except getopt.GetoptError, err: + print str(err) + usage() + sys.exit(2) + + client_session_id = None + filename = None + database = None + + for o, a in opts: + if o in ("-h", "--help"): + usage() + sys.exit(0) + elif o in ("-i", "--id"): + client_session_id = int(a) + elif o in ("-f", "--file"): + filename = a + else: + assert False, "unhandled option" + + if client_session_id == None: + print "Error: no client session id." + sys.exit(2) + + if filename == None: + print "Error: no status file." + sys.exit(2) + + # no database passed as argument + if len(args) != 1: + print "Error: no database file passed as argument." + sys.exit(2) + database = args[0] + + dbc = DatabaseCommander(database) + + # check for client_session_id, swarm_id, btclient_id + cursor = dbc.select_client_sessions_by_id(client_session_id) + if cursor == None: + print "Error: no client session id (%d) in database." % client_session_id + sys.exit(2) + for session_row in cursor: + pass + + swarm_id = session_row[1] + btclient_id = session_row[2] + + cursor = dbc.select_swarms(swarm_id) + if cursor == None: + print "Error: no swarm id (%d) in database." % swarm_id + sys.exit(2) + for swarm_row in cursor: + pass + + cursor = dbc.select_btclients(btclient_id) + if cursor == None: + print "Error: no client id (%d) in database." % btclient_id + sys.exit(2) + for btclient_row in cursor: + pass + + print "Client session row is: " + print " ", session_row + print "Swarm row is: " + print " ", swarm_row + print "Client row is: " + print " ", btclient_row + print "\nContinue parsing on file %s? (y/n) " % filename, + try: + ans = sys.stdin.readline().strip() + if ans != "y": + sys.exit(0) + except IOError: + print "Error reading standard input." + sys.exit(2) + print "" + + # parse status file + dbw = DatabaseWriter(database) + tribler_parse_status_file(dbw, client_session_id,filename) + +if __name__ == "__main__": + sys.exit(main_just_parse()) diff --git a/log-samples/tribler/log/test2.txt b/log-samples/tribler/log/test2.txt index 6681b5a..9bd9640 100644 --- a/log-samples/tribler/log/test2.txt +++ b/log-samples/tribler/log/test2.txt @@ -1,68 +1,68 @@ -03 Nov 2009 12:18:50 SingleDownload: save_as( u'aqua.mpeg' 94918656 '../p2p-dld/tribler/' False ) -03 Nov 2009 12:18:50 SingleDownload: perform_hashcheck() -03 Nov 2009 12:18:50 SingleDownload: hashcheck_done() -03 Nov 2009 12:18:50 connecter: Disabling overlay -03 Nov 2009 12:18:50 connecter: Disabling overlay -03 Nov 2009 12:18:50 Connecter: EXTEND: my dict {'Tr_LIVE_v1': 254, 'ut_pex': 1} -03 Nov 2009 12:18:50 aqua.mpeg DLSTATUS_HASHCHECKING 0.00% None up 0.00KB/s down 0.00KB/s eta 0.0 peers 0 -03 Nov 2009 12:18:51 choker: _rechoke: checkinternalbias 0 -03 Nov 2009 12:18:51 choker: _rechoke: NORMAL UNCHOKE [] +03 11 2009 12:18:50 SingleDownload: save_as( u'aqua.mpeg' 94918656 '../p2p-dld/tribler/' False ) +03 11 2009 12:18:50 SingleDownload: perform_hashcheck() +03 11 2009 12:18:50 SingleDownload: hashcheck_done() +03 11 2009 12:18:50 connecter: Disabling overlay +03 11 2009 12:18:50 connecter: Disabling overlay +03 11 2009 12:18:50 Connecter: EXTEND: my dict {'Tr_LIVE_v1': 254, 'ut_pex': 1} +03 11 2009 12:18:50 aqua.mpeg DLSTATUS_HASHCHECKING 0.00% None up 0.00KB/s down 0.00KB/s eta 0.0 peers 0 +03 11 2009 12:18:51 choker: _rechoke: checkinternalbias 0 +03 11 2009 12:18:51 choker: _rechoke: NORMAL UNCHOKE [] choker: _rechoke: G2G UNCHOKE [] -03 Nov 2009 12:18:51 choker: _rechoke: checkinternalbias 0 -03 Nov 2009 12:18:51 choker: _rechoke: NORMAL UNCHOKE [] +03 11 2009 12:18:51 choker: _rechoke: checkinternalbias 0 +03 11 2009 12:18:51 choker: _rechoke: NORMAL UNCHOKE [] choker: _rechoke: G2G UNCHOKE [] -03 Nov 2009 12:18:52 aqua.mpeg DLSTATUS_DOWNLOADING 6.85% None up 0.00KB/s down 2113.12KB/s eta 26.4850236947 peers 2 -03 Nov 2009 12:18:53 aqua.mpeg DLSTATUS_DOWNLOADING 14.53% None up 0.00KB/s down 3283.14KB/s eta 17.5126024978 peers 2 -03 Nov 2009 12:18:54 aqua.mpeg DLSTATUS_DOWNLOADING 22.37% None up 0.00KB/s down 3964.17KB/s eta 14.2589440652 peers 2 -03 Nov 2009 12:18:55 choker: _rechoke: checkinternalbias 0 -03 Nov 2009 12:18:55 choker: _rechoke: NORMAL UNCHOKE [] +03 11 2009 12:18:52 aqua.mpeg DLSTATUS_DOWNLOADING 6.85% None up 0.00KB/s down 2113.12KB/s eta 26.4850236947 peers 2 +03 11 2009 12:18:53 aqua.mpeg DLSTATUS_DOWNLOADING 14.53% None up 0.00KB/s down 3283.14KB/s eta 17.5126024978 peers 2 +03 11 2009 12:18:54 aqua.mpeg DLSTATUS_DOWNLOADING 22.37% None up 0.00KB/s down 3964.17KB/s eta 14.2589440652 peers 2 +03 11 2009 12:18:55 choker: _rechoke: checkinternalbias 0 +03 11 2009 12:18:55 choker: _rechoke: NORMAL UNCHOKE [] choker: _rechoke: G2G UNCHOKE [] -03 Nov 2009 12:18:55 aqua.mpeg DLSTATUS_DOWNLOADING 29.84% None up 0.00KB/s down 4414.39KB/s eta 12.0159059199 peers 2 -03 Nov 2009 12:18:56 aqua.mpeg DLSTATUS_DOWNLOADING 37.37% None up 0.00KB/s down 4680.87KB/s eta 10.9181039961 peers 2 -03 Nov 2009 12:18:57 aqua.mpeg DLSTATUS_DOWNLOADING 44.07% None up 0.00KB/s down 4842.35KB/s eta 9.91602307878 peers 2 -03 Nov 2009 12:18:58 aqua.mpeg DLSTATUS_DOWNLOADING 51.31% None up 0.00KB/s down 4970.70KB/s eta 7.88249513703 peers 2 -03 Nov 2009 12:18:59 aqua.mpeg DLSTATUS_DOWNLOADING 57.88% None up 0.00KB/s down 5060.50KB/s eta 6.88155028421 peers 2 -03 Nov 2009 12:19:00 choker: _rechoke: checkinternalbias 0 -03 Nov 2009 12:19:00 choker: _rechoke: NORMAL UNCHOKE [] +03 11 2009 12:18:55 aqua.mpeg DLSTATUS_DOWNLOADING 29.84% None up 0.00KB/s down 4414.39KB/s eta 12.0159059199 peers 2 +03 11 2009 12:18:56 aqua.mpeg DLSTATUS_DOWNLOADING 37.37% None up 0.00KB/s down 4680.87KB/s eta 10.9181039961 peers 2 +03 11 2009 12:18:57 aqua.mpeg DLSTATUS_DOWNLOADING 44.07% None up 0.00KB/s down 4842.35KB/s eta 9.91602307878 peers 2 +03 11 2009 12:18:58 aqua.mpeg DLSTATUS_DOWNLOADING 51.31% None up 0.00KB/s down 4970.70KB/s eta 7.88249513703 peers 2 +03 11 2009 12:18:59 aqua.mpeg DLSTATUS_DOWNLOADING 57.88% None up 0.00KB/s down 5060.50KB/s eta 6.88155028421 peers 2 +03 11 2009 12:19:00 choker: _rechoke: checkinternalbias 0 +03 11 2009 12:19:00 choker: _rechoke: NORMAL UNCHOKE [] choker: _rechoke: G2G UNCHOKE [] -03 Nov 2009 12:19:00 aqua.mpeg DLSTATUS_DOWNLOADING 64.60% None up 0.00KB/s down 5140.12KB/s eta 5.86957618791 peers 2 -03 Nov 2009 12:19:01 aqua.mpeg DLSTATUS_DOWNLOADING 71.95% None up 0.00KB/s down 5220.68KB/s eta 4.78167411882 peers 2 -03 Nov 2009 12:19:02 aqua.mpeg DLSTATUS_DOWNLOADING 79.18% None up 0.00KB/s down 5309.69KB/s eta 3.76926920014 peers 2 -03 Nov 2009 12:19:03 aqua.mpeg DLSTATUS_DOWNLOADING 86.62% None up 0.00KB/s down 5371.01KB/s eta 2.68195411759 peers 2 -03 Nov 2009 12:19:04 aqua.mpeg DLSTATUS_DOWNLOADING 93.89% None up 0.00KB/s down 5440.21KB/s eta 1.67072531777 peers 2 -03 Nov 2009 12:19:05 choker: _rechoke: checkinternalbias 0 -03 Nov 2009 12:19:05 choker: _rechoke: NORMAL UNCHOKE [] +03 11 2009 12:19:00 aqua.mpeg DLSTATUS_DOWNLOADING 64.60% None up 0.00KB/s down 5140.12KB/s eta 5.86957618791 peers 2 +03 11 2009 12:19:01 aqua.mpeg DLSTATUS_DOWNLOADING 71.95% None up 0.00KB/s down 5220.68KB/s eta 4.78167411882 peers 2 +03 11 2009 12:19:02 aqua.mpeg DLSTATUS_DOWNLOADING 79.18% None up 0.00KB/s down 5309.69KB/s eta 3.76926920014 peers 2 +03 11 2009 12:19:03 aqua.mpeg DLSTATUS_DOWNLOADING 86.62% None up 0.00KB/s down 5371.01KB/s eta 2.68195411759 peers 2 +03 11 2009 12:19:04 aqua.mpeg DLSTATUS_DOWNLOADING 93.89% None up 0.00KB/s down 5440.21KB/s eta 1.67072531777 peers 2 +03 11 2009 12:19:05 choker: _rechoke: checkinternalbias 0 +03 11 2009 12:19:05 choker: _rechoke: NORMAL UNCHOKE [] choker: _rechoke: G2G UNCHOKE [] -03 Nov 2009 12:19:05 SingleDownload::finishedfunc called: Download is complete ******************************* +03 11 2009 12:19:05 SingleDownload::finishedfunc called: Download is complete ******************************* bartercast: no overlay bridge found bartercast: no overlay bridge found -03 Nov 2009 12:19:06 aqua.mpeg DLSTATUS_SEEDING 100.00% None up 0.00KB/s down 0.00KB/s eta 0 peers 0 -03 Nov 2009 12:19:06 choker: _rechoke: checkinternalbias 0 -03 Nov 2009 12:19:06 choker: _rechoke: NORMAL UNCHOKE [] +03 11 2009 12:19:06 aqua.mpeg DLSTATUS_SEEDING 100.00% None up 0.00KB/s down 0.00KB/s eta 0 peers 0 +03 11 2009 12:19:06 choker: _rechoke: checkinternalbias 0 +03 11 2009 12:19:06 choker: _rechoke: NORMAL UNCHOKE [] choker: _rechoke: G2G UNCHOKE [] -03 Nov 2009 12:19:06 choker: _rechoke: checkinternalbias 0 -03 Nov 2009 12:19:06 choker: _rechoke: NORMAL UNCHOKE [] +03 11 2009 12:19:06 choker: _rechoke: checkinternalbias 0 +03 11 2009 12:19:06 choker: _rechoke: NORMAL UNCHOKE [] choker: _rechoke: G2G UNCHOKE [] bartercast: no overlay bridge found bartercast: no overlay bridge found -03 Nov 2009 12:19:07 aqua.mpeg DLSTATUS_SEEDING 100.00% None up 0.00KB/s down 0.00KB/s eta 0 peers 0 -03 Nov 2009 12:19:09 aqua.mpeg DLSTATUS_SEEDING 100.00% None up 0.00KB/s down 0.00KB/s eta 0 peers 0 -03 Nov 2009 12:19:10 choker: _rechoke: checkinternalbias 0 -03 Nov 2009 12:19:10 choker: _rechoke: NORMAL UNCHOKE [] +03 11 2009 12:19:07 aqua.mpeg DLSTATUS_SEEDING 100.00% None up 0.00KB/s down 0.00KB/s eta 0 peers 0 +03 11 2009 12:19:09 aqua.mpeg DLSTATUS_SEEDING 100.00% None up 0.00KB/s down 0.00KB/s eta 0 peers 0 +03 11 2009 12:19:10 choker: _rechoke: checkinternalbias 0 +03 11 2009 12:19:10 choker: _rechoke: NORMAL UNCHOKE [] choker: _rechoke: G2G UNCHOKE [] -03 Nov 2009 12:19:10 aqua.mpeg DLSTATUS_SEEDING 100.00% None up 0.00KB/s down 0.00KB/s eta 0 peers 0 -03 Nov 2009 12:19:12 aqua.mpeg DLSTATUS_SEEDING 100.00% None up 0.00KB/s down 0.00KB/s eta 0 peers 0 -03 Nov 2009 12:19:13 aqua.mpeg DLSTATUS_SEEDING 100.00% None up 0.00KB/s down 0.00KB/s eta 0 peers 0 +03 11 2009 12:19:10 aqua.mpeg DLSTATUS_SEEDING 100.00% None up 0.00KB/s down 0.00KB/s eta 0 peers 0 +03 11 2009 12:19:12 aqua.mpeg DLSTATUS_SEEDING 100.00% None up 0.00KB/s down 0.00KB/s eta 0 peers 0 +03 11 2009 12:19:13 aqua.mpeg DLSTATUS_SEEDING 100.00% None up 0.00KB/s down 0.00KB/s eta 0 peers 0 Traceback (most recent call last): File "Tribler/Tools/cmdlinedl.py", line 128, in main time.sleep(sys.maxint/2048) KeyboardInterrupt -03 Nov 2009 12:19:13 Session: checkpoint_shutdown -03 Nov 2009 12:19:14 SingleDownload: shutdown -03 Nov 2009 12:19:14 tlm: Number of threads still running 6 -03 Nov 2009 12:19:14 tlm: Thread still running MainThread daemon False -03 Nov 2009 12:19:14 tlm: Thread still running OverlayThread-4 daemon True -03 Nov 2009 12:19:14 tlm: Thread still running NetworkThread-3 daemon True -03 Nov 2009 12:19:14 tlm: Thread still running UPnPThread-5 daemon True -03 Nov 2009 12:19:14 tlm: Thread still running TrackerRerequestAThread-10 daemon True -03 Nov 2009 12:19:14 tlm: Thread still running TrackerRerequestBThread-11 daemon True +03 11 2009 12:19:13 Session: checkpoint_shutdown +03 11 2009 12:19:14 SingleDownload: shutdown +03 11 2009 12:19:14 tlm: Number of threads still running 6 +03 11 2009 12:19:14 tlm: Thread still running MainThread daemon False +03 11 2009 12:19:14 tlm: Thread still running OverlayThread-4 daemon True +03 11 2009 12:19:14 tlm: Thread still running NetworkThread-3 daemon True +03 11 2009 12:19:14 tlm: Thread still running UPnPThread-5 daemon True +03 11 2009 12:19:14 tlm: Thread still running TrackerRerequestAThread-10 daemon True +03 11 2009 12:19:14 tlm: Thread still running TrackerRerequestBThread-11 daemon True diff --git a/tribler-mod/scrips/modify.sh b/tribler-mod/scrips/modify.sh index baa1634..9fc19a2 100755 --- a/tribler-mod/scrips/modify.sh +++ b/tribler-mod/scrips/modify.sh @@ -1,2 +1,2 @@ -for i in $(find -name '*.py' -type f | grep -v '.svn'); do sed -i.bak 's/print >>sys.stderr,/print >>sys.stderr, strftime("%d %b %Y %H:%M:%S", localtime())," ",/g' $i; sed -i.bak '1i\from time import localtime, strftime\n' $i; done; +for i in $(find -name '*.py' -type f | grep -v '.svn'); do sed -i.bak 's/print >>sys.stderr,/print >>sys.stderr, strftime("%d %m %Y %H:%M:%S", localtime())," ",/g' $i; sed -i.bak '1i\from time import localtime, strftime\n' $i; done; diff --git a/tribler-mod/scrips/unmodify.sh b/tribler-mod/scrips/unmodify.sh index dc91381..cc6636a 100755 --- a/tribler-mod/scrips/unmodify.sh +++ b/tribler-mod/scrips/unmodify.sh @@ -1,2 +1,2 @@ -for i in $(find -name '*.py' -type f | grep -v '.svn'); do sed -i.bak 's/print >>sys.stderr, strftime("%d %b %Y %H:%M:%S", localtime())," ",/print >>sys.stderr, /g' $i; sed -i.bak 's/from time import localtime, strftime\n/ /g' $i; done; +for i in $(find -name '*.py' -type f | grep -v '.svn'); do sed -i.bak 's/print >>sys.stderr, strftime("%d %m %Y %H:%M:%S", localtime())," ",/print >>sys.stderr, /g' $i; sed -i.bak 's/from time import localtime, strftime\n/ /g' $i; done; -- 2.20.1