From: Razvan Deaconescu Date: Thu, 22 Apr 2010 09:56:14 +0000 (+0300) Subject: working version of libtorrent parser; logging issues to be fixed X-Git-Tag: getopt_long~60 X-Git-Url: http://p2p-next.cs.pub.ro/gitweb/?a=commitdiff_plain;h=39042dd6ae75ceab5ecfe16397b356dbe4d3be49;p=cs-p2p-next.git working version of libtorrent parser; logging issues to be fixed --- diff --git a/ppf/log-parser/generic/GenericStatusParser.py b/ppf/log-parser/generic/GenericStatusParser.py index 8310b0b..8d84717 100644 --- a/ppf/log-parser/generic/GenericStatusParser.py +++ b/ppf/log-parser/generic/GenericStatusParser.py @@ -24,55 +24,55 @@ class GenericStatusParser: # return boolean # - def is_status_line(line): + def is_status_line(self, line): return True # return integer # - def canon_num_peers(non_canon_value): + def canon_num_peers(self, non_canon_value): return 0 # return integer # - def canon_dht(non_canon_value): + def canon_dht(self, non_canon_value): return 0 # return integer # # 119.51kb/s -> 119 - def canon_download_speed(non_canon_value): + def canon_download_speed(self, non_canon_value): return 0 # return integer # # 12119.51kb/s -> 12119 - def canon_upload_speed(non_canon_value): + def canon_upload_speed(self, non_canon_value): return 0 # return timedelta object # # 698mb -> 698*1024*1024 - def canon_download_size(non_canon_value): + def canon_download_size(self, non_canon_value): return 0 # return timedelta object # # 492mb -> 492*1024*1024 - def canon_upload_size(non_canon_value): + def canon_upload_size(self, non_canon_value): return 0 # return timedelta object # # 1h 38m 37s -> [0, 1, 38, 37] # 3d 5h 24m 34s -> [3, 5, 24, 34] - def canon_eta(non_canon_value): + def canon_eta(self, non_canon_value): return datetime.timedelta() - def timedelta_to_seconds(delta): + def timedelta_to_seconds(self, delta): return delta.days * 24 * 3600 + delta.seconds - # return list of required - def parse_status_line(line): + # return list of required information + def parse_status_line(self, line): num_peers = 0 dht = 0 download_speed = 0 @@ -83,7 +83,7 @@ class GenericStatusParser: return (num_peers, dht, download_speed, upload_speed, download_size, upload_size, eta) - def parse_status_file(client_session_id, session_start, filename, callback_func, callback_arg = None): + def parse_status_file(self, client_session_id, session_start, filename, callback_func, callback_arg = None): message_time = session_start one_second = datetime.timedelta(0, 1) @@ -96,14 +96,14 @@ class GenericStatusParser: break line = line.strip() - if is_status_line(line) == False: + if self.is_status_line(line) == False: continue - (num_peers, dht, download_speed, upload_speed, download_size, upload_size, eta_seconds) = parse_status_line(line) + (num_peers, dht, download_speed, upload_speed, download_size, upload_size, eta_seconds) = self.parse_status_line(line) message_time = message_time + one_second - logging.debug("(%d, %s, %s, %d, %d kb/s, %d kb/s, %d bytes, %d bytes)" % (num_peers, date, time, dht, download_speed, upload_speed, download_size, upload_size)) + logging.debug("(%d, %d, %d kb/s, %d kb/s, %d bytes, %d bytes)" % (num_peers, dht, download_speed, upload_speed, download_size, upload_size)) if callback_arg == None: callback_func(client_session_id, message_time, @@ -112,7 +112,7 @@ class GenericStatusParser: download_size, upload_size, eta_seconds) else: - callback_func(client_session_id, message_time, + callback_func(callback_arg, client_session_id, message_time, num_peers, dht, download_speed, upload_speed, download_size, upload_size, diff --git a/ppf/log-parser/generic/LibtorrentStatusParser.py b/ppf/log-parser/generic/LibtorrentStatusParser.py index 8682517..b97fbdd 100644 --- a/ppf/log-parser/generic/LibtorrentStatusParser.py +++ b/ppf/log-parser/generic/LibtorrentStatusParser.py @@ -5,15 +5,16 @@ import getopt import re from DatabaseWriter import DatabaseWriter from DatabaseCommander import DatabaseCommander +from GenericStatusParser import GenericStatusParser import julian import datetime import logging # configure logging (change to logging.ERROR when no DEBUG required) -logging.basicConfig(level=logging.DEBUG) +logging.basicConfig(level=logging.ERROR) -class LibtorrentStatusParser: +class LibtorrentStatusParser(GenericStatusParser): """ Abstract-like parser class used for parsing BitTorrent log messages. Inherited by client-specific classes @@ -24,50 +25,50 @@ class LibtorrentStatusParser: # return boolean # - def is_status_line(line): - if re.match("^ps", line) == None: - return False - return True + def is_status_line(self, line): + if re.match("^ps", line) == None: + return False + return True # return integer # - def canon_num_peers(non_canon_value): + def canon_num_peers(self, non_canon_value): return int(non_canon_value) # return integer # - def canon_dht(non_canon_value): + def canon_dht(self, non_canon_value): return int(non_canon_value) # return integer # # 119.51kb/s -> 119 - def canon_download_speed(non_canon_value): + def canon_download_speed(self, non_canon_value): return int(float(non_canon_value.strip("kb/s"))) # return integer # # 12119.51kb/s -> 12119 - def canon_upload_speed(non_canon_value): + def canon_upload_speed(self, non_canon_value): return int(float(non_canon_value.strip("kb/s"))) # return timedelta object # # 698mb -> 698*1024*1024 - def canon_download_size(non_canon_value): + def canon_download_size(self, non_canon_value): return int(non_canon_value.strip("mb")) * 1024 * 1024 # return timedelta object # # 492mb -> 492*1024*1024 - def canon_upload_size(non_canon_value): + def canon_upload_size(self, non_canon_value): return int(non_canon_value.strip("mb")) * 1024 * 1024 # return timedelta object # # 1h 38m 37s -> [0, 1, 38, 37] # 3d 5h 24m 34s -> [3, 5, 24, 34] - def canon_eta(non_canon_value): + def canon_eta(self, non_canon_value): eta_string_array = re.split('\ *[dhms]\ *', non_canon_value) eta_string_array.remove('') eta = [] @@ -78,11 +79,8 @@ class LibtorrentStatusParser: return datetime.timedelta(eta[0], eta[3], 0, 0, eta[2], eta[1], 0) - def timedelta_to_seconds(delta): - return delta.days * 24 * 3600 + delta.seconds - # return list of required - def parse_status_line(line): + def parse_status_line(self, line): num_peers = 0 dht = 0 download_speed = 0 @@ -92,65 +90,137 @@ class LibtorrentStatusParser: eta = 0 string_array = re.split("\ *[,<>]+\ *", line) - if DEBUG == True: - print "string_array is: ", string_array + logging.debug("string_array is %s" % string_array) for string in string_array: pair = re.split("\ *:\ *", string) if pair[0] == "ps": - num_peers = libtorrent_canon_num_peers(pair[1]) + num_peers = self.canon_num_peers(pair[1]) if pair[0] == "dht": - dht = libtorrent_canon_dht(pair[1]) + dht = self.canon_dht(pair[1]) if pair[0] == "dl": - download_speed = libtorrent_canon_download_speed(pair[1]) + download_speed = self.canon_download_speed(pair[1]) if pair[0] == "ul": - upload_speed = libtorrent_canon_upload_speed(pair[1]) + upload_speed = self.canon_upload_speed(pair[1]) if pair[0] == "dld": - download_size = libtorrent_canon_download_size(pair[1]) + download_size = self.canon_download_size(pair[1]) if pair[0] == "uld": - upload_size = libtorrent_canon_upload_size(pair[1]) + upload_size = self.canon_upload_size(pair[1]) if pair[0] == "size": pass if pair[0] == "eta": - eta = libtorrent_canon_eta(pair[1]) - eta_seconds = eta.days * 24 * 3600 + eta.seconds + eta_seconds = self.timedelta_to_seconds(self.canon_eta(pair[1])) return (num_peers, dht, download_speed, upload_speed, download_size, upload_size, eta_seconds) - def parse_status_file(client_session_id, session_start, filename, callback_func, callback_arg = None): - - message_time = session_start - one_second = datetime.timedelta(0, 1) - - try: - fin = open(filename, "r") - while 1: - line = fin.readline() - if not line: - break - - line = line.strip() - if is_status_line(line) == False: - continue - - (num_peers, dht, download_speed, upload_speed, download_size, upload_size, eta_seconds) = parse_status_line(line) - - message_time = message_time + one_second - - logging.debug("(%d, %s, %s, %d, %d kb/s, %d kb/s, %d bytes, %d bytes)" % (num_peers, date, time, dht, download_speed, upload_speed, download_size, upload_size)) - - if callback_arg == None: - callback_func(client_session_id, message_time, +def db_write(dbw, client_session_id, message_time, num_peers, dht, download_speed, upload_speed, download_size, upload_size, - eta_seconds) - else: - callback_func(client_session_id, message_time, - num_peers, dht, - download_speed, upload_speed, - download_size, upload_size, - eta_seconds) + eta_seconds): + pass + +def usage(): + print "Usage: python StatusParser.py -i|--id id status_file" + print "id:" + print "\t--id" + print "\t-i\t\tclient_session_id" + print "\tstatus_file:" + print "\t--file" + print "\t-f\t\tstatus_file for libtorrent" + print "\tdatabase\t\tSQLite database file" + print "\t--help" + print "\t-h\t\t\tprint this help screen" + +def main(): + try: + opts, args = getopt.getopt(sys.argv[1:], "hi:f:", ["help", + "id=", "file="]) + except getopt.GetoptError, err: + print str(err) + usage() + sys.exit(2) + + client_session_id = None + filename = None + database = None + + for o, a in opts: + if o in ("-h", "--help"): + usage() + sys.exit(0) + elif o in ("-i", "--id"): + client_session_id = int(a) + elif o in ("-f", "--file"): + filename = a + else: + assert False, "unhandled option" + + if client_session_id == None: + print "Error: no client session id." + sys.exit(2) + + if filename == None: + print "Error: no status file." + sys.exit(2) + + # no database passed as argument + if len(args) != 1: + print "Error: no database file passed as argument." + sys.exit(2) + database = args[0] + + dbc = DatabaseCommander(database) + + # check for client_session_id, swarm_id, btclient_id + cursor = dbc.select_client_sessions_by_id(client_session_id) + if cursor == None: + print "Error: no client session id (%d) in database." % client_session_id + sys.exit(2) + for session_row in cursor: + pass + + swarm_id = session_row[1] + btclient_id = session_row[2] + + cursor = dbc.select_swarms(swarm_id) + if cursor == None: + print "Error: no swarm id (%d) in database." % swarm_id + sys.exit(2) + for swarm_row in cursor: + pass + + cursor = dbc.select_btclients(btclient_id) + if cursor == None: + print "Error: no client id (%d) in database." % btclient_id + sys.exit(2) + for btclient_row in cursor: + pass - except IOError: - logging.error("Error processing file %s." % filename) + print "Client session row is: " + print " ", session_row + print "Swarm row is: " + print " ", swarm_row + print "Client row is: " + print " ", btclient_row + + print "\nContinue parsing on file %s? (y/n) " % filename, + try: + ans = sys.stdin.readline().strip() + if ans != "y": + sys.exit(0) + except IOError: + print "Error reading standard input." + sys.exit(2) + print "" + + session_start = julian.julianToDatetime(session_row[11]) + + # parse status file + dbw = DatabaseWriter(database) + sp = LibtorrentStatusParser() + sp.parse_status_file(client_session_id, session_start, filename, db_write, dbw) + + +if __name__ == "__main__": + sys.exit(main())