From 19f7cb3e4550ca25d12d54827753ee0f9c3ef43b Mon Sep 17 00:00:00 2001 From: Mariana Marasoiu Date: Tue, 16 Aug 2011 18:50:31 +0300 Subject: [PATCH] ppf: Modify libtorrent/LogParser.py and libtorrent/StatusParser.py to use new SQL schema and configuration file. --- ppf/log-parser-mysql/libtorrent/LogParser.py | 95 ++++++++++--------- .../libtorrent/StatusParser.py | 22 ++--- 2 files changed, 59 insertions(+), 58 deletions(-) diff --git a/ppf/log-parser-mysql/libtorrent/LogParser.py b/ppf/log-parser-mysql/libtorrent/LogParser.py index f6417c6..75650f1 100644 --- a/ppf/log-parser-mysql/libtorrent/LogParser.py +++ b/ppf/log-parser-mysql/libtorrent/LogParser.py @@ -21,11 +21,11 @@ import string import os # the names used by Tribler for the BitTorrent messages -bt_msg_types = {"CHOKE": 0, "UNCHOKE": 1, "INTERESTED": 2, - "NOT_INTERESTED": 3, "HAVE": 4, "BITFIELD": 5, - "REQUEST": 6, "PIECE": 7, "CANCEL": 8, "DHT_PORT": 9} +bt_msg_types = {"CHOKE": 1, "UNCHOKE": 2, "INTERESTED": 3, + "NOT_INTERESTED": 4, "HAVE": 5, "BITFIELD": 6, + "REQUEST": 7, "PIECE": 8, "CANCEL": 9, "DHT_PORT": 10} -log_msg_dir = {"RECEIVE": 0, "SEND": 1} +log_msg_dir = {"RECEIVE": 1, "SEND": 2} DEBUG = False @@ -53,9 +53,9 @@ def string_to_timestamp(date_string): def libtorrent_parse_choke(line): if string.find(line, "<== CHOKE") != -1: - direction = log_msg_dir["RECEIVE"] + direction_id = log_msg_dir["RECEIVE"] elif string.find(line, "==> CHOKE") != -1: - direction = log_msg_dir["SEND"] + direction_id = log_msg_dir["SEND"] else: return None @@ -71,7 +71,7 @@ def libtorrent_parse_choke(line): length = 0 port = 0 - return (timestamp, direction, msg_type, index, begin, length, port) + return (timestamp, direction_id, msg_type, index, begin, length, port) # # parse unchoke line in libtorrent log file @@ -141,14 +141,14 @@ def libtorrent_parse_not_interested(line): if string.find(line, "<== NOT_INTERESTED") != -1: direction = log_msg_dir["RECEIVE"] elif string.find(line, "==> NOT_INTERESTED") != -1: - direction = log_msg_dir["SEND"] + transfer_direction_id = log_msg_dir["SEND"] else: return None if DEBUG == True: print "--- match NOT_INTERESTED" - msg_type = bt_msg_types["NOT_INTERESTED"] + msg_type_id = bt_msg_types["NOT_INTERESTED"] parts = re.split("[<=>]+", line) timestamp = string_to_timestamp(parts[0].strip()) @@ -157,7 +157,7 @@ def libtorrent_parse_not_interested(line): length = 0 port = 0 - return (timestamp, direction, msg_type, index, begin, length, port) + return (timestamp, transfer_direction_id, msg_type_id, index, begin, length, port) # # parse have line in libtorrent log file @@ -168,16 +168,16 @@ def libtorrent_parse_not_interested(line): def libtorrent_parse_have(line): if string.find(line, "<== HAVE ") != -1: - direction = log_msg_dir["RECEIVE"] + transfer_direction_id = log_msg_dir["RECEIVE"] elif string.find(line, "==> HAVE ") != -1: - direction = log_msg_dir["SEND"] + transfer_direction_id = log_msg_dir["SEND"] else: return None if DEBUG == True: print "--- match HAVE" - msg_type = bt_msg_types["HAVE"] + msg_type_id = bt_msg_types["HAVE"] parts = re.split("[\[\]<=>]+", line) timestamp = string_to_timestamp(parts[0].strip()) @@ -186,7 +186,7 @@ def libtorrent_parse_have(line): length = 0 port = 0 - return (timestamp, direction, msg_type, index, begin, length, port) + return (timestamp, transfer_direction_id, msg_type_id, index, begin, length, port) # # parse bitfield line in libtorrent log file @@ -197,16 +197,16 @@ def libtorrent_parse_have(line): def libtorrent_parse_bitfield(line): if string.find(line, "<== BITFIELD ") != -1: - direction = log_msg_dir["RECEIVE"] + transfer_direction_id = log_msg_dir["RECEIVE"] elif string.find(line, "==> BITFIELD ") != -1: - direction = log_msg_dir["SEND"] + transfer_direction_id = log_msg_dir["SEND"] else: return None if DEBUG == True: print "--- match BITFIELD" - msg_type = bt_msg_types["BITFIELD"] + msg_type_id = bt_msg_types["BITFIELD"] parts = re.split("[<=>]+", line) timestamp = string_to_timestamp(parts[0].strip()) @@ -215,7 +215,7 @@ def libtorrent_parse_bitfield(line): length = 0 port = 0 - return (timestamp, direction, msg_type, index, begin, length, port) + return (timestamp, transfer_direction_id, msg_type_id, index, begin, length, port) # # parse request line in libtorrent log file @@ -226,16 +226,16 @@ def libtorrent_parse_bitfield(line): def libtorrent_parse_request(line): if string.find(line, "<== REQUEST ") != -1: - direction = log_msg_dir["RECEIVE"] + transfer_direction_id = log_msg_dir["RECEIVE"] elif string.find(line, "==> REQUEST ") != -1: - direction = log_msg_dir["SEND"] + transfer_direction_id = log_msg_dir["SEND"] else: return None if DEBUG == True: print "--- match REQUEST" - msg_type = bt_msg_types["REQUEST"] + msg_type_id = bt_msg_types["REQUEST"] parts = re.split("[\[\]|<=>]+", line) timestamp = string_to_timestamp(parts[0].strip()) @@ -245,7 +245,7 @@ def libtorrent_parse_request(line): port = 0 - return (timestamp, direction, msg_type, index, begin, length, port) + return (timestamp, transfer_direction_id, msg_type_id, index, begin, length, port) # # parse piece line in libtorrent log file @@ -256,16 +256,16 @@ def libtorrent_parse_request(line): def libtorrent_parse_piece(line): if string.find(line, "<== PIECE ") != -1: - direction = log_msg_dir["RECEIVE"] + transfer_direction_id = log_msg_dir["RECEIVE"] elif string.find(line, "==> PIECE ") != -1: - direction = log_msg_dir["SEND"] + transfer_direction_id = log_msg_dir["SEND"] else: return None if DEBUG == True: print "--- match PIECE" - msg_type = bt_msg_types["PIECE"] + msg_type_id = bt_msg_types["PIECE"] parts = re.split("[\[\]|<=>]+", line) timestamp = string_to_timestamp(parts[0].strip()) @@ -274,7 +274,7 @@ def libtorrent_parse_piece(line): length = int("0x" + re.split(":", parts[4])[1].strip(), 16) port = 0 - return (timestamp, direction, msg_type, index, begin, length, port) + return (timestamp, transfer_direction_id, msg_type_id, index, begin, length, port) # # no cancel line in libtorrent log files @@ -292,16 +292,16 @@ def libtorrent_parse_cancel(line): def libtorrent_parse_allowed_fast(line): if string.find(line, "<== ALLOWED_FAST ") != -1: - direction = log_msg_dir["RECEIVE"] + transfer_direction_id = log_msg_dir["RECEIVE"] elif string.find(line, "==> ALLOWED_FAST ") != -1: - direction = log_msg_dir["SEND"] + transfer_direction_id = log_msg_dir["SEND"] else: return None if DEBUG == True: print "--- match ALLOWED_FAST" - msg_type = bt_msg_types["ALLOWED_FAST"] + msg_type_id = bt_msg_types["ALLOWED_FAST"] parts = re.split("[\[\]<=>]+", line) timestamp = string_to_timestamp(parts[0].strip()) @@ -310,7 +310,7 @@ def libtorrent_parse_allowed_fast(line): length = 0 port = int(parts[2].strip()) - return (timestamp, direction, msg_type, index, begin, length, port) + return (timestamp, transfer_direction_id, msg_type_id, index, begin, length, port) # # parse DHT port line in libtorrent log file @@ -321,28 +321,28 @@ def libtorrent_parse_allowed_fast(line): def libtorrent_parse_port(line): if string.find(line, "<== DHT_PORT ") != -1: - direction = log_msg_dir["RECEIVE"] + transfer_direction_id = log_msg_dir["RECEIVE"] elif string.find(line, "==> DHT_PORT ") != -1: - direction = log_msg_dir["SEND"] + transfer_direction_id = log_msg_dir["SEND"] else: return None if DEBUG == True: print "--- match DHT_PORT" - msg_type = bt_msg_types["DHT_PORT"] + msg_type_id = bt_msg_types["DHT_PORT"] parts = re.split("[\[\]<=>]+", line) timestamp = string_to_timestamp(parts[0].strip()) index = 0 begin = 0 length = 0 - if direction == log_msg_dir["RECEIVE"]: + if transfer_direction_id == log_msg_dir["RECEIVE"]: port = int("0x" + re.split(":", parts[2])[1].strip(), 16) else: port = int(parts[2].strip()) - return (timestamp, direction, msg_type, index, begin, length, port) + return (timestamp, transfer_direction_id, msg_type_id, index, begin, length, port) # @@ -448,26 +448,26 @@ def libtorrent_parse_log_file(dbw, client_session_id, logfile): if result == None: continue - (timestamp, direction, msg_type, index, begin, length, listen_port) = result + (timestamp, transfer_direction_id, msg_type_id, index, begin, length, listen_port) = result if DEBUG == True: print result dbw.add_verbose_message_datetime(client_session_id, timestamp, - direction, peer_ip, peer_port, msg_type, + transfer_direction_id, peer_ip, peer_port, msg_type_id, index, begin,length, listen_port) except IOError: print "Error processing file %s." %logfile def usage(): - print "Usage: python StatusParser.py -i|--id id -f|--file log_file database" + print "Usage: python StatusParser.py -i|--id id -f|--file log_file dbconf_file" print "id:" print "\t--id" print "\t-i\t\tclient_session_id" print "\tstatus_file:" print "\t--file" - print "\t-f\t\tstatus_file for tribler" - print "\tdatabase\t\tSQLite database file" + print "\t-f\t\tstatus_file for libtorrent" + print "\tdbconf_file\t\tMySQL database configuration file" print "\t--help" print "\t-h\t\t\tprint this help screen" @@ -475,7 +475,7 @@ def usage(): def main_just_parse(): filename = sys.argv[1] client_session_id = 1 - tribler_parse_status_file(None, 1, filename) + libtorrent_parse_status_file(None, 1, filename) def main_with_DB(): @@ -490,7 +490,7 @@ def main_with_DB(): client_session_id = None filename = None - database = None + dbconf = None for o, a in opts: if o in ("-h", "--help"): @@ -511,19 +511,20 @@ def main_with_DB(): print "Error: no status file." sys.exit(2) - # no database passed as argument + # no database configuration file passed as argument if len(args) != 1: - print "Error: no database file passed as argument." + print "Error: no database configuration file passed as argument." sys.exit(2) - database = args[0] + dbconf = args[0] - dbc = DatabaseCommander(database) + dbc = DatabaseCommander(dbconf) # check for client_session_id, swarm_id, btclient_id cursor = dbc.select_client_sessions_by_id(client_session_id) if cursor == None: print "Error: no client session id (%d) in database." % client_session_id sys.exit(2) + for session_row in cursor: pass @@ -561,7 +562,7 @@ def main_with_DB(): print "" # parse log file - dbw = DatabaseWriter(database) + dbw = DatabaseWriter(dbconf) libtorrent_parse_log_file(dbw, client_session_id, filename) diff --git a/ppf/log-parser-mysql/libtorrent/StatusParser.py b/ppf/log-parser-mysql/libtorrent/StatusParser.py index 4639d3c..0bc3490 100644 --- a/ppf/log-parser-mysql/libtorrent/StatusParser.py +++ b/ppf/log-parser-mysql/libtorrent/StatusParser.py @@ -18,7 +18,7 @@ def usage(): print "\tstatus_file:" print "\t--file" print "\t-f\t\tstatus_file for libtorrent" - print "\tdatabase\t\tSQLite database file" + print "\tdbconf_file\t\tMySQL database configuration file" print "\t--help" print "\t-h\t\t\tprint this help screen" @@ -86,7 +86,7 @@ def libtorrent_parse_status_line(line): if pair[0] == "ps": num_peers = libtorrent_canon_num_peers(pair[1]) if pair[0] == "dht": - dht = libtorrent_canon_dht(pair[1]) + num_dht_peers = libtorrent_canon_dht(pair[1]) if pair[0] == "dl": download_speed = libtorrent_canon_download_speed(pair[1]) if pair[0] == "ul": @@ -101,7 +101,7 @@ def libtorrent_parse_status_line(line): eta = libtorrent_canon_eta(pair[1]) eta_seconds = eta.days * 24 * 3600 + eta.seconds - return (num_peers, dht, download_speed, upload_speed, download_size, upload_size, eta_seconds) + return (num_peers, num_dht_peers, download_speed, upload_speed, download_size, upload_size, eta_seconds) def libtorrent_parse_status_file(dbw, client_session_id, session_start, filename): @@ -119,14 +119,14 @@ def libtorrent_parse_status_file(dbw, client_session_id, session_start, filename if libtorrent_is_status_line(line) == False: continue - (num_peers, dht, download_speed, upload_speed, download_size, upload_size, eta_seconds) = libtorrent_parse_status_line(line) + (num_peers, num_dht_peers, download_speed, upload_speed, download_size, upload_size, eta_seconds) = libtorrent_parse_status_line(line) message_time = message_time + one_second if DEBUG == True: - print "(%d, %s, %s, %d, %d kb/s, %d kb/s, %d bytes, %d bytes)" % (num_peers, date, time, dht, download_speed, upload_speed, download_size, upload_size) + print "(%d, %s, %s, %d, %d kb/s, %d kb/s, %d bytes, %d bytes)" % (num_peers, date, time, num_dht_peers, download_speed, upload_speed, download_size, upload_size) - dbw.add_status_message_datetime(client_session_id, message_time, num_peers, dht, download_speed, upload_speed, download_size, upload_size, eta_seconds) + dbw.add_status_message_datetime(client_session_id, message_time, num_peers, num_dht_peers, download_speed, upload_speed, download_size, upload_size, eta_seconds) except IOError: print "Error processing file %s." %filename @@ -163,13 +163,13 @@ def main(): print "Error: no status file." sys.exit(2) - # no database passed as argument + # no database configuration file passed as argument if len(args) != 1: - print "Error: no database file passed as argument." + print "Error: no database configuration file passed as argument." sys.exit(2) - database = args[0] + dbconf = args[0] - dbc = DatabaseCommander(database) + dbc = DatabaseCommander(dbconf) # check for client_session_id, swarm_id, btclient_id cursor = dbc.select_client_sessions_by_id(client_session_id) @@ -216,7 +216,7 @@ def main(): # session_start = julian.julianToDatetime(session_row[11]) session_start = session_row[11] # parse status file - dbw = DatabaseWriter(database) + dbw = DatabaseWriter(dbconf) libtorrent_parse_status_file(dbw, client_session_id, session_start, filename) -- 2.20.1