ppf: Modify libtorrent/LogParser.py and libtorrent/StatusParser.py to

use new SQL schema and configuration file.

diff --git a/ppf/log-parser-mysql/libtorrent/LogParser.py b/ppf/log-parser-mysql/libtorrent/LogParser.py
index f6417c6..75650f1 100644 (file)
--- a/ppf/log-parser-mysql/libtorrent/LogParser.py
+++ b/ppf/log-parser-mysql/libtorrent/LogParser.py
@@ -21,11 +21,11 @@ import string
 import os
 
 # the names used by Tribler for the BitTorrent messages 
-bt_msg_types = {"CHOKE": 0, "UNCHOKE": 1, "INTERESTED": 2,
-        "NOT_INTERESTED": 3, "HAVE": 4, "BITFIELD": 5,
-        "REQUEST": 6, "PIECE": 7, "CANCEL": 8, "DHT_PORT": 9}
+bt_msg_types = {"CHOKE": 1, "UNCHOKE": 2, "INTERESTED": 3,
+        "NOT_INTERESTED": 4, "HAVE": 5, "BITFIELD": 6,
+        "REQUEST": 7, "PIECE": 8, "CANCEL": 9, "DHT_PORT": 10}
 
-log_msg_dir = {"RECEIVE": 0, "SEND": 1}
+log_msg_dir = {"RECEIVE": 1, "SEND": 2}
 
 DEBUG = False
 
@@ -53,9 +53,9 @@ def string_to_timestamp(date_string):
 
 def libtorrent_parse_choke(line):
     if string.find(line, "<== CHOKE") != -1:
-        direction = log_msg_dir["RECEIVE"]
+        direction_id = log_msg_dir["RECEIVE"]
     elif string.find(line, "==> CHOKE") != -1:
-        direction = log_msg_dir["SEND"]
+        direction_id = log_msg_dir["SEND"]
     else:
         return None
 
@@ -71,7 +71,7 @@ def libtorrent_parse_choke(line):
     length = 0
     port = 0
 
-    return (timestamp, direction, msg_type, index, begin, length, port)
+    return (timestamp, direction_id, msg_type, index, begin, length, port)
 
 #
 # parse unchoke line in libtorrent log file
@@ -141,14 +141,14 @@ def libtorrent_parse_not_interested(line):
     if string.find(line, "<== NOT_INTERESTED") != -1:
         direction = log_msg_dir["RECEIVE"]
     elif string.find(line, "==> NOT_INTERESTED") != -1:
-        direction = log_msg_dir["SEND"]
+        transfer_direction_id = log_msg_dir["SEND"]
     else:
         return None
 
     if DEBUG == True:
         print "--- match NOT_INTERESTED"
 
-    msg_type = bt_msg_types["NOT_INTERESTED"]
+    msg_type_id = bt_msg_types["NOT_INTERESTED"]
     parts = re.split("[<=>]+", line)
 
     timestamp = string_to_timestamp(parts[0].strip())
@@ -157,7 +157,7 @@ def libtorrent_parse_not_interested(line):
     length = 0
     port = 0
 
-    return (timestamp, direction, msg_type, index, begin, length, port)
+    return (timestamp, transfer_direction_id, msg_type_id, index, begin, length, port)
 
 #
 # parse have line in libtorrent log file
@@ -168,16 +168,16 @@ def libtorrent_parse_not_interested(line):
 
 def libtorrent_parse_have(line):
     if string.find(line, "<== HAVE ") != -1:
-        direction = log_msg_dir["RECEIVE"]
+        transfer_direction_id = log_msg_dir["RECEIVE"]
     elif string.find(line, "==> HAVE ") != -1:
-        direction = log_msg_dir["SEND"]
+        transfer_direction_id = log_msg_dir["SEND"]
     else:
         return None
 
     if DEBUG == True:
         print "--- match HAVE"
 
-    msg_type = bt_msg_types["HAVE"]
+    msg_type_id = bt_msg_types["HAVE"]
     parts = re.split("[\[\]<=>]+", line)
 
     timestamp = string_to_timestamp(parts[0].strip())
@@ -186,7 +186,7 @@ def libtorrent_parse_have(line):
     length = 0
     port = 0
 
-    return (timestamp, direction, msg_type, index, begin, length, port)
+    return (timestamp, transfer_direction_id, msg_type_id, index, begin, length, port)
 
 #
 # parse bitfield line in libtorrent log file
@@ -197,16 +197,16 @@ def libtorrent_parse_have(line):
 
 def libtorrent_parse_bitfield(line):
     if string.find(line, "<== BITFIELD ") != -1:
-        direction = log_msg_dir["RECEIVE"]
+        transfer_direction_id = log_msg_dir["RECEIVE"]
     elif string.find(line, "==> BITFIELD ") != -1:
-        direction = log_msg_dir["SEND"]
+        transfer_direction_id = log_msg_dir["SEND"]
     else:
         return None
 
     if DEBUG == True:
         print "--- match BITFIELD"
 
-    msg_type = bt_msg_types["BITFIELD"]
+    msg_type_id = bt_msg_types["BITFIELD"]
     parts = re.split("[<=>]+", line)
 
     timestamp = string_to_timestamp(parts[0].strip())
@@ -215,7 +215,7 @@ def libtorrent_parse_bitfield(line):
     length = 0
     port = 0
 
-    return (timestamp, direction, msg_type, index, begin, length, port)
+    return (timestamp, transfer_direction_id, msg_type_id, index, begin, length, port)
 
 #
 # parse request line in libtorrent log file
@@ -226,16 +226,16 @@ def libtorrent_parse_bitfield(line):
 
 def libtorrent_parse_request(line):
     if string.find(line, "<== REQUEST ") != -1:
-        direction = log_msg_dir["RECEIVE"]
+        transfer_direction_id = log_msg_dir["RECEIVE"]
     elif string.find(line, "==> REQUEST ") != -1:
-        direction = log_msg_dir["SEND"]
+        transfer_direction_id = log_msg_dir["SEND"]
     else:
         return None
 
     if DEBUG == True:
         print "--- match REQUEST"
 
-    msg_type = bt_msg_types["REQUEST"]
+    msg_type_id = bt_msg_types["REQUEST"]
     parts = re.split("[\[\]|<=>]+", line)
 
     timestamp = string_to_timestamp(parts[0].strip())
@@ -245,7 +245,7 @@ def libtorrent_parse_request(line):
 
     port = 0
 
-    return (timestamp, direction, msg_type, index, begin, length, port)
+    return (timestamp, transfer_direction_id, msg_type_id, index, begin, length, port)
 
 #
 # parse piece line in libtorrent log file
@@ -256,16 +256,16 @@ def libtorrent_parse_request(line):
 
 def libtorrent_parse_piece(line):
     if string.find(line, "<== PIECE ") != -1:
-        direction = log_msg_dir["RECEIVE"]
+        transfer_direction_id = log_msg_dir["RECEIVE"]
     elif string.find(line, "==> PIECE ") != -1:
-        direction = log_msg_dir["SEND"]
+        transfer_direction_id = log_msg_dir["SEND"]
     else:
         return None
 
     if DEBUG == True:
         print "--- match PIECE"
 
-    msg_type = bt_msg_types["PIECE"]
+    msg_type_id = bt_msg_types["PIECE"]
     parts = re.split("[\[\]|<=>]+", line)
 
     timestamp = string_to_timestamp(parts[0].strip())
@@ -274,7 +274,7 @@ def libtorrent_parse_piece(line):
     length = int("0x" + re.split(":", parts[4])[1].strip(), 16)
     port = 0
 
-    return (timestamp, direction, msg_type, index, begin, length, port)
+    return (timestamp, transfer_direction_id, msg_type_id, index, begin, length, port)
 
 #
 # no cancel line in libtorrent log files
@@ -292,16 +292,16 @@ def libtorrent_parse_cancel(line):
 
 def libtorrent_parse_allowed_fast(line):
     if string.find(line, "<== ALLOWED_FAST ") != -1:
-        direction = log_msg_dir["RECEIVE"]
+        transfer_direction_id = log_msg_dir["RECEIVE"]
     elif string.find(line, "==> ALLOWED_FAST ") != -1:
-        direction = log_msg_dir["SEND"]
+        transfer_direction_id = log_msg_dir["SEND"]
     else:
         return None
 
     if DEBUG == True:
         print "--- match ALLOWED_FAST"
 
-    msg_type = bt_msg_types["ALLOWED_FAST"]
+    msg_type_id = bt_msg_types["ALLOWED_FAST"]
     parts = re.split("[\[\]<=>]+", line)
 
     timestamp = string_to_timestamp(parts[0].strip())
@@ -310,7 +310,7 @@ def libtorrent_parse_allowed_fast(line):
     length = 0
     port = int(parts[2].strip())
 
-    return (timestamp, direction, msg_type, index, begin, length, port)
+    return (timestamp, transfer_direction_id, msg_type_id, index, begin, length, port)
 
 #
 # parse DHT port line in libtorrent log file
@@ -321,28 +321,28 @@ def libtorrent_parse_allowed_fast(line):
 
 def libtorrent_parse_port(line):
     if string.find(line, "<== DHT_PORT ") != -1:
-        direction = log_msg_dir["RECEIVE"]
+        transfer_direction_id = log_msg_dir["RECEIVE"]
     elif string.find(line, "==> DHT_PORT ") != -1:
-        direction = log_msg_dir["SEND"]
+        transfer_direction_id = log_msg_dir["SEND"]
     else:
         return None
 
     if DEBUG == True:
         print "--- match DHT_PORT"
 
-    msg_type = bt_msg_types["DHT_PORT"]
+    msg_type_id = bt_msg_types["DHT_PORT"]
     parts = re.split("[\[\]<=>]+", line)
 
     timestamp = string_to_timestamp(parts[0].strip())
     index = 0
     begin = 0
     length = 0
-    if direction == log_msg_dir["RECEIVE"]:
+    if transfer_direction_id == log_msg_dir["RECEIVE"]:
         port = int("0x" + re.split(":", parts[2])[1].strip(), 16)
     else:
         port = int(parts[2].strip())
 
-    return (timestamp, direction, msg_type, index, begin, length, port)
+    return (timestamp, transfer_direction_id, msg_type_id, index, begin, length, port)
 
 
 #
@@ -448,26 +448,26 @@ def libtorrent_parse_log_file(dbw, client_session_id, logfile):
             if result == None:
                 continue
 
-            (timestamp, direction, msg_type, index, begin, length, listen_port) = result
+            (timestamp, transfer_direction_id, msg_type_id, index, begin, length, listen_port) = result
             if DEBUG == True:
                 print result 
 
             dbw.add_verbose_message_datetime(client_session_id, timestamp,
-                    direction, peer_ip, peer_port, msg_type,
+                    transfer_direction_id, peer_ip, peer_port, msg_type_id,
                     index, begin,length, listen_port)
 
     except IOError:
         print "Error processing file %s." %logfile
 
 def usage():
-    print "Usage: python StatusParser.py -i|--id id -f|--file log_file database"
+    print "Usage: python StatusParser.py -i|--id id -f|--file log_file dbconf_file"
     print "id:"
     print "\t--id"
     print "\t-i\t\tclient_session_id"
     print "\tstatus_file:"
     print "\t--file"
-    print "\t-f\t\tstatus_file for tribler"
-    print "\tdatabase\t\tSQLite database file"
+    print "\t-f\t\tstatus_file for libtorrent"
+    print "\tdbconf_file\t\tMySQL database configuration file"
     print "\t--help"
     print "\t-h\t\t\tprint this help screen"
 
@@ -475,7 +475,7 @@ def usage():
 def main_just_parse():
     filename = sys.argv[1]
     client_session_id = 1
-    tribler_parse_status_file(None, 1, filename)
+    libtorrent_parse_status_file(None, 1, filename)
 
 
 def main_with_DB():
@@ -490,7 +490,7 @@ def main_with_DB():
 
     client_session_id = None
     filename = None
-    database = None
+    dbconf = None
 
     for o, a in opts:
         if o in ("-h", "--help"):
@@ -511,19 +511,20 @@ def main_with_DB():
         print "Error: no status file."
         sys.exit(2)
 
-    # no database passed as argument
+    # no database configuration file passed as argument
     if len(args) != 1:
-        print "Error: no database file passed as argument."
+        print "Error: no database configuration file passed as argument."
         sys.exit(2)
-    database = args[0]
+    dbconf = args[0]
 
-    dbc = DatabaseCommander(database)
+    dbc = DatabaseCommander(dbconf)
 
     # check for client_session_id, swarm_id, btclient_id
     cursor = dbc.select_client_sessions_by_id(client_session_id)
     if cursor == None:
         print "Error: no client session id (%d) in database." % client_session_id
         sys.exit(2)
+
     for session_row in cursor:
         pass
 
@@ -561,7 +562,7 @@ def main_with_DB():
     print ""
 
     # parse log file
-    dbw = DatabaseWriter(database)
+    dbw = DatabaseWriter(dbconf)
     libtorrent_parse_log_file(dbw, client_session_id, filename)
 
 

diff --git a/ppf/log-parser-mysql/libtorrent/StatusParser.py b/ppf/log-parser-mysql/libtorrent/StatusParser.py
index 4639d3c..0bc3490 100644 (file)
--- a/ppf/log-parser-mysql/libtorrent/StatusParser.py
+++ b/ppf/log-parser-mysql/libtorrent/StatusParser.py
@@ -18,7 +18,7 @@ def usage():
     print "\tstatus_file:"
     print "\t--file"
     print "\t-f\t\tstatus_file for libtorrent"
-    print "\tdatabase\t\tSQLite database file"
+    print "\tdbconf_file\t\tMySQL database configuration file"
     print "\t--help"
     print "\t-h\t\t\tprint this help screen"
 
@@ -86,7 +86,7 @@ def libtorrent_parse_status_line(line):
         if pair[0] == "ps":
             num_peers = libtorrent_canon_num_peers(pair[1])
         if pair[0] == "dht":
-            dht = libtorrent_canon_dht(pair[1])
+            num_dht_peers = libtorrent_canon_dht(pair[1])
         if pair[0] == "dl":
             download_speed = libtorrent_canon_download_speed(pair[1])
         if pair[0] == "ul":
@@ -101,7 +101,7 @@ def libtorrent_parse_status_line(line):
             eta = libtorrent_canon_eta(pair[1])
             eta_seconds = eta.days * 24 * 3600 + eta.seconds
 
-    return (num_peers, dht, download_speed, upload_speed, download_size, upload_size, eta_seconds)
+    return (num_peers, num_dht_peers, download_speed, upload_speed, download_size, upload_size, eta_seconds)
 
 def libtorrent_parse_status_file(dbw, client_session_id, session_start, filename):
 
@@ -119,14 +119,14 @@ def libtorrent_parse_status_file(dbw, client_session_id, session_start, filename
             if libtorrent_is_status_line(line) == False:
                 continue
 
-            (num_peers, dht, download_speed, upload_speed, download_size, upload_size, eta_seconds) = libtorrent_parse_status_line(line)
+            (num_peers, num_dht_peers, download_speed, upload_speed, download_size, upload_size, eta_seconds) = libtorrent_parse_status_line(line)
 
             message_time = message_time + one_second
 
             if DEBUG == True:
-                print "(%d, %s, %s, %d, %d kb/s, %d kb/s, %d bytes, %d bytes)" % (num_peers, date, time, dht, download_speed, upload_speed, download_size, upload_size)
+                print "(%d, %s, %s, %d, %d kb/s, %d kb/s, %d bytes, %d bytes)" % (num_peers, date, time, num_dht_peers, download_speed, upload_speed, download_size, upload_size)
             
-            dbw.add_status_message_datetime(client_session_id, message_time, num_peers, dht, download_speed, upload_speed, download_size, upload_size, eta_seconds)
+            dbw.add_status_message_datetime(client_session_id, message_time, num_peers, num_dht_peers, download_speed, upload_speed, download_size, upload_size, eta_seconds)
 
     except IOError:
         print "Error processing file %s." %filename
@@ -163,13 +163,13 @@ def main():
         print "Error: no status file."
         sys.exit(2)
 
-    # no database passed as argument
+    # no database configuration file passed as argument
     if len(args) != 1:
-        print "Error: no database file passed as argument."
+        print "Error: no database configuration file passed as argument."
         sys.exit(2)
-    database = args[0]
+    dbconf = args[0]
 
-    dbc = DatabaseCommander(database)
+    dbc = DatabaseCommander(dbconf)
     
     # check for client_session_id, swarm_id, btclient_id
     cursor = dbc.select_client_sessions_by_id(client_session_id)
@@ -216,7 +216,7 @@ def main():
 #    session_start = julian.julianToDatetime(session_row[11])
     session_start = session_row[11]
     # parse status file
-    dbw = DatabaseWriter(database)
+    dbw = DatabaseWriter(dbconf)
     libtorrent_parse_status_file(dbw, client_session_id, session_start, filename)