Tribler: LogParser for choke/unchoke, have, piece, request, bitfied, cancel messages

diff --git a/log-parser/tribler/LogParser.py b/log-parser/tribler/LogParser.py
index dc45555..8b603b6 100644 (file)
--- a/log-parser/tribler/LogParser.py
+++ b/log-parser/tribler/LogParser.py
@@ -13,8 +13,14 @@ import getopt
 import re
 
 # the names used by Tribler for the BitTorrent messages 
-msg_types = {"BT_REQUEST": "new_request", "BT_CHOKE": "_rechoke", "BT_HAVE": "got_have"} 
-msg_db_code = {"BT_CHOKE" : 0, "BT_HAVE" : 4, "BT_REQUEST" : 6}
+msg_types = {"BT_REQUEST": "new_request", "BT_CHOKE": "CHOKE from", "BT_UNCHOKE": "UNCHOKE from",
+            "BT_HAVE": "HAVE(", "BT_PIECE": "PIECE(", "BT_BITFIELD": "BITFIELD from", 
+            "BT_CANCEL": "sent cancel"} 
+
+msg_db_code = {"BT_CHOKE" : 0, "BT_UNCHOKE": 1, "BT_INTERESTED": 2, "BT_NOT_INTERESTED": 3, 
+            "BT_HAVE" : 4, "BT_BITFIELD": 5, "BT_REQUEST" : 6, "BT_PIECE": 7, "BT_CANCEL": 8}
+
+log_msg_dir = {"RECEIVE": 0, "SEND": 1}
 
 DEBUG = True
 
@@ -45,9 +51,95 @@ def tribler_parse_timestamp(date, time):
                                     int(time_array[0]), int(time_array[1]), int(time_array[2])) #hour, min, sec
     return timestamp
 
-""" line format example: 20-10-2009 12:56:39   Downloader: new_request 52 98304 16384 to 141.85.37.41 14398
+"""
+    Parses a line that contains choke/unchoke messages.
+    line format: 
+        date time connecter: Got CHOKE from peer_ip 
+        date time connecter: Got UNCHOKE from peer_ip 
+"""
+def tribler_parse_choke_msg(line):
+    is_choke = line.find(msg_types["BT_CHOKE"]) 
+    is_unchoke = line.find(msg_types["BT_UNCHOKE"])
+    if is_choke == -1 or is_unchoke == -1:
+        return None
+    
+    line_parts = re.split(" *", line)
+    timestamp = tribler_parse_timestamp(line_parts[0], line_parts[1])
+    
+    if timestamp == None:
+        print "Error: invalid date & time format for Connecter. "
+        return None
+    
+    nr_parts = len(line_parts)
+    if nr_parts < 7 : 
+        print "Error: invalid line format for Connecter. ", line
+        return None
+
+    peer_ip = line_parts[nr_parts-1];
+    direction = log_msg_dir["RECEIVE"];
+    if is_choke != -1:
+        return (timestamp, direction, peer_ip, None, msg_db_code["BT_CHOKE"], None, None, None, 0) 
+
+    if is_unchoke != -1:
+        return (timestamp, direction, peer_ip, None, msg_db_code["BT_UNCHOKE"], None, None, None, 0) 
+
+"""
+    sample line: 14-11-2009 23:11:13   connecter: Got HAVE( 14 ) from 141.85.37.41
+    BitTorrent message format: have <id> <index>
+"""
+def tribler_parse_have_msg(line):
+   
+    if line.find(msg_types["BT_HAVE"]) == -1:
+        return None
+
+    # the messages can also have the format: Downloader: got_have <piece_number> is invalid piece
+    #if line.find("invalid") != -1:
+    #    return None # didn't decide yet what to do with it
+    
+    line_parts = re.split(" *", line)
+    timestamp = tribler_parse_timestamp(line_parts[0], line_parts[1])
+    
+    if timestamp == None:
+        print "Error: invalid date & time format for Connecter."
+        return None
+    
+    nr_parts = len(line_parts)
+    if nr_parts < 8 : 
+        print "Error: invalid line format for Connecter.", line
+        return None
+
+    index = int(line_parts[5])
+    peer_ip = line_parts[nr_parts-1];
+    direction = log_msg_dir["RECEIVE"];
+    return (timestamp, direction, peer_ip, None, msg_db_code["BT_HAVE"], index, None, None, 0) 
+
+"""
+    sample line: 14-11-2009 23:11:25   connecter: Got BITFIELD from 141.85.37.41
+"""
+def tribler_parse_bitfield_msg(line):
+
+    if line.find(msg_types["BT_BITFIELD"]) == -1:
+        return None
+
+    line_parts = re.split(" *", line)
+    timestamp = tribler_parse_timestamp(line_parts[0], line_parts[1])
+   
+    if timestamp == None:
+        print "Error: invalid date & time format for Connecter."
+        return None
+    
+    nr_parts = len(line_parts)
+    if nr_parts < 7 : 
+        print "Error: invalid line format for Connecter.", line
+        return None
+
+    peer_ip = line_parts[nr_parts-1];
+    direction = log_msg_dir["RECEIVE"];
+    return (timestamp, direction, peer_ip, None, msg_db_code["BT_BITFIELD"], None, None, None, 0) 
+
+
+""" sample line: 20-10-2009 12:56:39   Downloader: new_request 52 98304 16384 to 141.85.37.41 14398
     BitTorrent protocol message: request: <len=0013><id=6><index><begin><length>
-    Returns a list (timestamp, peer_ip, peer_port, message_type, index, begin, length, listen_port)
 """
 def tribler_parse_request_msg(line):
     if line.find(msg_types["BT_REQUEST"]) == -1:
@@ -71,68 +163,85 @@ def tribler_parse_request_msg(line):
     peer_ip = line_parts[8]
     peer_port = int(line_parts[9])
 
-    return (timestamp, peer_ip, peer_port, msg_db_code["BT_REQUEST"], index, begin, length, 0)
+    direction = log_msg_dir["SEND"];
+    return (timestamp, direction, peer_ip, peer_port, msg_db_code["BT_REQUEST"], index, begin, length, 0)
 
 
 """
-    line format example: 20-10-2009 12:56:39 Downloader: got_have 12
-    BitTorrent message format: have <id> <index>
-    Returns a list :...
+    sample line: 14-11-2009 23:11:13   connecter: Got PIECE( 141 ) from 141.85.37.41
+    BitTorrent message format: piece <id=7><index><begin><block>
 """
-def tribler_parse_have_msg(line):
-   
-    if line.find(msg_types["BT_HAVE"]) == -1:
+def tribler_parse_piece_msg(line):
+
+    if line.find(msg_types["BT_PIECE"]) == -1:
         return None
 
-    # the messages can also have the format: Downloader: got_have <piece_number> is invalid piece
-    if line.find("invalid") != -1:
-        return None # didn't decide yet what to do with it
-    
     line_parts = re.split(" *", line)
     timestamp = tribler_parse_timestamp(line_parts[0], line_parts[1])
     
     if timestamp == None:
-        print "Error: invalid date & time format for Downloader."
+        print "Error: invalid date & time format for Connecter."
         return None
     
-    if len(line_parts) < 5 : 
-        print "Error: invalid line format for Downloader."
+    nr_parts = len(line_parts)
+    if nr_parts < 8 : 
+        print "Error: invalid line format for Connecter.", line
         return None
 
-    index = int(line_parts[4])
-    return (timestamp, None, None, msg_db_code["BT_HAVE"], index, None, None, 0) 
-
+    index = int(line_parts[5])
+    peer_ip = line_parts[nr_parts-1];
+    direction = log_msg_dir["RECEIVE"];
+    return (timestamp, direction, peer_ip, None, msg_db_code["BT_PIECE"], index, None, None, 0) 
 
 
 """
-    Parses a line that contains choke/unchoke messages.
-    line format: choker: _rechoke
+    sample line: 14-11-2009 23:11:24   sent cancel: 130: 114688-131072
+    BitTorrent massage format: <id=8><index><begin><length>
 """
-def tribler_parse_choke_msg(line):
-    if line.find(msg_types["BT_CHOKE"]) == -1:
+def tribler_parse_cancel_msg(line):
+
+    if line.find(msg_types["BT_CANCEL"]) == -1:
         return None
-    pass
+
+    line_parts = re.split(" *", line)
+    timestamp = tribler_parse_timestamp(line_parts[0], line_parts[1])
+   
+    if timestamp == None:
+        print "Error: invalid date & time format for Connecter."
+        return None
+    
+    nr_parts = len(line_parts)
+    if nr_parts < 6 : 
+        print "Error: invalid line format for Connecter.", line
+        return None
+    
+    line_parts = re.split("[ :-]*", line)
+    index = int(line_parts[8])
+    begin = int(line_parts[9])
+    length = int(line_parts[10]) - begin
+    direction = log_msg_dir["SEND"];
+    return (timestamp, direction, None, None,  msg_db_code["BT_CANCEL"], index, begin, length, 0) 
 
 
-def tribler_parse_piece_msg(line):
-    pass
 
 def tribler_parse_line(line):
 
-    result = tribler_parse_request_msg(line)
+    result = tribler_parse_choke_msg(line)
     if result != None:
            return result
-
     result = tribler_parse_have_msg(line)
-   
     if result != None:
            return result
-    
-    result = tribler_parse_choke_msg(line)
+    result = tribler_parse_bitfield_msg(line)
+    if result != None:
+           return result
+    result = tribler_parse_request_msg(line)
     if result != None:
            return result
- 
     result = tribler_parse_piece_msg(line)
+    if result != None:
+           return result
+    result = tribler_parse_cancel_msg(line)
     return result;
 
 def tribler_parse_status_file(dbw, client_session_id, filename):
@@ -150,12 +259,11 @@ def tribler_parse_status_file(dbw, client_session_id, filename):
             if result == None:
                 continue
 
-            (timestamp, peer_ip, peer_port, msg_type, index, begin, length, listen_port) = result
+            (timestamp, direction, peer_ip, peer_port, msg_type, index, begin, length, listen_port) = result
             if DEBUG == True:
                 print result 
-            
-            dbw.add_verbose_message(client_session_id, timestamp, peer_ip, peer_port, msg_type, 
-                                    index, begin, length, listen_port)
+            dbw.add_verbose_message_datetime(client_session_id, timestamp, direction, peer_ip, peer_port, 
+                                msg_type, index, begin, length, listen_port)
             
     except IOError:
         print "Error processing file %s." %filename