ppf/new: Add peer message parsing support for libtorrent.

diff --git a/ppf/new/parsing.py b/ppf/new/parsing.py
index 41cea1f..654e129 100644 (file)
--- a/ppf/new/parsing.py
+++ b/ppf/new/parsing.py
@@ -104,7 +104,6 @@ class LibtorrentLogParser(SessionLogParser):
         # In case status file doesn't exist, skip it.
         if os.access(status_file_path, os.F_OK) and \
                 os.access(status_file_path, os.R_OK):
-            logger.debug("Status file exists: %s." %(status_file_path))
             # List functions as a stack. First files go to the end.
             if priority == "verbose":
                 self.to_parse.insert(0, status_file_path)
@@ -289,13 +288,13 @@ class LibtorrentLogParser(SessionLogParser):
             elif pair[0] == "dht":
                 msg.num_dht_peers = self.canon_dht(pair[1])
             elif pair[0] == "dl":
-                msg.download_speed = self.canon_download_speed(pair[1])
+                msg.download_speed = self.canon_speed(pair[1])
             elif pair[0] == "ul":
-                msg.upload_speed = self.canon_upload_speed(pair[1])
+                msg.upload_speed = self.canon_speed(pair[1])
             elif pair[0] == "dld":
-                msg.download_size = self.canon_download_size(pair[1])
+                msg.download_size = self.canon_size(pair[1])
             elif pair[0] == "uld":
-                msg.upload_size = self.canon_upload_size(pair[1])
+                msg.upload_size = self.canon_size(pair[1])
             elif pair[0] == "size":
                 pass
             elif pair[0] == "eta":
@@ -303,6 +302,7 @@ class LibtorrentLogParser(SessionLogParser):
                 msg.eta_seconds = eta.days * 24 * 3600 + eta.seconds
 
         # Add to message_list.
+        logger.debug("Add new status message.")
         self.message_list.append(msg)
 
     def canon_num_peers(self, non_canon_value):
@@ -311,22 +311,14 @@ class LibtorrentLogParser(SessionLogParser):
     def canon_dht(self, non_canon_value):
         return int(non_canon_value)
 
-    def canon_download_speed(self, non_canon_value):
+    def canon_speed(self, non_canon_value):
         """119.51kb/s -> 119"""
         return int(float(non_canon_value.strip("kb/s")))
 
-    def canon_upload_speed(self, non_canon_value):
-        """12119.51kb/s -> 12119"""
-        return int(float(non_canon_value.strip("kb/s")))
-
-    def canon_download_size(self, non_canon_value):
+    def canon_size(self, non_canon_value):
         """698mb -> 698*1024"""
         return int(non_canon_value.strip("mb")) * 1024
 
-    def canon_upload_size(self, non_canon_value):
-        """492mb -> 492*1024"""
-        return int(non_canon_value.strip("mb")) * 1024
-
     def canon_eta(self, non_canon_value):
         """1h 38m 37s -> [0, 1, 38, 37]; 3d 5h 24m 34s -> [3, 5, 24, 34]"""
         eta_string_array = re.split('\ *[dhms]\ *', non_canon_value)
@@ -342,7 +334,42 @@ class LibtorrentLogParser(SessionLogParser):
         return eta_td
 
     def parse_peer_status_log_line(self, line):
-        pass
+        """Parse Libtorrent status line.
+
+        Sample:
+            --Peers: (Aug 14 17:22:40) [ ip: 10.1.7.5:33289, dl: 23.6kb/s, ul: 94.91kb/s ][ ip: 10.1.6.5:47254, dl: 55.61kb/s, ul: 100.72kb/s ][ ip: 10.1.4.5:6881, dl: 93.99kb/s, ul: 100.11kb/s ][ ip: 10.1.10.5:48923, dl: 57.42kb/s, ul: 205.74kb/s ]")
+        """
+        parts = re.split(r'[\(\)\[\]]+', line)
+
+        date_string = parts[1].strip() + " %s" %(self.start_time.year)
+        timestamp = datetime.datetime.strptime(date_string,
+                "%b %d %H:%M:%S %Y")
+
+        for i in range(3, len(parts)):
+            msg = self.get_peer_status_message_from_part(re.sub(r'\s', '', parts[i]))
+            if msg:
+                msg.timestamp = timestamp
+                logger.debug("Add new peer status message.")
+                self.message_list.append(msg)
+
+    def get_peer_status_message_from_part(self, part):
+        """Part sample ip:10.1.7.5:33289,dl:23.6kb/s,ul:94.91kb/s."""
+        subparts = re.split(r',', part.strip())
+        if len(subparts) != 3:
+            return None
+
+        msg = storage.PeerStatusMessage()
+        for s in subparts:
+            pair = re.split(r':', s)
+            if pair[0] == 'ip':
+                msg.peer_ip = pair[1]
+                msg.peer_port = pair[2]
+            elif pair[0] == 'dl':
+                msg.download_speed = self.canon_speed(pair[1])
+            elif pair[0] == 'ul':
+                msg.upload_speed = self.canon_speed(pair[1])
+
+        return msg
 
     def parse_verbose_log_line(self, line, message_type):
         if self.verbose_log_ip == None or self.verbose_log_port == None:
@@ -357,6 +384,7 @@ class LibtorrentLogParser(SessionLogParser):
         (msg.index, msg.begin, msg.length, msg.listen_port) = self.get_verbose_log_line_noncommon_fields(line, message_type)
 
         # Add to message_list.
+        logger.debug("Add new verbose message.")
         self.message_list.append(msg)
 
     def get_verbose_log_line_common_fields(self, line):

diff --git a/ppf/new/tests/test_parsing.py b/ppf/new/tests/test_parsing.py
index 7e98e39..b1ad43e 100644 (file)
--- a/ppf/new/tests/test_parsing.py
+++ b/ppf/new/tests/test_parsing.py
@@ -108,9 +108,9 @@ class LibtorrentLogParserTest(unittest.TestCase):
     def add_peer_status_line(self, filename):
         """Add peer status line to file."""
         if self.peer_status_line_num_calls == 0:
-            self.append_line(filename, "--Peers: (Aug 14 17:22:35) [ ip: 10.1.7.5:33289, dl: 86.63kb/s, ul: 175.07kb/s ][ ip: 10.1.9.5:55611, dl: 14.63kb/s, ul: 110.79kb/s ][ ip: 10.1.8.5:6881, dl: 2.96kb/s, ul: 92.85kb/s ][ ip: 10.1.6.5:47254, dl: 80.1kb/s, ul: 176.44kb/s ][ ip: 10.1.4.5:6881, dl: 115.16kb/s, ul: 167.83kb/s ][ ip: 10.1.5.5:45222, dl: 108.39kb/s, ul: 158.11kb/s ][ ip: 10.1.10.5:48923, dl: 66.16kb/s, ul: 121.92kb/s ][ ip: 10.1.1.5:51146, dl: 198.66kb/s, ul: 6.11kb/s ]")
+            self.append_line(filename, "--Peers: (Aug 14 17:22:35) [ ip: 10.1.7.5:33289, dl: 86.63kb/s, ul: 175.07kb/s ][ ip: 10.1.9.5:55611, dl: 14.63kb/s, ul: 110.79kb/s ]")
         else:
-            self.append_line(filename, "--Peers: (Aug 14 17:22:40) [ ip: 10.1.7.5:33289, dl: 23.6kb/s, ul: 94.91kb/s ][ ip: 10.1.6.5:47254, dl: 55.61kb/s, ul: 100.72kb/s ][ ip: 10.1.4.5:6881, dl: 93.99kb/s, ul: 100.11kb/s ][ ip: 10.1.10.5:48923, dl: 57.42kb/s, ul: 205.74kb/s ]")
+            self.append_line(filename, "--Peers: (Aug 14 17:22:40) [ ip: 10.1.7.5:33289, dl: 23.6kb/s, ul: 94.91kb/s ][ ip: 10.1.6.5:47254, dl: 55.61kb/s, ul: 100.72kb/s ][ ip: 10.1.4.5:6881, dl: 93.99kb/s, ul: 100.11kb/s ]")
         self.peer_status_line_num_calls = self.peer_status_line_num_calls + 1
 
     def add_verbose_line(self, filename):
@@ -378,11 +378,17 @@ class LibtorrentLogParserTest(unittest.TestCase):
                 "2009-01-01 12:13:14", priority="verbose")
         msg1 = p.get_next_message()
         msg2 = p.get_next_message()
+        msg3 = p.get_next_message()
+        msg4 = p.get_next_message()
+        msg5 = p.get_next_message()
 
         self.add_message_to_count(msg1)
         self.add_message_to_count(msg2)
+        self.add_message_to_count(msg3)
+        self.add_message_to_count(msg4)
+        self.add_message_to_count(msg5)
 
-        self.assertEqual(self.pstatmsg_count, 2)
+        self.assertEqual(self.pstatmsg_count, 5)
 
     def test_get_peer_status_messages_different_files(self):
         """Return multiple peer status messages from different log files."""
@@ -393,11 +399,17 @@ class LibtorrentLogParserTest(unittest.TestCase):
                 "2009-01-01 12:13:14", priority="verbose")
         msg1 = p.get_next_message()
         msg2 = p.get_next_message()
+        msg3 = p.get_next_message()
+        msg4 = p.get_next_message()
+        msg5 = p.get_next_message()
 
         self.add_message_to_count(msg1)
         self.add_message_to_count(msg2)
+        self.add_message_to_count(msg3)
+        self.add_message_to_count(msg4)
+        self.add_message_to_count(msg5)
 
-        self.assertEqual(self.pstatmsg_count, 2)
+        self.assertEqual(self.pstatmsg_count, 5)
 
     def test_get_verbose_message(self):
         """Return a verbose message from verbose log file."""
@@ -452,13 +464,15 @@ class LibtorrentLogParserTest(unittest.TestCase):
         msg1 = p.get_next_message()
         msg2 = p.get_next_message()
         msg3 = p.get_next_message()
+        msg4 = p.get_next_message()
 
         self.add_message_to_count(msg1)
         self.add_message_to_count(msg2)
         self.add_message_to_count(msg3)
+        self.add_message_to_count(msg4)
 
         self.assertEqual(self.statmsg_count, 1)
-        self.assertEqual(self.pstatmsg_count, 1)
+        self.assertEqual(self.pstatmsg_count, 2)
         self.assertEqual(self.verbmsg_count, 1)
 
     def test_get_mixed_messages_own_file(self):
@@ -478,6 +492,9 @@ class LibtorrentLogParserTest(unittest.TestCase):
         msg4 = p.get_next_message()
         msg5 = p.get_next_message()
         msg6 = p.get_next_message()
+        msg7 = p.get_next_message()
+        msg8 = p.get_next_message()
+        msg9 = p.get_next_message()
 
         self.add_message_to_count(msg1)
         self.add_message_to_count(msg2)
@@ -485,9 +502,12 @@ class LibtorrentLogParserTest(unittest.TestCase):
         self.add_message_to_count(msg4)
         self.add_message_to_count(msg5)
         self.add_message_to_count(msg6)
+        self.add_message_to_count(msg7)
+        self.add_message_to_count(msg8)
+        self.add_message_to_count(msg9)
 
         self.assertEqual(self.statmsg_count, 2)
-        self.assertEqual(self.pstatmsg_count, 2)
+        self.assertEqual(self.pstatmsg_count, 5)
         self.assertEqual(self.verbmsg_count, 2)
 
     def test_get_mixed_messages_different_files(self):
@@ -507,6 +527,9 @@ class LibtorrentLogParserTest(unittest.TestCase):
         msg4 = p.get_next_message()
         msg5 = p.get_next_message()
         msg6 = p.get_next_message()
+        msg7 = p.get_next_message()
+        msg8 = p.get_next_message()
+        msg9 = p.get_next_message()
 
         self.add_message_to_count(msg1)
         self.add_message_to_count(msg2)
@@ -514,9 +537,12 @@ class LibtorrentLogParserTest(unittest.TestCase):
         self.add_message_to_count(msg4)
         self.add_message_to_count(msg5)
         self.add_message_to_count(msg6)
+        self.add_message_to_count(msg7)
+        self.add_message_to_count(msg8)
+        self.add_message_to_count(msg9)
 
         self.assertEqual(self.statmsg_count, 2)
-        self.assertEqual(self.pstatmsg_count, 2)
+        self.assertEqual(self.pstatmsg_count, 5)
         self.assertEqual(self.verbmsg_count, 2)
 
 
@@ -537,5 +563,6 @@ if __name__ == "__main__":
 #    suite.addTest(LibtorrentLogParserTest('test_get_verbose_message'))
 #    suite.addTest(LibtorrentLogParserTest('test_get_verbose_messages'))
 #    suite.addTest(LibtorrentLogParserTest('test_get_verbose_messages_different_files'))
+#    suite.addTest(LibtorrentLogParserTest('test_get_mixed_messages_same_file'))
 #    unittest.TextTestRunner().run(suite)
     unittest.main()