From: Razvan Deaconescu Date: Sun, 28 Aug 2011 05:56:10 +0000 (+0300) Subject: ppf/new: Add peer message parsing support for libtorrent. X-Git-Url: http://p2p-next.cs.pub.ro/gitweb/?a=commitdiff_plain;h=cef4d051278087dc0051f1f44821ed8515237628;p=cs-p2p-next.git ppf/new: Add peer message parsing support for libtorrent. --- diff --git a/ppf/new/parsing.py b/ppf/new/parsing.py index 41cea1f..654e129 100644 --- a/ppf/new/parsing.py +++ b/ppf/new/parsing.py @@ -104,7 +104,6 @@ class LibtorrentLogParser(SessionLogParser): # In case status file doesn't exist, skip it. if os.access(status_file_path, os.F_OK) and \ os.access(status_file_path, os.R_OK): - logger.debug("Status file exists: %s." %(status_file_path)) # List functions as a stack. First files go to the end. if priority == "verbose": self.to_parse.insert(0, status_file_path) @@ -289,13 +288,13 @@ class LibtorrentLogParser(SessionLogParser): elif pair[0] == "dht": msg.num_dht_peers = self.canon_dht(pair[1]) elif pair[0] == "dl": - msg.download_speed = self.canon_download_speed(pair[1]) + msg.download_speed = self.canon_speed(pair[1]) elif pair[0] == "ul": - msg.upload_speed = self.canon_upload_speed(pair[1]) + msg.upload_speed = self.canon_speed(pair[1]) elif pair[0] == "dld": - msg.download_size = self.canon_download_size(pair[1]) + msg.download_size = self.canon_size(pair[1]) elif pair[0] == "uld": - msg.upload_size = self.canon_upload_size(pair[1]) + msg.upload_size = self.canon_size(pair[1]) elif pair[0] == "size": pass elif pair[0] == "eta": @@ -303,6 +302,7 @@ class LibtorrentLogParser(SessionLogParser): msg.eta_seconds = eta.days * 24 * 3600 + eta.seconds # Add to message_list. + logger.debug("Add new status message.") self.message_list.append(msg) def canon_num_peers(self, non_canon_value): @@ -311,22 +311,14 @@ class LibtorrentLogParser(SessionLogParser): def canon_dht(self, non_canon_value): return int(non_canon_value) - def canon_download_speed(self, non_canon_value): + def canon_speed(self, non_canon_value): """119.51kb/s -> 119""" return int(float(non_canon_value.strip("kb/s"))) - def canon_upload_speed(self, non_canon_value): - """12119.51kb/s -> 12119""" - return int(float(non_canon_value.strip("kb/s"))) - - def canon_download_size(self, non_canon_value): + def canon_size(self, non_canon_value): """698mb -> 698*1024""" return int(non_canon_value.strip("mb")) * 1024 - def canon_upload_size(self, non_canon_value): - """492mb -> 492*1024""" - return int(non_canon_value.strip("mb")) * 1024 - def canon_eta(self, non_canon_value): """1h 38m 37s -> [0, 1, 38, 37]; 3d 5h 24m 34s -> [3, 5, 24, 34]""" eta_string_array = re.split('\ *[dhms]\ *', non_canon_value) @@ -342,7 +334,42 @@ class LibtorrentLogParser(SessionLogParser): return eta_td def parse_peer_status_log_line(self, line): - pass + """Parse Libtorrent status line. + + Sample: + --Peers: (Aug 14 17:22:40) [ ip: 10.1.7.5:33289, dl: 23.6kb/s, ul: 94.91kb/s ][ ip: 10.1.6.5:47254, dl: 55.61kb/s, ul: 100.72kb/s ][ ip: 10.1.4.5:6881, dl: 93.99kb/s, ul: 100.11kb/s ][ ip: 10.1.10.5:48923, dl: 57.42kb/s, ul: 205.74kb/s ]") + """ + parts = re.split(r'[\(\)\[\]]+', line) + + date_string = parts[1].strip() + " %s" %(self.start_time.year) + timestamp = datetime.datetime.strptime(date_string, + "%b %d %H:%M:%S %Y") + + for i in range(3, len(parts)): + msg = self.get_peer_status_message_from_part(re.sub(r'\s', '', parts[i])) + if msg: + msg.timestamp = timestamp + logger.debug("Add new peer status message.") + self.message_list.append(msg) + + def get_peer_status_message_from_part(self, part): + """Part sample ip:10.1.7.5:33289,dl:23.6kb/s,ul:94.91kb/s.""" + subparts = re.split(r',', part.strip()) + if len(subparts) != 3: + return None + + msg = storage.PeerStatusMessage() + for s in subparts: + pair = re.split(r':', s) + if pair[0] == 'ip': + msg.peer_ip = pair[1] + msg.peer_port = pair[2] + elif pair[0] == 'dl': + msg.download_speed = self.canon_speed(pair[1]) + elif pair[0] == 'ul': + msg.upload_speed = self.canon_speed(pair[1]) + + return msg def parse_verbose_log_line(self, line, message_type): if self.verbose_log_ip == None or self.verbose_log_port == None: @@ -357,6 +384,7 @@ class LibtorrentLogParser(SessionLogParser): (msg.index, msg.begin, msg.length, msg.listen_port) = self.get_verbose_log_line_noncommon_fields(line, message_type) # Add to message_list. + logger.debug("Add new verbose message.") self.message_list.append(msg) def get_verbose_log_line_common_fields(self, line): diff --git a/ppf/new/tests/test_parsing.py b/ppf/new/tests/test_parsing.py index 7e98e39..b1ad43e 100644 --- a/ppf/new/tests/test_parsing.py +++ b/ppf/new/tests/test_parsing.py @@ -108,9 +108,9 @@ class LibtorrentLogParserTest(unittest.TestCase): def add_peer_status_line(self, filename): """Add peer status line to file.""" if self.peer_status_line_num_calls == 0: - self.append_line(filename, "--Peers: (Aug 14 17:22:35) [ ip: 10.1.7.5:33289, dl: 86.63kb/s, ul: 175.07kb/s ][ ip: 10.1.9.5:55611, dl: 14.63kb/s, ul: 110.79kb/s ][ ip: 10.1.8.5:6881, dl: 2.96kb/s, ul: 92.85kb/s ][ ip: 10.1.6.5:47254, dl: 80.1kb/s, ul: 176.44kb/s ][ ip: 10.1.4.5:6881, dl: 115.16kb/s, ul: 167.83kb/s ][ ip: 10.1.5.5:45222, dl: 108.39kb/s, ul: 158.11kb/s ][ ip: 10.1.10.5:48923, dl: 66.16kb/s, ul: 121.92kb/s ][ ip: 10.1.1.5:51146, dl: 198.66kb/s, ul: 6.11kb/s ]") + self.append_line(filename, "--Peers: (Aug 14 17:22:35) [ ip: 10.1.7.5:33289, dl: 86.63kb/s, ul: 175.07kb/s ][ ip: 10.1.9.5:55611, dl: 14.63kb/s, ul: 110.79kb/s ]") else: - self.append_line(filename, "--Peers: (Aug 14 17:22:40) [ ip: 10.1.7.5:33289, dl: 23.6kb/s, ul: 94.91kb/s ][ ip: 10.1.6.5:47254, dl: 55.61kb/s, ul: 100.72kb/s ][ ip: 10.1.4.5:6881, dl: 93.99kb/s, ul: 100.11kb/s ][ ip: 10.1.10.5:48923, dl: 57.42kb/s, ul: 205.74kb/s ]") + self.append_line(filename, "--Peers: (Aug 14 17:22:40) [ ip: 10.1.7.5:33289, dl: 23.6kb/s, ul: 94.91kb/s ][ ip: 10.1.6.5:47254, dl: 55.61kb/s, ul: 100.72kb/s ][ ip: 10.1.4.5:6881, dl: 93.99kb/s, ul: 100.11kb/s ]") self.peer_status_line_num_calls = self.peer_status_line_num_calls + 1 def add_verbose_line(self, filename): @@ -378,11 +378,17 @@ class LibtorrentLogParserTest(unittest.TestCase): "2009-01-01 12:13:14", priority="verbose") msg1 = p.get_next_message() msg2 = p.get_next_message() + msg3 = p.get_next_message() + msg4 = p.get_next_message() + msg5 = p.get_next_message() self.add_message_to_count(msg1) self.add_message_to_count(msg2) + self.add_message_to_count(msg3) + self.add_message_to_count(msg4) + self.add_message_to_count(msg5) - self.assertEqual(self.pstatmsg_count, 2) + self.assertEqual(self.pstatmsg_count, 5) def test_get_peer_status_messages_different_files(self): """Return multiple peer status messages from different log files.""" @@ -393,11 +399,17 @@ class LibtorrentLogParserTest(unittest.TestCase): "2009-01-01 12:13:14", priority="verbose") msg1 = p.get_next_message() msg2 = p.get_next_message() + msg3 = p.get_next_message() + msg4 = p.get_next_message() + msg5 = p.get_next_message() self.add_message_to_count(msg1) self.add_message_to_count(msg2) + self.add_message_to_count(msg3) + self.add_message_to_count(msg4) + self.add_message_to_count(msg5) - self.assertEqual(self.pstatmsg_count, 2) + self.assertEqual(self.pstatmsg_count, 5) def test_get_verbose_message(self): """Return a verbose message from verbose log file.""" @@ -452,13 +464,15 @@ class LibtorrentLogParserTest(unittest.TestCase): msg1 = p.get_next_message() msg2 = p.get_next_message() msg3 = p.get_next_message() + msg4 = p.get_next_message() self.add_message_to_count(msg1) self.add_message_to_count(msg2) self.add_message_to_count(msg3) + self.add_message_to_count(msg4) self.assertEqual(self.statmsg_count, 1) - self.assertEqual(self.pstatmsg_count, 1) + self.assertEqual(self.pstatmsg_count, 2) self.assertEqual(self.verbmsg_count, 1) def test_get_mixed_messages_own_file(self): @@ -478,6 +492,9 @@ class LibtorrentLogParserTest(unittest.TestCase): msg4 = p.get_next_message() msg5 = p.get_next_message() msg6 = p.get_next_message() + msg7 = p.get_next_message() + msg8 = p.get_next_message() + msg9 = p.get_next_message() self.add_message_to_count(msg1) self.add_message_to_count(msg2) @@ -485,9 +502,12 @@ class LibtorrentLogParserTest(unittest.TestCase): self.add_message_to_count(msg4) self.add_message_to_count(msg5) self.add_message_to_count(msg6) + self.add_message_to_count(msg7) + self.add_message_to_count(msg8) + self.add_message_to_count(msg9) self.assertEqual(self.statmsg_count, 2) - self.assertEqual(self.pstatmsg_count, 2) + self.assertEqual(self.pstatmsg_count, 5) self.assertEqual(self.verbmsg_count, 2) def test_get_mixed_messages_different_files(self): @@ -507,6 +527,9 @@ class LibtorrentLogParserTest(unittest.TestCase): msg4 = p.get_next_message() msg5 = p.get_next_message() msg6 = p.get_next_message() + msg7 = p.get_next_message() + msg8 = p.get_next_message() + msg9 = p.get_next_message() self.add_message_to_count(msg1) self.add_message_to_count(msg2) @@ -514,9 +537,12 @@ class LibtorrentLogParserTest(unittest.TestCase): self.add_message_to_count(msg4) self.add_message_to_count(msg5) self.add_message_to_count(msg6) + self.add_message_to_count(msg7) + self.add_message_to_count(msg8) + self.add_message_to_count(msg9) self.assertEqual(self.statmsg_count, 2) - self.assertEqual(self.pstatmsg_count, 2) + self.assertEqual(self.pstatmsg_count, 5) self.assertEqual(self.verbmsg_count, 2) @@ -537,5 +563,6 @@ if __name__ == "__main__": # suite.addTest(LibtorrentLogParserTest('test_get_verbose_message')) # suite.addTest(LibtorrentLogParserTest('test_get_verbose_messages')) # suite.addTest(LibtorrentLogParserTest('test_get_verbose_messages_different_files')) +# suite.addTest(LibtorrentLogParserTest('test_get_mixed_messages_same_file')) # unittest.TextTestRunner().run(suite) unittest.main()