From e2ddd50414db44cbad8775e84282ea95042ef710 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Mariana=20M=C4=83r=C4=83=C8=99oiu?= Date: Sun, 28 Aug 2011 22:37:20 +0300 Subject: [PATCH] ppf/new: Add functions and tests for config. Add example files used for tests. --- ppf/new/config.py | 176 ++++++++++++++++++---- ppf/new/tests/access_example.ini | 14 ++ ppf/new/tests/config_example.ini | 102 +++++++++++++ ppf/new/tests/test_config.py | 244 ++++++++++++++++++++++++++++--- 4 files changed, 484 insertions(+), 52 deletions(-) create mode 100644 ppf/new/tests/access_example.ini create mode 100644 ppf/new/tests/config_example.ini diff --git a/ppf/new/config.py b/ppf/new/config.py index ba59068..5c2ee4e 100644 --- a/ppf/new/config.py +++ b/ppf/new/config.py @@ -7,6 +7,10 @@ Configuration class for P2P logging information. import os import os.path import logging +import ConfigParser +from StringIO import StringIO + +import storage # # Logging code heavily inspired by Logging HOWTO documentation: @@ -30,42 +34,154 @@ ch.setFormatter(formatter) # Add console handler to logger. logger.addHandler(ch) - class SwarmDescription(object): def __init__(self): - pass - - def load(self, ini_file): - pass + super(SwarmDescription, self).__init__() + + def load(self, swarm_config_file): + """ Load configuration file and use ConfigParser. + Expect configuration file without first section for swarm information. + + """ + NOSECTION = 'swarm' + try: + text = open(swarm_config_file).read() + except IOError: + logger.error("No such file: %s" %swarm_config_file) + text = ''.join(text.split('\t')) + f = StringIO("[%s]\n" % NOSECTION + text) + self.data = ConfigParser.ConfigParser() + self.data.readfp(f) def store(self, ini_file): - pass - - def add(self, section, option, value): - pass - - def get(self, section): - pass - - def set(self, section, option, value): - pass - + """ Write configuration information to file. """ + f = open(ini_file, 'w') + self.data.write(f) + f.close() + + def get_swarm(self): + """ Return Swarm object containing information from config file. """ + swarm_data = dict(self.data.items('swarm')) + swarm = storage.Swarm(swarm_data['torrent_filename'], + swarm_data['data_size'], + swarm_data['description']) + return swarm + + def get_session_entries(self): + """ Return list of SwarmDescription instances. """ + session_names = self.data.sections() + session_names.remove('swarm') + session_list = [] + for session in session_names: + entry = SwarmDescription() + entry.data = ConfigParser.ConfigParser() + entry.data.add_section(session) + session_items = dict(self.data.items(session)) + for key in session_items: + entry.data.set(session, key, session_items[key]) + session_list.append(entry) + return session_list + + def get_session(self): + """ Return name of the session. + If it contains more than one session, return None. + + """ + # TODO: May need rethinking. It may be better if entry is a separate + # class or instance of SwarmDescription. + session_list = self.data.sections() + if 'swarm' in session_list: + return None + else: + cs_data = dict(self.data.items(session_list[0])) + cs = storage.ClientSession( + btclient = cs_data['bittorrent_client'], + system_os = cs_data['system_os'], + system_os_version = cs_data['system_os_version'], + system_ram = cs_data['system_ram'], + system_cpu = cs_data['system_cpu'], + public_ip = cs_data['public_ip'], + public_port = cs_data['public_port'], + ds_limit = cs_data['ds_limit'], + us_limit = cs_data['us_limit'], + start_time = cs_data['start_time'], + dht_enabled = cs_data['dht_enabled'], + pxe_enabled = cs_data['pxe_enabled'], + streaming_enabled = cs_data['streaming_enabled'], + description = cs_data['description']) + return cs + + def update_session_entry_id(self, session_entry, cs_id): + """ Add or modify client session id. """ + session_name = session_entry.data.sections()[0] + self.data.set(session_name, 'client_session_id', str(cs_id)) + session_entry.data.set(session_name, 'client_session_id', str(cs_id)) + + def get_session_id(self): + """ Return client session id corresponding to the archive file.""" + section = self.data.sections()[0] + cs_id = self.data.get(section, 'client_session_id') + return cs_id + + def get_file_archives(self): + """ Return a list containing all archives from swarm. """ + archives = [] + for section in self.data.sections(): + try: + archives.append(self.data.get(section, 'log_file')) + except ConfigParser.NoOptionError: + pass + return archives class AccessConfig(object): def __init__(self): - pass - - def load(self, ini_file): - pass + super(AccessConfig, self).__init__() + + def load(self, access_config_file): + """ Load configuration file and use ConfigParser. """ + try: + text = open(access_config_file).read() + except IOError: + logger.error("No such file: %s" %swarm_config_file) + text = ''.join(text.split('\t')) + f = StringIO(text) + self.data = ConfigParser.ConfigParser() + self.data.readfp(f) def store(self, ini_file): - pass - - def add(self, section, option, value): - pass - - def get(self, section): - pass - - def set(self, section, option, value): - pass + """ Write configuration information to file. """ + f = open(ini_file, 'w') + self.data.write(f) + f.close() + + def get_swarm_writer(self): + """ Return storage.SwarmWriter instance. + For each section create coresponding storage.*Access instance + and add to SwarmWriter.handlers list. + + """ + sw = storage.SwarmWriter() + for section in self.data.sections(): + if section == 'mysql': + config = dict(self.data.items('mysql')) + + # If host and port options are not defined, use defaults. + if 'host' not in config: + config['host'] = 'localhost' + if 'port' not in config: + config['port'] = 3305 + mysql = storage.MySQLDatabaseAccess(config) + mysql.connect() + sw.add_access_handle(mysql) + + elif section == "sqlite": + database = dict(self.data.items('sqlite'))['database'] + sqlite = storage.SQLiteDatabaseAccess(database) + sqlite.connect() + sw.add_access_handle(sqlite) + + elif section == "treetextfile": + path = dict(self.data.items('treetextfile'))['path'] + treetextfile = storage.TreeTextFileAccess(path) + sw.add_access_handle(treetextfile) + return sw diff --git a/ppf/new/tests/access_example.ini b/ppf/new/tests/access_example.ini new file mode 100644 index 0000000..4b9cffb --- /dev/null +++ b/ppf/new/tests/access_example.ini @@ -0,0 +1,14 @@ +; +; Sample access configuration file for log parsing. +; +[mysql] +; necessary configuration entries +database = p2p_test +user = root +password = p2p4th3m45535 + +; host and port are optional configuration entries +host = localhost + +[sqlite] +database = test.db diff --git a/ppf/new/tests/config_example.ini b/ppf/new/tests/config_example.ini new file mode 100644 index 0000000..5eea384 --- /dev/null +++ b/ppf/new/tests/config_example.ini @@ -0,0 +1,102 @@ +; +; Sample swarm description file for log parsing. +; Swarm consists of 4 peers. +; Top level parameters describe swarm features. +; Second level parameters describe per-peer information +; + + +; .torrent file +torrent_filename = fedora-11-cds.torrent + +; data size in bytes, use 0 for live streaming +data_size = 12345678 + +; swarm description +description = BitTorrent Distribution Experiment, Fedora 11, i386 netinst + +; +; Use an unique identifier for peer section name. +; Possible BitTorrent clients: Tribler, NextShare, libtorrent-rasterbar, Vuze, +; Transmission, Aria, Mainline +; + +[p2p-next-01] + bittorrent_client = libtorrent-rasterbar + system_os = Linux + system_os_version = 2.6.26 + ; system RAM in MB + system_ram = 1999 + ; system CPU in MHz + system_cpu = 2992 + public_ip = 141.85.224.201 + public_port = 6881 + ; Use 0 (KB/s) for no download/upload speed limitation. + ds_limit = 0 + us_limit = 0 + start_time = Thu Aug 11 15:36:35 EEST 2010 + dht_enabled = true + pxe_enabled = false + streaming_enabled = false + description = NCIT Cluster System + log_file = p2p-next-01.grid.pub.ro/btex-fedora-11/dbex-fedora-cds.tar.gz + +[p2p-next-03] + bittorrent_client = libtorrent-rasterbar + system_os = Linux + system_os_version = 2.6.26 + ; system RAM in MB + system_ram = 2007 + ; system CPU in MHz + system_cpu = 2992 + public_ip = 141.85.224.203 + public_port = 6881 + ; Use 0 (KB/s) for no download/upload speed limitation. + ds_limit = 0 + us_limit = 0 + start_time = Thu Aug 11 15:36:35 EEST 2010 + dht_enabled = true + pxe_enabled = false + streaming_enabled = false + description = NCIT Cluster System + log_file = p2p-next-03.grid.pub.ro/btex-fedora-11/dbex-fedora-cds.tar.gz + +[p2p-next-04] + bittorrent_client = libtorrent-rasterbar + system_os = Linux + system_os_version = 2.6.26 + ; system RAM in MB + system_ram = 2007 + ; system CPU in MHz + system_cpu = 2992 + public_ip = 141.85.224.204 + public_port = 6881 + ; Use 0 (KB/s) for no download/upload speed limitation. + ds_limit = 0 + us_limit = 0 + start_time = Thu Aug 11 15:36:35 EEST 2010 + dht_enabled = true + pxe_enabled = false + streaming_enabled = false + description = NCIT Cluster System + log_file = p2p-next-04.grid.pub.ro/btex-fedora-11/dbex-fedora-cds.tar.gz + +[p2p-next-09] + bittorrent_client = libtorrent-rasterbar + system_os = Linux + system_os_version = 2.6.26 + ; system RAM in MB + system_ram = 1999 + ; system CPU in MHz + system_cpu = 2992 + public_ip = 141.85.224.209 + public_port = 6881 + ; Use 0 (KB/s) for no download/upload speed limitation. + ds_limit = 0 + us_limit = 0 + start_time = Thu Aug 11 15:36:35 EEST 2010 + dht_enabled = true + pxe_enabled = false + streaming_enabled = false + description = NCIT Cluster System + log_file = p2p-next-09.grid.pub.ro/btex-fedora-11/dbex-fedora-cds.tar.gz diff --git a/ppf/new/tests/test_config.py b/ppf/new/tests/test_config.py index 089eb3a..66a9b84 100644 --- a/ppf/new/tests/test_config.py +++ b/ppf/new/tests/test_config.py @@ -7,50 +7,250 @@ Test suite for config. Uses unittest module. import unittest import os import os.path -import shutil +import datetime import sys import config +import storage -class SwarmDescriptionTest(unittest.TestCase): +class SessionDescriptionTest(unittest.TestCase): """ - Test suite for SwarmDescription class in config.py. + Test suite for SessionDescription class in config.py. """ + + # Class specific variables. Initialized in setUp, used throughout tests. + config_file = "config_example.ini" + test_data = None + sd = None + def setUp(self): - pass + self.test_data = {'swarm': {'torrent_filename': 'fedora-11-cds.torrent', + 'data_size': '12345678', + 'description': 'BitTorrent Distribution Experiment, Fedora 11, i386 netinst'}, + 'p2p-next-01': {'bittorrent_client': 'libtorrent-rasterbar', + 'system_os': 'Linux', + 'system_os_version': '2.6.26', + 'system_ram': '1999', + 'system_cpu': '2992', + 'public_ip': '141.85.224.201', + 'public_port': '6881', + 'ds_limit': '0', + 'us_limit': '0', + 'start_time': 'Thu Aug 11 15:36:35 EEST 2010', + 'dht_enabled': 'true', + 'pxe_enabled': 'false', + 'streaming_enabled': 'false', + 'description': 'NCIT Cluster System', + 'log_file': 'p2p-next-01.grid.pub.ro/btex-fedora-11/dbex-fedora-cds.tar.gz'}, + 'p2p-next-03': {'bittorrent_client': 'libtorrent-rasterbar', + 'system_os': 'Linux', + 'system_os_version': '2.6.26', + 'system_ram': '2007', + 'system_cpu': '2992', + 'public_ip': '141.85.224.209', + 'public_port': '6881', + 'ds_limit': '0', + 'us_limit': '0', + 'start_time': 'Thu Aug 11 15:36:35 EEST 2010', + 'dht_enabled': 'true', + 'pxe_enabled': 'false', + 'streaming_enabled': 'false', + 'description': 'NCIT Cluster System', + 'log_file': 'p2p-next-03.grid.pub.ro/btex-fedora-11/dbex-fedora-cds.tar.gz'}, + 'p2p-next-04': {'bittorrent_client': 'libtorrent-rasterbar', + 'system_os': 'Linux', + 'system_os_version': '2.6.26', + 'system_ram': '2007', + 'system_cpu': '2992', + 'public_ip': '141.85.224.204', + 'public_port': '6881', + 'ds_limit': '0', + 'us_limit': '0', + 'start_time': 'Thu Aug 11 15:36:35 EEST 2010', + 'dht_enabled': 'true', + 'pxe_enabled': 'false', + 'streaming_enabled': 'false', + 'description': 'NCIT Cluster System', + 'log_file': 'p2p-next-04.grid.pub.ro/btex-fedora-11/dbex-fedora-cds.tar.gz'}, + 'p2p-next-09': {'bittorrent_client': 'libtorrent-rasterbar', + 'system_os': 'Linux', + 'system_os_version': '2.6.26', + 'system_ram': '1999', + 'system_cpu': '2992', + 'public_ip': '141.85.224.209', + 'public_port': '6881', + 'ds_limit': '0', + 'us_limit': '0', + 'start_time': 'Thu Aug 11 15:36:35 EEST 2010', + 'dht_enabled': 'true', + 'pxe_enabled': 'false', + 'streaming_enabled': 'false', + 'description': 'NCIT Cluster System', + 'log_file': 'p2p-next-09.grid.pub.ro/btex-fedora-11/dbex-fedora-cds.tar.gz'}} + + config_file_path = os.path.join(os.path.dirname(__file__), + self.config_file) + self.sd = config.SwarmDescription() + self.sd.load(config_file_path) + + def test_load(self): + # Check for the same sections and items. + # Use sort() method because self.test_data is unordered. + self.assertEqual(self.sd.data.sections().sort(), + list(self.test_data.keys()).sort()) + for section in self.sd.data.sections(): + self.assertEqual(self.sd.data.items(section).sort(), + list(self.test_data[section]).sort()) + + def test_store(self): + self.sd.store("session_config.ini") + sd2 = config.SwarmDescription() + sd2.load("session_config.ini") + + # Check for the same sections and items. + # Use sort() method because self.test_data is unordered. + self.assertEqual(sd2.data.sections().sort(), + list(self.test_data.keys()).sort()) + for section in sd2.data.sections(): + self.assertEqual(sd2.data.items(section).sort(), + list(self.test_data[section]).sort()) + os.remove("session_config.ini") + + def test_get_swarm(self): + swarm = self.sd.get_swarm() + expected_swarm = storage.Swarm( + self.test_data['swarm']['torrent_filename'], + self.test_data['swarm']['data_size'], + self.test_data['swarm']['description']) + + self.assertEqual(swarm.torrent_filename, + expected_swarm.torrent_filename) + self.assertEqual(swarm.data_size, expected_swarm.data_size) + self.assertEqual(swarm.description, expected_swarm.description) + + def test_get_session_entries(self): + session_entries = self.sd.get_session_entries() + expected_session_list = self.test_data.keys() + expected_session_list.remove('swarm') + + for entry in session_entries: + section = entry.data.sections()[0] + self.assertEqual(entry.data.items(section).sort(), + list(self.test_data[section]).sort()) - def tearDown(self): - pass + def test_get_session(self): + session_entries = self.sd.get_session_entries() + entry = session_entries[0] + cs = entry.get_session() - def test_add(self): - self.assertEqual(True, False) + self.assertTrue(type(cs) is storage.ClientSession) + self.assertEqual(cs.system_os, 'Linux') - def test_get(self): - self.assertEqual(True, False) + def test_update_session_entry_id(self): + session_entries = self.sd.get_session_entries() + entry = session_entries[0] + cs_id = '1' + self.sd.update_session_entry_id(entry, cs_id) - def test_other(self): - self.assertEqual(True, False) + self.assertEqual(entry.get_session_id(), cs_id) + + def test_get_file_archives(self): + archives = self.sd.get_file_archives() + expected_archives = [ + 'p2p-next-01.grid.pub.ro/btex-fedora-11/dbex-fedora-cds.tar.gz', + 'p2p-next-03.grid.pub.ro/btex-fedora-11/dbex-fedora-cds.tar.gz', + 'p2p-next-04.grid.pub.ro/btex-fedora-11/dbex-fedora-cds.tar.gz', + 'p2p-next-09.grid.pub.ro/btex-fedora-11/dbex-fedora-cds.tar.gz'] + + self.assertEqual(archives.sort(),expected_archives.sort()) class AccessConfigTest(unittest.TestCase): """ - Test suite for AccessConfig class in config.py. + Test suite for SessionDescription class in config.py. """ + + # Class specific variables. Initialized in setUp, used throughout tests. + access_file = "access_example.ini" + test_data = None + ac = None + def setUp(self): - pass + self.test_data = {'mysql': {'user': 'root', + 'database': 'p2p_test', + 'password': 'p2p4th3m45535', + 'host': 'localhost'}, + 'sqlite': {'database': 'test.db'}} + + access_file_path = os.path.join(os.path.dirname(__file__), + self.access_file) + self.ac = config.AccessConfig() + self.ac.load(access_file_path) + + def test_load(self): + # Check for the same sections. + # Use sort() method because self.test_data is unordered. + self.assertEqual(self.ac.data.sections().sort(), + list(self.test_data.keys()).sort()) + for section in self.ac.data.sections(): + self.assertEqual(self.ac.data.items(section).sort(), + list(self.test_data[section]).sort()) + + def test_store(self): + self.ac.store("access_config.ini") + ac2 = config.AccessConfig() + ac2.load("access_config.ini") + + # Check for the same sections and items. + # Use sort() method because self.test_data is unordered. + self.assertEqual(ac2.data.sections().sort(), + list(self.test_data.keys()).sort()) + for section in ac2.data.sections(): + self.assertEqual(ac2.data.items(section).sort(), + list(self.test_data[section]).sort()) + os.remove("access_config.ini") + + def test_get_swarm_writer(self): + sw = self.ac.get_swarm_writer() + + # Add swarm. + s = storage.Swarm(torrent_filename="fedora.torrent", data_size=102400) + sw.add_swarm(s) - def tearDown(self): - pass + # Add client session. + cs = storage.ClientSession( + swarm_id=1, btclient="Tribler", system_os="Linux", + system_os_version="2.6.26", system_ram=2048, + system_cpu=3000, public_ip="141.85.224.201", + public_port="50500", ds_limit=300, us_limit=200) + sw.add_client_session(cs) - def test_add(self): - self.assertEqual(True, False) + ts = datetime.datetime.strptime("2010-09-12 08:43:15", + "%Y-%m-%d %H:%M:%S") + # Add status message. + sm = storage.StatusMessage( + client_session_id=1, timestamp=ts, num_peers=10, + num_dht_peers=3, download_speed=102, upload_speed=99, + download_size=10213, upload_size=3301) + sw.add_status_message(sm) - def test_get(self): - self.assertEqual(True, False) + # Add peer status message. + psm = storage.PeerStatusMessage( + client_session_id=1, timestamp=ts, + peer_ip="141.85.224.202", peer_port="12345", + download_speed=13, upload_speed=98) + sw.add_peer_status_message(psm) - def test_other(self): - self.assertEqual(True, False) + # Add verbose message. + vm = storage.VerboseMessage( + client_session_id=1, timestamp=ts, + transfer_direction="send", peer_ip="141.85.224.202", + peer_port="12345", message_type="CHOKE") + sw.add_verbose_message(vm) + expected_handlers = 2 + handlers = len(sw.handlers) + self.assertEqual(expected_handlers, handlers) if __name__ == "__main__": unittest.main() -- 2.20.1