From: Razvan Deaconescu Date: Fri, 30 Oct 2009 18:46:15 +0000 (+0200) Subject: working version of DatabaseWriter; added read standard input facility; small fixes... X-Git-Tag: getopt_long~291 X-Git-Url: http://p2p-next.cs.pub.ro/gitweb/?a=commitdiff_plain;h=07744a5bf728976dcd81a0a423efe34466b45d3b;p=cs-p2p-next.git working version of DatabaseWriter; added read standard input facility; small fixes to DatabaseAccess; documented DatabaseWriter, DatabaseAccess and db_init in README --- diff --git a/auto/db/DatabaseAccess.py b/auto/db/DatabaseAccess.py index 3ec8188..81b7a50 100644 --- a/auto/db/DatabaseAccess.py +++ b/auto/db/DatabaseAccess.py @@ -3,6 +3,8 @@ import sys import sqlite3 +DEBUG = False + class DatabaseAccess: """ Low-level class for database access: insert, update, delete, @@ -169,6 +171,8 @@ class DatabaseAccess: print ("[client_sessions]An error ocurred: ", e.args[0]) def insert_status_messages_row(self, row): + if DEBUG == True: + print "[status_messages] insert row", row try: self.cursor.execute("insert into status_messages values (?,?,?,?,?,?,?,?,?)", row) self.conn.commit() @@ -195,12 +199,13 @@ class DatabaseAccess: self.cursor.execute("delete from status_messages") else: self.cursor.execute("delete from status_messages where cs_id=?", (cs_id, )) - for row in self.cursor: - print row + self.conn.commit() except sqlite3.Error, e: print("[status_messages]An error ocurred: ", e.args[0]) def insert_verbose_messages_row(self, row): + if DEBUG == True: + print "[verbose_messages] insert row", row try: self.cursor.execute("insert into verbose_messages values (?,?,?,?,?,?,?,?,?)", row) self.conn.commit() @@ -227,8 +232,7 @@ class DatabaseAccess: self.cursor.execute("delete from verbose_messages") else: self.cursor.execute("delete from verbose_messages where cs_id=?", (cs_id, )) - for row in self.cursor: - print row + self.conn.commit() except sqlite3.Error, e: print("[status_messages]An error ocurred: ", e.args[0]) diff --git a/auto/db/DatabaseWriter.py b/auto/db/DatabaseWriter.py index 14f3af9..162d32e 100644 --- a/auto/db/DatabaseWriter.py +++ b/auto/db/DatabaseWriter.py @@ -4,6 +4,7 @@ import sys import getopt import julian import datetime +import re # regular expression support from DatabaseAccess import DatabaseAccess class DatabaseWriter: @@ -15,11 +16,12 @@ class DatabaseWriter: def add_status_message(self, cs_id, date, time, peer_num, dht, download_speed, upload_speed, download_size, upload_size, eta_time): timestamp = float(julian.datetimeToJulian(date, time)); eta = datetime.timedelta(eta_time[0], eta_time[3], 0, 0, eta_time[2], eta_time[1], 0) - dba.insert_status_message(cs_id, timestamp, peer_num, dht, download_speed, upload_speed, download_size, upload_size, eta): + eta_seconds = eta.days * 24 * 3600 + eta.seconds + self.dba.insert_status_messages(cs_id, timestamp, peer_num, dht, download_speed, upload_speed, download_size, upload_size, eta_seconds) def add_verbose_message(self, cs_id, date, time, peer_ip, peer_port, message_type, index, begin, length, listen_port): timestamp = float(julian.datetimeToJulian(date, time)); - dba.insert_verbose_message(cs_id, timestamp, peer_ip, peer_port, message_type, index, begin, length, listen_port): + self.dba.insert_verbose_messages(cs_id, timestamp, peer_ip, peer_port, message_type, index, begin, length, listen_port) def show_status_messages(self, cs_id = -1): self.dba.select_status_messages(cs_id) @@ -32,3 +34,242 @@ class DatabaseWriter: def delete_verbose_messages(self, cs_id = -1): self.dba.delete_verbose_messages(cs_id) + + +def usage(): + print "Usage: python DatabaseWriter.py action target [-i|--id id] database" + print "action:" + print "\t--add" + print "\t-a\t\tadd entry to database" + print "\t\t\t\t(information is read from standard input)" + print "\t--list" + print "\t-l\t\tlist entry/entries from database" + print "\t--delete" + print "\t-d\t\tdelete entry from database" + print "\t--read" + print "\t-r\t\tread information from standard input" + print "target:" + print "\tstatus\t\tstatus messages" + print "\tverbose\t\tverbose messages" + print "id:" + print "\t--id" + print "\t-i\t\tclient_session_id" + print "\tdatabase\t\tSQLite database file" + print "\t--help" + print "\t-h\t\t\tprint this help screen" + +def read_status_messages(dbw, sep = None): + if sep == None: + sep = ',' + + while 1: + line = sys.stdin.readline().strip() + if line: + message_array = line.split(sep) + + cs_id = int(message_array[0].strip()) + timestamp = message_array[1].strip().split(' ') + date = timestamp[0] + time = timestamp[1] + num_peers = int(message_array[2].strip()) + dht = int(message_array[3].strip()) + download_speed = int(message_array[4].strip()) + upload_speed = int(message_array[5].strip()) + download_size = int(message_array[6].strip()) + upload_size = int(message_array[7].strip()) + eta_string_array = re.split('[dhms]', message_array[8].strip()) + eta_string_array.remove('') + eta = [] + for i in range(0, len(eta_string_array)): + eta.append(int(eta_string_array[i])) + for i in range(len(eta_string_array), 4): + eta.insert(0, 0) + + dbw.add_status_message(cs_id, date, time, num_peers, dht, download_speed, upload_speed, download_size, upload_size, eta) + else: + break + +def read_verbose_messages(dbw, sep = None): + if sep == None: + sep = ',' + + while 1: + line = sys.stdin.readline().strip() + if line: + message_array = line.split(sep) + + cs_id = int(message_array[0].strip()) + timestamp = message_array[1].strip().split(' ') + date = timestamp[0] + time = timestamp[1] + peer_ip = message_array[2].strip() + peer_port = int(message_array[3].strip()) + message_type = int(message_array[4].strip()) + index = int(message_array[5].strip()) + begin = int(message_array[6].strip()) + length = int(message_array[7].strip()) + listen_port = int(message_array[8].strip()) + + dbw.add_verbose_message(cs_id, date, time, peer_ip, peer_port, + message_type, index, begin, length, listen_port) + else: + break + +def main(): + """ + Command line interface for database handling. Allows insertion, + deletion and selection of rows in status_messages and verbose_messages + tables. + If id == -1, all rows in target table are deleted/selected. + Uses getopt for command line parsing. + Last argument must be a database file. + Please check README for details and running examples. + """ + + try: + opts, args = getopt.getopt(sys.argv[1:], "ha:l:d:r:i:s:", ["help", + "add=", "list=", "delete=", "read=", "id=", "separator="]) + except getopt.GetoptError, err: + print str(err) + usage() + sys.exit(2) + + action = None + target = None + id = None + sep = None + + for o, a in opts: + if o in ("-h", "--help"): + usage() + sys.exit(0) + elif o in ("-a", "--add"): + action = "add" + target = a + elif o in ("-d", "--delete"): + action = "delete" + target = a + elif o in ("-l", "--list"): + action = "list" + target = a + elif o in ("-r", "--read"): + action = "read" + target = a + elif o in ("-i", "--id"): + try: + id = int(a) + except TypeError, err: + print str(err) + sys.exit(2) + elif o in ("-s", "--sep"): + sep = a + else: + assert False, "unhandled option" + + # no database file passed as argument + if len(args) != 1: + print "Error: no database file passed as argument." + sys.exit(2) + database = args[0] + + if action == None: + print "Error: no action specified." + sys.exit(2) + + if target != "status" and target != "verbose": + print "Error: invalid target", target, "." + sys.exit(2) + + if id != None and (action == "add" or action == "read"): + print "Error:", action, "action doesn't use an id field." + sys.exit(2) + + if id == None and (action == "delete" or action == "list"): + print "Error: no id for", action, "action." + sys.exit(2) + + if sep != None and action != "read": + print "Error:", action, "action doesn't use a separator argument." + sys.exit(2) + + dbw = DatabaseWriter(database) + + + if target == "status": + if action == "add": + print "client session id: ", + cs_id = int(sys.stdin.readline().strip()) + print "timestamp (YYYY-MM-DD HH:MM:SS.ss): ", + timestamp = sys.stdin.readline().strip().split(' ') + date = timestamp[0] + time = timestamp[1] + print "number of peers: ", + num_peers = int(sys.stdin.readline().strip()) + print "dht: ", + dht = int(sys.stdin.readline().strip()) + print "current download speed (KB/s): ", + download_speed = int(sys.stdin.readline().strip()) + print "current upload speed (KB/s): ", + upload_speed = int(sys.stdin.readline().strip()) + print "download size (bytes): ", + download_size = int(sys.stdin.readline().strip()) + print "upload size (bytes): ", + upload_size = int(sys.stdin.readline().strip()) + print "eta (XdYhZmWs; e.g. 1d12h34m12s): ", + eta_string_array = re.split('[dhms]', sys.stdin.readline().strip()) + eta_string_array.remove('') + eta = [] + for i in range(0, len(eta_string_array)): + eta.append(int(eta_string_array[i])) + for i in range(len(eta_string_array), 4): + eta.insert(0, 0) + + dbw.add_status_message(cs_id, date, time, num_peers, dht, download_speed, upload_speed, download_size, upload_size, eta) + + if action == "delete": + dbw.delete_status_messages(id) + + if action == "list": + dbw.show_status_messages(id) + + if action == "read": + read_status_messages(dbw, sep) + + if target == "verbose": + if action == "add": + print "client session id: ", + cs_id = int(sys.stdin.readline().strip()) + print "timestamp (YYYY-MM-DD HH:MM:SS.ss): ", + timestamp = sys.stdin.readline().strip().split(' ') + date = timestamp[0] + time = timestamp[1] + print "peer IP address (X.Y.Z.T): ", + peer_ip = sys.stdin.readline().strip() + print "peer port: ", + peer_port = int(sys.stdin.readline().strip()) + print "message_type (0, 1, 2, 3, 4, 5, 6): ", + message_type = int(sys.stdin.readline().strip()) + print "index: ", + index = int(sys.stdin.readline().strip()) + print "begin: ", + begin = int(sys.stdin.readline().strip()) + print "length: ", + length = int(sys.stdin.readline().strip()) + print "listen_port: ", + listen_port = int(sys.stdin.readline().strip()) + + dbw.add_verbose_message(cs_id, date, time, peer_ip, peer_port, + message_type, index, begin, length, listen_port) + + if action == "delete": + dbw.delete_verbose_messages(id) + + if action == "list": + dbw.show_verbose_messages(id) + + if action == "read": + read_verbose_messages(dbw, sep) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/auto/db/README b/auto/db/README index 3455ea0..e4ede0d 100644 --- a/auto/db/README +++ b/auto/db/README @@ -1,13 +1,64 @@ +== db_init == + +db_init is a simple shell script that creates a SQLite BitTorrent message +logging database file. It uses the ../sql/p2p-log-sqlite.sql script. The +database tables are empty except for the btclients table. The latter is +initialized to standard BitTorrent clients information in the SQL script. + +The user needs to pass the database file name to the db_init script: + +razvan@valhalla:~/projects/p2p-next/cs-p2p-next/auto/db$ ./db_init p2p-next.db + +== DatabaseAccess.py == + +DatabaseAccess.py is a Python script that implements low-level Python +operations for managing a BitTorrent message logging database (created +through the use of the db_init script). + +DatabaseAccess.py exports methods allowing insertion, selection and deletion +of table entries in the database. This methods are directly used by the +DatabaseCommander.py and DatabaseWriter.py scripts (see below). + +DatabaseAccess.py can be called from the command line (i.e. it implements a +main method). This enables a small test case of the exported functions and +fills the database initial entries. + +--- +razvan@valhalla:~/projects/p2p-next/cs-p2p-next/auto/db$ python DatabaseAccess.py +('[swarms]An error ocurred: ', 'constraint failed') +('[btclients]An error ocurred: ', 'constraint failed') +('[status_messages]An error ocurred: ', 'constraint failed') +('[verbose_messages]An error ocurred: ', 'constraint failed') +(1, u'DarkKnight', 123000, u'experiment', u'TVTorrents') +(2, u'Fedora', 1024, u'experiment', u'local') +(3, u'Pulp Fiction', 102400, u'streaming', u'isohunt') +(1, u'Tribler', u'Python', 1, 1) +(2, u'libtorrent', u'C++', 1, 0) +(3, u'Vuze', u'Java', 1, 1) +(4, u'Transmission', u'C', 1, 0) +(5, u'Aria', u'C', 1, 0) +(6, u'Mainline', u'Python', 1, 0) +(7, u'Tribler', u'Python', 1, 1) +(8, u'libtorrent', u'C++', 1, 0) +(9, u'Vuze', u'Java', 1, 0) +(1, 1, 2, u'Linux', u'2.6.30', 256, 1833, u'0.0.0.0', 6969, 256, 96, 123131.1231) +(2, 3, 4, u'Linux', u'2.6.30', 256, 1833, u'0.0.0.0', 6969, 256, 96, 123131.1231) +(1, 2455128.1000000001, 222, 0, 213, 56, 200, 300, 121.324) +(1, 2455128.1212958111, u'127.0.0.1', 1345, 0, 3, 4, 13, 777) +(1, u'Tribler', u'Python', 1, 1) +(7, u'Tribler', u'Python', 1, 1) +--- + == DatabaseCommander.py == -DatabaseCommander.py is a python script that allows command line acces to and +DatabaseCommander.py is a Python script that allows command line access to and interraction with the BitTorrent information database. It allows adding, deleting and showing entries in the swarms and client_sessions table. The database has to be created using the db_init script and has to be used as a final argument to the command line program. -If swarm/session id -1 all swarms are deleted/shown. +If swarm/session id is -1 all swarms are deleted/shown. If action is "add", standard input is used for reading required arguments. @@ -15,6 +66,7 @@ DatabaseCommander.py calls DatabaseAccess.py. A sample run is shown below: +--- razvan@valhalla:~/projects/p2p-next/cs-p2p-next/auto/db$ python DatabaseCommander.py --add swarm p2p-next.db swarm name (torrent filename without .torrent extension): TestSwarm file size (in bytes): 12345678 @@ -57,3 +109,105 @@ swarm id (swarm identifier in database): 4 razvan@valhalla:~/projects/p2p-next/cs-p2p-next/auto/db$ python DatabaseCommander.py --list session --id -1 p2p-next.db (3, 4, 1, u'Linux', u'2.6.30', 2048, 1600, u'141.85.37.1', 6789, 512, 64, 2455132.8624999998) +--- + +== DatabaseWriter.py == + +DatabaseWriter.py is a python script that enables access to the +status_messages and verbose_messages tables. It offer a programmer interface +and a command line interface for adding, deleting and listing entries in +the two tables. + +The programmer interface exports six methods for adding, deletetin and listing +entries (three methods for each table). + +The command line interface is similar to the one belonging to the +DatabaseCommander. It's main actions are add (-a, --add), list (-l, --list) +and delete (-d, --delete). Entries are selected by their client session id. +This meas there is no possibility (due to the database design) to remove a +specific entry, but entries belonging to a particular client session. + +The database has to be created using the db_init script and has to be used +as a final argument to the command line program. + +One of the more interesting and useful features of DatabaseWriter.py +is the read action (-r, --read) allowing to read entries from standard input. +A separator may be specified through the -s (--sep) option. Comma (,) is +used as the default separator. This allows easy integration with a +non-Python message parser, similar to the below command + ./my_parser log_file | python DatabaseWriter.py -r verbose p2p-next.db + +DatabaseWriter.py uses DatabaseAccess.py + +A sample run of the command line interface for DatabaseWriter.py is +shown below: + +--- +razvan@valhalla:~/projects/p2p-next/cs-p2p-next/auto/db$ python DatabaseWriter.py -l status -i -1 p2p-next.db +(1, 2455128.1000000001, 222, 0, 213, 56, 200, 300, 121.324) +razvan@valhalla:~/projects/p2p-next/cs-p2p-next/auto/db$ python DatabaseWriter.py -l status -i 1 p2p-next.db +(1, 2455128.1000000001, 222, 0, 213, 56, 200, 300, 121.324) + +razvan@valhalla:~/projects/p2p-next/cs-p2p-next/auto/db$ python DatabaseWriter.py -l verbose -i 1 p2p-next.db +(1, 2455128.1212958111, u'127.0.0.1', 1345, 0, 3, 4, 13, 777) +razvan@valhalla:~/projects/p2p-next/cs-p2p-next/auto/db$ python DatabaseWriter.py -l verbose -i -1 p2p-next.db +(1, 2455128.1212958111, u'127.0.0.1', 1345, 0, 3, 4, 13, 777) + +razvan@valhalla:~/projects/p2p-next/cs-p2p-next/auto/db$ python DatabaseWriter.py -a verbose p2p-next.db +client session id: 1 + timestamp (YYYY-MM-DD HH:MM:SS.ss): 2009-10-30 12:34:12 + peer IP address (X.Y.Z.T): 10.1.1.2 + peer port: 34567 + message_type (0, 1, 2, 3, 4, 5, 6): 3 + index: 12 + begin: 45 + length: 1024 + listen_port: 0 + +razvan@valhalla:~/projects/p2p-next/cs-p2p-next/auto/db$ python DatabaseWriter.py -l verbose -i -1 p2p-next.db +(1, 2455128.1212958111, u'127.0.0.1', 1345, 0, 3, 4, 13, 777) +(1, 2455135.0237500002, u'10.1.1.2', 34567, 3, 12, 45, 1024, 0) + +razvan@valhalla:~/projects/p2p-next/cs-p2p-next/auto/db$ python DatabaseWriter.py -a status p2p-next.db +client session id: 1 + timestamp (YYYY-MM-DD HH:MM:SS.ss): 2009-10-30 12:12:12 + number of peers: 34 + dht: 4 + current download speed (KB/s): 456 + current upload speed (KB/s): 23 + download size (bytes): 123456 + upload size (bytes): 3321 + eta (XdYhZmWs; e.g. 1d12h34m12s): 13m12s + +razvan@valhalla:~/projects/p2p-next/cs-p2p-next/auto/db$ python DatabaseWriter.py -l status -i -1 p2p-next.db +(1, 2455128.1000000001, 222, 0, 213, 56, 200, 300, 121.324) +(1, 2455135.0084722224, 34, 4, 456, 23, 123456, 3321, 792) + +razvan@valhalla:~/projects/p2p-next/cs-p2p-next/auto/db$ python DatabaseWriter.py -d status -i 1 p2p-next.db +razvan@valhalla:~/projects/p2p-next/cs-p2p-next/auto/db$ python DatabaseWriter.py -l status -i -1 p2p-next.db + +razvan@valhalla:~/projects/p2p-next/cs-p2p-next/auto/db$ python DatabaseWriter.py -d verbose -i 1 p2p-next.db +razvan@valhalla:~/projects/p2p-next/cs-p2p-next/auto/db$ python DatabaseWriter.py -l verbose -i -1 p2p-next.db + +razvan@valhalla:~/projects/p2p-next/cs-p2p-next/auto/db$ cat status_messages.sample.txt | python DatabaseWriter.py -r status p2p-next.db +razvan@valhalla:~/projects/p2p-next/cs-p2p-next/auto/db$ python DatabaseWriter.py -l status -i -1 p2p-next.db +(1, 2455135.342037037, 23, 1, 324, 43, 121323, 12132, 852) +(1, 2455135.342048611, 23, 1, 324, 43, 122323, 12232, 852) +(1, 2455135.342060185, 23, 1, 323, 43, 123323, 12332, 852) +(1, 2455135.342071759, 24, 2, 322, 44, 124323, 12432, 852) +(1, 2455135.3420833335, 24, 3, 320, 45, 125323, 12532, 852) +(1, 2455135.3420949075, 25, 3, 319, 49, 126323, 12632, 852) +(1, 2455135.3421064815, 26, 2, 306, 48, 127323, 12732, 852) +(1, 2455135.3421180556, 25, 2, 301, 48, 128323, 12932, 852) + +razvan@valhalla:~/projects/p2p-next/cs-p2p-next/auto/db$ cat verbose_messages.sample.txt | python DatabaseWriter.py -r verbose p2p-next.db +razvan@valhalla:~/projects/p2p-next/cs-p2p-next/auto/db$ python DatabaseWriter.py -l verbose -i -1 p2p-next.db +(1, 2455135.0226182872, u'10.0.1.2', 5678, 1, 12, 16, 1024, 0) +(1, 2455135.0226196758, u'10.0.1.2', 5678, 2, 12, 16, 1024, 0) +(1, 2455135.0226287036, u'10.0.1.2', 5678, 3, 12, 16, 1024, 0) +(1, 2455135.0226305556, u'10.0.1.2', 5678, 2, 12, 16, 1024, 0) +(1, 2455135.0226306715, u'10.0.1.2', 5678, 4, 12, 16, 1024, 0) +(1, 2455135.0226402776, u'10.0.1.2', 5678, 5, 12, 16, 1024, 0) +(1, 2455135.0226541664, u'10.0.1.2', 5678, 3, 12, 16, 1024, 0) +(1, 2455135.0226603011, u'10.0.1.2', 5678, 2, 12, 16, 1024, 0) +---