From: Razvan Deaconescu Date: Sun, 19 Sep 2010 08:28:53 +0000 (+0300) Subject: instrumentation: add next-share/ X-Git-Url: http://p2p-next.cs.pub.ro/gitweb/?a=commitdiff_plain;h=e89f96796a5ecf5c5328d39f0f859b137a33c7a1;p=cs-p2p-next.git instrumentation: add next-share/ Next-Share M32 release (also currently - 19.09.2010 - in trunk/) --- diff --git a/instrumentation/next-share/BaseLib/Category/Category.py b/instrumentation/next-share/BaseLib/Category/Category.py new file mode 100644 index 0000000..aabdcb2 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Category/Category.py @@ -0,0 +1,383 @@ +# written by Yuan Yuan, Jelle Roozenburg +# see LICENSE.txt for license information + +import os, re +from BaseLib.Category.init_category import getCategoryInfo +from FamilyFilter import XXXFilter +from traceback import print_exc + +import sys + +from BaseLib.__init__ import LIBRARYNAME + +DEBUG=False +category_file = "category.conf" + + +class Category: + + # Code to make this a singleton + __single = None + __size_change = 1024 * 1024 + + def __init__(self, install_dir='.'): + + if Category.__single: + raise RuntimeError, "Category is singleton" + filename = os.path.join(install_dir,LIBRARYNAME, 'Category', category_file) + Category.__single = self + self.utility = None + #self.torrent_db = TorrentDBHandler.getInstance() # Arno, 2009-01-30: apparently unused + try: + self.category_info = getCategoryInfo(filename) + self.category_info.sort(rankcmp) + except: + self.category_info = [] + if DEBUG: + print_exc() + + self.xxx_filter = XXXFilter(install_dir) + + + if DEBUG: + print >>sys.stderr,"category: Categories defined by user",self.getCategoryNames() + + + # return Category instance + def getInstance(*args, **kw): + if Category.__single is None: + Category(*args, **kw) + return Category.__single + getInstance = staticmethod(getInstance) + + def register(self,metadata_handler): + self.metadata_handler = metadata_handler + + def init_from_main(self, utility): + self.utility = utility + self.set_family_filter(None) # init family filter to saved state + + +# # check to see whether need to resort torrent file +# # return bool +# def checkResort(self, data_manager): +# data = data_manager.data +# #=============================================================================== +# # if not data: +# # data = data_manager.torrent_db.getRecommendedTorrents(all = True) +# #=============================================================================== +# if not data: +# return False + +# # data = data_manager.torrent_db.getRecommendedTorrents(all = True) +# # self.reSortAll(data) +# # return True +# torrent = data[0] +# if torrent["category"] == ["?"]: +# #data = data_manager.torrent_db.getRecommendedTorrents(all = True) +# self.reSortAll(data) +# # del data +# return True + +# begin = time() +# for item in data: +# if len(item['category']) > 1: +# #data = data_manager.torrent_db.getRecommendedTorrents(all = True) +# self.reSortAll(data) +# # del data +# return True +# if DEBUG: +# print >>sys.stderr,'torrcoll: Checking of %d torrents costs: %f s' % (len(data), time() - begin) +# return False + +# # recalculate category of all torrents, remove torrents from db if not existed +# def reSortAll(self, data, parent = None): + +# max = len(data) +# if max == 0: +# return +# import wx +# dlgHolder = [] +# event = Event() +# def makeDialog(): +# dlg = wx.ProgressDialog("Upgrading Database", +# "Upgrading Old Database to New Database", +# maximum = max, +# parent = None, +# style = wx.PD_AUTO_HIDE +# | wx.PD_ELAPSED_TIME +# | wx.PD_REMAINING_TIME +# ) +# dlgHolder.append(dlg) +# event.set() + + +# wx.CallAfter(makeDialog) + +# # Wait for dialog to be ready +# event.wait() +# dlg = dlgHolder[0] + +# count = 0 +# step = int(float(max) / 20) + 1 + +# # sort each torrent file +# for i in xrange(len(data)): +# count += 1 +# if count % step == 0: +# wx.CallAfter(dlg.Update, [count]) +# try: +# # try alternative dir if bsddb doesnt match with current Tribler install +# rec = data[i] +# (torrent_dir,torrent_name) = self.metadata_handler.get_std_torrent_dir_name(rec) + +# # read the torrent file +# filesrc = os.path.join(torrent_dir,torrent_name) + +# # print filesrc +# f = open(filesrc, "rb") +# torrentdata = f.read() # torrent decoded string +# f.close() +# except IOError: # torrent file not found +# # delete the info from db +# self.torrent_db.deleteTorrent(data[i]['infohash']) +# continue + +# # decode the data +# torrent_dict = bencode.bdecode(torrentdata) +# content_name = dunno2unicode(torrent_dict["info"].get('name', '?')) + +# category_belong = [] +# category_belong = self.calculateCategory(torrent_dict, content_name) + +# if (category_belong == []): +# category_belong = ['other'] + +# data[i]['category'] = category_belong # should have updated self.data +# self.torrent_db.updateTorrent(data[i]['infohash'], updateFlag=False, category=category_belong) +# self.torrent_db.sync() +# wx.CallAfter(dlg.Destroy) + + def getCategoryKeys(self): + if self.category_info is None: + return [] + keys = [] + keys.append("All") + keys.append("other") + for category in self.category_info: + keys.append(category['name']) + keys.sort() + return keys + + def getCategoryNames(self): + if self.category_info is None: + return [] + keys = [] + for category in self.category_info: + rank = category['rank'] + if rank == -1: + break + keys.append((category['name'],category['displayname'])) + return keys + + def hasActiveCategory(self, torrent): + try: + name = torrent['category'][0] + except: + print >> sys.stderr, 'Torrent: %s has no valid category' % `torrent['content_name']` + return False + for category in [{'name':'other', 'rank':1}]+self.category_info: + rank = category['rank'] + if rank == -1: + break + if name.lower() == category['name'].lower(): + return True + #print >> sys.stderr, 'Category: %s was not in %s' % (name.lower(), [a['name'].lower() for a in self.category_info if a['rank'] != -1]) + return False + + def getCategoryRank(self,cat): + for category in self.category_info: + if category['name'] == cat: + return category['rank'] + return None + + # calculate the category for a given torrent_dict of a torrent file + # return list + def calculateCategory(self, torrent_dict, display_name): + # torrent_dict is the dict of + # a torrent file + # return value: list of category the torrent belongs to + torrent_category = None + + files_list = [] + try: + # the multi-files mode + for ifiles in torrent_dict['info']["files"]: + files_list.append((ifiles['path'][-1], ifiles['length'] / float(self.__size_change))) + except KeyError: + # single mode + files_list.append((torrent_dict['info']["name"],torrent_dict['info']['length'] / float(self.__size_change))) + + # Check xxx + try: + tracker = torrent_dict.get('announce') + if not tracker: + tracker = torrent_dict.get('announce-list',[['']])[0][0] + if self.xxx_filter.isXXXTorrent(files_list, display_name, torrent_dict.get('announce'), torrent_dict.get('comment')): + return ['xxx'] + except: + print >> sys.stderr, 'Category: Exception in explicit terms filter in torrent: %s' % torrent_dict + print_exc() + + # filename_list ready + strongest_cat = 0.0 + for category in self.category_info: # for each category + (decision, strength) = self.judge(category, files_list, display_name) + if decision and (strength > strongest_cat): + torrent_category = [category['name']] + strongest_cat = strength + + if torrent_category == None: + torrent_category = ['other'] + + return torrent_category + + # judge whether a torrent file belongs to a certain category + # return bool + def judge(self, category, files_list, display_name = ''): + + # judge file keywords + display_name = display_name.lower() + factor = 1.0 + fileKeywords = self._getWords(display_name) + + for ikeywords in category['keywords'].keys(): + try: + fileKeywords.index(ikeywords) + factor *= 1 - category['keywords'][ikeywords] + except: + pass + if (1 - factor) > 0.5: + if 'strength' in category: + return (True, category['strength']) + else: + return (True, (1- factor)) + + # judge each file + matchSize = 0 + totalSize = 1e-19 + for name, length in files_list: + totalSize += length + # judge file size + if ( length < category['minfilesize'] ) or \ + (category['maxfilesize'] > 0 and length > category['maxfilesize'] ): + continue + + # judge file suffix + OK = False + for isuffix in category['suffix']: + if name.lower().endswith( isuffix ): + OK = True + break + if OK: + matchSize += length + continue + + # judge file keywords + factor = 1.0 + fileKeywords = self._getWords(name.lower()) + + for ikeywords in category['keywords'].keys(): +# pass + try: + fileKeywords.index(ikeywords) + #print ikeywords + factor *= 1 - category['keywords'][ikeywords] + except: + pass + if factor < 0.5: + # print filename_list[index] + '#######################' + matchSize += length + + # match file + if (matchSize / totalSize) >= category['matchpercentage']: + if 'strength' in category: + return (True, category['strength']) + else: + return (True, (matchSize/ totalSize)) + + return (False, 0) + + + WORDS_REGEXP = re.compile('[a-zA-Z0-9]+') + def _getWords(self, string): + return self.WORDS_REGEXP.findall(string) + + + def family_filter_enabled(self): + """ + Return is xxx filtering is enabled in this client + """ + if self.utility is None: + return False + state = self.utility.config.Read('family_filter') + if state in ('1', '0'): + return state == '1' + else: + self.utility.config.Write('family_filter', '1') + self.utility.config.Flush() + return True + + def set_family_filter(self, b=None): + assert b in (True, False, None) + old = self.family_filter_enabled() + if b != old or b is None: # update category data if initial call, or if state changes + if b is None: + b=old + if self.utility is None: + return + #print >> sys.stderr , b + if b: + self.utility.config.Write('family_filter', '1') + else: + self.utility.config.Write('family_filter', '0') + self.utility.config.Flush() + # change category data + for category in self.category_info: + if category['name'] == 'xxx': + if b: + category['old-rank'] = category['rank'] + category['rank'] = -1 + elif category['rank'] == -1: + category['rank'] = category['old-rank'] + break + + + def get_family_filter_sql(self, _getCategoryID, table_name=''): + if self.family_filter_enabled(): + forbiddencats = [cat['name'] for cat in self.category_info if cat['rank'] == -1] + if table_name: + table_name+='.' + if forbiddencats: + return " and %scategory_id not in (%s)" % (table_name, ','.join([str(_getCategoryID([cat])) for cat in forbiddencats])) + return '' + + + + +def rankcmp(a,b): + if not ('rank' in a): + return 1 + elif not ('rank' in b): + return -1 + elif a['rank'] == -1: + return 1 + elif b['rank'] == -1: + return -1 + elif a['rank'] == b['rank']: + return 0 + elif a['rank'] < b['rank']: + return -1 + else: + return 1 + diff --git a/instrumentation/next-share/BaseLib/Category/FamilyFilter.py b/instrumentation/next-share/BaseLib/Category/FamilyFilter.py new file mode 100644 index 0000000..3bd79b1 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Category/FamilyFilter.py @@ -0,0 +1,111 @@ +# Written by Jelle Roozenburg +# see LICENSE.txt for license information + +import re, sys, os +from traceback import print_exc + +from BaseLib.__init__ import LIBRARYNAME + +WORDS_REGEXP = re.compile('[a-zA-Z0-9]+') +DEBUG = False + +class XXXFilter: + def __init__(self, install_dir): + termfilename = os.path.join(install_dir, LIBRARYNAME, 'Category','filter_terms.filter') + self.xxx_terms, self.xxx_searchterms = self.initTerms(termfilename) + + def initTerms(self, filename): + terms = set() + searchterms = set() + + try: + f = file(filename, 'r') + lines = f.read().lower().splitlines() + + for line in lines: + if line.startswith('*'): + searchterms.add(line[1:]) + else: + terms.add(line) + f.close() + except: + if DEBUG: + print_exc() + + if DEBUG: + print 'Read %d XXX terms from file %s' % (len(terms)+len(searchterms), filename) + return terms, searchterms + + def _getWords(self, string): + return [a.lower() for a in WORDS_REGEXP.findall(string)] + + + def isXXXTorrent(self, files_list, torrent_name, tracker, comment=None): + if tracker: + tracker = tracker.lower().replace('http://', '').replace('announce','') + else: + tracker = '' + terms = [a[0].lower() for a in files_list] + is_xxx = (len(filter(self.isXXX, terms)) > 0 or + self.isXXX(torrent_name, False) or + self.isXXX(tracker, False) or + (comment and self.isXXX(comment, False)) + ) + if DEBUG: + if is_xxx: + print 'Torrent is XXX: %s %s' % (torrent_name, tracker) + else: + print 'Torrent is NOT XXX: %s %s' % (torrent_name, tracker) + return is_xxx + + + def isXXX(self, s, isFilename=True): + s = s.lower() + if self.isXXXTerm(s): # We have also put some full titles in the filter file + return True + if not self.isAudio(s) and self.foundXXXTerm(s): + return True + words = self._getWords(s) + words2 = [' '.join(words[i:i+2]) for i in xrange(0, len(words)-1)] + num_xxx = len([w for w in words+words2 if self.isXXXTerm(w, s)]) + if isFilename and self.isAudio(s): + return num_xxx > 2 # almost never classify mp3 as porn + else: + return num_xxx > 0 + + def foundXXXTerm(self, s): + for term in self.xxx_searchterms: + if term in s: + if DEBUG: + print 'XXXFilter: Found term "%s" in %s' % (term, s) + return True + return False + + def isXXXTerm(self, s, title=None): + # check if term-(e)s is in xxx-terms + s = s.lower() + if s in self.xxx_terms: + if DEBUG: + print 'XXXFilter: "%s" is dirty%s' % (s, title and ' in %s' % title or '') + return True + if s.endswith('es'): + if s[:-2] in self.xxx_terms: + if DEBUG: + print 'XXXFilter: "%s" is dirty%s' % (s[:-2], title and ' in %s' % title or '') + return True + elif s.endswith('s') or s.endswith('n'): + if s[:-1] in self.xxx_terms: + if DEBUG: + print 'XXXFilter: "%s" is dirty%s' % (s[:-1], title and ' in %s' % title or '') + return True + + return False + + audio_extensions = ['cda', 'flac', 'm3u', 'mp2', 'mp3', 'md5', 'vorbis', 'wav', 'wma', 'ogg'] + def isAudio(self, s): + return s[s.rfind('.')+1:] in self.audio_extensions + + + + + diff --git a/instrumentation/next-share/BaseLib/Category/TestCategory.py b/instrumentation/next-share/BaseLib/Category/TestCategory.py new file mode 100644 index 0000000..c923e96 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Category/TestCategory.py @@ -0,0 +1,148 @@ +# Written by Yuan Yuan +# see LICENSE.txt for license information + +import sys, os +execpath = os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])), '..', '..') +sys.path.append(execpath) +#print sys.path +from Utility.utility import getMetainfo +from BaseLib.Category.Category import Category + +DEBUG = False + +def testFilter(catfilename, torrentpath): + readCategorisationFile(catfilename) + #print 'Install_dir is %s' % execpath + c = Category.getInstance(execpath, None) + total = porn = fn = fp = 0 + for tfilename,isporn in tdict.items(): + torrent = getMetainfo(os.path.join(torrentpath,tfilename)) + name = torrent['info']['name'] + cat = c.calculateCategory(torrent, name) + fporn = (cat == ['xxx']) + total+= 1 + porn += int(isporn) + if isporn == fporn: + if DEBUG: + print (isporn, fporn), 'good', name + + elif isporn and not fporn: + fn+=1 + print 'FALSE NEGATIVE' + showTorrent(os.path.join(torrentpath,tfilename)) + elif not isporn and fporn: + fp +=1 + print 'FALSE POSITIVE' + showTorrent(os.path.join(torrentpath,tfilename)) + + print """ + Total torrents: %(total)d + XXX torrents: %(porn)d + Correct filtered: %(good)d + False negatives: %(fn)d + False positives: %(fp)d + """ % {'total':total, 'porn':porn, 'fn':fn,'fp':fp,'good':total-fn-fp} + +def readCategorisationFile(filename): + global tdict + tdict = {} + try: + f = file(filename, 'r') + lines = f.read().splitlines() + for line in lines: + if line: + parts = line.split('\t') + tdict[parts[0]] = bool(int(parts[1])) + f.close() + except IOError: + print 'No file %s found, starting with empty file' % filename + +def getTorrentData(path, max_num=-1): + torrents= [] + i = 0 + for fname in os.listdir(path): + if fname.endswith('.torrent'): + torrents.append(os.path.join(path,fname)) + if i%1000 == 0 and i: + print 'Loaded: %d torrents' % i + if i == int(max_num): + break + i+=1 + print 'Loaded %d torrents' % len(torrents) + return torrents + +def showTorrent(path): + torrent = getMetainfo(os.path.join(path)) + name = torrent['info']['name'] + print '------------------------------' + print '\tfiles :' + files_list = [] + __size_change = 1024 + try: + # the multi-files mode + for ifiles in torrent['info']["files"]: + files_list.append((ifiles['path'][-1], ifiles['length'] / float(__size_change))) + except KeyError: + # single mode + files_list.append((torrent['info']["name"],torrent['info']['length'] / float(__size_change))) + for fname, fsize in files_list: + print'\t\t%s\t%d kb' % (fname, fsize) + print 'Torrent name: %s' % name + print '\ttracker:%s' % torrent['announce'] + print '------------------------------' + +def createTorrentDataSet(filename, torrentpath): + initSaveFile(filename) + f_out = file(filename, 'a') + torrents = getTorrentData(torrentpath) + for torrent in torrents: + if os.path.split(torrent)[-1] in tset: # already done + continue + showTorrent(torrent) + ans = None + while ans not in ['q', 'y','n']: + print 'Is this torrent porn? (y/n/q)' + ans = sys.stdin.readline()[:-1].lower() + if ans == 'q': + break + else: + saveTorrent(f_out, torrent, (ans=='y')) + f_out.close() + +def saveTorrent(f_out, torrent, boolean): + if torrent in tset: + return + tfilename = os.path.split(torrent)[-1] + assert tfilename + f_out.write('%s\t%d\n' % (tfilename, int(boolean))) + f_out.flush() + tset.add(torrent) + +def initSaveFile(filename): + global tset + tset = set() + try: + f = file(filename, 'r') + lines = f.read().splitlines() + for line in lines: + tset.add(line.split('\t')[0]) + f.close() + except IOError: + print 'No file %s found, starting with empty file' % filename + + + +def main(args): + if len(args) != 4 or args[1] not in ['categorise', 'test']: + print 'Usage 1: %s categorise [torrent-dir] [torrent-data-file]' % args[0] + print 'Usage 2: %s test [torrent-dir] [torrent-data-file]' % args[0] + sys.exit(1) + if args[1] == 'categorise': + createTorrentDataSet(args[3], args[2]) + elif args[1] == 'test': + testFilter(args[3], args[2]) + print 'ready' + + +if __name__ == '__main__': + main(sys.argv) diff --git a/instrumentation/next-share/BaseLib/Category/__init__.py b/instrumentation/next-share/BaseLib/Category/__init__.py new file mode 100644 index 0000000..316dcff --- /dev/null +++ b/instrumentation/next-share/BaseLib/Category/__init__.py @@ -0,0 +1,2 @@ +# Written by Yuan Yuan +# see LICENSE.txt for license information diff --git a/instrumentation/next-share/BaseLib/Category/category.conf b/instrumentation/next-share/BaseLib/Category/category.conf new file mode 100644 index 0000000..4f9000f --- /dev/null +++ b/instrumentation/next-share/BaseLib/Category/category.conf @@ -0,0 +1,62 @@ +[xxx] +rank = 10 +displayname = XXX +matchpercentage = 0.001 +strength = 1.1 +# Keywords are in seperate file: filter_content.filter + + +[Video] +rank = 1 +displayname = Video Files +suffix = asf, asp, avi, flc, fli, flic, mkv, mov, movie, mpeg, mpg, qicktime, ram, rm, rmvb, rpm, vob, wma, wmv +minfilesize = 50 +maxfilesize = 10000000 +matchpercentage = 0.5 + +*divx = 1 +*xvid = 1 +*rmvb = 1 + +[VideoClips] +rank = 2 +displayname = Video Clips +suffix = asf, asp, avi, flc, fli, flic, mkv, mov, movie, mpeg, mpg, qicktime, ram, rm, rmvb, rpm, vob, wma, wmv, mp4, flv +minfilesize = 0 +maxfilesize = 50 +matchpercentage = 0.5 + +[Audio] +rank = 3 +displayname = Audio +suffix = cda, flac, m3u, mp2, mp3, vorbis, wav, wma, ogg, ape +matchpercentage = 0.8 + +[Document] +rank = 5 +displayname = Documents +suffix = doc, pdf, ppt, ps, tex, txt, vsd +matchpercentage = 0.8 + +[Compressed] +rank = 4 +displayname = Compressed +suffix = ace, bin, bwt, cab, ccd, cdi, cue, gzip, iso, jar, mdf, mds, nrg, rar, tar, vcd, z, zip +matchpercentage = 0.8 + +*.r0 = 1 +*.r1 = 1 +*.r2 = 1 +*.r3 = 1 +*.r4 = 1 +*.r5 = 1 +*.r6 = 1 +*.r7 = 1 +*.r8 = 1 +*.r9 = 1 + +[Picture] +rank = 6 +displayname = Pictures +suffix = bmp, dib, dwg, gif, ico, jpeg, jpg, pic, png, swf, tif, tiff +matchpercentage = 0.8 diff --git a/instrumentation/next-share/BaseLib/Category/init_category.py b/instrumentation/next-share/BaseLib/Category/init_category.py new file mode 100644 index 0000000..1d8edfb --- /dev/null +++ b/instrumentation/next-share/BaseLib/Category/init_category.py @@ -0,0 +1,57 @@ +# Written by Yuan Yuan +# see LICENSE.txt for license information + +# give the initial category information + +import ConfigParser + +def splitList(string): + l = [] + for word in string.split(","): + word = word.strip() + l.append(word) + return l + +init_fun = {} +init_fun["minfilenumber"] = int +init_fun["maxfilenumber"] = int +init_fun["minfilesize"] = int +init_fun["maxfilesize"] = int +init_fun["suffix"] = splitList +init_fun["matchpercentage"] = float +init_fun["keywords"] = float +init_fun["strength"] = float +init_fun["displayname"] = str +init_fun["rank"] = int + +def getDefault(): + category = {} + category["name"] = "" + category["keywords"] ={} + category["suffix"] = [] + category["minfilesize"] = 0 + category["maxfilesize"] = -1 + return category + +def getCategoryInfo(filename): + config = ConfigParser.ConfigParser() + config.readfp(open(filename)) + + cate_list = [] + sections = config.sections() + + for isection in sections: + category = getDefault() + category["name"] = isection + for (name, value) in config.items(isection): + if name[0] != "*": + category[name] = init_fun[name](value) + else: + name = name[1:] + name = name.strip() + category["keywords"][name] = init_fun["keywords"](value) + cate_list.append(category) + +# print cate_list + return cate_list + diff --git a/instrumentation/next-share/BaseLib/Core/API.py b/instrumentation/next-share/BaseLib/Core/API.py new file mode 100644 index 0000000..63368aa --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/API.py @@ -0,0 +1,160 @@ +# Written by Arno Bakker +# see LICENSE.txt for license information +# +# To use the Tribler Core just do: +# from BaseLib.Core.API import * +# +""" Tribler Core API v1.0.7, Aug 2010. Import this to use the API """ + +# History: +# +# 1.0.7 Released with Next-Share M32 +# +# 1.0.7rc2 Added: get_peer_id() to Download, returning the BT peer ID when +# the download is not stopped. +# +# 1.0.7rc1 Added: set_proxy_mode/get_proxy_mode to DownloadConfig and +# DownloadRuntimeConfig, set_proxyservice_status/ +# get_proxyservice_status to SessionConfig and SessionRuntimeConfig +# +# 1.0.6 Released with Next-Share M30 +# +# 1.0.6rc4 Added: TorrentDef.set_initial_peers() for magnet: links support. +# +# Added: Session.get_subtitles_support_facade() to get access to +# the subtitle-gossiping subsystem. +# +# 1.0.6rc3 Added: TorrentDef.set_live_ogg_headers() for live streams +# in Ogg container format. +# +# 1.0.5 Released with Next-Share M24.2 +# +# 1.0.5rc4 Added: TorrentDef.set_metadata() for including Next-Share Core +# metadata in .tstream files. +# +# 1.0.5rc3 Added: restartstatefilename to DownloadConfig.set_video_source() +# +# 1.0.5rc2 Added: TorrentDef.set_live_ogg_headers() for live streams +# in Ogg container format. +# +# 1.0.5rc1 Session.query_connected_peers() returns all names as Unicode +# strings. +# +# 1.0.4 Released with Next-Share M24 +# +# 1.0.4rc7 Added: DLMODE_SVC for Scalable Video Coding support following +# P2P-Next WP6's design. +# +# 1.0.4rc6 Added: SIMPLE+METADATA query. +# +# 1.0.4rc5 Added: DLSTATUS_REPEXING. +# +# Added: initialdlstatus parameter to Session.start_download() to +# restart a Download in a particular state, in particular, +# DLSTATUS_REPEXING. +# +# Added: initialdlstatus parameter to Download.restart() to +# restart a Download in a particular state. +# +# Added: get_swarmcache() to DownloadState. +# +# 1.0.4rc4 Added: get_total_transferred() to the DownloadState to +# retrieve the total amount of bytes that are up or +# downloaded for a single Download. +# +# Removed: get_peerid() and get_videoinfo() from DownloadState, +# the former is not a state parameter and the latter exposed internal +# state. +# +# 1.0.4rc3 Added "CHANNEL" queries to query_connected_peers() to Session +# class for making queries for the new channel concept. +# +# Removed: ModerationCast configuration parameters from SessionConfig. +# Added: ChannelCast configuration parameters to SessionConfig. +# Added: VoteCast configuration parameters to SessionConfig. +# +# 1.0.4rc2 TorrentDef now supports P2P URLs. +# +# 1.0.4rc1 Added: torrent_size (size of the .torrent file) to the remote +# torrent search response, see Session.query_connected_peers(). +# +# Timeline disruption: API v1.0.3 was released with Next-Share M16 on April 30. +# 1.0.2rc6 was released with Tribler 5.1.0. Reintroduced as 1.0.4rc1 +# +# 1.0.3 Released with Next-Share M16 +# +# 1.0.3rc1 Added: [s/g]et_multicast_local_peer_discovery() to Session API. +# Added: [s/g]et_moderation_cast_promote_own() to aggressively +# promote your own moderations (to be run by a moderator) +# Removed: [s/g]et_rss_*(). These were not Core/Session parameters. +# Removed: [s/g]et_moderationcast_upload/download_bandwidth_limit(), +# no longer used. +# +# 1.0.2 Released with Tribler 5.0.0 Preview1 +# +# 1.0.2rc5 Added: [s/g]et_moderationcast_*() to configure ModerationCast. +# +# 1.0.2rc4 Added: Session.get_externally_reachable() which tells whether the +# listen port is reachable from the Internet. +# +# 1.0.2rc3 Added: Session.has_shutdown() which tells whether it is already +# safe to quit the process the Session was running in. +# +# 1.0.2rc2 Removed: [s/g]et_puncturing_coordinators in SessionConfig. +# Bugfix: [s/g]et_puncturing_private_port in SessionConfig renamed to +# [s/g]et_puncturing_internal_port. +# +# 1.0.2rc1 Added: set_same_nat_try_internal(). If set Tribler will +# check if other Tribler peers it meets in a swarm are behind the +# same NAT and if so, replace the connection with an connection over +# the internal network. Also added set_unchoke_bias_for_internal() +# +# 1.0.1 Released with Tribler 4.5.0 +# +# 1.0.1rc4 Added: friendship extension to Session API. +# Added: 'gracetime' parameter to Session shutdown. +# +# 1.0.1rc3 Bugfix: [s/g]et_internaltracker in SessionRuntimeConfig renamed to +# [s/g]et_internal_tracker. +# +# Added/bugfix: [s/g]et_mainline_dht in SessionConfigInterface to +# control whether mainline DHT support is activated. +# +# 1.0.1rc2 Added: set_seeding_policy() to Download class to dynamically set +# different seeding policies. +# +# Added: Methods to SessionConfigInterface for Network Address +# Translator detection, see also Session.get_nat_type() +# +# 1.0.1rc1 Bugfix: The query passed to the callback function for +# query_connected_peers() is now the original query, rather than +# the query with "SIMPLE " stripped off. +# +# 1.0.0 Released with SwarmPlayer 1.0 +# +# 1.0.0rc5 Added option to define auxiliary seeding servers for live stream +# (=these servers are always unchoked at the source server). +# +# 1.0.0rc4 Changed DownloadConfig.set_vod_start_callback() to a generic +# event-driven interface. + + +from BaseLib.Core.simpledefs import * +from BaseLib.Core.Base import * +from BaseLib.Core.Session import * +from BaseLib.Core.SessionConfig import * +from BaseLib.Core.Download import * +from BaseLib.Core.DownloadConfig import * +from BaseLib.Core.DownloadState import * +from BaseLib.Core.exceptions import * +try: + from BaseLib.Core.RequestPolicy import * +except ImportError: + pass +from BaseLib.Core.TorrentDef import * +try: + import M2Crypto + from BaseLib.Core.LiveSourceAuthConfig import * +except ImportError: + pass + diff --git a/instrumentation/next-share/BaseLib/Core/APIImplementation/DownloadImpl.py b/instrumentation/next-share/BaseLib/Core/APIImplementation/DownloadImpl.py new file mode 100644 index 0000000..baa94a7 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/APIImplementation/DownloadImpl.py @@ -0,0 +1,649 @@ +# Written by Arno Bakker +# see LICENSE.txt for license information + +import sys +import os +import copy +from traceback import print_exc,print_stack +from threading import RLock,Condition,Event,Thread,currentThread + +from BaseLib.Core.DownloadState import DownloadState +from BaseLib.Core.DownloadConfig import DownloadStartupConfig +from BaseLib.Core.simpledefs import * +from BaseLib.Core.exceptions import * +from BaseLib.Core.osutils import * +from BaseLib.Core.APIImplementation.SingleDownload import SingleDownload +import BaseLib.Core.APIImplementation.maketorrent as maketorrent + +DEBUG = False + +class DownloadImpl: + + def __init__(self,session,tdef): + self.dllock = RLock() + # just enough so error saving and get_state() works + self.error = None + self.sd = None # hack + # To be able to return the progress of a stopped torrent, how far it got. + self.progressbeforestop = 0.0 + self.filepieceranges = [] + self.pstate_for_restart = None # h4x0r to remember resumedata + + # Copy tdef, so we get an infohash + self.session = session + self.tdef = tdef.copy() + self.tdef.readonly = True + + # + # Creating a Download + # + def setup(self,dcfg=None,pstate=None,initialdlstatus=None,lmcreatedcallback=None,lmvodeventcallback=None): + """ + Create a Download object. Used internally by Session. + @param dcfg DownloadStartupConfig or None (in which case + a new DownloadConfig() is created and the result + becomes the runtime config of this Download. + """ + # Called by any thread + try: + self.dllock.acquire() # not really needed, no other threads know of this object + + torrentdef = self.get_def() + metainfo = torrentdef.get_metainfo() + # H4xor this so the 'name' field is safe + self.correctedinfoname = fix_filebasename(torrentdef.get_name_as_unicode()) + + if DEBUG: + print >>sys.stderr,"Download: setup: piece size",metainfo['info']['piece length'] + + # See if internal tracker used + itrackerurl = self.session.get_internal_tracker_url() + #infohash = self.tdef.get_infohash() + metainfo = self.tdef.get_metainfo() + usingitracker = False + + if DEBUG: + print >>sys.stderr,"Download: setup: internal tracker?",metainfo['announce'],itrackerurl,"#" + + if itrackerurl.endswith('/'): + slashless = itrackerurl[:-1] + else: + slashless = itrackerurl + if metainfo['announce'] == itrackerurl or metainfo['announce'] == slashless: + usingitracker = True + elif 'announce-list' in metainfo: + for tier in metainfo['announce-list']: + if itrackerurl in tier or slashless in tier: + usingitracker = True + break + + if usingitracker: + if DEBUG: + print >>sys.stderr,"Download: setup: Using internal tracker" + # Copy .torrent to state_dir/itracker so the tracker thread + # finds it and accepts peer registrations for it. + # + self.session.add_to_internal_tracker(self.tdef) + elif DEBUG: + print >>sys.stderr,"Download: setup: Not using internal tracker" + + # Copy dlconfig, from default if not specified + if dcfg is None: + cdcfg = DownloadStartupConfig() + else: + cdcfg = dcfg + self.dlconfig = copy.copy(cdcfg.dlconfig) + + + # Copy sessconfig into dlconfig, such that BitTornado.BT1.Connecter, etc. + # knows whether overlay is on, etc. + # + for (k,v) in self.session.get_current_startup_config_copy().sessconfig.iteritems(): + self.dlconfig.setdefault(k,v) + self.set_filepieceranges(metainfo) + + # Things that only exist at runtime + self.dlruntimeconfig= {} + self.dlruntimeconfig['max_desired_upload_rate'] = 0 + self.dlruntimeconfig['max_desired_download_rate'] = 0 + + if DEBUG: + print >>sys.stderr,"DownloadImpl: setup: initialdlstatus",`self.tdef.get_name_as_unicode()`,initialdlstatus + + # Closed swarms config + self.dlconfig['cs_keys'] = self.tdef.get_cs_keys_as_ders() + self.dlconfig['permid'] = self.session.get_permid() + if self.dlconfig['cs_keys']: + print >> sys.stderr,"DownloadImpl: setup: This is a closed swarm" + #if dcfg.get_poa(): + # self.dlconfig['poa'] = dcfg.get_poa() + #else: + # print >> sys.stderr,"POA not available - seeding?" + + # Set progress + if pstate is not None and pstate.has_key('dlstate'): + self.progressbeforestop = pstate['dlstate'].get('progress', 0.0) + + # Note: initialdlstatus now only works for STOPPED + if initialdlstatus != DLSTATUS_STOPPED: + if pstate is None or pstate['dlstate']['status'] != DLSTATUS_STOPPED: + # Also restart on STOPPED_ON_ERROR, may have been transient + self.create_engine_wrapper(lmcreatedcallback,pstate,lmvodeventcallback,initialdlstatus) # RePEX: propagate initialdlstatus + + self.pstate_for_restart = pstate + + self.dllock.release() + except Exception,e: + print_exc() + self.set_error(e) + self.dllock.release() + + def create_engine_wrapper(self,lmcreatedcallback,pstate,lmvodeventcallback,initialdlstatus=None): + """ Called by any thread, assume dllock already acquired """ + if DEBUG: + print >>sys.stderr,"Download: create_engine_wrapper()" + + # all thread safe + infohash = self.get_def().get_infohash() + metainfo = copy.deepcopy(self.get_def().get_metainfo()) + + # H4xor this so the 'name' field is safe + metainfo['info']['name'] = self.correctedinfoname + if 'name.utf-8' in metainfo['info']: + metainfo['info']['name.utf-8'] = self.correctedinfoname + + multihandler = self.session.lm.multihandler + listenport = self.session.get_listen_port() + vapath = self.session.get_video_analyser_path() + + # Note: BT1Download is started with copy of d.dlconfig, not direct access + kvconfig = copy.copy(self.dlconfig) + + # RePEX: extend kvconfig with initialdlstatus + kvconfig['initialdlstatus'] = initialdlstatus + + # Define which file to DL in VOD mode + live = self.get_def().get_live() + vodfileindex = { + 'index':-1, + 'inpath':None, + 'bitrate':0.0, + 'live':live, + 'usercallback':None, + 'userevents': [], + 'outpath':None} + + # --- streaming settings + if self.dlconfig['mode'] == DLMODE_VOD or self.dlconfig['video_source']: + # video file present which is played or produced + multi = False + if 'files' in metainfo['info']: + multi = True + + # Determine bitrate + if multi and len(self.dlconfig['selected_files']) == 0: + # Multi-file torrent, but no file selected + raise VODNoFileSelectedInMultifileTorrentException() + + if not multi: + # single-file torrent + file = self.get_def().get_name() + idx = -1 + bitrate = self.get_def().get_bitrate() + else: + # multi-file torrent + file = self.dlconfig['selected_files'][0] + idx = self.get_def().get_index_of_file_in_files(file) + bitrate = self.get_def().get_bitrate(file) + + # Determine MIME type + mimetype = self.get_mimetype(file) + # Arno: don't encode mimetype in lambda, allow for dynamic + # determination by videoanalyser + vod_usercallback_wrapper = lambda event,params:self.session.uch.perform_vod_usercallback(self,self.dlconfig['vod_usercallback'],event,params) + + vodfileindex['index'] = idx + vodfileindex['inpath'] = file + vodfileindex['bitrate'] = bitrate + vodfileindex['mimetype'] = mimetype + vodfileindex['usercallback'] = vod_usercallback_wrapper + vodfileindex['userevents'] = self.dlconfig['vod_userevents'][:] + elif live: + # live torrents must be streamed or produced, but not just downloaded + raise LiveTorrentRequiresUsercallbackException() + # Ric: added svc case TODO + elif self.dlconfig['mode'] == DLMODE_SVC: + # video file present which is played or produced + multi = False + if 'files' in metainfo['info']: + multi = True + + # Determine bitrate + if multi and len(self.dlconfig['selected_files']) == 0: + # Multi-file torrent, but no file selected + raise VODNoFileSelectedInMultifileTorrentException() + + # multi-file torrent + # Ric: the selected files are already ordered + files = self.dlconfig['selected_files'] + + idx = [] + for file in files: + idx.append( self.get_def().get_index_of_file_in_files(file) ) + + bitrate = self.get_def().get_bitrate(files[0]) + + # Determine MIME type + mimetype = self.get_mimetype(file) + # Arno: don't encode mimetype in lambda, allow for dynamic + # determination by videoanalyser + vod_usercallback_wrapper = lambda event,params:self.session.uch.perform_vod_usercallback(self,self.dlconfig['vod_usercallback'],event,params) + + vodfileindex['index'] = idx + vodfileindex['inpath'] = files + vodfileindex['bitrate'] = bitrate + vodfileindex['mimetype'] = mimetype + vodfileindex['usercallback'] = vod_usercallback_wrapper + vodfileindex['userevents'] = self.dlconfig['vod_userevents'][:] + + else: + vodfileindex['mimetype'] = 'application/octet-stream' + + if DEBUG: + print >>sys.stderr,"Download: create_engine_wrapper: vodfileindex",`vodfileindex` + + # Delegate creation of engine wrapper to network thread + network_create_engine_wrapper_lambda = lambda:self.network_create_engine_wrapper(infohash,metainfo,kvconfig,multihandler,listenport,vapath,vodfileindex,lmcreatedcallback,pstate,lmvodeventcallback) + self.session.lm.rawserver.add_task(network_create_engine_wrapper_lambda,0) + + + def network_create_engine_wrapper(self,infohash,metainfo,kvconfig,multihandler,listenport,vapath,vodfileindex,lmcallback,pstate,lmvodeventcallback): + """ Called by network thread """ + self.dllock.acquire() + try: + self.sd = SingleDownload(infohash,metainfo,kvconfig,multihandler,self.session.lm.get_ext_ip,listenport,vapath,vodfileindex,self.set_error,pstate,lmvodeventcallback,self.session.lm.hashcheck_done) + sd = self.sd + exc = self.error + if lmcallback is not None: + lmcallback(self,sd,exc,pstate) + finally: + self.dllock.release() + + # + # Public method + # + def get_def(self): + # No lock because attrib immutable and return value protected + return self.tdef + + # + # Retrieving DownloadState + # + def set_state_callback(self,usercallback,getpeerlist=False): + """ Called by any thread """ + self.dllock.acquire() + try: + network_get_state_lambda = lambda:self.network_get_state(usercallback,getpeerlist) + # First time on general rawserver + self.session.lm.rawserver.add_task(network_get_state_lambda,0.0) + finally: + self.dllock.release() + + + def network_get_state(self,usercallback,getpeerlist,sessioncalling=False): + """ Called by network thread """ + self.dllock.acquire() + try: + # RePEX: get last stored SwarmCache, if any: + swarmcache = None + if self.pstate_for_restart is not None and self.pstate_for_restart.has_key('dlstate'): + swarmcache = self.pstate_for_restart['dlstate'].get('swarmcache',None) + + if self.sd is None: + if DEBUG: + print >>sys.stderr,"DownloadImpl: network_get_state: Download not running" + ds = DownloadState(self,DLSTATUS_STOPPED,self.error,self.progressbeforestop,swarmcache=swarmcache) + else: + # RePEX: try getting the swarmcache from SingleDownload or use our last known swarmcache: + swarmcache = self.sd.get_swarmcache() or swarmcache + + (status,stats,logmsgs,coopdl_helpers,coopdl_coordinator) = self.sd.get_stats(getpeerlist) + ds = DownloadState(self,status,self.error,0.0,stats=stats,filepieceranges=self.filepieceranges,logmsgs=logmsgs,coopdl_helpers=coopdl_helpers,coopdl_coordinator=coopdl_coordinator,swarmcache=swarmcache) + self.progressbeforestop = ds.get_progress() + + if sessioncalling: + return ds + + # Invoke the usercallback function via a new thread. + # After the callback is invoked, the return values will be passed to + # the returncallback for post-callback processing. + self.session.uch.perform_getstate_usercallback(usercallback,ds,self.sesscb_get_state_returncallback) + finally: + self.dllock.release() + + + def sesscb_get_state_returncallback(self,usercallback,when,newgetpeerlist): + """ Called by SessionCallbackThread """ + self.dllock.acquire() + try: + if when > 0.0: + # Schedule next invocation, either on general or DL specific + # TODO: ensure this continues when dl is stopped. Should be OK. + network_get_state_lambda = lambda:self.network_get_state(usercallback,newgetpeerlist) + if self.sd is None: + self.session.lm.rawserver.add_task(network_get_state_lambda,when) + else: + self.sd.dlrawserver.add_task(network_get_state_lambda,when) + finally: + self.dllock.release() + + # + # Download stop/resume + # + def stop(self): + """ Called by any thread """ + self.stop_remove(removestate=False,removecontent=False) + + def stop_remove(self,removestate=False,removecontent=False): + """ Called by any thread """ + if DEBUG: + print >>sys.stderr,"DownloadImpl: stop_remove:",`self.tdef.get_name_as_unicode()`,"state",removestate,"content",removecontent + self.dllock.acquire() + try: + network_stop_lambda = lambda:self.network_stop(removestate,removecontent) + self.session.lm.rawserver.add_task(network_stop_lambda,0.0) + finally: + self.dllock.release() + + def network_stop(self,removestate,removecontent): + """ Called by network thread """ + if DEBUG: + print >>sys.stderr,"DownloadImpl: network_stop",`self.tdef.get_name_as_unicode()` + self.dllock.acquire() + try: + infohash = self.tdef.get_infohash() + pstate = self.network_get_persistent_state() + if self.sd is not None: + pstate['engineresumedata'] = self.sd.shutdown() + self.sd = None + self.pstate_for_restart = pstate + else: + # This method is also called at Session shutdown, where one may + # choose to checkpoint its Download. If the Download was + # stopped before, pstate_for_restart contains its resumedata. + # and that should be written into the checkpoint. + # + if self.pstate_for_restart is not None: + if DEBUG: + print >>sys.stderr,"DownloadImpl: network_stop: Reusing previously saved engineresume data for checkpoint" + # Don't copy full pstate_for_restart, as the torrent + # may have gone from e.g. HASHCHECK at startup to STOPPED + # now, at shutdown. In other words, it was never active + # in this session and the pstate_for_restart still says + # HASHCHECK. + pstate['engineresumedata'] = self.pstate_for_restart['engineresumedata'] + + # Offload the removal of the content and other disk cleanup to another thread + if removestate: + contentdest = self.get_content_dest() + self.session.uch.perform_removestate_callback(infohash,contentdest,removecontent) + + return (infohash,pstate) + finally: + self.dllock.release() + + + def restart(self, initialdlstatus=None): + """ Restart the Download. Technically this action does not need to be + delegated to the network thread, but does so removes some concurrency + problems. By scheduling both stops and restarts via the network task + queue we ensure that they are executed in the order they were called. + + Note that when a Download is downloading or seeding, calling restart + is a no-op. If a Download is performing some other task, it is left + up to the internal running SingleDownload to determine what a restart + means. Often it means SingleDownload will abort its current task and + switch to downloading/seeding. + + Called by any thread """ + # RePEX: added initialdlstatus parameter + # RePEX: TODO: Should we mention the initialdlstatus behaviour in the docstring? + if DEBUG: + print >>sys.stderr,"DownloadImpl: restart:",`self.tdef.get_name_as_unicode()` + self.dllock.acquire() + try: + network_restart_lambda = lambda:self.network_restart(initialdlstatus) + self.session.lm.rawserver.add_task(network_restart_lambda,0.0) + finally: + self.dllock.release() + + def network_restart(self,initialdlstatus=None): + """ Called by network thread """ + # Must schedule the hash check via lm. In some cases we have batch stops + # and restarts, e.g. we have stop all-but-one & restart-all for VOD) + + # RePEX: added initialdlstatus parameter + if DEBUG: + print >>sys.stderr,"DownloadImpl: network_restart",`self.tdef.get_name_as_unicode()` + self.dllock.acquire() + try: + if self.sd is None: + self.error = None # assume fatal error is reproducible + # h4xor: restart using earlier loaded resumedata + # RePEX: propagate initialdlstatus + self.create_engine_wrapper(self.session.lm.network_engine_wrapper_created_callback,pstate=self.pstate_for_restart,lmvodeventcallback=self.session.lm.network_vod_event_callback,initialdlstatus=initialdlstatus) + else: + if DEBUG: + print >>sys.stderr,"DownloadImpl: network_restart: SingleDownload already running",`self` + # RePEX: leave decision what to do to SingleDownload + self.sd.restart(initialdlstatus) + + # No exception if already started, for convenience + finally: + self.dllock.release() + + + # + # Config parameters that only exists at runtime + # + def set_max_desired_speed(self,direct,speed): + if DEBUG: + print >>sys.stderr,"Download: set_max_desired_speed",direct,speed + #if speed < 10: + # print_stack() + + self.dllock.acquire() + if direct == UPLOAD: + self.dlruntimeconfig['max_desired_upload_rate'] = speed + else: + self.dlruntimeconfig['max_desired_download_rate'] = speed + self.dllock.release() + + def get_max_desired_speed(self,direct): + self.dllock.acquire() + try: + if direct == UPLOAD: + return self.dlruntimeconfig['max_desired_upload_rate'] + else: + return self.dlruntimeconfig['max_desired_download_rate'] + finally: + self.dllock.release() + + def get_dest_files(self, exts=None): + """ We could get this from BT1Download.files (see BT1Download.saveAs()), + but that object is the domain of the network thread. + You can give a list of extensions to return. If None: return all dest_files + """ + + def get_ext(filename): + (prefix,ext) = os.path.splitext(filename) + if ext != '' and ext[0] == '.': + ext = ext[1:] + return ext + + self.dllock.acquire() + try: + f2dlist = [] + metainfo = self.tdef.get_metainfo() + if 'files' not in metainfo['info']: + # single-file torrent + diskfn = self.get_content_dest() + f2dtuple = (None, diskfn) + ext = get_ext(diskfn) + if exts is None or ext in exts: + f2dlist.append(f2dtuple) + else: + # multi-file torrent + if len(self.dlconfig['selected_files']) > 0: + fnlist = self.dlconfig['selected_files'] + else: + fnlist = self.tdef.get_files(exts=exts) + + for filename in fnlist: + filerec = maketorrent.get_torrentfilerec_from_metainfo(filename,metainfo) + savepath = maketorrent.torrentfilerec2savefilename(filerec) + diskfn = maketorrent.savefilenames2finaldest(self.get_content_dest(),savepath) + ext = get_ext(diskfn) + if exts is None or ext in exts: + f2dtuple = (filename,diskfn) + f2dlist.append(f2dtuple) + + return f2dlist + finally: + self.dllock.release() + + + + + + # + # Persistence + # + def network_checkpoint(self): + """ Called by network thread """ + self.dllock.acquire() + try: + pstate = self.network_get_persistent_state() + if self.sd is None: + resdata = None + else: + resdata = self.sd.checkpoint() + pstate['engineresumedata'] = resdata + return (self.tdef.get_infohash(),pstate) + finally: + self.dllock.release() + + + def network_get_persistent_state(self): + """ Assume dllock already held """ + pstate = {} + pstate['version'] = PERSISTENTSTATE_CURRENTVERSION + pstate['metainfo'] = self.tdef.get_metainfo() # assumed immutable + dlconfig = copy.copy(self.dlconfig) + # Reset unpicklable params + dlconfig['vod_usercallback'] = None + dlconfig['mode'] = DLMODE_NORMAL # no callback, no VOD + pstate['dlconfig'] = dlconfig + + pstate['dlstate'] = {} + #ds = self.network_get_state(None,False,sessioncalling=True) + ds = self.network_get_state(None,True,sessioncalling=True) # RePEX: get peerlist in case of running Download + pstate['dlstate']['status'] = ds.get_status() + pstate['dlstate']['progress'] = ds.get_progress() + pstate['dlstate']['swarmcache'] = ds.get_swarmcache() # RePEX: store SwarmCache + + if DEBUG: + print >>sys.stderr,"Download: netw_get_pers_state: status",dlstatus_strings[ds.get_status()],"progress",ds.get_progress() + + pstate['engineresumedata'] = None + return pstate + + # + # Coop download + # + def get_coopdl_role_object(self,role): + """ Called by network thread """ + role_object = None + self.dllock.acquire() + try: + if self.sd is not None: + role_object = self.sd.get_coopdl_role_object(role) + finally: + self.dllock.release() + return role_object + + + + # + # Internal methods + # + def set_error(self,e): + self.dllock.acquire() + self.error = e + self.dllock.release() + + + def set_filepieceranges(self,metainfo): + """ Determine which file maps to which piece ranges for progress info """ + + if DEBUG: + print >>sys.stderr,"Download: set_filepieceranges:",self.dlconfig['selected_files'] + (length,self.filepieceranges) = maketorrent.get_length_filepieceranges_from_metainfo(metainfo,self.dlconfig['selected_files']) + + def get_content_dest(self): + """ Returns the file (single-file torrent) or dir (multi-file torrent) + to which the downloaded content is saved. """ + return os.path.join(self.dlconfig['saveas'],self.correctedinfoname) + + # ARNOCOMMENT: better if we removed this from Core, user knows which + # file he selected to play, let him figure out MIME type + def get_mimetype(self,file): + (prefix,ext) = os.path.splitext(file) + ext = ext.lower() + mimetype = None + if sys.platform == 'win32': + # TODO: Use Python's mailcap facility on Linux to find player + try: + from BaseLib.Video.utils import win32_retrieve_video_play_command + + [mimetype,playcmd] = win32_retrieve_video_play_command(ext,file) + if DEBUG: + print >>sys.stderr,"DownloadImpl: Win32 reg said MIME type is",mimetype + except: + if DEBUG: + print_exc() + pass + else: + try: + import mimetypes + # homedir = os.path.expandvars('${HOME}') + homedir = get_home_dir() + homemapfile = os.path.join(homedir,'.mimetypes') + mapfiles = [homemapfile] + mimetypes.knownfiles + mimetypes.init(mapfiles) + (mimetype,encoding) = mimetypes.guess_type(file) + + if DEBUG: + print >>sys.stderr,"DownloadImpl: /etc/mimetypes+ said MIME type is",mimetype,file + except: + print_exc() + + # if auto detect fails + if mimetype is None: + if ext == '.avi': + # Arno, 2010-01-08: Hmmm... video/avi is not official registered at IANA + mimetype = 'video/avi' + elif ext == '.mpegts' or ext == '.ts': + mimetype = 'video/mp2t' + elif ext == '.mkv': + mimetype = 'video/x-matroska' + elif ext in ('.ogg', '.ogv'): + mimetype = 'video/ogg' + elif ext in ('.oga'): + mimetype = 'audio/ogg' + elif ext == '.webm': + mimetype = 'video/webm' + else: + mimetype = 'video/mpeg' + return mimetype + diff --git a/instrumentation/next-share/BaseLib/Core/APIImplementation/DownloadRuntimeConfig.py b/instrumentation/next-share/BaseLib/Core/APIImplementation/DownloadRuntimeConfig.py new file mode 100644 index 0000000..5472bd1 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/APIImplementation/DownloadRuntimeConfig.py @@ -0,0 +1,586 @@ +# Written by Arno Bakker, George Milescu +# see LICENSE.txt for license information + +import sys + +from BaseLib.Core.simpledefs import * +from BaseLib.Core.DownloadConfig import DownloadConfigInterface +from BaseLib.Core.exceptions import OperationNotPossibleAtRuntimeException + +DEBUG = False + +# 10/02/10 Boudewijn: pylint points out that member variables used in +# DownloadRuntimeConfig do not exist. This is because they are set in +# BaseLib.Core.Download which is a subclass of DownloadRuntimeConfig. +# +# We disable this error +# pylint: disable-msg=E1101 + +class DownloadRuntimeConfig(DownloadConfigInterface): + """ + Implements the BaseLib.Core.DownloadConfig.DownloadConfigInterface + + Use these to change the download config at runtime. + + DownloadConfigInterface: All methods called by any thread + """ + def set_max_speed(self,direct,speed): + if DEBUG: + print >>sys.stderr,"Download: set_max_speed",`self.get_def().get_metainfo()['info']['name']`,direct,speed + #print_stack() + + self.dllock.acquire() + try: + # Don't need to throw an exception when stopped, we then just save the new value and + # use it at (re)startup. + if self.sd is not None: + set_max_speed_lambda = lambda:self.sd is not None and self.sd.set_max_speed(direct,speed,None) + self.session.lm.rawserver.add_task(set_max_speed_lambda,0) + + # At the moment we can't catch any errors in the engine that this + # causes, so just assume it always works. + DownloadConfigInterface.set_max_speed(self,direct,speed) + finally: + self.dllock.release() + + def get_max_speed(self,direct): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_max_speed(self,direct) + finally: + self.dllock.release() + + def set_dest_dir(self,path): + raise OperationNotPossibleAtRuntimeException() + + def set_video_event_callback(self,usercallback,dlmode=DLMODE_VOD): + """ Note: this currently works only when the download is stopped. """ + self.dllock.acquire() + try: + DownloadConfigInterface.set_video_event_callback(self,usercallback,dlmode=dlmode) + finally: + self.dllock.release() + + def set_video_events(self,events): + """ Note: this currently works only when the download is stopped. """ + self.dllock.acquire() + try: + DownloadConfigInterface.set_video_events(self,events) + finally: + self.dllock.release() + + def set_mode(self,mode): + """ Note: this currently works only when the download is stopped. """ + self.dllock.acquire() + try: + DownloadConfigInterface.set_mode(self,mode) + finally: + self.dllock.release() + + def get_mode(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_mode(self) + finally: + self.dllock.release() + + def get_video_event_callback(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_video_event_callback(self) + finally: + self.dllock.release() + + def get_video_events(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_video_events(self) + finally: + self.dllock.release() + + def set_selected_files(self,files): + """ Note: this currently works only when the download is stopped. """ + self.dllock.acquire() + try: + DownloadConfigInterface.set_selected_files(self,files) + self.set_filepieceranges(self.tdef.get_metainfo()) + finally: + self.dllock.release() + + + def get_selected_files(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_selected_files(self) + finally: + self.dllock.release() + + def set_max_conns_to_initiate(self,nconns): + self.dllock.acquire() + try: + if self.sd is not None: + set_max_conns2init_lambda = lambda:self.sd is not None and self.sd.set_max_conns_to_initiate(nconns,None) + self.session.lm.rawserver.add_task(set_max_conns2init_lambda,0.0) + DownloadConfigInterface.set_max_conns_to_initiate(self,nconns) + finally: + self.dllock.release() + + def get_max_conns_to_initiate(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_max_conns_to_initiate(self) + finally: + self.dllock.release() + + def set_max_conns(self,nconns): + self.dllock.acquire() + try: + if self.sd is not None: + set_max_conns_lambda = lambda:self.sd is not None and self.sd.set_max_conns(nconns,None) + self.session.lm.rawserver.add_task(set_max_conns_lambda,0.0) + DownloadConfigInterface.set_max_conns(self,nconns) + finally: + self.dllock.release() + + def get_max_conns(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_max_conns(self) + finally: + self.dllock.release() + + # + # Advanced download parameters + # + def set_max_uploads(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_max_uploads(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_max_uploads(self) + finally: + self.dllock.release() + + def set_keepalive_interval(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_keepalive_interval(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_keepalive_interval(self) + finally: + self.dllock.release() + + def set_download_slice_size(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_download_slice_size(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_download_slice_size(self) + finally: + self.dllock.release() + + def set_upload_unit_size(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_upload_unit_size(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_upload_unit_size(self) + finally: + self.dllock.release() + + def set_request_backlog(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_request_backlog(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_request_backlog(self) + finally: + self.dllock.release() + + def set_max_message_length(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_max_message_length(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_max_message_length(self) + finally: + self.dllock.release() + + def set_max_slice_length(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_max_slice_length(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_max_slice_length(self) + finally: + self.dllock.release() + + def set_max_rate_period(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_max_rate_period(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_max_rate_period(self) + finally: + self.dllock.release() + + def set_upload_rate_fudge(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_upload_rate_fudge(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_upload_rate_fudge(self) + finally: + self.dllock.release() + + def set_tcp_ack_fudge(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tcp_ack_fudge(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_tcp_ack_fudge(self) + finally: + self.dllock.release() + + def set_rerequest_interval(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_rerequest_interval(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_rerequest_interval(self) + finally: + self.dllock.release() + + def set_min_peers(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_min_peers(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_min_peers(self) + finally: + self.dllock.release() + + def set_http_timeout(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_http_timeout(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_http_timeout(self) + finally: + self.dllock.release() + + def set_check_hashes(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_check_hashes(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_check_hashes(self) + finally: + self.dllock.release() + + def set_alloc_type(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_alloc_type(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_alloc_type(self) + finally: + self.dllock.release() + + def set_alloc_rate(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_alloc_rate(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_alloc_rate(self) + finally: + self.dllock.release() + + def set_buffer_reads(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_buffer_reads(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_buffer_reads(self) + finally: + self.dllock.release() + + def set_write_buffer_size(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_write_buffer_size(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_write_buffer_size(self) + finally: + self.dllock.release() + + def set_breakup_seed_bitfield(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_breakup_seed_bitfield(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_breakup_seed_bitfield(self) + finally: + self.dllock.release() + + def set_snub_time(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_snub_time(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_snub_time(self) + finally: + self.dllock.release() + + def set_rarest_first_cutoff(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_rarest_first_cutoff(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_rarest_first_cutoff(self) + finally: + self.dllock.release() + + def set_rarest_first_priority_cutoff(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_rarest_first_priority_cutoff(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_rarest_first_priority_cutoff(self) + finally: + self.dllock.release() + + def set_min_uploads(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_min_uploads(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_min_uploads(self) + finally: + self.dllock.release() + + def set_max_files_open(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_max_files_open(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_max_files_open(self) + finally: + self.dllock.release() + + def set_round_robin_period(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_round_robin_period(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_round_robin_period(self) + finally: + self.dllock.release() + + def set_super_seeder(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_super_seeder(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_super_seeder(self) + finally: + self.dllock.release() + + def set_security(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_security(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_security(self) + finally: + self.dllock.release() + + def set_auto_kick(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_auto_kick(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_auto_kick(self) + finally: + self.dllock.release() + + def set_double_check_writes(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_double_check_writes(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_double_check_writes(self) + finally: + self.dllock.release() + + def set_triple_check_writes(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_triple_check_writes(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_triple_check_writes(self) + finally: + self.dllock.release() + + def set_lock_files(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_lock_files(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_lock_files(self) + finally: + self.dllock.release() + + def set_lock_while_reading(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_lock_while_reading(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_lock_while_reading(self) + finally: + self.dllock.release() + + def set_auto_flush(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_auto_flush(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_auto_flush(self) + finally: + self.dllock.release() + + def set_exclude_ips(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_exclude_ips(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_exclude_ips(self) + finally: + self.dllock.release() + + def set_ut_pex_max_addrs_from_peer(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_ut_pex_max_addrs_from_peer(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_ut_pex_max_addrs_from_peer(self) + finally: + self.dllock.release() + + def set_poa(self, poa): + self.dllock.acquire() + try: + DownloadConfigInterface.set_poa(self, poa) + finally: + self.dllock.release() + + + def get_poa(self, poa): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_poa(self) + finally: + self.dllock.release() + def set_same_nat_try_internal(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_same_nat_try_internal(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_same_nat_try_internal(self) + finally: + self.dllock.release() + + + def set_unchoke_bias_for_internal(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_unchoke_bias_for_internal(self): + self.dllock.acquire() + try: + return DownloadConfigInterface.get_unchoke_bias_for_internal(self) + finally: + self.dllock.release() + + # + # ProxyService_ + # + def set_proxy_mode(self,value): + """ Set the proxymode for current download + . + @param value: the proxyservice mode: PROXY_MODE_OFF, PROXY_MODE_PRIVATE or PROXY_MODE_SPEED + """ + self.dllock.acquire() + try: + DownloadConfigInterface.set_proxy_mode(self, value) + finally: + self.dllock.release() + + def get_proxy_mode(self): + """ Returns the proxymode of the client. + @return: one of the possible three values: PROXY_MODE_OFF, PROXY_MODE_PRIVATE, PROXY_MODE_SPEED + """ + self.dllock.acquire() + try: + return DownloadConfigInterface.get_proxy_mode(self) + finally: + self.dllock.release() + + def set_no_helpers(self,value): + """ Set the maximum number of helpers used for a download. + @param value: a positive integer number + """ + self.dllock.acquire() + try: + return DownloadConfigInterface.set_no_helpers(self, value) + finally: + self.dllock.release() + + def get_no_helpers(self): + """ Returns the maximum number of helpers used for a download. + @return: a positive integer number + """ + self.dllock.acquire() + try: + return DownloadConfigInterface.get_no_helpers(self) + finally: + self.dllock.release() + # + # _ProxyService + # + diff --git a/instrumentation/next-share/BaseLib/Core/APIImplementation/LaunchManyCore.py b/instrumentation/next-share/BaseLib/Core/APIImplementation/LaunchManyCore.py new file mode 100644 index 0000000..928ff57 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/APIImplementation/LaunchManyCore.py @@ -0,0 +1,907 @@ +# Written by Arno Bakker +# see LICENSE.txt for license information + +import sys +import os +import pickle +import socket +import binascii +import time as timemod +from threading import Event,Thread,enumerate +from traceback import print_exc, print_stack + +from BaseLib.__init__ import LIBRARYNAME +from BaseLib.Core.BitTornado.RawServer import RawServer +from BaseLib.Core.BitTornado.ServerPortHandler import MultiHandler +from BaseLib.Core.BitTornado.BT1.track import Tracker +from BaseLib.Core.BitTornado.HTTPHandler import HTTPHandler,DummyHTTPHandler +from BaseLib.Core.simpledefs import * +from BaseLib.Core.exceptions import * +from BaseLib.Core.Download import Download +from BaseLib.Core.DownloadConfig import DownloadStartupConfig +from BaseLib.Core.TorrentDef import TorrentDef +from BaseLib.Core.NATFirewall.guessip import get_my_wan_ip +from BaseLib.Core.NATFirewall.UPnPThread import UPnPThread +from BaseLib.Core.NATFirewall.UDPPuncture import UDPHandler +from BaseLib.Core.DecentralizedTracking import mainlineDHT +from BaseLib.Core.osutils import get_readable_torrent_name +from BaseLib.Core.DecentralizedTracking.MagnetLink.MagnetLink import MagnetHandler + +SPECIAL_VALUE=481 + +DEBUG = False +PROFILE = False + +# Internal classes +# + +class TriblerLaunchMany(Thread): + + def __init__(self): + """ Called only once (unless we have multiple Sessions) by MainThread """ + Thread.__init__(self) + self.setDaemon(True) + self.setName("Network"+self.getName()) + + def register(self,session,sesslock): + self.session = session + self.sesslock = sesslock + + self.downloads = {} + config = session.sessconfig # Should be safe at startup + + self.locally_guessed_ext_ip = self.guess_ext_ip_from_local_info() + self.upnp_ext_ip = None + self.dialback_ext_ip = None + self.yourip_ext_ip = None + self.udppuncture_handler = None + + # Orig + self.sessdoneflag = Event() + + # Following two attributes set/get by network thread ONLY + self.hashcheck_queue = [] + self.sdownloadtohashcheck = None + + # Following 2 attributes set/get by UPnPThread + self.upnp_thread = None + self.upnp_type = config['upnp_nat_access'] + self.nat_detect = config['nat_detect'] + + self.rawserver = RawServer(self.sessdoneflag, + config['timeout_check_interval'], + config['timeout'], + ipv6_enable = config['ipv6_enabled'], + failfunc = self.rawserver_fatalerrorfunc, + errorfunc = self.rawserver_nonfatalerrorfunc) + self.rawserver.add_task(self.rawserver_keepalive,1) + + self.listen_port = self.rawserver.find_and_bind(0, + config['minport'], config['maxport'], config['bind'], + reuse = True, + ipv6_socket_style = config['ipv6_binds_v4'], + randomizer = config['random_port']) + + if DEBUG: + print >>sys.stderr,"tlm: Got listen port", self.listen_port + + self.multihandler = MultiHandler(self.rawserver, self.sessdoneflag) + self.shutdownstarttime = None + + # do_cache -> do_overlay -> (do_buddycast, do_download_help) + if config['megacache']: + import BaseLib.Core.CacheDB.cachedb as cachedb + from BaseLib.Core.CacheDB.SqliteCacheDBHandler import MyDBHandler, PeerDBHandler, TorrentDBHandler, MyPreferenceDBHandler, PreferenceDBHandler, SuperPeerDBHandler, FriendDBHandler, BarterCastDBHandler, VoteCastDBHandler, SearchDBHandler,TermDBHandler, CrawlerDBHandler, ChannelCastDBHandler, SimilarityDBHandler, PopularityDBHandler + from BaseLib.Core.CacheDB.SqliteSeedingStatsCacheDB import SeedingStatsDBHandler, SeedingStatsSettingsDBHandler + from BaseLib.Core.CacheDB.SqliteFriendshipStatsCacheDB import FriendshipStatisticsDBHandler + from BaseLib.Category.Category import Category + + # 13-04-2010, Andrea: rich metadata (subtitle) db + from BaseLib.Core.CacheDB.MetadataDBHandler import MetadataDBHandler + + # init cache db + if config['nickname'] == '__default_name__': + config['nickname'] = socket.gethostname() + + if DEBUG: + print >>sys.stderr,'tlm: Reading Session state from',config['state_dir'] + + cachedb.init(config, self.rawserver_fatalerrorfunc) + + self.my_db = MyDBHandler.getInstance() + self.peer_db = PeerDBHandler.getInstance() + # Register observer to update connection opened/closed to peer_db_handler + self.peer_db.registerConnectionUpdater(self.session) + self.torrent_db = TorrentDBHandler.getInstance() + torrent_collecting_dir = os.path.abspath(config['torrent_collecting_dir']) + self.torrent_db.register(Category.getInstance(),torrent_collecting_dir) + self.mypref_db = MyPreferenceDBHandler.getInstance() + self.pref_db = PreferenceDBHandler.getInstance() + self.superpeer_db = SuperPeerDBHandler.getInstance() + self.superpeer_db.loadSuperPeers(config) + self.friend_db = FriendDBHandler.getInstance() + self.bartercast_db = BarterCastDBHandler.getInstance() + self.bartercast_db.registerSession(self.session) + self.votecast_db = VoteCastDBHandler.getInstance() + self.votecast_db.registerSession(self.session) + self.channelcast_db = ChannelCastDBHandler.getInstance() + self.channelcast_db.registerSession(self.session) + self.search_db = SearchDBHandler.getInstance() + self.term_db = TermDBHandler.getInstance() + self.simi_db = SimilarityDBHandler.getInstance() + self.pops_db = PopularityDBHandler.getInstance() + + # 13-04-2010, Andrea: rich metadata (subtitle) db + self.richmetadataDbHandler = MetadataDBHandler.getInstance() + + # Crawling + if config['crawler']: + # ARNOCOMMENT, 2009-10-02: Should be moved out of core, used in Main client only. + # initialize SeedingStats database + cachedb.init_seeding_stats(config, self.rawserver_fatalerrorfunc) + + # initialize VideoPlayback statistics database + cachedb.init_videoplayback_stats(config, self.rawserver_fatalerrorfunc) + + self.crawler_db = CrawlerDBHandler.getInstance() + self.crawler_db.loadCrawlers(config) + self.seedingstats_db = SeedingStatsDBHandler.getInstance() + self.seedingstatssettings_db = SeedingStatsSettingsDBHandler.getInstance() + + if config['socnet']: + # initialize Friendship statistics database + cachedb.init_friendship_stats(config, self.rawserver_fatalerrorfunc) + + self.friendship_statistics_db = FriendshipStatisticsDBHandler().getInstance() + else: + self.friendship_statistics_db = None + else: + self.crawler_db = None + self.seedingstats_db = None + self.friendship_statistics_db = None + + else: + config['overlay'] = 0 # turn overlay off + config['torrent_checking'] = 0 + self.my_db = None + self.peer_db = None + self.torrent_db = None + self.mypref_db = None + self.pref_db = None + self.superpeer_db = None + self.crawler_db = None + self.seedingstats_db = None + self.seedingstatssettings_db = None + self.friendship_statistics_db = None + self.friend_db = None + self.bartercast_db = None + self.votecast_db = None + self.channelcast_db = None + self.mm = None + # 13-04-2010, Andrea: rich metadata (subtitle) db + self.richmetadataDbHandler = None + + if config['overlay']: + from BaseLib.Core.Overlay.SecureOverlay import SecureOverlay + from BaseLib.Core.Overlay.OverlayThreadingBridge import OverlayThreadingBridge + from BaseLib.Core.Overlay.OverlayApps import OverlayApps + from BaseLib.Core.RequestPolicy import FriendsCoopDLOtherRQueryQuotumCrawlerAllowAllRequestPolicy + + self.secure_overlay = SecureOverlay.getInstance() + self.secure_overlay.register(self, config['overlay_max_message_length']) + + # Set policy for which peer requests (dl_helper, rquery) to answer and which to ignore + + self.overlay_apps = OverlayApps.getInstance() + # Default policy, override with Session.set_overlay_request_policy() + policy = FriendsCoopDLOtherRQueryQuotumCrawlerAllowAllRequestPolicy(self.session) + + # For the new DB layer we need to run all overlay apps in a + # separate thread instead of the NetworkThread as before. + + self.overlay_bridge = OverlayThreadingBridge.getInstance() + + self.overlay_bridge.register_bridge(self.secure_overlay,self.overlay_apps) + + self.overlay_apps.register(self.overlay_bridge,self.session,self,config,policy) + # It's important we don't start listening to the network until + # all higher protocol-handling layers are properly configured. + self.overlay_bridge.start_listening() + + if config['multicast_local_peer_discovery']: + self.setup_multicast_discovery() + + else: + self.secure_overlay = None + self.overlay_apps = None + config['buddycast'] = 0 + config['download_help'] = 0 + config['socnet'] = 0 + config['rquery'] = 0 + + try: + # Minimal to allow yourip external-IP address detection + from BaseLib.Core.NATFirewall.DialbackMsgHandler import DialbackMsgHandler + some_dialback_handler = DialbackMsgHandler.getInstance() + some_dialback_handler.register_yourip(self) + except: + if DEBUG: + print_exc() + pass + + + if config['megacache'] or config['overlay']: + # Arno: THINK! whoever added this should at least have made the + # config files configurable via SessionConfigInterface. + # + # TODO: see if we can move this out of the core. We could make the + # category a parameter to TorrentDB.addExternalTorrent(), but that + # will not work directly for MetadataHandler, which is part of the + # core. + + # Some author: First Category instantiation requires install_dir, so do it now + from BaseLib.Category.Category import Category + + Category.getInstance(config['install_dir']) + + # Internal tracker + self.internaltracker = None + if config['internaltracker']: + self.internaltracker = Tracker(config, self.rawserver) + self.httphandler = HTTPHandler(self.internaltracker.get, config['tracker_min_time_between_log_flushes']) + else: + self.httphandler = DummyHTTPHandler() + self.multihandler.set_httphandler(self.httphandler) + + + if config['mainline_dht']: + #import logging + # Arno,The equivalent of DEBUG=False for kadtracker + #logging.disable(logging.CRITICAL) + # New: see DecentralizedTracking/kadtracker/logging_conf.py + + # Start up KTH mainline DHT + #TODO: Can I get the local IP number? + mainlineDHT.init(('127.0.0.1', self.listen_port), config['state_dir']) + + + # add task for tracker checking + if config['torrent_checking']: + + if config['mainline_dht']: + # Create torrent-liveliness checker based on DHT + from BaseLib.Core.DecentralizedTracking.mainlineDHTChecker import mainlineDHTChecker + + c = mainlineDHTChecker.getInstance() + c.register(mainlineDHT.dht) + + self.torrent_checking_period = config['torrent_checking_period'] + #self.torrent_checking_period = 5 + self.rawserver.add_task(self.run_torrent_check, self.torrent_checking_period) + + # Gertjan's UDP code + # OFF in P2P-Next + if False and config['overlay'] and config['crawler']: + # Gertjan's UDP code + self.udppuncture_handler = UDPHandler(self.rawserver, config['overlay'] and config['crawler']) + + if config["magnetlink"]: + # initialise the first instance + MagnetHandler.get_instance(self.rawserver) + + + def add(self,tdef,dscfg,pstate=None,initialdlstatus=None): + """ Called by any thread """ + self.sesslock.acquire() + try: + if not tdef.is_finalized(): + raise ValueError("TorrentDef not finalized") + + infohash = tdef.get_infohash() + + # Check if running or saved on disk + if infohash in self.downloads: + raise DuplicateDownloadException() + + d = Download(self.session,tdef) + + if pstate is None and not tdef.get_live(): # not already resuming + pstate = self.load_download_pstate_noexc(infohash) + if pstate is not None: + if DEBUG: + print >>sys.stderr,"tlm: add: pstate is",dlstatus_strings[pstate['dlstate']['status']],pstate['dlstate']['progress'] + + # Store in list of Downloads, always. + self.downloads[infohash] = d + d.setup(dscfg,pstate,initialdlstatus,self.network_engine_wrapper_created_callback,self.network_vod_event_callback) + + if self.torrent_db != None and self.mypref_db != None: + raw_filename = tdef.get_name_as_unicode() + save_name = get_readable_torrent_name(infohash, raw_filename) + #print >> sys.stderr, 'tlm: add', save_name, self.session.sessconfig + torrent_dir = self.session.sessconfig['torrent_collecting_dir'] + save_path = os.path.join(torrent_dir, save_name) + if not os.path.exists(save_path): # save the torrent to the common torrent dir + tdef.save(save_path) + + # hack, make sure these torrents are always good so they show up + # in TorrentDBHandler.getTorrents() + extra_info = {'status':'good'} + + # 03/02/10 Boudewijn: addExternalTorrent now requires + # a torrentdef, consequently we provide the filename + # through the extra_info dictionary + extra_info['filename'] = save_name + + self.torrent_db.addExternalTorrent(tdef, source='',extra_info=extra_info) + dest_path = d.get_dest_dir() + # TODO: if user renamed the dest_path for single-file-torrent + data = {'destination_path':dest_path} + self.mypref_db.addMyPreference(infohash, data) + # BuddyCast is now notified of this new Download in our + # preferences via the Notifier mechanism. See BC.sesscb_ntfy_myprefs() + return d + finally: + self.sesslock.release() + + + def network_engine_wrapper_created_callback(self,d,sd,exc,pstate): + """ Called by network thread """ + if exc is None: + # Always need to call the hashcheck func, even if we're restarting + # a download that was seeding, this is just how the BT engine works. + # We've provided the BT engine with its resumedata, so this should + # be fast. + # + try: + if sd is not None: + self.queue_for_hashcheck(sd) + if pstate is None and not d.get_def().get_live(): + # Checkpoint at startup + (infohash,pstate) = d.network_checkpoint() + self.save_download_pstate(infohash,pstate) + else: + raise TriblerException("tlm: network_engine_wrapper_created_callback: sd is None!") + except Exception,e: + # There was a bug in queue_for_hashcheck that is now fixed. + # Leave this in place to catch unexpected errors. + print_exc() + d.set_error(e) + + + def remove(self,d,removecontent=False): + """ Called by any thread """ + self.sesslock.acquire() + try: + d.stop_remove(removestate=True,removecontent=removecontent) + infohash = d.get_def().get_infohash() + del self.downloads[infohash] + finally: + self.sesslock.release() + + def get_downloads(self): + """ Called by any thread """ + self.sesslock.acquire() + try: + return self.downloads.values() #copy, is mutable + finally: + self.sesslock.release() + + def download_exists(self,infohash): + self.sesslock.acquire() + try: + return infohash in self.downloads + finally: + self.sesslock.release() + + + def rawserver_fatalerrorfunc(self,e): + """ Called by network thread """ + if DEBUG: + print >>sys.stderr,"tlm: RawServer fatal error func called",e + print_exc() + + def rawserver_nonfatalerrorfunc(self,e): + """ Called by network thread """ + if DEBUG: + print >>sys.stderr,"tlm: RawServer non fatal error func called",e + print_exc() + # Could log this somewhere, or phase it out + + def _run(self): + """ Called only once by network thread """ + + try: + try: + self.start_upnp() + self.start_multicast() + self.multihandler.listen_forever() + except: + print_exc() + finally: + if self.internaltracker is not None: + self.internaltracker.save_state() + + self.stop_upnp() + self.rawserver.shutdown() + + def rawserver_keepalive(self): + """ Hack to prevent rawserver sleeping in select() for a long time, not + processing any tasks on its queue at startup time + + Called by network thread """ + self.rawserver.add_task(self.rawserver_keepalive,1) + + # + # TODO: called by TorrentMaker when new torrent added to itracker dir + # Make it such that when Session.add_torrent() is called and the internal + # tracker is used that we write a metainfo to itracker dir and call this. + # + def tracker_rescan_dir(self): + if self.internaltracker is not None: + self.internaltracker.parse_allowed(source='Session') + + # + # Torrent hash checking + # + def queue_for_hashcheck(self,sd): + """ Schedule a SingleDownload for integrity check of on-disk data + + Called by network thread """ + if hash: + self.hashcheck_queue.append(sd) + # Check smallest torrents first + self.hashcheck_queue.sort(singledownload_size_cmp) + + if not self.sdownloadtohashcheck: + self.dequeue_and_start_hashcheck() + + def dequeue_and_start_hashcheck(self): + """ Start integriy check for first SingleDownload in queue + + Called by network thread """ + self.sdownloadtohashcheck = self.hashcheck_queue.pop(0) + self.sdownloadtohashcheck.perform_hashcheck(self.hashcheck_done) + + def hashcheck_done(self,success=True): + """ Integrity check for first SingleDownload in queue done + + Called by network thread """ + if DEBUG: + print >>sys.stderr,"tlm: hashcheck_done, success",success + if success: + self.sdownloadtohashcheck.hashcheck_done() + if self.hashcheck_queue: + self.dequeue_and_start_hashcheck() + else: + self.sdownloadtohashcheck = None + + # + # State retrieval + # + def set_download_states_callback(self,usercallback,getpeerlist,when=0.0): + """ Called by any thread """ + network_set_download_states_callback_lambda = lambda:self.network_set_download_states_callback(usercallback,getpeerlist) + self.rawserver.add_task(network_set_download_states_callback_lambda,when) + + def network_set_download_states_callback(self,usercallback,getpeerlist): + """ Called by network thread """ + self.sesslock.acquire() + try: + # Even if the list of Downloads changes in the mean time this is + # no problem. For removals, dllist will still hold a pointer to the + # Download, and additions are no problem (just won't be included + # in list of states returned via callback. + # + dllist = self.downloads.values() + finally: + self.sesslock.release() + + dslist = [] + for d in dllist: + ds = d.network_get_state(None,getpeerlist,sessioncalling=True) + dslist.append(ds) + + # Invoke the usercallback function via a new thread. + # After the callback is invoked, the return values will be passed to + # the returncallback for post-callback processing. + self.session.uch.perform_getstate_usercallback(usercallback,dslist,self.sesscb_set_download_states_returncallback) + + def sesscb_set_download_states_returncallback(self,usercallback,when,newgetpeerlist): + """ Called by SessionCallbackThread """ + if when > 0.0: + # reschedule + self.set_download_states_callback(usercallback,newgetpeerlist,when=when) + + # + # Persistence methods + # + def load_checkpoint(self,initialdlstatus=None): + """ Called by any thread """ + self.sesslock.acquire() + try: + dir = self.session.get_downloads_pstate_dir() + filelist = os.listdir(dir) + for basename in filelist: + # Make this go on when a torrent fails to start + filename = os.path.join(dir,basename) + self.resume_download(filename,initialdlstatus) + finally: + self.sesslock.release() + + + def load_download_pstate_noexc(self,infohash): + """ Called by any thread, assume sesslock already held """ + try: + dir = self.session.get_downloads_pstate_dir() + basename = binascii.hexlify(infohash)+'.pickle' + filename = os.path.join(dir,basename) + return self.load_download_pstate(filename) + except Exception,e: + # TODO: remove saved checkpoint? + #self.rawserver_nonfatalerrorfunc(e) + return None + + def resume_download(self,filename,initialdlstatus=None): + try: + # TODO: filter for file not found explicitly? + pstate = self.load_download_pstate(filename) + + if DEBUG: + print >>sys.stderr,"tlm: load_checkpoint: pstate is",dlstatus_strings[pstate['dlstate']['status']],pstate['dlstate']['progress'] + if pstate['engineresumedata'] is None: + print >>sys.stderr,"tlm: load_checkpoint: resumedata None" + else: + print >>sys.stderr,"tlm: load_checkpoint: resumedata len",len(pstate['engineresumedata']) + + tdef = TorrentDef.load_from_dict(pstate['metainfo']) + + # Activate + dscfg = DownloadStartupConfig(dlconfig=pstate['dlconfig']) + self.add(tdef,dscfg,pstate,initialdlstatus) + except Exception,e: + # TODO: remove saved checkpoint? + self.rawserver_nonfatalerrorfunc(e) + + + def checkpoint(self,stop=False,checkpoint=True,gracetime=2.0): + """ Called by any thread, assume sesslock already held """ + # Even if the list of Downloads changes in the mean time this is + # no problem. For removals, dllist will still hold a pointer to the + # Download, and additions are no problem (just won't be included + # in list of states returned via callback. + # + dllist = self.downloads.values() + if DEBUG: + print >>sys.stderr,"tlm: checkpointing",len(dllist) + + network_checkpoint_callback_lambda = lambda:self.network_checkpoint_callback(dllist,stop,checkpoint,gracetime) + self.rawserver.add_task(network_checkpoint_callback_lambda,0.0) + # TODO: checkpoint overlayapps / friendship msg handler + + + def network_checkpoint_callback(self,dllist,stop,checkpoint,gracetime): + """ Called by network thread """ + if checkpoint: + for d in dllist: + # Tell all downloads to stop, and save their persistent state + # in a infohash -> pstate dict which is then passed to the user + # for storage. + # + if DEBUG: + print >>sys.stderr,"tlm: network checkpointing:",`d.get_def().get_name()` + if stop: + (infohash,pstate) = d.network_stop(False,False) + else: + (infohash,pstate) = d.network_checkpoint() + + try: + self.save_download_pstate(infohash,pstate) + except Exception,e: + self.rawserver_nonfatalerrorfunc(e) + + if stop: + # Some grace time for early shutdown tasks + if self.shutdownstarttime is not None: + now = timemod.time() + diff = now - self.shutdownstarttime + if diff < gracetime: + print >>sys.stderr,"tlm: shutdown: delaying for early shutdown tasks",gracetime-diff + delay = gracetime-diff + network_shutdown_callback_lambda = lambda:self.network_shutdown() + self.rawserver.add_task(network_shutdown_callback_lambda,delay) + return + + self.network_shutdown() + + def early_shutdown(self): + """ Called as soon as Session shutdown is initiated. Used to start + shutdown tasks that takes some time and that can run in parallel + to checkpointing, etc. + """ + self.shutdownstarttime = timemod.time() + if self.overlay_apps is not None: + self.overlay_bridge.add_task(self.overlay_apps.early_shutdown,0) + if self.udppuncture_handler is not None: + self.udppuncture_handler.shutdown() + + def network_shutdown(self): + try: + # Detect if megacache is enabled + if self.peer_db is not None: + from BaseLib.Core.CacheDB.sqlitecachedb import SQLiteCacheDB + + db = SQLiteCacheDB.getInstance() + db.commit() + + mainlineDHT.deinit() + + ts = enumerate() + print >>sys.stderr,"tlm: Number of threads still running",len(ts) + for t in ts: + print >>sys.stderr,"tlm: Thread still running",t.getName(),"daemon",t.isDaemon(), "instance:", t + except: + print_exc() + + # Stop network thread + self.sessdoneflag.set() + # Arno, 2010-08-09: Stop Session pool threads only after gracetime + self.session.uch.shutdown() + + def save_download_pstate(self,infohash,pstate): + """ Called by network thread """ + basename = binascii.hexlify(infohash)+'.pickle' + filename = os.path.join(self.session.get_downloads_pstate_dir(),basename) + + if DEBUG: + print >>sys.stderr,"tlm: network checkpointing: to file",filename + f = open(filename,"wb") + pickle.dump(pstate,f) + f.close() + + + def load_download_pstate(self,filename): + """ Called by any thread """ + f = open(filename,"rb") + pstate = pickle.load(f) + f.close() + return pstate + + # + # External IP address methods + # + def guess_ext_ip_from_local_info(self): + """ Called at creation time """ + ip = get_my_wan_ip() + if ip is None: + host = socket.gethostbyname_ex(socket.gethostname()) + ipaddrlist = host[2] + for ip in ipaddrlist: + return ip + return '127.0.0.1' + else: + return ip + + def run(self): + if PROFILE: + fname = "profile-%s" % self.getName() + import cProfile + cProfile.runctx( "self._run()", globals(), locals(), filename=fname ) + import pstats + print >>sys.stderr,"profile: data for %s" % self.getName() + pstats.Stats(fname,stream=sys.stderr).sort_stats("cumulative").print_stats(20) + else: + self._run() + + def start_upnp(self): + """ Arno: as the UPnP discovery and calls to the firewall can be slow, + do it in a separate thread. When it fails, it should report popup + a dialog to inform and help the user. Or report an error in textmode. + + Must save type here, to handle case where user changes the type + In that case we still need to delete the port mapping using the old mechanism + + Called by network thread """ + + if DEBUG: + print >>sys.stderr,"tlm: start_upnp()" + self.set_activity(NTFY_ACT_UPNP) + self.upnp_thread = UPnPThread(self.upnp_type,self.locally_guessed_ext_ip,self.listen_port,self.upnp_failed_callback,self.upnp_got_ext_ip_callback) + self.upnp_thread.start() + + def stop_upnp(self): + """ Called by network thread """ + if self.upnp_type > 0: + self.upnp_thread.shutdown() + + def upnp_failed_callback(self,upnp_type,listenport,error_type,exc=None,listenproto='TCP'): + """ Called by UPnP thread TODO: determine how to pass to API user + In principle this is a non fatal error. But it is one we wish to + show to the user """ + print >>sys.stderr,"UPnP mode "+str(upnp_type)+" request to firewall failed with error "+str(error_type)+" Try setting a different mode in Preferences. Listen port was "+str(listenport)+", protocol"+listenproto,exc + + def upnp_got_ext_ip_callback(self,ip): + """ Called by UPnP thread """ + self.sesslock.acquire() + self.upnp_ext_ip = ip + self.sesslock.release() + + def dialback_got_ext_ip_callback(self,ip): + """ Called by network thread """ + self.sesslock.acquire() + self.dialback_ext_ip = ip + self.sesslock.release() + + def yourip_got_ext_ip_callback(self,ip): + """ Called by network thread """ + self.sesslock.acquire() + self.yourip_ext_ip = ip + if DEBUG: + print >> sys.stderr,"tlm: yourip_got_ext_ip_callback: others think my IP address is",ip + self.sesslock.release() + + + def get_ext_ip(self,unknowniflocal=False): + """ Called by any thread """ + self.sesslock.acquire() + try: + if self.dialback_ext_ip is not None: + # more reliable + return self.dialback_ext_ip # string immutable + elif self.upnp_ext_ip is not None: + # good reliability, if known + return self.upnp_ext_ip + elif self.yourip_ext_ip is not None: + # majority vote, could be rigged + return self.yourip_ext_ip + else: + # slighly wild guess + if unknowniflocal: + return None + else: + return self.locally_guessed_ext_ip + finally: + self.sesslock.release() + + + def get_int_ip(self): + """ Called by any thread """ + self.sesslock.acquire() + try: + return self.locally_guessed_ext_ip + finally: + self.sesslock.release() + + + # + # Events from core meant for API user + # + def dialback_reachable_callback(self): + """ Called by overlay+network thread """ + self.session.uch.notify(NTFY_REACHABLE, NTFY_INSERT, None, '') + + + def set_activity(self,type, str = '', arg2=None): + """ Called by overlay + network thread """ + #print >>sys.stderr,"tlm: set_activity",type,str,arg2 + self.session.uch.notify(NTFY_ACTIVITIES, NTFY_INSERT, type, str, arg2) + + + def network_vod_event_callback(self,videoinfo,event,params): + """ Called by network thread """ + + if DEBUG: + print >>sys.stderr,"tlm: network_vod_event_callback: event %s, params %s" % (event,params) + + # Call Session threadpool to call user's callback + try: + videoinfo['usercallback'](event,params) + except: + print_exc() + + + def update_torrent_checking_period(self): + # dynamically change the interval: update at least once per day + if self.overlay_apps and self.overlay_apps.metadata_handler: + ntorrents = self.overlay_apps.metadata_handler.num_torrents + if ntorrents > 0: + self.torrent_checking_period = min(max(86400/ntorrents, 15), 300) + #print >> sys.stderr, "torrent_checking_period", self.torrent_checking_period + #self.torrent_checking_period = 1 ### DEBUG, remove it before release!! + + def run_torrent_check(self): + """ Called by network thread """ + + self.update_torrent_checking_period() + self.rawserver.add_task(self.run_torrent_check, self.torrent_checking_period) + # print "torrent_checking start" + try: + from BaseLib.TrackerChecking.TorrentChecking import TorrentChecking + + t = TorrentChecking() + t.start() + except Exception, e: + print_exc() + self.rawserver_nonfatalerrorfunc(e) + + def get_coopdl_role_object(self,infohash,role): + """ Called by network thread """ + role_object = None + self.sesslock.acquire() + try: + if infohash in self.downloads: + d = self.downloads[infohash] + role_object = d.get_coopdl_role_object(role) + finally: + self.sesslock.release() + return role_object + + + def h4xor_reset_init_conn_counter(self): + self.rawserver.add_task(self.network_h4xor_reset,0) + + def network_h4xor_reset(self): + from BaseLib.Core.BitTornado.BT1.Encrypter import incompletecounter + print >>sys.stderr,"tlm: h4x0r Resetting outgoing TCP connection rate limiter",incompletecounter.c,"===" + incompletecounter.c = 0 + + + def setup_multicast_discovery(self): + # Set up local node discovery here + # TODO: Fetch these from system configuration + mc_config = {'permid':self.session.get_permid(), + 'multicast_ipv4_address':'224.0.1.43', + 'multicast_ipv6_address':'ff02::4124:1261:ffef', + 'multicast_port':'32109', + 'multicast_enabled':True, + 'multicast_ipv4_enabled':True, + 'multicast_ipv6_enabled':False, + 'multicast_announce':True} + + from BaseLib.Core.Overlay.SecureOverlay import OLPROTO_VER_CURRENT + from BaseLib.Core.Multicast import Multicast + + self.mc_channel = Multicast(mc_config,self.overlay_bridge,self.listen_port,OLPROTO_VER_CURRENT,self.peer_db) + self.mc_channel.addAnnounceHandler(self.mc_channel.handleOVERLAYSWARMAnnounce) + + self.mc_sock = self.mc_channel.getSocket() + self.mc_sock.setblocking(0) + + def start_multicast(self): + if not self.session.get_overlay() or not self.session.get_multicast_local_peer_discovery(): + return + + self.rawserver.start_listening_udp(self.mc_sock, self.mc_channel) + + print >>sys.stderr,"mcast: Sending node announcement" + params = [self.session.get_listen_port(), self.secure_overlay.olproto_ver_current] + self.mc_channel.sendAnnounce(params) + + +def singledownload_size_cmp(x,y): + """ Method that compares 2 SingleDownload objects based on the size of the + content of the BT1Download (if any) contained in them. + """ + if x is None and y is None: + return 0 + elif x is None: + return 1 + elif y is None: + return -1 + else: + a = x.get_bt1download() + b = y.get_bt1download() + if a is None and b is None: + return 0 + elif a is None: + return 1 + elif b is None: + return -1 + else: + if a.get_datalength() == b.get_datalength(): + return 0 + elif a.get_datalength() < b.get_datalength(): + return -1 + else: + return 1 + diff --git a/instrumentation/next-share/BaseLib/Core/APIImplementation/SessionRuntimeConfig.py b/instrumentation/next-share/BaseLib/Core/APIImplementation/SessionRuntimeConfig.py new file mode 100644 index 0000000..0888c79 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/APIImplementation/SessionRuntimeConfig.py @@ -0,0 +1,959 @@ +# Written by Arno Bakker +# see LICENSE.txt for license information + +from __future__ import with_statement +import sys +from traceback import print_exc + +from BaseLib.Core.exceptions import * +from BaseLib.Core.SessionConfig import SessionConfigInterface +from BaseLib.Core.Subtitles.SubtitlesHandler import SubtitlesHandler + +# 10/02/10 Boudewijn: pylint points out that member variables used in +# SessionRuntimeConfig do not exist. This is because they are set in +# BaseLib.Core.Session which is a subclass of SessionRuntimeConfig. +# +# We disable this error +# pylint: disable-msg=E1101 + +class SessionRuntimeConfig(SessionConfigInterface): + """ + Implements the BaseLib.Core.API.SessionConfigInterface + + Use these to change the session config at runtime. + """ + def set_state_dir(self,statedir): + raise OperationNotPossibleAtRuntimeException() + + def get_state_dir(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_state_dir(self) + finally: + self.sesslock.release() + + def set_install_dir(self,statedir): + raise OperationNotPossibleAtRuntimeException() + + def get_install_dir(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_install_dir(self) + finally: + self.sesslock.release() + + def set_permid_keypair_filename(self,keypair): + raise OperationNotPossibleAtRuntimeException() + + def get_permid_keypair_filename(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_permid_keypair_filename(self) + finally: + self.sesslock.release() + + def set_listen_port(self,port): + raise OperationNotPossibleAtRuntimeException() + + def get_listen_port(self): + # To protect self.sessconfig + self.sesslock.acquire() + try: + return SessionConfigInterface.get_listen_port(self) + finally: + self.sesslock.release() + + def get_video_analyser_path(self): + # To protect self.sessconfig + self.sesslock.acquire() + try: + return SessionConfigInterface.get_video_analyser_path(self) + finally: + self.sesslock.release() + + def set_tracker_ip(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_ip(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_ip(self) + finally: + self.sesslock.release() + + def set_bind_to_addresses(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_bind_to_addresses(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_bind_to_addresses(self) + finally: + self.sesslock.release() + + def set_upnp_mode(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_upnp_mode(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_upnp_mode(self) + finally: + self.sesslock.release() + + def set_autoclose_timeout(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_autoclose_timeout(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_autoclose_timeout(self) + finally: + self.sesslock.release() + + def set_autoclose_check_interval(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_autoclose_check_interval(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_autoclose_check_interval(self) + finally: + self.sesslock.release() + + def set_megacache(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_megacache(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_megacache(self) + finally: + self.sesslock.release() + + def set_overlay(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_overlay(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_overlay(self) + finally: + self.sesslock.release() + + def set_buddycast(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_buddycast(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_buddycast(self) + finally: + self.sesslock.release() + + def set_start_recommender(self,value): + self.sesslock.acquire() + try: + from BaseLib.Core.Overlay.OverlayThreadingBridge import OverlayThreadingBridge + + SessionConfigInterface.set_start_recommender(self,value) + olbridge = OverlayThreadingBridge.getInstance() + task = lambda:self.olthread_set_start_recommender(value) + olbridge.add_task(task,0) + finally: + self.sesslock.release() + + def olthread_set_start_recommender(self,value): + from BaseLib.Core.BuddyCast.buddycast import BuddyCastFactory + bcfac = BuddyCastFactory.getInstance() + if value: + bcfac.restartBuddyCast() + else: + bcfac.pauseBuddyCast() + + def get_start_recommender(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_start_recommender(self) + finally: + self.sesslock.release() + + # + # ProxyService_ + # + def set_download_help(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_download_help(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_download_help(self) + finally: + self.sesslock.release() + + def set_proxyservice_status(self,value): + """ Set the status of the proxyservice (on or off). + + ProxyService off means the current node could not be used as a proxy. ProxyService on means other nodes will be able to use it as a proxy. + + @param value: one of the possible two values: PROXYSERVICE_OFF, PROXYSERVICE_ON + """ + self.sesslock.acquire() + try: + SessionConfigInterface.set_proxyservice_status(self, value) + finally: + self.sesslock.release() + + def get_proxyservice_status(self): + """ Returns the status of the proxyservice (on or off). + @return: one of the possible two values: PROXYSERVICE_OFF, PROXYSERVICE_ON + """ + self.sesslock.acquire() + try: + return SessionConfigInterface.get_proxyservice_status(self) + finally: + self.sesslock.release() + # + # _ProxyService + # + + + + def set_torrent_collecting(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_torrent_collecting(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_torrent_collecting(self) + finally: + self.sesslock.release() + + + def set_torrent_collecting_dir(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_torrent_collecting_dir(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_torrent_collecting_dir(self) + finally: + self.sesslock.release() + + def get_subtitles_collecting_dir(self): + with self.sesslock: + return SessionConfigInterface.get_subtitles_collecting_dir(self) + + def set_subtitles_upload_rate(self, value): + with self.sesslock: + SubtitlesHandler.getInstance().setUploadRate(value) + SessionConfigInterface.set_subtitles_uploade_rate(self, value) + + def get_subtitles_upload_rate(self): + with self.sesslock: + return SessionConfigInterface.get_subtitles_upload_rate(self) + + + + def set_superpeer(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_superpeer(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_superpeer(self) + finally: + self.sesslock.release() + + def set_overlay_log(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_overlay_log(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_overlay_log(self) + finally: + self.sesslock.release() + + def set_buddycast_interval(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_buddycast_interval(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_buddycast_interval(self) + finally: + self.sesslock.release() + + def set_torrent_collecting_max_torrents(self,value): + self.sesslock.acquire() + try: + from BaseLib.Core.Overlay.OverlayThreadingBridge import OverlayThreadingBridge + + SessionConfigInterface.set_torrent_collecting_max_torrents(self,value) + olbridge = OverlayThreadingBridge.getInstance() + task = lambda:self.olthread_set_torrent_collecting_max_torrents(value) + olbridge.add_task(task,0) + finally: + self.sesslock.release() + + def olthread_set_torrent_collecting_max_torrents(self,value): + from BaseLib.Core.Overlay.MetadataHandler import MetadataHandler + mh = MetadataHandler.getInstance() + mh.set_overflow(value) + mh.delayed_check_overflow(2) + + + def get_torrent_collecting_max_torrents(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_torrent_collecting_max_torrents(self) + finally: + self.sesslock.release() + + def set_buddycast_max_peers(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_buddycast_max_peers(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_buddycast_max_peers(self) + finally: + self.sesslock.release() + + def set_torrent_collecting_rate(self,value): + self.sesslock.acquire() + try: + from BaseLib.Core.Overlay.OverlayThreadingBridge import OverlayThreadingBridge + + SessionConfigInterface.set_torrent_collecting_rate(self,value) + olbridge = OverlayThreadingBridge.getInstance() + task = lambda:self.olthread_set_torrent_collecting_rate(value) + olbridge.add_task(task,0) + finally: + self.sesslock.release() + + def olthread_set_torrent_collecting_rate(self,value): + from BaseLib.Core.Overlay.MetadataHandler import MetadataHandler + mh = MetadataHandler.getInstance() + mh.set_rate(value) + + def get_torrent_collecting_rate(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_torrent_collecting_rate(self) + finally: + self.sesslock.release() + + def set_torrent_checking(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_torrent_checking(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_torrent_checking(self) + finally: + self.sesslock.release() + + def set_torrent_checking_period(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_torrent_checking_period(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_torrent_checking_period(self) + finally: + self.sesslock.release() + + def set_dialback(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_dialback(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_dialback(self) + finally: + self.sesslock.release() + + def set_social_networking(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_social_networking(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_social_networking(self) + finally: + self.sesslock.release() + + def set_remote_query(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_remote_query(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_remote_query(self) + finally: + self.sesslock.release() + + def set_stop_collecting_threshold(self,value): + self.sesslock.acquire() + try: + from BaseLib.Core.Overlay.OverlayThreadingBridge import OverlayThreadingBridge + + SessionConfigInterface.set_stop_collecting_threshold(self,value) + olbridge = OverlayThreadingBridge.getInstance() + task = lambda:self.olthread_set_stop_collecting_threshold(value) + olbridge.add_task(task,0) + finally: + self.sesslock.release() + + def olthread_set_stop_collecting_threshold(self,value): + from BaseLib.Core.Overlay.MetadataHandler import MetadataHandler + mh = MetadataHandler.getInstance() + mh.set_min_free_space(value) + mh.delayed_check_free_space(2) + + def get_stop_collecting_threshold(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_stop_collecting_threshold(self) + finally: + self.sesslock.release() + + def set_internal_tracker(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_internal_tracker(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_internal_tracker(self) + finally: + self.sesslock.release() + + def set_internal_tracker_url(self,value): + raise OperationNotPossibleAtRuntimeException() + + #def get_internal_tracker_url(self): + """ Implemented in Session.py """ + + def set_mainline_dht(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_mainline_dht(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_mainline_dht(self) + finally: + self.sesslock.release() + + def set_nickname(self,value): + self.sesslock.acquire() + try: + return SessionConfigInterface.set_nickname(self, value) + finally: + self.sesslock.release() + + def get_nickname(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_nickname(self) + finally: + self.sesslock.release() + + def set_mugshot(self,value, mime='image/jpeg'): + self.sesslock.acquire() + try: + return SessionConfigInterface.set_mugshot(self, value, mime) + finally: + self.sesslock.release() + + def get_mugshot(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_mugshot(self) + finally: + self.sesslock.release() + + + def set_tracker_dfile(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_dfile(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_dfile(self) + finally: + self.sesslock.release() + + def set_tracker_dfile_format(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_dfile_format(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_dfile_format(self) + finally: + self.sesslock.release() + + def set_tracker_socket_timeout(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_socket_timeout(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_socket_timeout(self) + finally: + self.sesslock.release() + + def set_tracker_save_dfile_interval(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_save_dfile_interval(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_save_dfile_interval(self) + finally: + self.sesslock.release() + + def set_tracker_timeout_downloaders_interval(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_timeout_downloaders_interval(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_timeout_downloaders_interval(self) + finally: + self.sesslock.release() + + def set_tracker_reannounce_interval(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_reannounce_interval(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_reannounce_interval(self) + finally: + self.sesslock.release() + + def set_tracker_response_size(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_response_size(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_response_size(self) + finally: + self.sesslock.release() + + def set_tracker_timeout_check_interval(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_timeout_check_interval(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_timeout_check_interval(self) + finally: + self.sesslock.release() + + def set_tracker_nat_check(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_nat_check(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_nat_check(self) + finally: + self.sesslock.release() + + def set_tracker_log_nat_checks(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_log_nat_checks(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_log_nat_checks(self) + finally: + self.sesslock.release() + + def set_tracker_min_time_between_log_flushes(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_min_time_between_log_flushes(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_min_time_between_log_flushes(self) + finally: + self.sesslock.release() + + def set_tracker_min_time_between_cache_refreshes(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_min_time_between_cache_refreshes(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_min_time_between_cache_refreshes(self) + finally: + self.sesslock.release() + + def set_tracker_allowed_dir(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_allowed_dir(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_allowed_dir(self) + finally: + self.sesslock.release() + + def set_tracker_allowed_list(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_allowed_list(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_allowed_list(self) + finally: + self.sesslock.release() + + def set_tracker_allowed_controls(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_allowed_controls(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_allowed_controls(self) + finally: + self.sesslock.release() + + def set_tracker_multitracker_enabled(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_multitracker_enabled(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_multitracker_enabled(self) + finally: + self.sesslock.release() + + def set_tracker_multitracker_allowed(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_multitracker_allowed(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_multitracker_allowed(self) + finally: + self.sesslock.release() + + def set_tracker_multitracker_reannounce_interval(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_multitracker_reannounce_interval(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_multitracker_reannounce_interval(self) + finally: + self.sesslock.release() + + def set_tracker_multitracker_maxpeers(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_multitracker_maxpeers(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_multitracker_maxpeers(self) + finally: + self.sesslock.release() + + def set_tracker_aggregate_forward(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_aggregate_forward(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_aggregate_forward(self) + finally: + self.sesslock.release() + + def set_tracker_aggregator(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_aggregator(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_aggregator(self) + finally: + self.sesslock.release() + + def set_tracker_hupmonitor(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_hupmonitor(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_hupmonitor(self) + finally: + self.sesslock.release() + + def set_tracker_multitracker_http_timeout(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_multitracker_http_timeout(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_multitracker_http_timeout(self) + finally: + self.sesslock.release() + + def set_tracker_parse_dir_interval(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_parse_dir_interval(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_parse_dir_interval(self) + finally: + self.sesslock.release() + + def set_tracker_show_infopage(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_show_infopage(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_show_infopage(self) + finally: + self.sesslock.release() + + def set_tracker_infopage_redirect(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_infopage_redirect(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_infopage_redirect(self) + finally: + self.sesslock.release() + + def set_tracker_show_names(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_show_names(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_show_names(self) + finally: + self.sesslock.release() + + def set_tracker_favicon(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_favicon(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_favicon(self) + finally: + self.sesslock.release() + + def set_tracker_allowed_ips(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_allowed_ips(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_allowed_ips(self) + finally: + self.sesslock.release() + + def set_tracker_banned_ips(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_banned_ips(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_banned_ips(self) + finally: + self.sesslock.release() + + def set_tracker_only_local_override_ip(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_only_local_override_ip(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_only_local_override_ip(self) + finally: + self.sesslock.release() + + def set_tracker_logfile(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_logfile(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_logfile(self) + finally: + self.sesslock.release() + + def set_tracker_allow_get(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_allow_get(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_allow_get(self) + finally: + self.sesslock.release() + + def set_tracker_keep_dead(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_keep_dead(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_keep_dead(self) + finally: + self.sesslock.release() + + def set_tracker_scrape_allowed(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_tracker_scrape_allowed(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_tracker_scrape_allowed(self) + finally: + self.sesslock.release() + + def set_overlay_max_message_length(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_overlay_max_message_length(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_overlay_max_message_length(self) + finally: + self.sesslock.release() + + def set_download_help_dir(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_download_help_dir(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_download_help_dir(self) + finally: + self.sesslock.release() + + def set_bartercast(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_bartercast(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_bartercast(self) + finally: + self.sesslock.release() + + def set_superpeer_file(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_superpeer_file(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_superpeer_file(self) + finally: + self.sesslock.release() + + def set_buddycast_collecting_solution(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_buddycast_collecting_solution(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_buddycast_collecting_solution(self) + finally: + self.sesslock.release() + + def set_peer_icon_path(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_peer_icon_path(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_peer_icon_path(self) + finally: + self.sesslock.release() + + # + # NAT Puncturing servers information setting/retrieval + # + def set_nat_detect(self,value): + raise OperationNotPossibleAtRuntimeException() + + def set_puncturing_private_port(self, puncturing_private_port): + raise OperationNotPossibleAtRuntimeException() + + def set_stun_servers(self, stun_servers): + raise OperationNotPossibleAtRuntimeException() + + def set_pingback_servers(self, pingback_servers): + raise OperationNotPossibleAtRuntimeException() + + def set_puncturing_coordinators(self, puncturing_coordinators): + raise OperationNotPossibleAtRuntimeException() + + def get_nat_detect(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_nat_detect(self) + finally: + self.sesslock.release() + + def get_puncturing_internal_port(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_puncturing_internal_port(self) + finally: + self.sesslock.release() + + def get_stun_servers(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_stun_servers(self) + finally: + self.sesslock.release() + + def get_pingback_servers(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_pingback_servers(self) + finally: + self.sesslock.release() + + # + # Crawler + # + def set_crawler(self, value): + raise OperationNotPossibleAtRuntimeException() + + def get_crawler(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_crawler(self) + finally: + self.sesslock.release() + + # + # Local Peer Discovery using IP Multicast + # + def set_multicast_local_peer_discovery(self,value): + raise OperationNotPossibleAtRuntimeException() + + def get_multicast_local_peer_discovery(self): + self.sesslock.acquire() + try: + return SessionConfigInterface.get_multicast_local_peer_discovery(self) + finally: + self.sesslock.release() + diff --git a/instrumentation/next-share/BaseLib/Core/APIImplementation/SingleDownload.py b/instrumentation/next-share/BaseLib/Core/APIImplementation/SingleDownload.py new file mode 100644 index 0000000..8daa781 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/APIImplementation/SingleDownload.py @@ -0,0 +1,425 @@ +# Written by Arno Bakker, George Milescu +# see LICENSE.txt for license information + +import sys +import os +import time +import copy +import pickle +import socket +import binascii +from base64 import b64encode +from types import StringType,ListType,IntType +from traceback import print_exc,print_stack +from threading import Event + +from BaseLib.Core.simpledefs import * +from BaseLib.Core.exceptions import * +from BaseLib.Core.BitTornado.__init__ import createPeerID +from BaseLib.Core.BitTornado.download_bt1 import BT1Download +from BaseLib.Core.BitTornado.bencode import bencode,bdecode +from BaseLib.Core.Video.VideoStatus import VideoStatus +from BaseLib.Core.Video.SVCVideoStatus import SVCVideoStatus +from BaseLib.Core.DecentralizedTracking.repex import RePEXer +from BaseLib.Core.Statistics.Status.Status import get_status_holder + + +SPECIAL_VALUE = 481 + +DEBUG = False + +class SingleDownload: + """ This class is accessed solely by the network thread """ + + def __init__(self,infohash,metainfo,kvconfig,multihandler,get_extip_func,listenport,videoanalyserpath,vodfileindex,set_error_func,pstate,lmvodeventcallback,lmhashcheckcompletecallback): + self.dow = None + self.set_error_func = set_error_func + self.videoinfo = None + self.videostatus = None + self.lmvodeventcallback = lmvodeventcallback + self.lmhashcheckcompletecallback = lmhashcheckcompletecallback + self.logmsgs = [] + self._hashcheckfunc = None + self._getstatsfunc = None + self.infohash = infohash + self.b64_infohash = b64encode(infohash) + self.repexer = None + try: + self.dldoneflag = Event() + self.dlrawserver = multihandler.newRawServer(infohash,self.dldoneflag) + self.lmvodeventcallback = lmvodeventcallback + + if pstate is not None: + self.hashcheckfrac = pstate['dlstate']['progress'] + else: + self.hashcheckfrac = 0.0 + + self.peerid = createPeerID() + + # LOGGING + event_reporter = get_status_holder("LivingLab") + event_reporter.create_and_add_event("peerid", [self.b64_infohash, b64encode(self.peerid)]) + + #print >>sys.stderr,"SingleDownload: __init__: My peer ID is",`peerid` + + self.dow = BT1Download(self.hashcheckprogressfunc, + self.finishedfunc, + self.fatalerrorfunc, + self.nonfatalerrorfunc, + self.logerrorfunc, + self.dldoneflag, + kvconfig, + metainfo, + infohash, + self.peerid, + self.dlrawserver, + get_extip_func, + listenport, + videoanalyserpath + ) + + file = self.dow.saveAs(self.save_as) + #if DEBUG: + # print >>sys.stderr,"SingleDownload: dow.saveAs returned",file + + # Set local filename in vodfileindex + if vodfileindex is not None: + # Ric: for SVC the index is a list of indexes + index = vodfileindex['index'] + if type(index) == ListType: + svc = len(index) > 1 + else: + svc = False + + if svc: + outpathindex = self.dow.get_dest(index[0]) + else: + if index == -1: + index = 0 + outpathindex = self.dow.get_dest(index) + + vodfileindex['outpath'] = outpathindex + self.videoinfo = vodfileindex + if 'live' in metainfo['info']: + authparams = metainfo['info']['live'] + else: + authparams = None + if svc: + self.videostatus = SVCVideoStatus(metainfo['info']['piece length'],self.dow.files,vodfileindex,authparams) + else: + self.videostatus = VideoStatus(metainfo['info']['piece length'],self.dow.files,vodfileindex,authparams) + self.videoinfo['status'] = self.videostatus + self.dow.set_videoinfo(vodfileindex,self.videostatus) + + #if DEBUG: + # print >>sys.stderr,"SingleDownload: setting vodfileindex",vodfileindex + + # RePEX: Start in RePEX mode + if kvconfig['initialdlstatus'] == DLSTATUS_REPEXING: + if pstate is not None and pstate.has_key('dlstate'): + swarmcache = pstate['dlstate'].get('swarmcache',{}) + else: + swarmcache = {} + self.repexer = RePEXer(self.infohash, swarmcache) + else: + self.repexer = None + + if pstate is None: + resumedata = None + else: + # Restarting download + resumedata=pstate['engineresumedata'] + self._hashcheckfunc = self.dow.initFiles(resumedata=resumedata) + + + except Exception,e: + self.fatalerrorfunc(e) + + def get_bt1download(self): + return self.dow + + def save_as(self,name,length,saveas,isdir): + """ Return the local filename to which to save the file 'name' in the torrent """ + if DEBUG: + print >>sys.stderr,"SingleDownload: save_as(",`name`,length,`saveas`,isdir,")" + try: + if not os.access(saveas,os.F_OK): + os.mkdir(saveas) + path = os.path.join(saveas,name) + if isdir and not os.path.isdir(path): + os.mkdir(path) + return path + except Exception,e: + self.fatalerrorfunc(e) + + def perform_hashcheck(self,complete_callback): + """ Called by any thread """ + if DEBUG: + print >>sys.stderr,"SingleDownload: perform_hashcheck()" # ,self.videoinfo + try: + """ Schedules actually hashcheck on network thread """ + self._getstatsfunc = SPECIAL_VALUE # signal we're hashchecking + # Already set, should be same + self.lmhashcheckcompletecallback = complete_callback + self._hashcheckfunc(self.lmhashcheckcompletecallback) + except Exception,e: + self.fatalerrorfunc(e) + + def hashcheck_done(self): + """ Called by LaunchMany when hashcheck complete and the Download can be + resumed + + Called by network thread + """ + if DEBUG: + print >>sys.stderr,"SingleDownload: hashcheck_done()" + try: + self.dow.startEngine(vodeventfunc = self.lmvodeventcallback) + self._getstatsfunc = self.dow.startStats() # not possible earlier + + # RePEX: don't start the Rerequester in RePEX mode + repexer = self.repexer + if repexer is None: + # ProxyService_ + # + # ProxyDevel + # If proxymode is PROXY_MODE_PRIVATE, deactivate the tracker support + download_config = self.get_bt1download().getConfig() + proxy_mode = download_config.get('proxy_mode',0) + + # Only activate the tracker if the proxy_mode is PROXY_MODE_OFF or PROXY_MODE_SPEED + if proxy_mode == PROXY_MODE_OFF or proxy_mode == PROXY_MODE_SPEED: + self.dow.startRerequester() + if DEBUG: + print "Tracker class has been activated." + str(proxy_mode) + else: + #self.dow.startRerequester() + if DEBUG: + print "Tracker class has not been activated." + str(proxy_mode) + # + #_ProxyService + else: + self.hook_repexer() + + self.dlrawserver.start_listening(self.dow.getPortHandler()) + except Exception,e: + self.fatalerrorfunc(e) + + + # DownloadConfigInterface methods + def set_max_speed(self,direct,speed,callback): + if self.dow is not None: + if DEBUG: + print >>sys.stderr,"SingleDownload: set_max_speed",`self.dow.response['info']['name']`,direct,speed + if direct == UPLOAD: + self.dow.setUploadRate(speed,networkcalling=True) + else: + self.dow.setDownloadRate(speed,networkcalling=True) + if callback is not None: + callback(direct,speed) + + def set_max_conns_to_initiate(self,nconns,callback): + if self.dow is not None: + if DEBUG: + print >>sys.stderr,"SingleDownload: set_max_conns_to_initiate",`self.dow.response['info']['name']` + self.dow.setInitiate(nconns,networkcalling=True) + if callback is not None: + callback(nconns) + + + def set_max_conns(self,nconns,callback): + if self.dow is not None: + if DEBUG: + print >>sys.stderr,"SingleDownload: set_max_conns",`self.dow.response['info']['name']` + self.dow.setMaxConns(nconns,networkcalling=True) + if callback is not None: + callback(nconns) + + + # + # For DownloadState + # + def get_stats(self,getpeerlist): + logmsgs = self.logmsgs[:] # copy + coopdl_helpers = [] + coopdl_coordinator = None + if self.dow is not None: + if not self.dow.helper is None: + coopdl_coordinator = self.dow.helper.get_coordinator_permid() + if self.dow.coordinator is not None: + # No coordinator when you're a helper + peerreclist = self.dow.coordinator.network_get_asked_helpers_copy() + for peerrec in peerreclist: + coopdl_helpers.append(peerrec['permid']) + if self._getstatsfunc is None: + return (DLSTATUS_WAITING4HASHCHECK,None,logmsgs,coopdl_helpers,coopdl_coordinator) + elif self._getstatsfunc == SPECIAL_VALUE: + stats = {} + stats['frac'] = self.hashcheckfrac + return (DLSTATUS_HASHCHECKING,stats,logmsgs,coopdl_helpers,coopdl_coordinator) + else: + # RePEX: if we're repexing, set our status + if self.repexer is not None: + status = DLSTATUS_REPEXING + else: + status = None + return (status,self._getstatsfunc(getpeerlist=getpeerlist),logmsgs,coopdl_helpers,coopdl_coordinator) + + def get_infohash(self): + return self.infohash + + # + # Persistent State + # + def checkpoint(self): + if self.dow is not None: + return self.dow.checkpoint() + else: + return None + + def shutdown(self): + if DEBUG: + print >>sys.stderr,"SingleDownload: shutdown" + resumedata = None + if self.dow is not None: + # RePEX: unhook and abort RePEXer + if self.repexer: + repexer = self.unhook_repexer() + repexer.repex_aborted(self.infohash, DLSTATUS_STOPPED) + + self.dldoneflag.set() + self.dlrawserver.shutdown() + resumedata = self.dow.shutdown() + self.dow = None + #if DEBUG: + # print >>sys.stderr,"SingleDownload: stopped dow" + + if self._getstatsfunc is None or self._getstatsfunc == SPECIAL_VALUE: + # Hashchecking or waiting for while being shutdown, signal LaunchMany + # so it can schedule a new one. + self.lmhashcheckcompletecallback(success=False) + + return resumedata + + # + # RePEX, Raynor Vliegendhart: + # Restarting a running Download previously was a NoOp according to + # DownloadImpl, but now the decision is left up to SingleDownload. + def restart(self, initialdlstatus=None): + """ + Called by network thread. Called when Download was already running + and Download.restart() was called. + """ + if self.repexer and initialdlstatus != DLSTATUS_REPEXING: + # kill the RePEX process + repexer = self.unhook_repexer() + repexer.repex_aborted(self.infohash, initialdlstatus) + else: + pass # NoOp, continue with download as before + + + # + # RePEX: get_swarmcache + # + def get_swarmcache(self): + """ + Returns the last stored swarmcache when RePEXing otherwise None. + + @return A dict mapping dns to a dict with at least 'last_seen' + and 'pex' keys. + """ + if self.repexer is not None: + return self.repexer.get_swarmcache()[0] + return None + + # + # RePEX: Hooking and unhooking the RePEXer + # + def hook_repexer(self): + repexer = self.repexer + if repexer is None: + return + self.dow.Pause() + + # create Rerequester in BT1D just to be sure, but don't start it + # (this makes sure that Encoder.rerequest != None) + self.dow.startRerequester(paused=True) + + connecter, encoder = self.dow.connecter, self.dow.encoder + connecter.repexer = repexer + encoder.repexer = repexer + rerequest = self.dow.createRerequester(repexer.rerequester_peers) + repexer.repex_ready(self.infohash, connecter, encoder, rerequest) + + def unhook_repexer(self): + repexer = self.repexer + if repexer is None: + return + self.repexer = None + if self.dow is not None: + connecter, encoder = self.dow.connecter, self.dow.encoder + connecter.repexer = None + encoder.repexer = None + self.dow.startRerequester() # not started, so start it. + self.dow.Unpause() + return repexer + + # + # Cooperative download + # + def ask_coopdl_helpers(self,peerreclist): + if self.dow is not None: + self.dow.coordinator.send_ask_for_help(peerreclist) + + def stop_coopdl_helpers(self,peerreclist): + if self.dow is not None: + self.dow.coordinator.send_stop_helping(peerreclist,force=True) + + def get_coopdl_role_object(self,role): + # Used by Coordinator/HelperMessageHandler indirectly + if self.dow is not None: + if role == COOPDL_ROLE_COORDINATOR: + return self.dow.coordinator + else: + return self.dow.helper + else: + return None + + # + # Internal methods + # + def hashcheckprogressfunc(self,activity = '', fractionDone = 0.0): + """ Allegedly only used by StorageWrapper during hashchecking """ + #print >>sys.stderr,"SingleDownload::statusfunc called",activity,fractionDone + self.hashcheckfrac = fractionDone + + def finishedfunc(self): + """ Download is complete """ + if DEBUG: + print >>sys.stderr,"SingleDownload::finishedfunc called: Download is complete *******************************" + pass + + def fatalerrorfunc(self,data): + print >>sys.stderr,"SingleDownload::fatalerrorfunc called",data + if type(data) == StringType: + print >>sys.stderr,"LEGACY CORE FATAL ERROR",data + print_stack() + self.set_error_func(TriblerLegacyException(data)) + else: + print_exc() + self.set_error_func(data) + self.shutdown() + + def nonfatalerrorfunc(self,e): + print >>sys.stderr,"SingleDownload::nonfatalerrorfunc called",e + # Could log this somewhere, or phase it out (only used in Rerequester) + + def logerrorfunc(self,msg): + t = time.time() + self.logmsgs.append((t,msg)) + + # Keep max 10 log entries, API user should save them if he wants + # complete history + if len(self.logmsgs) > 10: + self.logmsgs.pop(0) + diff --git a/instrumentation/next-share/BaseLib/Core/APIImplementation/ThreadPool.py b/instrumentation/next-share/BaseLib/Core/APIImplementation/ThreadPool.py new file mode 100644 index 0000000..99b0a10 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/APIImplementation/ThreadPool.py @@ -0,0 +1,180 @@ +# Written by Jelle Roozenburg, Arno Bakker +# see LICENSE.txt for license information + +import sys +import time +from traceback import print_exc +import threading + +class ThreadPool: + + """Flexible thread pool class. Creates a pool of threads, then + accepts tasks that will be dispatched to the next available + thread.""" + + def __init__(self, numThreads): + + """Initialize the thread pool with numThreads workers.""" + + self.__threads = [] + self.__resizeLock = threading.Condition(threading.Lock()) + self.__taskCond = threading.Condition(threading.Lock()) + self.__tasks = [] + self.__isJoiningStopQueuing = False + self.__isJoining = False + self.setThreadCount(numThreads) + + def setThreadCount(self, newNumThreads): + + """ External method to set the current pool size. Acquires + the resizing lock, then calls the internal version to do real + work.""" + + # Can't change the thread count if we're shutting down the pool! + if self.__isJoining: + return False + + self.__resizeLock.acquire() + try: + self.__setThreadCountNolock(newNumThreads) + finally: + self.__resizeLock.release() + return True + + def __setThreadCountNolock(self, newNumThreads): + + """Set the current pool size, spawning or terminating threads + if necessary. Internal use only; assumes the resizing lock is + held.""" + + # If we need to grow the pool, do so + while newNumThreads > len(self.__threads): + newThread = ThreadPoolThread(self) + self.__threads.append(newThread) + newThread.start() + # If we need to shrink the pool, do so + while newNumThreads < len(self.__threads): + self.__threads[0].goAway() + del self.__threads[0] + + def getThreadCount(self): + + """Return the number of threads in the pool.""" + + self.__resizeLock.acquire() + try: + return len(self.__threads) + finally: + self.__resizeLock.release() + + def queueTask(self, task, args=(), taskCallback=None): + + """Insert a task into the queue. task must be callable; + args and taskCallback can be None.""" + + if self.__isJoining == True or self.__isJoiningStopQueuing: + return False + if not callable(task): + return False + + self.__taskCond.acquire() + try: + self.__tasks.append((task, args, taskCallback)) + # Arno, 2010-04-07: Use proper notify()+wait() + self.__taskCond.notifyAll() + return True + finally: + self.__taskCond.release() + + def getNextTask(self): + + """ Retrieve the next task from the task queue. For use + only by ThreadPoolThread objects contained in the pool.""" + + self.__taskCond.acquire() + try: + while self.__tasks == [] and not self.__isJoining: + self.__taskCond.wait() + if self.__isJoining: + return (None, None, None) + else: + return self.__tasks.pop(0) + finally: + self.__taskCond.release() + + def joinAll(self, waitForTasks = True, waitForThreads = True): + + """ Clear the task queue and terminate all pooled threads, + optionally allowing the tasks and threads to finish.""" + + # Mark the pool as joining to prevent any more task queueing + self.__isJoiningStopQueuing = True + + # Wait for tasks to finish + if waitForTasks: + while self.__tasks != []: + time.sleep(.1) + + # Mark the pool as joining to make all threads stop executing tasks + self.__isJoining = True + + # Tell all the threads to quit + self.__resizeLock.acquire() + try: + self.__setThreadCountNolock(0) + self.__isJoining = True + + # Wait until all threads have exited + if waitForThreads: + for t in self.__threads: + t.join() + del t + + # Reset the pool for potential reuse + self.__isJoining = False + finally: + self.__resizeLock.release() + + + +class ThreadPoolThread(threading.Thread): + + """ Pooled thread class. """ + + def __init__(self, pool): + + """ Initialize the thread and remember the pool. """ + + threading.Thread.__init__(self) + self.setName('SessionPool'+self.getName()) + self.setDaemon(True) + self.__pool = pool + self.__isDying = False + + def run(self): + + """ Until told to quit, retrieve the next task and execute + it, calling the callback if any. """ + + # Arno, 2010-04-07: Dying only used when shrinking pool now. + while self.__isDying == False: + # Arno, 2010-01-28: add try catch block. Sometimes tasks lists grow, + # could be because all Threads are dying. + try: + cmd, args, callback = self.__pool.getNextTask() + if cmd is None: + break + elif callback is None: + cmd(*args) + else: + callback(cmd(args)) + except: + print_exc() + + + def goAway(self): + + """ Exit the run loop next time through.""" + + self.__isDying = True + \ No newline at end of file diff --git a/instrumentation/next-share/BaseLib/Core/APIImplementation/UserCallbackHandler.py b/instrumentation/next-share/BaseLib/Core/APIImplementation/UserCallbackHandler.py new file mode 100644 index 0000000..ef5b8f4 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/APIImplementation/UserCallbackHandler.py @@ -0,0 +1,133 @@ +# Written by Arno Bakker +# see LICENSE.txt for license information + +import sys +import os +import shutil +import binascii +from threading import currentThread +from traceback import print_exc + +from BaseLib.Core.simpledefs import * +from BaseLib.Core.APIImplementation.ThreadPool import ThreadPool +from BaseLib.Core.CacheDB.Notifier import Notifier + +DEBUG = False + +class UserCallbackHandler: + + def __init__(self,session): + self.session = session + self.sesslock = session.sesslock + self.sessconfig = session.sessconfig + + # Notifier for callbacks to API user + self.threadpool = ThreadPool(2) + self.notifier = Notifier.getInstance(self.threadpool) + + def shutdown(self): + # stop threadpool + self.threadpool.joinAll() + + def perform_vod_usercallback(self,d,usercallback,event,params): + """ Called by network thread """ + if DEBUG: + print >>sys.stderr,"Session: perform_vod_usercallback()",`d.get_def().get_name_as_unicode()` + def session_vod_usercallback_target(): + try: + usercallback(d,event,params) + except: + print_exc() + self.perform_usercallback(session_vod_usercallback_target) + + def perform_getstate_usercallback(self,usercallback,data,returncallback): + """ Called by network thread """ + if DEBUG: + print >>sys.stderr,"Session: perform_getstate_usercallback()" + def session_getstate_usercallback_target(): + try: + (when,getpeerlist) = usercallback(data) + returncallback(usercallback,when,getpeerlist) + except: + print_exc() + self.perform_usercallback(session_getstate_usercallback_target) + + + def perform_removestate_callback(self,infohash,contentdest,removecontent): + """ Called by network thread """ + if DEBUG: + print >>sys.stderr,"Session: perform_removestate_callback()" + def session_removestate_callback_target(): + if DEBUG: + print >>sys.stderr,"Session: session_removestate_callback_target called",currentThread().getName() + try: + self.sesscb_removestate(infohash,contentdest,removecontent) + except: + print_exc() + self.perform_usercallback(session_removestate_callback_target) + + def perform_usercallback(self,target): + self.sesslock.acquire() + try: + # TODO: thread pool, etc. + self.threadpool.queueTask(target) + + finally: + self.sesslock.release() + + def sesscb_removestate(self,infohash,contentdest,removecontent): + """ See DownloadImpl.setup(). + Called by SessionCallbackThread """ + if DEBUG: + print >>sys.stderr,"Session: sesscb_removestate called",`infohash`,`contentdest`,removecontent + self.sesslock.acquire() + try: + if self.session.lm.download_exists(infohash): + print >>sys.stderr,"Session: sesscb_removestate: Download is back, restarted? Canceling removal!",`infohash` + return + + dlpstatedir = os.path.join(self.sessconfig['state_dir'],STATEDIR_DLPSTATE_DIR) + finally: + self.sesslock.release() + + # See if torrent uses internal tracker + try: + self.session.remove_from_internal_tracker_by_infohash(infohash) + except: + # Show must go on + print_exc() + + # Remove checkpoint + hexinfohash = binascii.hexlify(infohash) + try: + basename = hexinfohash+'.pickle' + filename = os.path.join(dlpstatedir,basename) + if DEBUG: + print >>sys.stderr,"Session: sesscb_removestate: removing dlcheckpoint entry",filename + if os.access(filename,os.F_OK): + os.remove(filename) + except: + # Show must go on + print_exc() + + # Remove downloaded content from disk + if removecontent: + if DEBUG: + print >>sys.stderr,"Session: sesscb_removestate: removing saved content",contentdest + if not os.path.isdir(contentdest): + # single-file torrent + os.remove(contentdest) + else: + # multi-file torrent + shutil.rmtree(contentdest,True) # ignore errors + + + def notify(self, subject, changeType, obj_id, *args): + """ + Notify all interested observers about an event with threads from the pool + """ + if DEBUG: + print >>sys.stderr,"ucb: notify called:",subject,changeType,`obj_id`, args + self.notifier.notify(subject,changeType,obj_id,*args) + + diff --git a/instrumentation/next-share/BaseLib/Core/APIImplementation/__init__.py b/instrumentation/next-share/BaseLib/Core/APIImplementation/__init__.py new file mode 100644 index 0000000..395f8fb --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/APIImplementation/__init__.py @@ -0,0 +1,2 @@ +# Written by Arno Bakker +# see LICENSE.txt for license information diff --git a/instrumentation/next-share/BaseLib/Core/APIImplementation/maketorrent.py b/instrumentation/next-share/BaseLib/Core/APIImplementation/maketorrent.py new file mode 100644 index 0000000..0084480 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/APIImplementation/maketorrent.py @@ -0,0 +1,630 @@ +# Written by Arno Bakker, Bram Cohen +# multitracker extensions by John Hoffman +# modified for Merkle hashes and digital signatures by Arno Bakker +# see LICENSE.txt for license information + +import sys +import os +import md5 +import zlib + +from BaseLib.Core.Utilities.Crypto import sha +from copy import copy +from time import time +from traceback import print_exc +from types import LongType + +from BaseLib.Core.BitTornado.bencode import bencode +from BaseLib.Core.BitTornado.BT1.btformats import check_info +from BaseLib.Core.Merkle.merkle import MerkleTree +from BaseLib.Core.Utilities.unicode import str2unicode,bin2unicode +from BaseLib.Core.APIImplementation.miscutils import parse_playtime_to_secs,offset2piece +from BaseLib.Core.osutils import fix_filebasename +from BaseLib.Core.defaults import tdefdictdefaults + + +ignore = [] # Arno: was ['core', 'CVS'] + +DEBUG = False + +def make_torrent_file(input, userabortflag = None, userprogresscallback = lambda x: None): + """ Create a torrent file from the supplied input. + + Returns a (infohash,metainfo) pair, or (None,None) on userabort. """ + + (info,piece_length) = makeinfo(input,userabortflag,userprogresscallback) + if userabortflag is not None and userabortflag.isSet(): + return (None,None) + if info is None: + return (None,None) + + #if DEBUG: + # print >>sys.stderr,"mktorrent: makeinfo returned",`info` + + check_info(info) + metainfo = {'info': info, 'encoding': input['encoding'], 'creation date': long(time())} + + # http://www.bittorrent.org/DHT_protocol.html says both announce and nodes + # are not allowed, but some torrents (Azureus?) apparently violate this. + if input['nodes'] is None and input['announce'] is None: + raise ValueError('No tracker set') + + for key in ['announce','announce-list','nodes','comment','created by','httpseeds', 'url-list']: + if input[key] is not None and len(input[key]) > 0: + metainfo[key] = input[key] + if key == 'comment': + metainfo['comment.utf-8'] = uniconvert(input['comment'],'utf-8') + + # Assuming 1 file, Azureus format no support multi-file torrent with diff + # bitrates + bitrate = None + for file in input['files']: + if file['playtime'] is not None: + secs = parse_playtime_to_secs(file['playtime']) + bitrate = file['length']/secs + break + if input.get('bps') is not None: + bitrate = input['bps'] + break + + if bitrate is not None or input['thumb'] is not None: + mdict = {} + mdict['Publisher'] = 'Tribler' + if input['comment'] is None: + descr = '' + else: + descr = input['comment'] + mdict['Description'] = descr + + if bitrate is not None: + mdict['Progressive'] = 1 + mdict['Speed Bps'] = int(bitrate) # bencode fails for float + else: + mdict['Progressive'] = 0 + + mdict['Title'] = metainfo['info']['name'] + mdict['Creation Date'] = long(time()) + # Azureus client source code doesn't tell what this is, so just put in random value from real torrent + mdict['Content Hash'] = 'PT3GQCPW4NPT6WRKKT25IQD4MU5HM4UY' + mdict['Revision Date'] = long(time()) + if input['thumb'] is not None: + mdict['Thumbnail'] = input['thumb'] + cdict = {} + cdict['Content'] = mdict + metainfo['azureus_properties'] = cdict + + if input['torrentsigkeypairfilename'] is not None: + from BaseLib.Core.Overlay.permid import create_torrent_signature + + create_torrent_signature(metainfo,input['torrentsigkeypairfilename']) + + if 'url-compat' in input: + metainfo['info']['url-compat'] = input['url-compat'] + + # Arno, 2010-03-02: + # Theoretically should go into 'info' field, to get infohash protection + # because the video won't play without them. In the future we'll sign + # the whole .torrent IMHO so it won't matter. Keeping it out of 'info' + # at the moment makes the .tstream files more stable (in case you restart + # the live source, and the Ogg header generated contains some date or + # what not, we'd need a new .tstream to be distributed to all. + # + if 'ogg-headers' in input: + metainfo['ogg-headers'] = input['ogg-headers'] + + + # Two places where infohash calculated, here and in TorrentDef. + # Elsewhere: must use TorrentDef.get_infohash() to allow P2PURLs. + + infohash = sha(bencode(info)).digest() + return (infohash,metainfo) + + +def uniconvertl(l, e): + """ Convert a pathlist to a list of strings encoded in encoding "e" using + uniconvert. """ + r = [] + try: + for s in l: + r.append(uniconvert(s, e)) + except UnicodeError: + raise UnicodeError('bad filename: '+os.path.join(l)) + return r + +def uniconvert(s, enc): + """ Convert 's' to a string containing a Unicode sequence encoded using + encoding "enc". If 's' is not a Unicode object, we first try to convert + it to one, guessing the encoding if necessary. """ + if not isinstance(s, unicode): + try: + s = bin2unicode(s,enc) + except UnicodeError: + raise UnicodeError('bad filename: '+s) + return s.encode(enc) + + +def makeinfo(input,userabortflag,userprogresscallback): + """ Calculate hashes and create torrent file's 'info' part """ + encoding = input['encoding'] + + pieces = [] + sh = sha() + done = 0L + fs = [] + totalsize = 0L + totalhashed = 0L + + # 1. Determine which files should go into the torrent (=expand any dirs + # specified by user in input['files'] + subs = [] + for file in input['files']: + inpath = file['inpath'] + outpath = file['outpath'] + + if DEBUG: + print >>sys.stderr,"makeinfo: inpath",inpath,"outpath",outpath + + if os.path.isdir(inpath): + dirsubs = subfiles(inpath) + subs.extend(dirsubs) + else: + if outpath is None: + subs.append(([os.path.basename(inpath)],inpath)) + else: + subs.append((filename2pathlist(outpath,skipfirst=True),inpath)) + + subs.sort() + + # 2. Calc total size + newsubs = [] + for p, f in subs: + if 'live' in input: + size = input['files'][0]['length'] + else: + size = os.path.getsize(f) + totalsize += size + newsubs.append((p,f,size)) + subs = newsubs + + # 3. Calc piece length from totalsize if not set + if input['piece length'] == 0: + if input['createmerkletorrent']: + # used to be 15=32K, but this works better with slow python + piece_len_exp = 18 + else: + if totalsize > 8L*1024*1024*1024: # > 8 gig = + piece_len_exp = 21 # 2 meg pieces + elif totalsize > 2*1024*1024*1024: # > 2 gig = + piece_len_exp = 20 # 1 meg pieces + elif totalsize > 512*1024*1024: # > 512M = + piece_len_exp = 19 # 512K pieces + elif totalsize > 64*1024*1024: # > 64M = + piece_len_exp = 18 # 256K pieces + elif totalsize > 16*1024*1024: # > 16M = + piece_len_exp = 17 # 128K pieces + elif totalsize > 4*1024*1024: # > 4M = + piece_len_exp = 16 # 64K pieces + else: # < 4M = + piece_len_exp = 15 # 32K pieces + piece_length = 2 ** piece_len_exp + else: + piece_length = input['piece length'] + + # 4. Read files and calc hashes, if not live + if 'live' not in input: + for p, f, size in subs: + pos = 0L + + h = open(f, 'rb') + + if input['makehash_md5']: + hash_md5 = md5.new() + if input['makehash_sha1']: + hash_sha1 = sha() + if input['makehash_crc32']: + hash_crc32 = zlib.crc32('') + + while pos < size: + a = min(size - pos, piece_length - done) + + # See if the user cancelled + if userabortflag is not None and userabortflag.isSet(): + return (None,None) + + readpiece = h.read(a) + + # See if the user cancelled + if userabortflag is not None and userabortflag.isSet(): + return (None,None) + + sh.update(readpiece) + + if input['makehash_md5']: + # Update MD5 + hash_md5.update(readpiece) + + if input['makehash_crc32']: + # Update CRC32 + hash_crc32 = zlib.crc32(readpiece, hash_crc32) + + if input['makehash_sha1']: + # Update SHA1 + hash_sha1.update(readpiece) + + done += a + pos += a + totalhashed += a + + if done == piece_length: + pieces.append(sh.digest()) + done = 0 + sh = sha() + + if userprogresscallback is not None: + userprogresscallback(float(totalhashed) / float(totalsize)) + + newdict = {'length': num2num(size), + 'path': uniconvertl(p,encoding), + 'path.utf-8': uniconvertl(p, 'utf-8') } + + # Find and add playtime + for file in input['files']: + if file['inpath'] == f: + if file['playtime'] is not None: + newdict['playtime'] = file['playtime'] + break + + if input['makehash_md5']: + newdict['md5sum'] = hash_md5.hexdigest() + if input['makehash_crc32']: + newdict['crc32'] = "%08X" % hash_crc32 + if input['makehash_sha1']: + newdict['sha1'] = hash_sha1.digest() + + fs.append(newdict) + + h.close() + + if done > 0: + pieces.append(sh.digest()) + + # 5. Create info dict + if len(subs) == 1: + flkey = 'length' + flval = num2num(totalsize) + name = subs[0][0][0] + else: + flkey = 'files' + flval = fs + + outpath = input['files'][0]['outpath'] + l = filename2pathlist(outpath) + name = l[0] + + infodict = { 'piece length':num2num(piece_length), flkey: flval, + 'name': uniconvert(name,encoding), + 'name.utf-8': uniconvert(name,'utf-8')} + + if 'live' not in input: + + if input['createmerkletorrent']: + merkletree = MerkleTree(piece_length,totalsize,None,pieces) + root_hash = merkletree.get_root_hash() + infodict.update( {'root hash': root_hash } ) + else: + infodict.update( {'pieces': ''.join(pieces) } ) + else: + # With source auth, live is a dict + infodict['live'] = input['live'] + + if 'cs_keys' in input: + # This is a closed swarm - add torrent keys + infodict['cs_keys'] = input['cs_keys'] + + if 'ns-metadata' in input: + # This has P2P-Next metadata, store in info field to make it + # immutable. + infodict['ns-metadata'] = input['ns-metadata'] + + if len(subs) == 1: + # Find and add playtime + for file in input['files']: + if file['inpath'] == f: + if file['playtime'] is not None: + infodict['playtime'] = file['playtime'] + + return (infodict,piece_length) + + +def subfiles(d): + """ Return list of (pathlist,local filename) tuples for all the files in + directory 'd' """ + r = [] + stack = [([], d)] + while stack: + p, n = stack.pop() + if os.path.isdir(n): + for s in os.listdir(n): + if s not in ignore and s[:1] != '.': + stack.append((copy(p) + [s], os.path.join(n, s))) + else: + r.append((p, n)) + return r + + +def filename2pathlist(path,skipfirst=False): + """ Convert a filename to a 'path' entry suitable for a multi-file torrent + file """ + #if DEBUG: + # print >>sys.stderr,"mktorrent: filename2pathlist:",path,skipfirst + + h = path + l = [] + while True: + #if DEBUG: + # print >>sys.stderr,"mktorrent: filename2pathlist: splitting",h + + (h,t) = os.path.split(h) + if h == '' and t == '': + break + if h == '' and skipfirst: + continue + if t != '': # handle case where path ends in / (=path separator) + l.append(t) + + l.reverse() + #if DEBUG: + # print >>sys.stderr,"mktorrent: filename2pathlist: returning",l + + return l + + +def pathlist2filename(pathlist): + """ Convert a multi-file torrent file 'path' entry to a filename. """ + fullpath = '' + for elem in pathlist: + fullpath = os.path.join(fullpath,elem) + return fullpath + +def pathlist2savefilename(pathlist,encoding): + fullpath = u'' + for elem in pathlist: + u = bin2unicode(elem,encoding) + b = fix_filebasename(u) + fullpath = os.path.join(fullpath,b) + return fullpath + +def torrentfilerec2savefilename(filerec,length=None): + if length is None: + length = len(filerec['path']) + if 'path.utf-8' in filerec: + key = 'path.utf-8' + encoding = 'utf-8' + else: + key = 'path' + encoding = None + + return pathlist2savefilename(filerec[key][:length],encoding) + +def savefilenames2finaldest(fn1,fn2): + """ Returns the join of two savefilenames, possibly shortened + to adhere to OS specific limits. + """ + j = os.path.join(fn1,fn2) + if sys.platform == 'win32': + # Windows has a maximum path length of 260 + # http://msdn2.microsoft.com/en-us/library/aa365247.aspx + j = j[:259] # 260 don't work. + return j + + +def num2num(num): + """ Converts long to int if small enough to fit """ + if type(num) == LongType and num < sys.maxint: + return int(num) + else: + return num + +def get_torrentfilerec_from_metainfo(filename,metainfo): + info = metainfo['info'] + if filename is None: + return info + + if filename is not None and 'files' in info: + for i in range(len(info['files'])): + x = info['files'][i] + + intorrentpath = pathlist2filename(x['path']) + if intorrentpath == filename: + return x + + raise ValueError("File not found in torrent") + else: + raise ValueError("File not found in single-file torrent") + +def get_bitrate_from_metainfo(file,metainfo): + info = metainfo['info'] + if file is None: + bitrate = None + try: + playtime = None + if info.has_key('playtime'): + #print >>sys.stderr,"TorrentDef: get_bitrate: Bitrate in info field" + playtime = parse_playtime_to_secs(info['playtime']) + elif 'playtime' in metainfo: # HACK: encode playtime in non-info part of existing torrent + #print >>sys.stderr,"TorrentDef: get_bitrate: Bitrate in metainfo" + playtime = parse_playtime_to_secs(metainfo['playtime']) + elif 'azureus_properties' in metainfo: + azprop = metainfo['azureus_properties'] + if 'Content' in azprop: + content = metainfo['azureus_properties']['Content'] + if 'Speed Bps' in content: + bitrate = float(content['Speed Bps']) + #print >>sys.stderr,"TorrentDef: get_bitrate: Bitrate in Azureus metainfo",bitrate + if playtime is not None: + bitrate = info['length']/playtime + if DEBUG: + print >>sys.stderr,"TorrentDef: get_bitrate: Found bitrate",bitrate + except: + print_exc() + + return bitrate + + if file is not None and 'files' in info: + for i in range(len(info['files'])): + x = info['files'][i] + + intorrentpath = '' + for elem in x['path']: + intorrentpath = os.path.join(intorrentpath,elem) + bitrate = None + try: + playtime = None + if x.has_key('playtime'): + playtime = parse_playtime_to_secs(x['playtime']) + elif 'playtime' in metainfo: # HACK: encode playtime in non-info part of existing torrent + playtime = parse_playtime_to_secs(metainfo['playtime']) + elif 'azureus_properties' in metainfo: + azprop = metainfo['azureus_properties'] + if 'Content' in azprop: + content = metainfo['azureus_properties']['Content'] + if 'Speed Bps' in content: + bitrate = float(content['Speed Bps']) + #print >>sys.stderr,"TorrentDef: get_bitrate: Bitrate in Azureus metainfo",bitrate + + if playtime is not None: + bitrate = x['length']/playtime + except: + print_exc() + + if intorrentpath == file: + return bitrate + + raise ValueError("File not found in torrent") + else: + raise ValueError("File not found in single-file torrent: "+file) + + +def get_length_filepieceranges_from_metainfo(metainfo,selectedfiles): + + if 'files' not in metainfo['info']: + # single-file torrent + return (metainfo['info']['length'],None) + else: + # multi-file torrent + files = metainfo['info']['files'] + piecesize = metainfo['info']['piece length'] + + total = 0L + filepieceranges = [] + for i in xrange(len(files)): + path = files[i]['path'] + length = files[i]['length'] + filename = pathlist2filename(path) + + if length > 0 and (not selectedfiles or (selectedfiles and filename in selectedfiles)): + range = (offset2piece(total,piecesize), offset2piece(total + length,piecesize),filename) + filepieceranges.append(range) + total += length + return (total,filepieceranges) + + +def copy_metainfo_to_input(metainfo,input): + + keys = tdefdictdefaults.keys() + # Arno: For magnet link support + keys.append("initial peers") + for key in keys: + if key in metainfo: + input[key] = metainfo[key] + + infokeys = ['name','piece length','live','url-compat'] + for key in infokeys: + if key in metainfo['info']: + input[key] = metainfo['info'][key] + + # Note: don't know inpath, set to outpath + if 'length' in metainfo['info']: + outpath = metainfo['info']['name'] + if 'playtime' in metainfo['info']: + playtime = metainfo['info']['playtime'] + else: + playtime = None + length = metainfo['info']['length'] + d = {'inpath':outpath,'outpath':outpath,'playtime':playtime,'length':length} + input['files'].append(d) + else: # multi-file torrent + files = metainfo['info']['files'] + for file in files: + outpath = pathlist2filename(file['path']) + if 'playtime' in file: + playtime = file['playtime'] + else: + playtime = None + length = file['length'] + d = {'inpath':outpath,'outpath':outpath,'playtime':playtime,'length':length} + input['files'].append(d) + + if 'azureus_properties' in metainfo: + azprop = metainfo['azureus_properties'] + if 'Content' in azprop: + content = metainfo['azureus_properties']['Content'] + if 'Thumbnail' in content: + input['thumb'] = content['Thumbnail'] + + if 'live' in metainfo['info']: + input['live'] = metainfo['info']['live'] + + if 'cs_keys' in metainfo['info']: + input['cs_keys'] = metainfo['info']['cs_keys'] + + if 'url-compat' in metainfo['info']: + input['url-compat'] = metainfo['info']['url-compat'] + + if 'ogg-headers' in metainfo: + input['ogg-headers'] = metainfo['ogg-headers'] + + if 'ns-metadata' in metainfo['info']: + input['ns-metadata'] = metainfo['info']['ns-metadata'] + + # Diego : we want web seeding + if 'url-list' in metainfo: + input['url-list'] = metainfo['url-list'] + + if 'httpseeds' in metainfo: + input['httpseeds'] = metainfo['httpseeds'] + + +def get_files(metainfo,exts): + # 01/02/10 Boudewijn: now returns (file, length) tuples instead of files + + videofiles = [] + if 'files' in metainfo['info']: + # Multi-file torrent + files = metainfo['info']['files'] + for file in files: + + p = file['path'] + #print >>sys.stderr,"TorrentDef: get_files: file is",p + filename = '' + for elem in p: + #print >>sys.stderr,"TorrentDef: get_files: elem is",elem + filename = os.path.join(filename,elem) + + #print >>sys.stderr,"TorrentDef: get_files: composed filename is",filename + (prefix,ext) = os.path.splitext(filename) + if ext != '' and ext[0] == '.': + ext = ext[1:] + #print >>sys.stderr,"TorrentDef: get_files: ext",ext + if exts is None or ext.lower() in exts: + videofiles.append((filename, file['length'])) + else: + #print >>sys.stderr,"TorrentDef: get_files: Single-torrent file" + + filename = metainfo['info']['name'] # don't think we need fixed name here + (prefix,ext) = os.path.splitext(filename) + if ext != '' and ext[0] == '.': + ext = ext[1:] + if exts is None or ext.lower() in exts: + videofiles.append((filename, metainfo['info']['length'])) + return videofiles diff --git a/instrumentation/next-share/BaseLib/Core/APIImplementation/makeurl.py b/instrumentation/next-share/BaseLib/Core/APIImplementation/makeurl.py new file mode 100644 index 0000000..34a5ae8 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/APIImplementation/makeurl.py @@ -0,0 +1,354 @@ +# Written by Arno Bakker +# see LICENSE.txt for license information +# +# TODO: +# * Test suite +# * Tracker support: how do they determine which files to seed. +# +# * Reverse support for URL-compat: URLs that do use infohash. +# - Make sure internal tracker understands URL-compat torrentfiles +# - Make sure internal tracker understands P2P URLs +# +# ISSUE: what if trackers have query parts? Is that officially/practically allowed? + + +import sys +import urlparse +import urllib +import math +if sys.platform != "win32": + import curses.ascii +from types import IntType, LongType +from struct import pack, unpack +from base64 import b64encode, b64decode +from M2Crypto import Rand # TODO REMOVE FOR LICHT +from traceback import print_exc,print_stack + +from BaseLib.Core.simpledefs import * +from BaseLib.Core.Utilities.Crypto import sha + + +DEBUG = False + + +def metainfo2p2purl(metainfo): + """ metainfo must be a Merkle torrent or a live torrent with an + 'encoding' field set. + @return URL + """ + info = metainfo['info'] + + bitrate = None + if 'azureus_properties' in metainfo: + azprops = metainfo['azureus_properties'] + if 'Content' in azprops: + content = metainfo['azureus_properties']['Content'] + if 'Speed Bps' in content: + bitrate = content['Speed Bps'] + + if 'encoding' not in metainfo: + encoding = 'utf-8' + else: + encoding = metainfo['encoding'] + + urldict = {} + + urldict['s'] = p2purl_encode_piecelength(info['piece length']) + # Warning: mbcs encodings sometimes don't work well under python! + urldict['n'] = p2purl_encode_name2url(info['name'],encoding) + + if info.has_key('length'): + urldict['l'] = p2purl_encode_nnumber(info['length']) + else: + raise ValueError("Multi-file torrents currently not supported") + #list = [] + #for filedict in info['files']: + # newdict = {} + # newdict['p'] = list_filename_escape(filedict['path']) + # newdict['l'] = p2purl_encode_nnumber(filedict['length']) + # list.append(newdict) + #urldict['f'] = '' # TODO bencode(list) + if info.has_key('root hash'): + urldict['r'] = b64urlencode(info['root hash']) + elif info.has_key('live'): + urldict['k'] = b64urlencode(info['live']['pubkey']) + urldict['a'] = info['live']['authmethod'] + else: + raise ValueError("url-compat and Merkle torrent must be on to create URL") + + if bitrate is not None: + urldict['b'] = p2purl_encode_nnumber(bitrate) + + query = '' + for k in ['n','r','k','l','s','a','b']: + if k in urldict: + if query != "": + query += '&' + v = urldict[k] + if k == 'n': + s = v + else: + s = k+"="+v + query += s + + sidx = metainfo['announce'].find(":") + hierpart = metainfo['announce'][sidx+1:] + url = P2PURL_SCHEME+':'+hierpart+"?"+query + return url + + + +def p2purl2metainfo(url): + """ Returns (metainfo,swarmid) """ + + if DEBUG: + print >>sys.stderr,"p2purl2metainfo: URL",url + + # Python's urlparse only supports a defined set of schemes, if not + # recognized, everything becomes path. Handy. + colidx = url.find(":") + scheme = url[0:colidx] + qidx = url.find("?") + if qidx == -1: + # Compact form, no authority part and path rootless + authority = None + path = None + query = url[colidx+1:] + fragment = None + else: + # Long form, with authority + authoritypath = url[colidx+3:qidx] + pidx = authoritypath.find("/") + authority = authoritypath[0:pidx] + path = authoritypath[pidx:] + fidx = url.find("#") + if fidx == -1: + # No fragment + query = url[qidx+1:] + fragment = None + else: + query = url[qidx+1:fidx] + fragment = url[fidx:] + + # Check port no. + csbidx = authority.find("]") + if authority.startswith("[") and csbidx != -1: + # Literal IPv6 address + if csbidx == len(authority)-1: + port = None + else: + port = authority[csbidx+1:] + else: + cidx = authority.find(":") + if cidx != -1: + port = authority[cidx+1:] + else: + port = None + if port is not None and not port.isdigit(): + raise ValueError("Port not int") + + + if scheme != P2PURL_SCHEME: + raise ValueError("Unknown scheme "+P2PURL_SCHEME) + + metainfo = {} + if authority and path: + metainfo['announce'] = 'http://'+authority+path + # Check for malformedness + result = urlparse.urlparse(metainfo['announce']) + if result[0] != "http": + raise ValueError("Malformed tracker URL") + + + reqinfo = p2purl_parse_query(query) + metainfo.update(reqinfo) + + swarmid = metainfo2swarmid(metainfo) + + if DEBUG: + print >>sys.stderr,"p2purl2metainfo: parsed",`metainfo` + + + return (metainfo,swarmid) + +def metainfo2swarmid(metainfo): + if 'live' in metainfo['info']: + swarmid = pubkey2swarmid(metainfo['info']['live']) + else: + swarmid = metainfo['info']['root hash'] + return swarmid + + +def p2purl_parse_query(query): + if DEBUG: + print >>sys.stderr,"p2purl_parse_query: query",query + + gotname = False + gotkey = False + gotrh = False + gotlen = False + gotps = False + gotam = False + gotbps = False + + reqinfo = {} + reqinfo['info'] = {} + + # Hmmm... could have used urlparse.parse_qs + kvs = query.split('&') + for kv in kvs: + if '=' not in kv: + # Must be name + reqinfo['info']['name'] = p2purl_decode_name2utf8(kv) + reqinfo['encoding'] = 'UTF-8' + gotname = True + continue + + k,v = kv.split('=') + + if k =='k' or k == 'a' and not ('live' in reqinfo['info']): + reqinfo['info']['live'] = {} + + if k == 'n': + reqinfo['info']['name'] = p2purl_decode_name2utf8(v) + reqinfo['encoding'] = 'UTF-8' + gotname = True + elif k == 'r': + reqinfo['info']['root hash'] = p2purl_decode_base64url(v) + gotrh = True + elif k == 'k': + reqinfo['info']['live']['pubkey'] = p2purl_decode_base64url(v) + # reqinfo['info']['live']['authmethod'] = pubkey2authmethod(reqinfo['info']['live']['pubkey']) + gotkey = True + elif k == 'l': + reqinfo['info']['length'] = p2purl_decode_nnumber(v) + gotlen = True + elif k == 's': + reqinfo['info']['piece length'] = p2purl_decode_piecelength(v) + gotps = True + elif k == 'a': + reqinfo['info']['live']['authmethod'] = v + gotam = True + elif k == 'b': + bitrate = p2purl_decode_nnumber(v) + reqinfo['azureus_properties'] = {} + reqinfo['azureus_properties']['Content'] = {} + reqinfo['azureus_properties']['Content']['Speed Bps'] = bitrate + gotbps = True + + if not gotname: + raise ValueError("Missing name field") + if not gotrh and not gotkey: + raise ValueError("Missing root hash or live pub key field") + if gotrh and gotkey: + raise ValueError("Found both root hash and live pub key field") + if not gotlen: + raise ValueError("Missing length field") + if not gotps: + raise ValueError("Missing piece size field") + if gotkey and not gotam: + raise ValueError("Missing live authentication method field") + if gotrh and gotam: + raise ValueError("Inconsistent: root hash and live authentication method field") + + if not gotbps: + raise ValueError("Missing bitrate field") + + return reqinfo + + +def pubkey2swarmid(livedict): + """ Calculate SHA1 of pubkey (or cert). + Make X.509 Subject Key Identifier compatible? + """ + if DEBUG: + print >>sys.stderr,"pubkey2swarmid:",livedict.keys() + + if livedict['authmethod'] == "None": + # No live-source auth + return Rand.rand_bytes(20) + else: + return sha(livedict['pubkey']).digest() + + +def p2purl_decode_name2utf8(v): + """ URL decode name to UTF-8 encoding """ + if sys.platform != "win32": + for c in v: + if not curses.ascii.isascii(c): + raise ValueError("Name contains unescaped 8-bit value "+`c`) + return urllib.unquote_plus(v) + +def p2purl_encode_name2url(name,encoding): + """ Encode name in specified encoding to URL escaped UTF-8 """ + + if encoding.lower() == 'utf-8': + utf8name = name + else: + uname = unicode(name, encoding) + utf8name = uname.encode('utf-8') + return urllib.quote_plus(utf8name) + + + +def p2purl_decode_base64url(v): + return b64urldecode(v) + +# +# Convert Python number to binary value of sufficient bytes, +# in network-byte order and BASE64-URL encode that binary value, or vice versa. +# +def p2purl_decode_nnumber(s): + b = b64urldecode(s) + if len(b) == 2: + format = "H" + elif len(b) == 4: + format = "l" + else: + format = "Q" + format = "!"+format # network-byte order + return unpack(format,b)[0] + +def p2purl_encode_nnumber(s): + if type(s) == IntType: + if s < 2 ** 16: + format = "H" + elif s < 2 ** 32: + format = "l" + else: + format = "Q" + format = "!"+format # network-byte order + return b64urlencode(pack(format,s)) + + +# +# Convert Python power-of-two piecelength to text value, or vice versa. +# +def p2purl_decode_piecelength(s): + return int(math.pow(2.0,float(s))) + +def p2purl_encode_piecelength(s): + return str(int(math.log(float(s),2.0))) + +# +# "Modified BASE64 for URL" as informally specified in +# http://en.wikipedia.org/wiki/Base64#URL_applications +# +def b64urlencode(input): + output = b64encode(input) + output = output.rstrip('=') + output = output.replace('+','-') + output = output.replace('/','_') + return output + +def b64urldecode(input): + inter = input[:] + # readd padding. + padlen = 4 - (len(inter) - ((len(inter) / 4) * 4)) + padstr = '=' * padlen + inter += padstr + inter = inter.replace('-','+') + inter = inter.replace('_','/') + output = b64decode(inter) + return output + diff --git a/instrumentation/next-share/BaseLib/Core/APIImplementation/miscutils.py b/instrumentation/next-share/BaseLib/Core/APIImplementation/miscutils.py new file mode 100644 index 0000000..40d3f37 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/APIImplementation/miscutils.py @@ -0,0 +1,43 @@ +# Written by Arno Bakker +# see LICENSE.txt for license information + +import sys +import re +from threading import Timer + +DEBUG = False + +def parse_playtime_to_secs(hhmmss): + if DEBUG: + print >>sys.stderr,"miscutils: Playtime is",hhmmss + r = re.compile("([0-9\.]+):*") + occ = r.findall(hhmmss) + t = None + if len(occ) > 0: + if len(occ) == 3: + # hours as well + t = int(occ[0])*3600 + int(occ[1])*60 + float(occ[2]) + elif len(occ) == 2: + # minutes and seconds + t = int(occ[0])*60 + float(occ[1]) + elif len(occ) == 1: + # seconds + t = float(occ[0]) + # Arno, 2010-07-05: Bencode doesn't support floats + return int(t) + + +def offset2piece(offset,piecesize): + + p = offset / piecesize + if offset % piecesize > 0: + p += 1 + return p + + + +def NamedTimer(*args,**kwargs): + t = Timer(*args,**kwargs) + t.setDaemon(True) + t.setName("NamedTimer"+t.getName()) + return t diff --git a/instrumentation/next-share/BaseLib/Core/Base.py b/instrumentation/next-share/BaseLib/Core/Base.py new file mode 100644 index 0000000..5a1ac93 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/Base.py @@ -0,0 +1,30 @@ +# Written by Arno Bakker +# see LICENSE.txt for license information +""" Base classes for the Core API """ + +from BaseLib.Core.exceptions import * + +DEBUG = False + +# +# Tribler API base classes +# +class Serializable: + """ + Interface to signal that the object is pickleable. + """ + def __init__(self): + pass + +class Copyable: + """ + Interface for copying an instance (or rather signaling that it can be + copied) + """ + def copy(self): + """ + Copies the instance. + @param self an unbound instance of the class + @return Returns a copy of "self" + """ + raise NotYetImplementedException() diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/Choker.py b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/Choker.py new file mode 100644 index 0000000..515ea7b --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/Choker.py @@ -0,0 +1,269 @@ +# Written by Bram Cohen, Pawel Garbacki, Boxun Zhang +# see LICENSE.txt for license information + +from random import randrange, shuffle +import sys + +from BaseLib.Core.BitTornado.clock import clock + +try: + True +except: + True = 1 + False = 0 + +DEBUG = False + +class Choker: + def __init__(self, config, schedule, picker, seeding_selector, done = lambda: False): + self.config = config + self.round_robin_period = config['round_robin_period'] + self.schedule = schedule + self.picker = picker + self.connections = [] + self.last_preferred = 0 + self.last_round_robin = clock() + self.done = done + self.super_seed = False + self.paused = False + schedule(self._round_robin, 5) + + # SelectiveSeeding + self.seeding_manager = None + + + def set_round_robin_period(self, x): + self.round_robin_period = x + + def _round_robin(self): + self.schedule(self._round_robin, 5) + if self.super_seed: + cons = range(len(self.connections)) + to_close = [] + count = self.config['min_uploads']-self.last_preferred + if count > 0: # optimization + shuffle(cons) + for c in cons: + # SelectiveSeeding + if self.seeding_manager is None or self.seeding_manager.is_conn_eligible(c): + + i = self.picker.next_have(self.connections[c], count > 0) + if i is None: + continue + if i < 0: + to_close.append(self.connections[c]) + continue + self.connections[c].send_have(i) + count -= 1 + else: + # Drop non-eligible connections + to_close.append(self.connections[c]) + for c in to_close: + c.close() + if self.last_round_robin + self.round_robin_period < clock(): + self.last_round_robin = clock() + for i in xrange(1, len(self.connections)): + c = self.connections[i] + + # SelectiveSeeding + if self.seeding_manager is None or self.seeding_manager.is_conn_eligible(c): + u = c.get_upload() + if u.is_choked() and u.is_interested(): + self.connections = self.connections[i:] + self.connections[:i] + break + self._rechoke() + + def _rechoke(self): + # 2fast + helper = self.picker.helper + if helper is not None and helper.coordinator is None and helper.is_complete(): + for c in self.connections: + if not c.connection.is_coordinator_con(): + u = c.get_upload() + u.choke() + return + + if self.paused: + for c in self.connections: + c.get_upload().choke() + return + + # NETWORK AWARE + if 'unchoke_bias_for_internal' in self.config: + checkinternalbias = self.config['unchoke_bias_for_internal'] + else: + checkinternalbias = 0 + + if DEBUG: + print >>sys.stderr,"choker: _rechoke: checkinternalbias",checkinternalbias + + # 0. Construct candidate list + preferred = [] + maxuploads = self.config['max_uploads'] + if maxuploads > 1: + + # 1. Get some regular candidates + for c in self.connections: + + # g2g: unchoke some g2g peers later + if c.use_g2g: + continue + + # SelectiveSeeding + if self.seeding_manager is None or self.seeding_manager.is_conn_eligible(c): + u = c.get_upload() + if not u.is_interested(): + continue + if self.done(): + r = u.get_rate() + else: + d = c.get_download() + r = d.get_rate() + if r < 1000 or d.is_snubbed(): + continue + + # NETWORK AWARENESS + if checkinternalbias and c.na_get_address_distance() == 0: + r += checkinternalbias + if DEBUG: + print >>sys.stderr,"choker: _rechoke: BIASING",c.get_ip(),c.get_port() + + preferred.append((-r, c)) + + self.last_preferred = len(preferred) + preferred.sort() + del preferred[maxuploads-1:] + if DEBUG: + print >>sys.stderr,"choker: _rechoke: NORMAL UNCHOKE",preferred + preferred = [x[1] for x in preferred] + + # 2. Get some g2g candidates + g2g_preferred = [] + for c in self.connections: + if not c.use_g2g: + continue + + # SelectiveSeeding + if self.seeding_manager is None or self.seeding_manager.is_conn_eligible(c): + + u = c.get_upload() + if not u.is_interested(): + continue + + r = c.g2g_score() + if checkinternalbias and c.na_get_address_distance() == 0: + r[0] += checkinternalbias + r[1] += checkinternalbias + if DEBUG: + print >>sys.stderr,"choker: _rechoke: G2G BIASING",c.get_ip(),c.get_port() + + g2g_preferred.append((-r[0], -r[1], c)) + + g2g_preferred.sort() + del g2g_preferred[maxuploads-1:] + if DEBUG: + print >>sys.stderr,"choker: _rechoke: G2G UNCHOKE",g2g_preferred + g2g_preferred = [x[2] for x in g2g_preferred] + + preferred += g2g_preferred + + + # + count = len(preferred) + hit = False + to_unchoke = [] + + # 3. The live source must always unchoke its auxiliary seeders + # LIVESOURCE + if 'live_aux_seeders' in self.config: + + for hostport in self.config['live_aux_seeders']: + for c in self.connections: + if c.get_ip() == hostport[0]: + u = c.get_upload() + to_unchoke.append(u) + #print >>sys.stderr,"Choker: _rechoke: LIVE: Permanently unchoking aux seed",hostport + + # 4. Select from candidate lists, aux seeders always selected + for c in self.connections: + u = c.get_upload() + if c in preferred: + to_unchoke.append(u) + else: + if count < maxuploads or not hit: + if self.seeding_manager is None or self.seeding_manager.is_conn_eligible(c): + to_unchoke.append(u) + if u.is_interested(): + count += 1 + if DEBUG and not hit: print >>sys.stderr,"choker: OPTIMISTIC UNCHOKE",c + hit = True + + else: + if not c.connection.is_coordinator_con() and not c.connection.is_helper_con(): + u.choke() + elif u.is_choked(): + to_unchoke.append(u) + + # 5. Unchoke selected candidates + for u in to_unchoke: + u.unchoke() + + + def add_connection(self, connection, p = None): + """ + Just add a connection, do not start doing anything yet + Must call "start_connection" later! + """ + print >>sys.stderr, "Added connection",connection + if p is None: + p = randrange(-2, len(self.connections) + 1) + connection.get_upload().choke() + self.connections.insert(max(p, 0), connection) + self.picker.got_peer(connection) + self._rechoke() + + def start_connection(self, connection): + connection.get_upload().unchoke() + + def connection_made(self, connection, p = None): + if p is None: + p = randrange(-2, len(self.connections) + 1) + self.connections.insert(max(p, 0), connection) + self.picker.got_peer(connection) + self._rechoke() + + def connection_lost(self, connection): + """ connection is a Connecter.Connection """ + # Raynor Vliegendhart, RePEX: + # The RePEX code can close a connection right after the handshake + # but before the Choker has been informed via connection_made. + # However, Choker.connection_lost is still called when a connection + # is closed, so we should check whether Choker knows the connection: + if connection in self.connections: + self.connections.remove(connection) + self.picker.lost_peer(connection) + if connection.get_upload().is_interested() and not connection.get_upload().is_choked(): + self._rechoke() + + def interested(self, connection): + if not connection.get_upload().is_choked(): + self._rechoke() + + def not_interested(self, connection): + if not connection.get_upload().is_choked(): + self._rechoke() + + def set_super_seed(self): + while self.connections: # close all connections + self.connections[0].close() + self.picker.set_superseed() + self.super_seed = True + + def pause(self, flag): + self.paused = flag + self._rechoke() + + # SelectiveSeeding + def set_seeding_manager(self, manager): + # When seeding starts, a non-trivial seeding manager will be set + self.seeding_manager = manager diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/Connecter.py b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/Connecter.py new file mode 100644 index 0000000..e24929c --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/Connecter.py @@ -0,0 +1,1698 @@ +# Written by Bram Cohen, Pawel Garbacki, Arno Bakker and Njaal Borch, George Milescu +# see LICENSE.txt for license information + +import time +import sys +from types import DictType,IntType,LongType,ListType,StringType +from random import shuffle +from traceback import print_exc,print_stack +from math import ceil +import socket +import urlparse + +from threading import Event # Wait for CS to complete + +from BaseLib.Core.BitTornado.bitfield import Bitfield +from BaseLib.Core.BitTornado.clock import clock +from BaseLib.Core.BitTornado.bencode import bencode,bdecode +from BaseLib.Core.BitTornado.__init__ import version_short,decodePeerID,TRIBLER_PEERID_LETTER +from BaseLib.Core.BitTornado.BT1.convert import tobinary,toint + +from BaseLib.Core.BitTornado.BT1.MessageID import * +from BaseLib.Core.DecentralizedTracking.MagnetLink.__init__ import * + +from BaseLib.Core.DecentralizedTracking.ut_pex import * +from BaseLib.Core.BitTornado.BT1.track import compact_ip,decompact_ip + +from BaseLib.Core.BitTornado.CurrentRateMeasure import Measure +from BaseLib.Core.ClosedSwarm import ClosedSwarm +from BaseLib.Core.Statistics.Status import Status + +KICK_OLD_CLIENTS=False + +DEBUG = False +DEBUG_NORMAL_MSGS = False +DEBUG_UT_PEX = False +DEBUG_MESSAGE_HANDLING = False +DEBUG_CS = False # Debug closed swarms + +UNAUTH_PERMID_PERIOD = 3600 + +# allow FACTOR times the metadata to be uploaded each PERIOD. +# Example: +# FACTOR = 2 and PERIOD = 60 will allow all the metadata to be +# uploaded 2 times every 60 seconds. +UT_METADATA_FLOOD_FACTOR = 1 +UT_METADATA_FLOOD_PERIOD = 5 * 60 * 60 + +""" +Arno: 2007-02-16: +uTorrent and Bram's BitTorrent now support an extension to the protocol, +documented on http://www.bittorrent.org/beps/bep_0010.html (previously +http://www.rasterbar.com/products/libtorrent/extension_protocol.html) + +The problem is that the bit they use in the options field of the BT handshake +is the same as we use to indicate a peer supports the overlay-swarm connection. +The new clients will send an EXTEND message with ID 20 after the handshake to +inform the otherside what new messages it supports. + +As a result, Tribler <= 3.5.0 clients won't be confused, but can't talk to these +new clients either or vice versa. The new client will think we understand the +message, send it. But because we don't know that message ID, we will close +the connection. Our attempts to establish a new overlay connection with the new +client will gracefully fail, as the new client will not know of infohash=00000... +and close the connection. + +We solve this conflict by adding support for the EXTEND message. We are now be +able to receive it, and send our own. Our message will contain one method name, +i.e. Tr_OVERLAYSWARM=253. Processing is now as follows: + +* If bit 43 is set and the peerID is from an old Tribler (<=3.5.0) + peer, we initiate an overlay-swarm connection. +* If bit 43 is set and the peer's EXTEND hs message contains method Tr_OVERLAYSWARM, + it's a new Tribler peer, and we initiate an overlay-swarm connection. +* If bit 43 is set, and the EXTEND hs message does not contain Tr_OVERLAYSWARM + it's not a Tribler client and we do not initiate an overlay-swarm + connection. + +N.B. The EXTEND message is poorly designed, it lacks protocol versioning +support which is present in the Azureus Extended Messaging Protocol +and our overlay-swarm protocol. + +""" +EXTEND_MSG_HANDSHAKE_ID = chr(0) +EXTEND_MSG_OVERLAYSWARM = 'Tr_OVERLAYSWARM' +EXTEND_MSG_G2G_V1 = 'Tr_G2G' +EXTEND_MSG_G2G_V2 = 'Tr_G2G_v2' +EXTEND_MSG_HASHPIECE = 'Tr_hashpiece' +EXTEND_MSG_CS = 'NS_CS' + +CURRENT_LIVE_VERSION=1 +EXTEND_MSG_LIVE_PREFIX = 'Tr_LIVE_v' +LIVE_FAKE_MESSAGE_ID = chr(254) + + + +G2G_CALLBACK_INTERVAL = 4 + +def show(s): + text = [] + for i in xrange(len(s)): + text.append(ord(s[i])) + return text + + +class Connection: + def __init__(self, connection, connecter): + self.connection = connection + self.connecter = connecter + self.got_anything = False + self.next_upload = None + self.outqueue = [] + self.partial_message = None + self.download = None + self.upload = None + self.send_choke_queued = False + self.just_unchoked = None + self.unauth_permid = None + self.looked_for_permid = UNAUTH_PERMID_PERIOD-3 + self.closed = False + self.extend_hs_dict = {} # what extended messages does this peer support + self.initiated_overlay = False + + # G2G + self.use_g2g = False # set to true if both sides use G2G, indicated by self.connector.use_g2g + self.g2g_version = None + self.perc_sent = {} + # batch G2G_XFER information and periodically send it out. + self.last_perc_sent = {} + + config = self.connecter.config + self.forward_speeds = [Measure(config['max_rate_period'], config['upload_rate_fudge']), + Measure(config['max_rate_period'], config['upload_rate_fudge'])] + + # BarterCast counters + self.total_downloaded = 0 + self.total_uploaded = 0 + + self.ut_pex_first_flag = True # first time we sent a ut_pex to this peer? + self.na_candidate_ext_ip = None + + self.na_candidate_ext_ip = None + + # RePEX counters and repexer instance field + self.pex_received = 0 # number of PEX messages received + + # Closed swarm stuff + # Closed swarms + self.is_closed_swarm = False + self.cs_complete = False # Arno, 2010-08-24; no need for thread safety + self.remote_is_authenticated = False + self.remote_supports_cs = False + status = Status.get_status_holder("LivingLab") + self.cs_status = status.create_event("CS_protocol") + # This is a total for all + self.cs_status_unauth_requests = status.get_or_create_status_element("unauthorized_requests", 0) + self.cs_status_supported = status.get_or_create_status_element("nodes_supporting_cs", 0) + self.cs_status_not_supported = status.get_or_create_status_element("nodes_not_supporting_cs", 0) + + if not self.connecter.is_closed_swarm: + self.cs_complete = True # Don't block anything if we're not a CS + + if self.connecter.is_closed_swarm: + if DEBUG_CS: + print >>sys.stderr,"connecter: conn: CS: This is a closed swarm" + self.is_closed_swarm = True + if 'poa' in self.connecter.config: + try: + from base64 import decodestring + #poa = self.connecter.config.get_poa() + poa = ClosedSwarm.POA.deserialize(decodestring(self.connecter.config['poa'])) + #poa = self.connecter.config['poa'] + except Exception,e: + print_exc() + poa = None + else: + print >>sys.stderr,"connecter: conn: CS: Missing POA" + poa = None + + # Need to also get the rest of the info, like my keys + # and my POA + my_keypair = ClosedSwarm.read_cs_keypair(self.connecter.config['eckeypairfilename']) + self.closed_swarm_protocol = ClosedSwarm.ClosedSwarm(my_keypair, + self.connecter.infohash, + self.connecter.config['cs_keys'], + poa) + if DEBUG_CS: + print >>sys.stderr,"connecter: conn: CS: Closed swarm ready to start handshake" + + + def get_myip(self, real=False): + return self.connection.get_myip(real) + + def get_myport(self, real=False): + return self.connection.get_myport(real) + + def get_ip(self, real=False): + return self.connection.get_ip(real) + + def get_port(self, real=False): + return self.connection.get_port(real) + + def get_id(self): + return self.connection.get_id() + + def get_readable_id(self): + return self.connection.get_readable_id() + + def can_send_to(self): + if self.is_closed_swarm and not self.remote_is_authenticated: + return False + return True + + def close(self): + if DEBUG: + if self.get_ip() == self.connecter.tracker_ip: + print >>sys.stderr,"connecter: close: live: WAAH closing SOURCE" + + self.connection.close() + self.closed = True + + + def is_closed(self): + return self.closed + + def is_locally_initiated(self): + return self.connection.is_locally_initiated() + + def send_interested(self): + self._send_message(INTERESTED) + + def send_not_interested(self): + self._send_message(NOT_INTERESTED) + + def send_choke(self): + if self.partial_message: + self.send_choke_queued = True + else: + self._send_message(CHOKE) + self.upload.choke_sent() + self.just_unchoked = 0 + + def send_unchoke(self): + if not self.cs_complete: + if DEBUG_CS: + print >> sys.stderr, 'Connection: send_unchoke: Not sending UNCHOKE, closed swarm handshanke not done' + return False + + if self.send_choke_queued: + self.send_choke_queued = False + if DEBUG_NORMAL_MSGS: + print >>sys.stderr,'Connection: send_unchoke: CHOKE SUPPRESSED' + else: + self._send_message(UNCHOKE) + if (self.partial_message or self.just_unchoked is None + or not self.upload.interested or self.download.active_requests): + self.just_unchoked = 0 + else: + self.just_unchoked = clock() + return True + + def send_request(self, index, begin, length): + self._send_message(REQUEST + tobinary(index) + + tobinary(begin) + tobinary(length)) + if DEBUG_NORMAL_MSGS: + print >>sys.stderr,"sending REQUEST to",self.get_ip() + print >>sys.stderr,'sent request: '+str(index)+': '+str(begin)+'-'+str(begin+length) + + def send_cancel(self, index, begin, length): + self._send_message(CANCEL + tobinary(index) + + tobinary(begin) + tobinary(length)) + if DEBUG_NORMAL_MSGS: + print >>sys.stderr,'sent cancel: '+str(index)+': '+str(begin)+'-'+str(begin+length) + + def send_bitfield(self, bitfield): + if not self.cs_complete: + print >> sys.stderr, "Connection: send_bitfield: Not sending bitfield - CS handshake not done" + return + + if self.can_send_to(): + self._send_message(BITFIELD + bitfield) + else: + self.cs_status_unauth_requests.inc() + print >>sys.stderr,"Connection: send_bitfield: Sending empty bitfield to unauth node" + self._send_message(BITFIELD + Bitfield(self.connecter.numpieces).tostring()) + + + def send_have(self, index): + if self.can_send_to(): + self._send_message(HAVE + tobinary(index)) + #elif DEBUG_CS: + # print >>sys.stderr,"Supressing HAVE messages" + + def send_keepalive(self): + self._send_message('') + + def _send_message(self, s): + s = tobinary(len(s))+s + if self.partial_message: + self.outqueue.append(s) + else: + self.connection.send_message_raw(s) + + def send_partial(self, bytes): + if self.connection.closed: + return 0 + if not self.can_send_to(): + return 0 + if self.partial_message is None: + s = self.upload.get_upload_chunk() + if s is None: + return 0 + # Merkle: send hashlist along with piece in HASHPIECE message + index, begin, hashlist, piece = s + + if self.use_g2g: + # ----- G2G: record who we send this to + self.g2g_sent_piece_part( self, index, begin, hashlist, piece ) + + # ---- G2G: we are uploading len(piece) data of piece #index + for c in self.connecter.connections.itervalues(): + if not c.use_g2g: + continue + + # include sending to self, because it should not be excluded from the statistics + + c.queue_g2g_piece_xfer( index, begin, piece ) + + if self.connecter.merkle_torrent: + hashpiece_msg_id = self.his_extend_msg_name_to_id(EXTEND_MSG_HASHPIECE) + bhashlist = bencode(hashlist) + if hashpiece_msg_id is None: + # old Tribler <= 4.5.2 style + self.partial_message = ''.join(( + tobinary(1+4+4+4+len(bhashlist)+len(piece)), HASHPIECE, + tobinary(index), tobinary(begin), tobinary(len(bhashlist)), bhashlist, piece.tostring() )) + else: + # Merkle BEP + self.partial_message = ''.join(( + tobinary(2+4+4+4+len(bhashlist)+len(piece)), EXTEND, hashpiece_msg_id, + tobinary(index), tobinary(begin), tobinary(len(bhashlist)), bhashlist, piece.tostring() )) + + else: + self.partial_message = ''.join(( + tobinary(len(piece) + 9), PIECE, + tobinary(index), tobinary(begin), piece.tostring())) + if DEBUG_NORMAL_MSGS: + print >>sys.stderr,'sending chunk: '+str(index)+': '+str(begin)+'-'+str(begin+len(piece)) + + if bytes < len(self.partial_message): + self.connection.send_message_raw(self.partial_message[:bytes]) + self.partial_message = self.partial_message[bytes:] + return bytes + + q = [self.partial_message] + self.partial_message = None + if self.send_choke_queued: + self.send_choke_queued = False + self.outqueue.append(tobinary(1)+CHOKE) + self.upload.choke_sent() + self.just_unchoked = 0 + q.extend(self.outqueue) + self.outqueue = [] + q = ''.join(q) + self.connection.send_message_raw(q) + return len(q) + + def get_upload(self): + return self.upload + + def get_download(self): + return self.download + + def set_download(self, download): + self.download = download + + def backlogged(self): + return not self.connection.is_flushed() + + def got_request(self, i, p, l): + self.upload.got_request(i, p, l) + if self.just_unchoked: + self.connecter.ratelimiter.ping(clock() - self.just_unchoked) + self.just_unchoked = 0 + + # + # Extension protocol support + # + def supports_extend_msg(self,msg_name): + if 'm' in self.extend_hs_dict: + return msg_name in self.extend_hs_dict['m'] + else: + return False + + def got_extend_handshake(self,d): + if DEBUG: + print >>sys.stderr,"connecter: Got EXTEND handshake:",d + if 'm' in d: + if type(d['m']) != DictType: + raise ValueError('Key m does not map to a dict') + m = d['m'] + newm = {} + for key,val in m.iteritems(): + if type(val) != IntType: + # Fix for BitTorrent 4.27.2e + if type(val) == StringType: + newm[key]= ord(val) + continue + else: + raise ValueError('Message ID in m-dict not int') + newm[key]= val + + if not 'm' in self.extend_hs_dict: + self.extend_hs_dict['m'] = {} + # Note: we store the dict without converting the msg IDs to bytes. + self.extend_hs_dict['m'].update(newm) + if self.connecter.overlay_enabled and EXTEND_MSG_OVERLAYSWARM in self.extend_hs_dict['m']: + # This peer understands our overlay swarm extension + if self.connection.locally_initiated: + if DEBUG: + print >>sys.stderr,"connecter: Peer supports Tr_OVERLAYSWARM, attempt connection" + self.connect_overlay() + + if EXTEND_MSG_CS in self.extend_hs_dict['m']: + self.remote_supports_cs = True + self.cs_status_supported.inc() + if DEBUG_CS: + print >>sys.stderr,"connecter: Peer supports Closed swarms" + + if self.is_closed_swarm and self.connection.locally_initiated: + if DEBUG_CS: + print >>sys.stderr,"connecter: Initiating Closed swarm handshake" + self.start_cs_handshake() + else: + self.remote_supports_cs = False + self.cs_status_not_supported.inc() + if DEBUG_CS: + print >>sys.stderr,"connecter: conn: Remote node does not support CS, flagging CS as done" + self.connecter.cs_handshake_completed() + status = Status.get_status_holder("LivingLab") + status.add_event(self.cs_status) + self.cs_status = status.create_event("CS_protocol") + + + if self.connecter.use_g2g and (EXTEND_MSG_G2G_V1 in self.extend_hs_dict['m'] or EXTEND_MSG_G2G_V2 in self.extend_hs_dict['m']): + # Both us and the peer want to use G2G + if self.connection.locally_initiated: + if DEBUG: + print >>sys.stderr,"connecter: Peer supports Tr_G2G" + + self.use_g2g = True + if EXTEND_MSG_G2G_V2 in self.extend_hs_dict['m']: + self.g2g_version = EXTEND_MSG_G2G_V2 + else: + self.g2g_version = EXTEND_MSG_G2G_V1 + + # LIVEHACK + if KICK_OLD_CLIENTS: + peerhaslivekey = False + for key in self.extend_hs_dict['m']: + if key.startswith(EXTEND_MSG_LIVE_PREFIX): + peerhaslivekey = True + livever = int(key[len(EXTEND_MSG_LIVE_PREFIX):]) + if livever < CURRENT_LIVE_VERSION: + raise ValueError("Too old LIVE VERSION "+livever) + else: + print >>sys.stderr,"Connecter: live: Keeping connection to up-to-date peer v",livever,self.get_ip() + + if not peerhaslivekey: + if self.get_ip() == self.connecter.tracker_ip: + # Keep connection to tracker / source + print >>sys.stderr,"Connecter: live: Keeping connection to SOURCE",self.connecter.tracker_ip + else: + raise ValueError("Kicking old LIVE peer "+self.get_ip()) + + # 'p' is peer's listen port, 'v' is peer's version, all optional + # 'e' is used by uTorrent to show it prefers encryption (whatever that means) + # See http://www.bittorrent.org/beps/bep_0010.html + for key in ['p','e', 'yourip','ipv4','ipv6','reqq']: + if key in d: + self.extend_hs_dict[key] = d[key] + + #print >>sys.stderr,"connecter: got_extend_hs: keys",d.keys() + + # If he tells us our IP, record this and see if we get a majority vote on it + if 'yourip' in d: + try: + yourip = decompact_ip(d['yourip']) + + try: + from BaseLib.Core.NATFirewall.DialbackMsgHandler import DialbackMsgHandler + dmh = DialbackMsgHandler.getInstance() + dmh.network_btengine_extend_yourip(yourip) + except: + if DEBUG: + print_exc() + pass + + if 'same_nat_try_internal' in self.connecter.config and self.connecter.config['same_nat_try_internal']: + if 'ipv4' in d: + self.na_check_for_same_nat(yourip) + except: + print_exc() + + # RePEX: Tell repexer we have received an extended handshake + repexer = self.connecter.repexer + if repexer: + try: + version = d.get('v',None) + repexer.got_extend_handshake(self, version) + except: + print_exc() + + def his_extend_msg_name_to_id(self,ext_name): + """ returns the message id (byte) for the given message name or None """ + val = self.extend_hs_dict['m'].get(ext_name) + if val is None: + return val + else: + return chr(val) + + def get_extend_encryption(self): + return self.extend_hs_dict.get('e',0) + + def get_extend_listenport(self): + return self.extend_hs_dict.get('p') + + def is_tribler_peer(self): + client, version = decodePeerID(self.connection.id) + return client == TRIBLER_PEERID_LETTER + + def send_extend_handshake(self): + + # NETWORK AWARE + hisip = self.connection.get_ip(real=True) + ipv4 = None + if self.connecter.config.get('same_nat_try_internal',0): + is_tribler_peer = self.is_tribler_peer() + print >>sys.stderr,"connecter: send_extend_hs: Peer is Tribler client",is_tribler_peer + if is_tribler_peer: + # If we're connecting to a Tribler peer, show our internal IP address + # as 'ipv4'. + ipv4 = self.get_ip(real=True) + + # See: http://www.bittorrent.org/beps/bep_0010.html + d = {} + d['m'] = self.connecter.EXTEND_HANDSHAKE_M_DICT + d['p'] = self.connecter.mylistenport + ver = version_short.replace('-',' ',1) + d['v'] = ver + d['e'] = 0 # Apparently this means we don't like uTorrent encryption + d['yourip'] = compact_ip(hisip) + if ipv4 is not None: + # Only send IPv4 when necessary, we prefer this peer to use this addr. + d['ipv4'] = compact_ip(ipv4) + if self.connecter.ut_metadata_enabled: + # todo: set correct size if known + d['metadata_size'] = self.connecter.ut_metadata_size + + self._send_message(EXTEND + EXTEND_MSG_HANDSHAKE_ID + bencode(d)) + if DEBUG: + print >>sys.stderr,'connecter: sent extend: id=0+',d,"yourip",hisip,"ipv4",ipv4 + + # + # ut_pex support + # + def got_ut_pex(self,d): + if DEBUG_UT_PEX: + print >>sys.stderr,"connecter: Got uTorrent PEX:",d + (same_added_peers,added_peers,dropped_peers) = check_ut_pex(d) + + # RePEX: increase counter + self.pex_received += 1 + + # RePEX: for now, we pass the whole PEX dict to the repexer and + # let it decode it. The reason is that check_ut_pex's interface + # has recently changed, currently returning a triple to prefer + # Tribler peers. The repexer, however, might have different + # interests (e.g., storinng all flags). To cater to both interests, + # check_ut_pex needs to be rewritten. + repexer = self.connecter.repexer + if repexer: + try: + repexer.got_ut_pex(self, d) + except: + print_exc() + return + + # DoS protection: we're accepting IP addresses from + # an untrusted source, so be a bit careful + mx = self.connecter.ut_pex_max_addrs_from_peer + if DEBUG_UT_PEX: + print >>sys.stderr,"connecter: Got",len(added_peers),"peers via uTorrent PEX, using max",mx + + # for now we have a strong bias towards Tribler peers + if self.is_tribler_peer(): + shuffle(same_added_peers) + shuffle(added_peers) + sample_peers = same_added_peers + sample_peers.extend(added_peers) + else: + sample_peers = same_added_peers + sample_peers.extend(added_peers) + shuffle(sample_peers) + + # Take random sample of mx peers + sample_added_peers_with_id = [] + + # Put the sample in the format desired by Encoder.start_connections() + for dns in sample_peers[:mx]: + peer_with_id = (dns, 0) + sample_added_peers_with_id.append(peer_with_id) + if len(sample_added_peers_with_id) > 0: + if DEBUG_UT_PEX: + print >>sys.stderr,"connecter: Starting ut_pex conns to",len(sample_added_peers_with_id) + self.connection.Encoder.start_connections(sample_added_peers_with_id) + + def send_extend_ut_pex(self,payload): + msg = EXTEND+self.his_extend_msg_name_to_id(EXTEND_MSG_UTORRENT_PEX)+payload + self._send_message(msg) + + def first_ut_pex(self): + if self.ut_pex_first_flag: + self.ut_pex_first_flag = False + return True + else: + return False + + def _send_cs_message(self, cs_list): + blist = bencode(cs_list) + self._send_message(EXTEND + self.his_extend_msg_name_to_id(EXTEND_MSG_CS) + blist) + + def got_cs_message(self, cs_list): + if not self.is_closed_swarm: + raise Exception("Got ClosedSwarm message, but this swarm is not closed") + + # Process incoming closed swarm messages + t = cs_list[0] + if t == CS_CHALLENGE_A: + if DEBUG_CS: + print >>sys.stderr,"connecter: conn: CS: Got initial challenge" + # Got a challenge to authenticate to participate in a closed swarm + try: + response = self.closed_swarm_protocol.b_create_challenge(cs_list) + self._send_cs_message(response) + except Exception,e: + self.cs_status.add_value("CS_bad_initial_challenge") + if DEBUG_CS: + print >>sys.stderr,"connecter: conn: CS: Bad initial challenge:",e + elif t == CS_CHALLENGE_B: + if DEBUG_CS: + print >>sys.stderr,"connecter: conn: CS: Got return challenge" + try: + response = self.closed_swarm_protocol.a_provide_poa_message(cs_list) + if DEBUG_CS and not response: + print >> sys.stderr, "connecter: I'm not intererested in data" + self._send_cs_message(response) + except Exception,e: + self.cs_status.add_value("CS_bad_return_challenge") + if DEBUG_CS: + print >>sys.stderr,"connecter: conn: CS: Bad return challenge",e + print_exc() + + elif t == CS_POA_EXCHANGE_A: + if DEBUG_CS: + print >>sys.stderr,"connecter: conn: CS:Got POA from A" + try: + response = self.closed_swarm_protocol.b_provide_poa_message(cs_list) + self.remote_is_authenticated = self.closed_swarm_protocol.is_remote_node_authorized() + if DEBUG_CS: + print >>sys.stderr,"connecter: conn: CS: Remote node authorized:",self.remote_is_authenticated + if response: + self._send_cs_message(response) + except Exception,e: + self.cs_status.add_value("CS_bad_POA_EXCHANGE_A") + if DEBUG_CS: + print >>sys.stderr,"connecter: conn: CS: Bad POA from A:",e + + elif t == CS_POA_EXCHANGE_B: + try: + self.closed_swarm_protocol.a_check_poa_message(cs_list) + self.remote_is_authenticated = self.closed_swarm_protocol.is_remote_node_authorized() + if DEBUG_CS: + print >>sys.stderr,"connecter: conn: CS: Remote node authorized:",self.remote_is_authenticated + except Exception,e: + self.cs_status.add_value("CS_bad_POA_EXCHANGE_B") + if DEBUG_CS: + print >>sys.stderr,"connecter: conn: CS: Bad POA from B:",e + + if not self.closed_swarm_protocol.is_incomplete(): + self.connecter.cs_handshake_completed() + status = Status.get_status_holder("LivingLab") + self.cs_complete = True # Flag CS as completed + # Don't need to add successful CS event + + # + # Give-2-Get + # + def g2g_sent_piece_part( self, c, index, begin, hashlist, piece ): + """ Keeps a record of the fact that we sent piece index[begin:begin+chunk]. """ + + wegaveperc = float(len(piece))/float(self.connecter.piece_size) + if index in self.perc_sent: + self.perc_sent[index] = self.perc_sent[index] + wegaveperc + else: + self.perc_sent[index] = wegaveperc + + + def queue_g2g_piece_xfer(self,index,begin,piece): + """ Queue the fact that we sent piece index[begin:begin+chunk] for + tranmission to peers + """ + if self.g2g_version == EXTEND_MSG_G2G_V1: + self.send_g2g_piece_xfer_v1(index,begin,piece) + return + + perc = float(len(piece))/float(self.connecter.piece_size) + if index in self.last_perc_sent: + self.last_perc_sent[index] = self.last_perc_sent[index] + perc + else: + self.last_perc_sent[index] = perc + + def dequeue_g2g_piece_xfer(self): + """ Send queued information about pieces we sent to peers. Called + periodically. + """ + psf = float(self.connecter.piece_size) + ppdict = {} + + #print >>sys.stderr,"connecter: g2g dq: orig",self.last_perc_sent + + for index,perc in self.last_perc_sent.iteritems(): + # due to rerequests due to slow pieces the sum can be above 1.0 + capperc = min(1.0,perc) + percb = chr(int((100.0 * capperc))) + # bencode can't deal with int keys + ppdict[str(index)] = percb + self.last_perc_sent = {} + + #print >>sys.stderr,"connecter: g2g dq: dest",ppdict + + if len(ppdict) > 0: + self.send_g2g_piece_xfer_v2(ppdict) + + def send_g2g_piece_xfer_v1(self,index,begin,piece): + """ Send fact that we sent piece index[begin:begin+chunk] to a peer + to all peers (G2G V1). + """ + self._send_message(self.his_extend_msg_name_to_id(EXTEND_MSG_G2G_V1) + tobinary(index) + tobinary(begin) + tobinary(len(piece))) + + def send_g2g_piece_xfer_v2(self,ppdict): + """ Send list of facts that we sent pieces to all peers (G2G V2). """ + blist = bencode(ppdict) + self._send_message(EXTEND + self.his_extend_msg_name_to_id(EXTEND_MSG_G2G_V2) + blist) + + def got_g2g_piece_xfer_v1(self,index,begin,length): + """ Got a G2G_PIECE_XFER message in V1 format. """ + hegaveperc = float(length)/float(self.connecter.piece_size) + self.g2g_peer_forwarded_piece_part(index,hegaveperc) + + def got_g2g_piece_xfer_v2(self,ppdict): + """ Got a G2G_PIECE_XFER message in V2 format. """ + for indexstr,hegavepercb in ppdict.iteritems(): + index = int(indexstr) + hegaveperc = float(ord(hegavepercb))/100.0 + self.g2g_peer_forwarded_piece_part(index,hegaveperc) + + def g2g_peer_forwarded_piece_part(self,index,hegaveperc): + """ Processes this peer forwarding piece i[begin:end] to a grandchild. """ + # Reward for forwarding data in general + length = ceil(hegaveperc * float(self.connecter.piece_size)) + self.forward_speeds[1].update_rate(length) + + if index not in self.perc_sent: + # piece came from disk + return + + # Extra reward if its data we sent + wegaveperc = self.perc_sent[index] + overlapperc = wegaveperc * hegaveperc + overlap = ceil(overlapperc * float(self.connecter.piece_size)) + if overlap > 0: + self.forward_speeds[0].update_rate( overlap ) + + def g2g_score( self ): + return [x.get_rate() for x in self.forward_speeds] + + + # + # SecureOverlay support + # + def connect_overlay(self): + if DEBUG: + print >>sys.stderr,"connecter: Initiating overlay connection" + if not self.initiated_overlay: + from BaseLib.Core.Overlay.SecureOverlay import SecureOverlay + + self.initiated_overlay = True + so = SecureOverlay.getInstance() + so.connect_dns(self.connection.dns,self.network_connect_dns_callback) + + def network_connect_dns_callback(self,exc,dns,permid,selversion): + # WARNING: WILL BE CALLED BY NetworkThread + if exc is not None: + print >>sys.stderr,"connecter: peer",dns,"said he supported overlay swarm, but we can't connect to him",exc + + def start_cs_handshake(self): + try: + if DEBUG_CS: + print >>sys.stderr,"connecter: conn: CS: Initiating Closed Swarm Handshake" + challenge = self.closed_swarm_protocol.a_create_challenge() + self._send_cs_message(challenge) + except Exception,e: + print >>sys.stderr,"connecter: conn: CS: Bad initial challenge:",e + + + # + # NETWORK AWARE + # + def na_check_for_same_nat(self,yourip): + """ See if peer is local, e.g. behind same NAT, same AS or something. + If so, try to optimize: + - Same NAT -> reconnect to use internal network + """ + hisip = self.connection.get_ip(real=True) + if hisip == yourip: + # Do we share the same NAT? + myextip = self.connecter.get_extip_func(unknowniflocal=True) + myintip = self.get_ip(real=True) + + if DEBUG: + print >>sys.stderr,"connecter: na_check_for_same_nat: his",hisip,"myext",myextip,"myint",myintip + + if hisip != myintip or hisip == '127.0.0.1': # to allow testing + # He can't fake his source addr, so we're not running on the + # same machine, + + # He may be quicker to determine we should have a local + # conn, so prepare for his connection in advance. + # + if myextip is None: + # I don't known my external IP and he's not on the same + # machine as me. yourip could be our real external IP, test. + if DEBUG: + print >>sys.stderr,"connecter: na_check_same_nat: Don't know my ext ip, try to loopback to",yourip,"to see if that's me" + self.na_start_loopback_connection(yourip) + elif hisip == myextip: + # Same NAT. He can't fake his source addr. + # Attempt local network connection + if DEBUG: + print >>sys.stderr,"connecter: na_check_same_nat: Yes, trying to connect via internal" + self.na_start_internal_connection() + else: + # hisip != myextip + # He claims we share the same IP, but I think my ext IP + # is something different. Either he is lying or I'm + # mistaken, test + if DEBUG: + print >>sys.stderr,"connecter: na_check_same_nat: Maybe, me thinks not, try to loopback to",yourip + self.na_start_loopback_connection(yourip) + + + def na_start_loopback_connection(self,yourip): + """ Peer claims my external IP is "yourip". Try to connect back to myself """ + if DEBUG: + print >>sys.stderr,"connecter: na_start_loopback: Checking if my ext ip is",yourip + self.na_candidate_ext_ip = yourip + + dns = (yourip,self.connecter.mylistenport) + self.connection.Encoder.start_connection(dns,0,forcenew=True) + + def na_got_loopback(self,econnection): + """ Got a connection with my peer ID. Check that this is indeed me looping + back to myself. No man-in-the-middle attacks protection. This is complex + if we're also connecting to ourselves because of a stale tracker + registration. Window of opportunity is small. + """ + himismeip = econnection.get_ip(real=True) + if DEBUG: + print >>sys.stderr,"connecter: conn: na_got_loopback:",himismeip,self.na_candidate_ext_ip + if self.na_candidate_ext_ip == himismeip: + self.na_start_internal_connection() + + + def na_start_internal_connection(self): + """ Reconnect to peer using internal network """ + if DEBUG: + print >>sys.stderr,"connecter: na_start_internal_connection" + + # Doesn't really matter who initiates. Letting other side do it makes + # testing easier. + if not self.is_locally_initiated(): + + hisip = decompact_ip(self.extend_hs_dict['ipv4']) + hisport = self.extend_hs_dict['p'] + + # For testing, see Tribler/Test/test_na_extend_hs.py + if hisip == '224.4.8.1' and hisport == 4810: + hisip = '127.0.0.1' + hisport = 4811 + + self.connection.na_want_internal_conn_from = hisip + + hisdns = (hisip,hisport) + if DEBUG: + print >>sys.stderr,"connecter: na_start_internal_connection to",hisdns + self.connection.Encoder.start_connection(hisdns,0) + + def na_get_address_distance(self): + return self.connection.na_get_address_distance() + + def is_live_source(self): + if self.connecter.live_streaming: + if self.get_ip() == self.connecter.tracker_ip: + return True + return False + + +class Connecter: +# 2fastbt_ + def __init__(self, metadata, make_upload, downloader, choker, numpieces, piece_size, + totalup, config, ratelimiter, merkle_torrent, sched = None, + coordinator = None, helper = None, get_extip_func = lambda: None, mylistenport = None, use_g2g = False, infohash=None, tracker=None, live_streaming = False): + + self.downloader = downloader + self.make_upload = make_upload + self.choker = choker + self.numpieces = numpieces + self.piece_size = piece_size + self.config = config + self.ratelimiter = ratelimiter + self.rate_capped = False + self.sched = sched + self.totalup = totalup + self.rate_capped = False + self.connections = {} + self.external_connection_made = 0 + self.merkle_torrent = merkle_torrent + self.use_g2g = use_g2g + # 2fastbt_ + self.coordinator = coordinator + self.helper = helper + self.round = 0 + self.get_extip_func = get_extip_func + self.mylistenport = mylistenport + self.infohash = infohash + self.live_streaming = live_streaming + self.tracker = tracker + self.tracker_ip = None + if self.live_streaming: + try: + (scheme, netloc, path, pars, query, _fragment) = urlparse.urlparse(self.tracker) + host = netloc.split(':')[0] + self.tracker_ip = socket.getaddrinfo(host,None)[0][4][0] + except: + print_exc() + self.tracker_ip = None + #print >>sys.stderr,"Connecter: live: source/tracker is",self.tracker_ip + self.overlay_enabled = 0 + if self.config['overlay']: + self.overlay_enabled = True + + if DEBUG: + if self.overlay_enabled: + print >>sys.stderr,"connecter: Enabling overlay" + else: + print >>sys.stderr,"connecter: Disabling overlay" + + self.ut_pex_enabled = 0 + if 'ut_pex_max_addrs_from_peer' in self.config: + self.ut_pex_max_addrs_from_peer = self.config['ut_pex_max_addrs_from_peer'] + self.ut_pex_enabled = self.ut_pex_max_addrs_from_peer > 0 + self.ut_pex_previous_conns = [] # last value of 'added' field for all peers + + self.ut_metadata_enabled = self.config["magnetlink"] + if self.ut_metadata_enabled: + # metadata (or self.responce as its called in download_bt1) is + # a dic containing the metadata. Ut_metadata shares the + # bencoded 'info' part of this metadata in 16kb pieces. + infodata = bencode(metadata["info"]) + self.ut_metadata_size = len(infodata) + self.ut_metadata_list = [infodata[index:index+16*1024] for index in xrange(0, len(infodata), 16*1024)] + # history is a list containing previous request served (to + # limit our bandwidth usage) + self.ut_metadata_history = [] + if DEBUG: print >> sys.stderr,"connecter.__init__: Enable ut_metadata" + + if DEBUG_UT_PEX: + if self.ut_pex_enabled: + print >>sys.stderr,"connecter: Enabling uTorrent PEX",self.ut_pex_max_addrs_from_peer + else: + print >>sys.stderr,"connecter: Disabling uTorrent PEX" + + # The set of messages we support. Note that the msg ID is an int not a byte in + # this dict. + self.EXTEND_HANDSHAKE_M_DICT = {} + + # Say in the EXTEND handshake that we support Closed swarms + if DEBUG: + print >>sys.stderr,"connecter: I support Closed Swarms" + d = {EXTEND_MSG_CS:ord(CS_CHALLENGE_A)} + self.EXTEND_HANDSHAKE_M_DICT.update(d) + + if self.overlay_enabled: + # Say in the EXTEND handshake we support the overlay-swarm ext. + d = {EXTEND_MSG_OVERLAYSWARM:ord(CHALLENGE)} + self.EXTEND_HANDSHAKE_M_DICT.update(d) + if self.ut_pex_enabled: + # Say in the EXTEND handshake we support uTorrent's peer exchange ext. + d = {EXTEND_MSG_UTORRENT_PEX:ord(EXTEND_MSG_UTORRENT_PEX_ID)} + self.EXTEND_HANDSHAKE_M_DICT.update(d) + self.sched(self.ut_pex_callback,6) + if self.use_g2g: + # Say in the EXTEND handshake we want to do G2G. + d = {EXTEND_MSG_G2G_V2:ord(G2G_PIECE_XFER)} + self.EXTEND_HANDSHAKE_M_DICT.update(d) + self.sched(self.g2g_callback,G2G_CALLBACK_INTERVAL) + if self.merkle_torrent: + d = {EXTEND_MSG_HASHPIECE:ord(HASHPIECE)} + self.EXTEND_HANDSHAKE_M_DICT.update(d) + if self.ut_metadata_enabled: + d = {EXTEND_MSG_METADATA:ord(EXTEND_MSG_METADATA_ID)} + self.EXTEND_HANDSHAKE_M_DICT.update(d) + + + # LIVEHACK + livekey = EXTEND_MSG_LIVE_PREFIX+str(CURRENT_LIVE_VERSION) + d = {livekey:ord(LIVE_FAKE_MESSAGE_ID)} + self.EXTEND_HANDSHAKE_M_DICT.update(d) + + if DEBUG: + print >>sys.stderr,"Connecter: EXTEND: my dict",self.EXTEND_HANDSHAKE_M_DICT + + # BarterCast + if config['overlay']: + from BaseLib.Core.Overlay.OverlayThreadingBridge import OverlayThreadingBridge + + self.overlay_bridge = OverlayThreadingBridge.getInstance() + else: + self.overlay_bridge = None + + # RePEX + self.repexer = None # Should this be called observer instead? + + # Closed Swarm stuff + self.is_closed_swarm = False + self.cs_post_func = None + if 'cs_keys' in self.config: + if self.config['cs_keys'] != None: + if len(self.config['cs_keys']) == 0: + if DEBUG_CS: + print >>sys.stderr, "connecter: cs_keys is empty" + else: + if DEBUG_CS: + print >>sys.stderr, "connecter: This is a closed swarm - has cs_keys" + self.is_closed_swarm = True + + + def how_many_connections(self): + return len(self.connections) + + def connection_made(self, connection): + + assert connection + c = Connection(connection, self) + self.connections[connection] = c + + # RePEX: Inform repexer connection is made + repexer = self.repexer + if repexer: + try: + repexer.connection_made(c,connection.supports_extend_messages()) + if c.closed: + # The repexer can close the connection in certain cases. + # If so, we abort further execution of this function. + return c + except: + print_exc() + + if connection.supports_extend_messages(): + # The peer either supports our overlay-swarm extension or + # the utorrent extended protocol. + + [client,version] = decodePeerID(connection.id) + + if DEBUG: + print >>sys.stderr,"connecter: Peer is client",client,"version",version,c.get_ip(),c.get_port() + + if self.overlay_enabled and client == TRIBLER_PEERID_LETTER and version <= '3.5.0' and connection.locally_initiated: + # Old Tribler, establish overlay connection< + if DEBUG: + print >>sys.stderr,"connecter: Peer is previous Tribler version, attempt overlay connection" + c.connect_overlay() + elif self.ut_pex_enabled: + # EXTEND handshake must be sent just after BT handshake, + # before BITFIELD even + c.send_extend_handshake() + + #TODO: overlay swarm also needs upload and download to control transferring rate + # If this is a closed swarm, don't do this now - will be done on completion of the CS protocol! + c.upload = self.make_upload(c, self.ratelimiter, self.totalup) + c.download = self.downloader.make_download(c) + if not self.is_closed_swarm: + if DEBUG_CS: + print >>sys.stderr,"connecter: connection_made: Freeing choker!" + self.choker.connection_made(c) + else: + if DEBUG_CS: + print >>sys.stderr,"connecter: connection_made: Will free choker later" + self.choker.add_connection(c) + #self.cs_post_func = lambda:self.choker.connection_made(c) + #self.cs_post_func = lambda:self.choker.start_connection(c) + self.cs_post_func = lambda:self._cs_completed(c) + + return c + + def connection_lost(self, connection): + c = self.connections[connection] + + # RePEX: inform repexer of closed connection + repexer = self.repexer + if repexer: + try: + repexer.connection_closed(c) + except: + print_exc() + + ###################################### + # BarterCast + if self.overlay_bridge is not None: + ip = c.get_ip(False) + port = c.get_port(False) + down_kb = int(c.total_downloaded / 1024) + up_kb = int(c.total_uploaded / 1024) + + if DEBUG: + print >> sys.stderr, "bartercast: attempting database update, adding olthread" + + olthread_bartercast_conn_lost_lambda = lambda:olthread_bartercast_conn_lost(ip,port,down_kb,up_kb) + self.overlay_bridge.add_task(olthread_bartercast_conn_lost_lambda,0) + else: + if DEBUG: + print >> sys.stderr, "bartercast: no overlay bridge found" + + ######################### + + if DEBUG: + if c.get_ip() == self.tracker_ip: + print >>sys.stderr,"connecter: connection_lost: live: WAAH2 closing SOURCE" + + del self.connections[connection] + if c.download: + c.download.disconnected() + self.choker.connection_lost(c) + + def connection_flushed(self, connection): + conn = self.connections[connection] + if conn.next_upload is None and (conn.partial_message is not None + or conn.upload.buffer): + self.ratelimiter.queue(conn) + + def got_piece(self, i): + for co in self.connections.values(): + co.send_have(i) + + def our_extend_msg_id_to_name(self,ext_id): + """ find the name for the given message id (byte) """ + for key,val in self.EXTEND_HANDSHAKE_M_DICT.iteritems(): + if val == ord(ext_id): + return key + return None + + def get_ut_pex_conns(self): + conns = [] + for conn in self.connections.values(): + if conn.get_extend_listenport() is not None: + conns.append(conn) + return conns + + def get_ut_pex_previous_conns(self): + return self.ut_pex_previous_conns + + def set_ut_pex_previous_conns(self,conns): + self.ut_pex_previous_conns = conns + + def ut_pex_callback(self): + """ Periocially send info about the peers you know to the other peers """ + if DEBUG_UT_PEX: + print >>sys.stderr,"connecter: Periodic ut_pex update" + + currconns = self.get_ut_pex_conns() + (addedconns,droppedconns) = ut_pex_get_conns_diff(currconns,self.get_ut_pex_previous_conns()) + self.set_ut_pex_previous_conns(currconns) + if DEBUG_UT_PEX: + for conn in addedconns: + print >>sys.stderr,"connecter: ut_pex: Added",conn.get_ip(),conn.get_extend_listenport() + for conn in droppedconns: + print >>sys.stderr,"connecter: ut_pex: Dropped",conn.get_ip(),conn.get_extend_listenport() + + for c in currconns: + if c.supports_extend_msg(EXTEND_MSG_UTORRENT_PEX): + try: + if DEBUG_UT_PEX: + print >>sys.stderr,"connecter: ut_pex: Creating msg for",c.get_ip(),c.get_extend_listenport() + if c.first_ut_pex(): + aconns = currconns + dconns = [] + else: + aconns = addedconns + dconns = droppedconns + payload = create_ut_pex(aconns,dconns,c) + c.send_extend_ut_pex(payload) + except: + print_exc() + self.sched(self.ut_pex_callback,60) + + def g2g_callback(self): + try: + self.sched(self.g2g_callback,G2G_CALLBACK_INTERVAL) + for c in self.connections.itervalues(): + if not c.use_g2g: + continue + + c.dequeue_g2g_piece_xfer() + except: + print_exc() + + def got_ut_metadata(self, connection, dic, message): + """ + CONNECTION: The connection instance where we received this message + DIC: The bdecoded dictionary + MESSAGE: The entire message: + """ + if DEBUG: print >> sys.stderr, "connecter.got_ut_metadata:", dic + + msg_type = dic.get("msg_type", None) + if not type(msg_type) in (int, long): + raise ValueError("Invalid ut_metadata.msg_type") + piece = dic.get("piece", None) + if not type(piece) in (int, long): + raise ValueError("Invalid ut_metadata.piece type") + if not 0 <= piece < len(self.ut_metadata_list): + raise ValueError("Invalid ut_metadata.piece value") + + if msg_type == 0: # request + if DEBUG: print >> sys.stderr, "connecter.got_ut_metadata: Received request for piece", piece + + # our flood protection policy is to upload all metadata + # once every n minutes. + now = time.time() + deadline = now - UT_METADATA_FLOOD_PERIOD + # remove old history + self.ut_metadata_history = [timestamp for timestamp in self.ut_metadata_history if timestamp > deadline] + + if len(self.ut_metadata_history) > UT_METADATA_FLOOD_FACTOR * len(self.ut_metadata_list): + # refuse to upload at this time + reply = bencode({"msg_type":2, "piece":piece}) + else: + reply = bencode({"msg_type":1, "piece":piece, "data":self.ut_metadata_list[piece]}) + self.ut_metadata_history.append(now) + connection._send_message(EXTEND + connection.his_extend_msg_name_to_id(EXTEND_MSG_METADATA) + reply) + + elif msg_type == 1: # data + # at this point in the code we must assume that the + # metadata is already there, everything is designed in + # such a way that metadata is required. data replies can + # therefore never occur. + raise ValueError("Invalid ut_metadata: we did not request data") + + elif msg_type == 2: # reject + # at this point in the code we must assume that the + # metadata is already there, everything is designed in + # such a way that metadata is required. rejects can + # therefore never occur. + raise ValueError("Invalid ut_metadata: we did not request data that can be rejected") + + else: + raise ValueError("Invalid ut_metadata.msg_type value") + + def got_hashpiece(self, connection, message): + """ Process Merkle hashpiece message. Note: EXTEND byte has been + stripped, it starts with peer's Tr_hashpiece id for historic reasons ;-) + """ + try: + c = self.connections[connection] + + if len(message) <= 13: + if DEBUG: + print >>sys.stderr,"Close on bad HASHPIECE: msg len" + connection.close() + return + i = toint(message[1:5]) + if i >= self.numpieces: + if DEBUG: + print >>sys.stderr,"Close on bad HASHPIECE: index out of range" + connection.close() + return + begin = toint(message[5:9]) + len_hashlist = toint(message[9:13]) + bhashlist = message[13:13+len_hashlist] + hashlist = bdecode(bhashlist) + if not isinstance(hashlist, list): + raise AssertionError, "hashlist not list" + for oh in hashlist: + if not isinstance(oh,list) or \ + not (len(oh) == 2) or \ + not isinstance(oh[0],int) or \ + not isinstance(oh[1],str) or \ + not ((len(oh[1])==20)): \ + raise AssertionError, "hashlist entry invalid" + piece = message[13+len_hashlist:] + + if DEBUG_NORMAL_MSGS: + print >>sys.stderr,"connecter: Got HASHPIECE",i,begin + + if c.download.got_piece(i, begin, hashlist, piece): + self.got_piece(i) + except Exception,e: + if DEBUG: + print >>sys.stderr,"Close on bad HASHPIECE: exception",str(e) + print_exc() + connection.close() + + # NETWORK AWARE + def na_got_loopback(self,econnection): + if DEBUG: + print >>sys.stderr,"connecter: na_got_loopback: Got connection from",econnection.get_ip(),econnection.get_port() + for c in self.connections.itervalues(): + ret = c.na_got_loopback(econnection) + if ret is not None: + return ret + return False + + def na_got_internal_connection(self,origconn,newconn): + """ This is called only at the initiator side of the internal conn. + Doesn't matter, only one is enough to close the original connection. + """ + if DEBUG: + print >>sys.stderr,"connecter: na_got_internal: From",newconn.get_ip(),newconn.get_port() + + origconn.close() + + + def got_message(self, connection, message): + # connection: Encrypter.Connection; c: Connecter.Connection + c = self.connections[connection] + t = message[0] + # EXTEND handshake will be sent just after BT handshake, + # before BITFIELD even + + if DEBUG_MESSAGE_HANDLING: + st = time.time() + + if DEBUG_NORMAL_MSGS: + print >>sys.stderr,"connecter: Got",getMessageName(t),connection.get_ip() + + if t == EXTEND: + self.got_extend_message(connection,c,message,self.ut_pex_enabled) + return + + # If this is a closed swarm and we have not authenticated the + # remote node, we must NOT GIVE IT ANYTHING! + #if self.is_closed_swarm and c.closed_swarm_protocol.is_incomplete(): + #print >>sys.stderr, "connecter: Remote node not authorized, ignoring it" + #return + + if self.is_closed_swarm and c.can_send_to(): + c.got_anything = False # Is this correct or does it break something? + + if t == BITFIELD and c.got_anything: + if DEBUG: + print >>sys.stderr,"Close on BITFIELD" + connection.close() + return + c.got_anything = True + if (t in [CHOKE, UNCHOKE, INTERESTED, NOT_INTERESTED] and + len(message) != 1): + if DEBUG: + print >>sys.stderr,"Close on bad (UN)CHOKE/(NOT_)INTERESTED",t + connection.close() + return + if t == CHOKE: + if DEBUG_NORMAL_MSGS: + print >>sys.stderr,"connecter: Got CHOKE from",connection.get_ip() + c.download.got_choke() + elif t == UNCHOKE: + if DEBUG_NORMAL_MSGS: + print >>sys.stderr,"connecter: Got UNCHOKE from",connection.get_ip() + c.download.got_unchoke() + elif t == INTERESTED: + if DEBUG_NORMAL_MSGS: + print >>sys.stderr,"connecter: Got INTERESTED from",connection.get_ip() + if c.upload is not None: + c.upload.got_interested() + elif t == NOT_INTERESTED: + c.upload.got_not_interested() + elif t == HAVE: + if len(message) != 5: + if DEBUG: + print >>sys.stderr,"Close on bad HAVE: msg len" + connection.close() + return + i = toint(message[1:]) + if i >= self.numpieces: + if DEBUG: + print >>sys.stderr,"Close on bad HAVE: index out of range" + connection.close() + return + if DEBUG_NORMAL_MSGS: + print >>sys.stderr,"connecter: Got HAVE(",i,") from",connection.get_ip() + c.download.got_have(i) + elif t == BITFIELD: + if DEBUG_NORMAL_MSGS: + print >>sys.stderr,"connecter: Got BITFIELD from",connection.get_ip() + try: + b = Bitfield(self.numpieces, message[1:],calcactiveranges=self.live_streaming) + except ValueError: + if DEBUG: + print >>sys.stderr,"Close on bad BITFIELD" + connection.close() + return + if c.download is not None: + c.download.got_have_bitfield(b) + elif t == REQUEST: + if not c.can_send_to(): + c.cs_status_unauth_requests.inc() + print >> sys.stderr,"Got REQUEST but remote node is not authenticated" + return # TODO: Do this better + + if len(message) != 13: + if DEBUG: + print >>sys.stderr,"Close on bad REQUEST: msg len" + connection.close() + return + i = toint(message[1:5]) + if i >= self.numpieces: + if DEBUG: + print >>sys.stderr,"Close on bad REQUEST: index out of range" + connection.close() + return + if DEBUG_NORMAL_MSGS: + print >>sys.stderr,"connecter: Got REQUEST(",i,") from",connection.get_ip() + c.got_request(i, toint(message[5:9]), toint(message[9:])) + elif t == CANCEL: + if len(message) != 13: + if DEBUG: + print >>sys.stderr,"Close on bad CANCEL: msg len" + connection.close() + return + i = toint(message[1:5]) + if i >= self.numpieces: + if DEBUG: + print >>sys.stderr,"Close on bad CANCEL: index out of range" + connection.close() + return + c.upload.got_cancel(i, toint(message[5:9]), + toint(message[9:])) + elif t == PIECE: + if len(message) <= 9: + if DEBUG: + print >>sys.stderr,"Close on bad PIECE: msg len" + connection.close() + return + i = toint(message[1:5]) + if i >= self.numpieces: + if DEBUG: + print >>sys.stderr,"Close on bad PIECE: msg len" + connection.close() + return + if DEBUG_NORMAL_MSGS: # or connection.get_ip().startswith("192"): + print >>sys.stderr,"connecter: Got PIECE(",i,") from",connection.get_ip() + #if connection.get_ip().startswith("192"): + # print >>sys.stderr,"@", + try: + if c.download.got_piece(i, toint(message[5:9]), [], message[9:]): + self.got_piece(i) + except Exception,e: + if DEBUG: + print >>sys.stderr,"Close on bad PIECE: exception",str(e) + print_exc() + connection.close() + return + + elif t == HASHPIECE: + # Merkle: Handle pieces with hashes, old Tribler<= 4.5.2 style + self.got_hashpiece(connection,message) + + elif t == G2G_PIECE_XFER: + # EXTEND_MSG_G2G_V1 only, V2 is proper EXTEND msg + if len(message) <= 12: + if DEBUG: + print >>sys.stderr,"Close on bad G2G_PIECE_XFER: msg len" + connection.close() + return + if not c.use_g2g: + if DEBUG: + print >>sys.stderr,"Close on receiving G2G_PIECE_XFER over non-g2g connection" + connection.close() + return + + index = toint(message[1:5]) + begin = toint(message[5:9]) + length = toint(message[9:13]) + c.got_g2g_piece_xfer_v1(index,begin,length) + + else: + connection.close() + + if DEBUG_MESSAGE_HANDLING: + et = time.time() + diff = et - st + if diff > 0.1: + print >>sys.stderr,"connecter: $$$$$$$$$$$$",getMessageName(t),"took",diff + + + def got_extend_message(self,connection,c,message,ut_pex_enabled): + # connection: Encrypter.Connection; c: Connecter.Connection + if DEBUG: + print >>sys.stderr,"connecter: Got EXTEND message, len",len(message) + print >>sys.stderr,"connecter: his handshake",c.extend_hs_dict,c.get_ip() + + try: + if len(message) < 4: + if DEBUG: + print >>sys.stderr,"Close on bad EXTEND: msg len" + connection.close() + return + ext_id = message[1] + if DEBUG: + print >>sys.stderr,"connecter: Got EXTEND message, id",ord(ext_id) + if ext_id == EXTEND_MSG_HANDSHAKE_ID: + # Message is Handshake + d = bdecode(message[2:]) + if type(d) == DictType: + c.got_extend_handshake(d) + else: + if DEBUG: + print >>sys.stderr,"Close on bad EXTEND: payload of handshake is not a bencoded dict" + connection.close() + return + else: + # Message is regular message e.g ut_pex + ext_msg_name = self.our_extend_msg_id_to_name(ext_id) + if ext_msg_name is None: + if DEBUG: + print >>sys.stderr,"Close on bad EXTEND: peer sent ID we didn't define in handshake" + connection.close() + return + elif ext_msg_name == EXTEND_MSG_OVERLAYSWARM: + if DEBUG: + print >>sys.stderr,"Not closing EXTEND+CHALLENGE: peer didn't read our spec right, be liberal" + elif ext_msg_name == EXTEND_MSG_UTORRENT_PEX and ut_pex_enabled: + d = bdecode(message[2:]) + if type(d) == DictType: + c.got_ut_pex(d) + else: + if DEBUG: + print >>sys.stderr,"Close on bad EXTEND: payload of ut_pex is not a bencoded dict" + connection.close() + return + elif ext_msg_name == EXTEND_MSG_METADATA: + if DEBUG: + print >> sys.stderr, "Connecter.got_extend_message() ut_metadata" + # bdecode sloppy will make bdecode ignore the data + # in message that is placed -after- the bencoded + # data (this is the case for a data message) + d = bdecode(message[2:], sloppy=1) + if type(d) == DictType: + self.got_ut_metadata(c, d, message) + else: + if DEBUG: + print >> sys.stderr, "Connecter.got_extend_message() close on bad ut_metadata message" + connection.close() + return + elif ext_msg_name == EXTEND_MSG_G2G_V2 and self.use_g2g: + ppdict = bdecode(message[2:]) + if type(ppdict) != DictType: + if DEBUG: + print >>sys.stderr,"Close on bad EXTEND+G2G: payload not dict" + connection.close() + return + for k,v in ppdict.iteritems(): + if type(k) != StringType or type(v) != StringType: + if DEBUG: + print >>sys.stderr,"Close on bad EXTEND+G2G: key,value not of type int,char" + connection.close() + return + try: + int(k) + except: + if DEBUG: + print >>sys.stderr,"Close on bad EXTEND+G2G: key not int" + connection.close() + return + if ord(v) > 100: + if DEBUG: + print >>sys.stderr,"Close on bad EXTEND+G2G: value too big",ppdict,v,ord(v) + connection.close() + return + + c.got_g2g_piece_xfer_v2(ppdict) + + elif ext_msg_name == EXTEND_MSG_HASHPIECE and self.merkle_torrent: + # Merkle: Handle pieces with hashes, Merkle BEP + oldmsg = message[1:] + self.got_hashpiece(connection,oldmsg) + + elif ext_msg_name == EXTEND_MSG_CS: + cs_list = bdecode(message[2:]) + c.got_cs_message(cs_list) + + else: + if DEBUG: + print >>sys.stderr,"Close on bad EXTEND: peer sent ID that maps to name we don't support",ext_msg_name,`ext_id`,ord(ext_id) + connection.close() + return + return + except Exception,e: + if not DEBUG: + print >>sys.stderr,"Close on bad EXTEND: exception:",str(e),`message[2:]` + print_exc() + connection.close() + return + + def _cs_completed(self, connection): + """ + When completed, this is a callback function to reset the connection + """ + connection.cs_complete = True # Flag CS as completed + + try: + # Can't send bitfield here, must loop and send a bunch of HAVEs + # Get the bitfield from the uploader + have_list = connection.upload.storage.get_have_list() + bitfield = Bitfield(self.numpieces, have_list) + connection.send_bitfield(bitfield.tostring()) + connection.got_anything = False + self.choker.start_connection(connection) + except Exception,e: + print >> sys.stderr,"connecter: CS: Error restarting after CS handshake:",e + + def cs_handshake_completed(self): + if DEBUG_CS: + print >>sys.stderr,"connecter: Closed swarm handshake completed!" + if self.cs_post_func: + self.cs_post_func() + elif DEBUG_CS: + print >>sys.stderr,"connecter: CS: Woops, don't have post function" + + +def olthread_bartercast_conn_lost(ip,port,down_kb,up_kb): + """ Called by OverlayThread to store information about the peer to + whom the connection was just closed in the (slow) databases. """ + + from BaseLib.Core.CacheDB.CacheDBHandler import PeerDBHandler, BarterCastDBHandler + + peerdb = PeerDBHandler.getInstance() + bartercastdb = BarterCastDBHandler.getInstance() + + if bartercastdb: + + permid = peerdb.getPermIDByIP(ip) + my_permid = bartercastdb.my_permid + + if DEBUG: + print >> sys.stderr, "bartercast: (Connecter): Up %d down %d peer %s:%s (PermID = %s)" % (up_kb, down_kb, ip, port, `permid`) + + # Save exchanged KBs in BarterCastDB + changed = False + if permid is not None: + #name = bartercastdb.getName(permid) + + if down_kb > 0: + new_value = bartercastdb.incrementItem((my_permid, permid), 'downloaded', down_kb, commit=False) + changed = True + + if up_kb > 0: + new_value = bartercastdb.incrementItem((my_permid, permid), 'uploaded', up_kb, commit=False) + changed = True + + # For the record: save KBs exchanged with non-tribler peers + else: + if down_kb > 0: + new_value = bartercastdb.incrementItem((my_permid, 'non-tribler'), 'downloaded', down_kb, commit=False) + changed = True + + if up_kb > 0: + new_value = bartercastdb.incrementItem((my_permid, 'non-tribler'), 'uploaded', up_kb, commit=False) + changed = True + + if changed: + bartercastdb.commit() + + else: + if DEBUG: + print >> sys.stderr, "BARTERCAST: No bartercastdb instance" + + + diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/Downloader.py b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/Downloader.py new file mode 100644 index 0000000..35e79c1 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/Downloader.py @@ -0,0 +1,1196 @@ +# Written by Bram Cohen and Pawel Garbacki, George Milescu +# see LICENSE.txt for license information + +import sys +import time +from BaseLib.Core.BitTornado.CurrentRateMeasure import Measure +from BaseLib.Core.BitTornado.bitfield import Bitfield +from random import shuffle +from base64 import b64encode +from BaseLib.Core.BitTornado.clock import clock +from BaseLib.Core.Statistics.Status.Status import get_status_holder + +#ProxyService_ +# +try: + from BaseLib.Core.ProxyService.Helper import SingleDownloadHelperInterface +except ImportError: + class SingleDownloadHelperInterface: + + def __init__(self): + pass +# +#_ProxyService + +try: + True +except: + True = 1 + False = 0 + +DEBUG = False +DEBUGBF = False +DEBUG_CHUNKS = False # set DEBUG_CHUNKS in PiecePickerStreaming to True +EXPIRE_TIME = 60 * 60 + +# only define the following functions in __debug__. And only import +# them in this case. They are to expensive to have, and have no +# purpose, outside debug mode. +# +# Arno, 2009-06-15: Win32 binary versions have __debug__ True apparently, workaround. +# +if DEBUG_CHUNKS: + _ident_letters = {} + _ident_letter_pool = None + def get_ident_letter(download): + if not download.ip in _ident_letters: + global _ident_letter_pool + if not _ident_letter_pool: + _ident_letter_pool = [chr(c) for c in range(ord("a"), ord("z")+1)] + [chr(c) for c in range(ord("A"), ord("Z")+1)] + _ident_letters[download.ip] = _ident_letter_pool.pop(0) + return _ident_letters[download.ip] + + def print_chunks(downloader, pieces, before=(), after=(), compact=True): + """ + Print a line summery indicating completed/outstanding/non-requested chunks + + When COMPACT is True one character will represent one piece. + # --> downloaded + - --> no outstanding requests + 1-9 --> the number of outstanding requests (max 9) + + When COMPACT is False one character will requests one chunk. + # --> downloaded + - --> no outstanding requests + a-z --> requested at peer with that character (also capitals, duplicates may occur) + 1-9 --> requested multipile times (at n peers) + """ + if pieces: + do_I_have = downloader.storage.do_I_have + do_I_have_requests = downloader.storage.do_I_have_requests + inactive_requests = downloader.storage.inactive_requests + piece_size = downloader.storage.piece_length + chunk_size = downloader.storage.request_size + chunks_per_piece = int(piece_size / chunk_size) + + if compact: + request_map = {} + for download in downloader.downloads: + for piece, begin, length in download.active_requests: + if not piece in request_map: + request_map[piece] = 0 + request_map[piece] += 1 + + def print_chunks_helper(piece_id): + if do_I_have(piece_id): return "#" + if do_I_have_requests(piece_id): return "-" + if piece_id in request_map: return str(min(9, request_map[piece_id])) + return "?" + + else: + request_map = {} + for download in downloader.downloads: + + for piece, begin, length in download.active_requests: + if not piece in request_map: + request_map[piece] = ["-"] * chunks_per_piece + index = int(begin/chunk_size) + if request_map[piece][index] == "-": + request_map[piece][index] = get_ident_letter(download) + elif type(request_map[piece][index]) is str: + request_map[piece][index] = 2 + else: + request_map[piece][index] += 1 + request_map[piece][int(begin/chunk_size)] = get_ident_letter(download) + + def print_chunks_helper(piece_id): + if do_I_have(piece_id): return "#" * chunks_per_piece +# if do_I_have_requests(piece_id): return "-" * chunks_per_piece + if piece_id in request_map: + if piece_id in inactive_requests and type(inactive_requests[piece_id]) is list: + for begin, length in inactive_requests[piece_id]: + request_map[piece_id][int(begin/chunk_size)] = " " + return "".join([str(c) for c in request_map[piece_id]]) + return "-" * chunks_per_piece + + if before: + s_before = before[0] + else: + s_before = "" + + if after: + s_after = after[-1] + else: + s_after = "" + + print >>sys.stderr, "Outstanding %s:%d:%d:%s [%s|%s|%s]" % (s_before, pieces[0], pieces[-1], s_after, "".join(map(print_chunks_helper, before)), "".join(map(print_chunks_helper, pieces)), "".join(map(print_chunks_helper, after))) + + else: + print >>sys.stderr, "Outstanding 0:0 []" + +else: + def print_chunks(downloader, pieces, before=(), after=(), compact=True): + pass + + +class PerIPStats: + def __init__(self, ip): + self.numgood = 0 + self.bad = {} + self.numconnections = 0 + self.lastdownload = None + self.peerid = None + +class BadDataGuard: + def __init__(self, download): + self.download = download + self.ip = download.ip + self.downloader = download.downloader + self.stats = self.downloader.perip[self.ip] + self.lastindex = None + + def failed(self, index, bump = False): + self.stats.bad.setdefault(index, 0) + self.downloader.gotbaddata[self.ip] = 1 + self.stats.bad[index] += 1 + if len(self.stats.bad) > 1: + if self.download is not None: + self.downloader.try_kick(self.download) + elif self.stats.numconnections == 1 and self.stats.lastdownload is not None: + self.downloader.try_kick(self.stats.lastdownload) + if len(self.stats.bad) >= 3 and len(self.stats.bad) > int(self.stats.numgood/30): + self.downloader.try_ban(self.ip) + elif bump: + self.downloader.picker.bump(index) + + def good(self, index): + # lastindex is a hack to only increase numgood by one for each good + # piece, however many chunks come from the connection(s) from this IP + if index != self.lastindex: + self.stats.numgood += 1 + self.lastindex = index + +# 2fastbt_ +class SingleDownload(SingleDownloadHelperInterface): +# _2fastbt + def __init__(self, downloader, connection): +# 2fastbt_ + SingleDownloadHelperInterface.__init__(self) +# _2fastbt + self.downloader = downloader + self.connection = connection + self.choked = True + self.interested = False + self.active_requests = [] + self.measure = Measure(downloader.max_rate_period) + self.peermeasure = Measure(downloader.max_rate_period) + self.have = Bitfield(downloader.numpieces) + self.last = -1000 + self.last2 = -1000 + self.example_interest = None + self.backlog = 2 + self.ip = connection.get_ip() + self.guard = BadDataGuard(self) +# 2fastbt_ + self.helper = downloader.picker.helper + self.proxy_have = Bitfield(downloader.numpieces) +# _2fastbt + + # boudewijn: VOD needs a download measurement that is not + # averaged over a 'long' period. downloader.max_rate_period is + # (by default) 20 seconds because this matches the unchoke + # policy. + self.short_term_measure = Measure(5) + + # boudewijn: each download maintains a counter for the number + # of high priority piece requests that did not get any + # responce within x seconds. + self.bad_performance_counter = 0 + + def _backlog(self, just_unchoked): + self.backlog = int(min( + 2+int(4*self.measure.get_rate()/self.downloader.chunksize), + (2*just_unchoked)+self.downloader.queue_limit() )) + if self.backlog > 50: + self.backlog = int(max(50, self.backlog * 0.075)) + return self.backlog + + def disconnected(self): + self.downloader.lost_peer(self) + + """ JD: obsoleted -- moved to picker.lost_peer + + if self.have.complete(): + self.downloader.picker.lost_seed() + else: + for i in xrange(len(self.have)): + if self.have[i]: + self.downloader.picker.lost_have(i) + """ + + if self.have.complete() and self.downloader.storage.is_endgame(): + self.downloader.add_disconnected_seed(self.connection.get_readable_id()) + self._letgo() + self.guard.download = None + + def _letgo(self): + if self.downloader.queued_out.has_key(self): + del self.downloader.queued_out[self] + if not self.active_requests: + return + if self.downloader.endgamemode: + self.active_requests = [] + return + lost = {} + for index, begin, length in self.active_requests: + self.downloader.storage.request_lost(index, begin, length) + lost[index] = 1 + lost = lost.keys() + self.active_requests = [] + if self.downloader.paused: + return + ds = [d for d in self.downloader.downloads if not d.choked] + shuffle(ds) + for d in ds: + d._request_more() + for d in self.downloader.downloads: + if d.choked and not d.interested: + for l in lost: + if d.have[l] and self.downloader.storage.do_I_have_requests(l): + d.send_interested() + break + + def got_choke(self): + if not self.choked: + self.choked = True + self._letgo() + + def got_unchoke(self): + if self.choked: + self.choked = False + if self.interested: + self._request_more(new_unchoke = True) + self.last2 = clock() + + def is_choked(self): + return self.choked + + def is_interested(self): + return self.interested + + def send_interested(self): + if not self.interested: + self.interested = True + self.connection.send_interested() + + def send_not_interested(self): + if self.interested: + self.interested = False + self.connection.send_not_interested() + + def got_piece(self, index, begin, hashlist, piece): + """ + Returns True if the piece is complete. + Note that in this case a -piece- means a chunk! + """ + + if self.bad_performance_counter: + self.bad_performance_counter -= 1 + if DEBUG: print >>sys.stderr, "decreased bad_performance_counter to", self.bad_performance_counter + + length = len(piece) + #if DEBUG: + # print >> sys.stderr, 'Downloader: got piece of length %d' % length + try: + self.active_requests.remove((index, begin, length)) + except ValueError: + self.downloader.discarded += length + return False + if self.downloader.endgamemode: + self.downloader.all_requests.remove((index, begin, length)) + if DEBUG: print >>sys.stderr, "Downloader: got_piece: removed one request from all_requests", len(self.downloader.all_requests), "remaining" + + self.last = clock() + self.last2 = clock() + self.measure.update_rate(length) + # Update statistic gatherer + status = get_status_holder("LivingLab") + s_download = status.get_or_create_status_element("downloaded",0) + s_download.inc(length) + + self.short_term_measure.update_rate(length) + self.downloader.measurefunc(length) + if not self.downloader.storage.piece_came_in(index, begin, hashlist, piece, self.guard): + self.downloader.piece_flunked(index) + return False + + # boudewijn: we need more accurate (if possibly invalid) + # measurements on current download speed + self.downloader.picker.got_piece(index, begin, length) + +# print "Got piece=", index, "begin=", begin, "len=", length + if self.downloader.storage.do_I_have(index): + self.downloader.picker.complete(index) + + if self.downloader.endgamemode: + for d in self.downloader.downloads: + if d is not self: + if d.interested: + if d.choked: + assert not d.active_requests + d.fix_download_endgame() + else: + try: + d.active_requests.remove((index, begin, length)) + except ValueError: + continue + d.connection.send_cancel(index, begin, length) + d.fix_download_endgame() + else: + assert not d.active_requests + self._request_more() + self.downloader.check_complete(index) + + # BarterCast counter + self.connection.total_downloaded += length + + return self.downloader.storage.do_I_have(index) + +# 2fastbt_ + def helper_forces_unchoke(self): + self.choked = False +# _2fastbt + + def _request_more(self, new_unchoke = False, slowpieces = []): +# 2fastbt_ + if DEBUG: + print >>sys.stderr,"Downloader: _request_more()" + if self.helper is not None and self.is_frozen_by_helper(): + if DEBUG: + print >>sys.stderr,"Downloader: _request_more: blocked, returning" + return +# _2fastbt + if self.choked: + if DEBUG: + print >>sys.stderr,"Downloader: _request_more: choked, returning" + return +# 2fastbt_ + # do not download from coordinator + if self.connection.connection.is_coordinator_con(): + if DEBUG: + print >>sys.stderr,"Downloader: _request_more: coordinator conn" + return +# _2fastbt + if self.downloader.endgamemode: + self.fix_download_endgame(new_unchoke) + if DEBUG: + print >>sys.stderr,"Downloader: _request_more: endgame mode, returning" + return + if self.downloader.paused: + if DEBUG: + print >>sys.stderr,"Downloader: _request_more: paused, returning" + return + if len(self.active_requests) >= self._backlog(new_unchoke): + if DEBUG: + print >>sys.stderr,"Downloader: more req than unchoke (active req: %d >= backlog: %d)" % (len(self.active_requests), self._backlog(new_unchoke)) + # Jelle: Schedule _request more to be called in some time. Otherwise requesting and receiving packages + # may stop, if they arrive to quickly + if self.downloader.download_rate: + wait_period = self.downloader.chunksize / self.downloader.download_rate / 2.0 + + # Boudewijn: when wait_period is 0.0 this will cause + # the the _request_more method to be scheduled + # multiple times (recursively), causing severe cpu + # problems. + # + # Therefore, only schedule _request_more to be called + # if the call will be made in the future. The minimal + # wait_period should be tweaked. + if wait_period > 1.0: + if DEBUG: + print >>sys.stderr,"Downloader: waiting for %f s to call _request_more again" % wait_period + self.downloader.scheduler(self._request_more, wait_period) + + if not (self.active_requests or self.backlog): + self.downloader.queued_out[self] = 1 + return + + #if DEBUG: + # print >>sys.stderr,"Downloader: _request_more: len act",len(self.active_requests),"back",self.backlog + + lost_interests = [] + while len(self.active_requests) < self.backlog: + #if DEBUG: + # print >>sys.stderr,"Downloader: Looking for interesting piece" + #st = time.time() + #print "DOWNLOADER self.have=", self.have.toboollist() + + # This is the PiecePicker call is the current client is a Coordinator + interest = self.downloader.picker.next(self.have, + self.downloader.storage.do_I_have_requests, + self, + self.downloader.too_many_partials(), + self.connection.connection.is_helper_con(), + slowpieces = slowpieces, connection = self.connection, proxyhave = self.proxy_have) + #et = time.time() + #diff = et-st + diff=-1 + if DEBUG: + print >>sys.stderr,"Downloader: _request_more: next() returned",interest,"took %.5f" % (diff) + if interest is None: + break + + if self.helper and self.downloader.storage.inactive_requests[interest] is None: + # The current node is a helper and received a request from a coordinator for a piece it has already downloaded + # Should send a Have message to the coordinator + self.connection.send_have(interest) + break + + if self.helper and self.downloader.storage.inactive_requests[interest] == []: + # The current node is a helper and received a request from a coordinator for a piece that is downloading + # (all blocks are requested to the swarm, and have not arrived yet) + break + + + self.example_interest = interest + self.send_interested() + loop = True + while len(self.active_requests) < self.backlog and loop: + + begin, length = self.downloader.storage.new_request(interest) + + if DEBUG: + print >>sys.stderr,"Downloader: new_request",interest,begin,length,"to",self.connection.connection.get_ip(),self.connection.connection.get_port() + + self.downloader.picker.requested(interest, begin, length) + self.active_requests.append((interest, begin, length)) + self.connection.send_request(interest, begin, length) + self.downloader.chunk_requested(length) + if not self.downloader.storage.do_I_have_requests(interest): + loop = False + lost_interests.append(interest) + if not self.active_requests: + self.send_not_interested() + if lost_interests: + for d in self.downloader.downloads: + if d.active_requests or not d.interested: + continue + if d.example_interest is not None and self.downloader.storage.do_I_have_requests(d.example_interest): + continue + for lost in lost_interests: + if d.have[lost]: + break + else: + continue +# 2fastbt_ + #st = time.time() + interest = self.downloader.picker.next(d.have, + self.downloader.storage.do_I_have_requests, + self, # Arno, 2008-05-22; self -> d? Original Pawel code + self.downloader.too_many_partials(), + self.connection.connection.is_helper_con(), willrequest=False,connection=self.connection, proxyhave = self.proxy_have) + #et = time.time() + #diff = et-st + diff=-1 + if DEBUG: + print >>sys.stderr,"Downloader: _request_more: next()2 returned",interest,"took %.5f" % (diff) + + if interest is not None: + # The helper has at least one piece that the coordinator requested + if self.helper and self.downloader.storage.inactive_requests[interest] is None: + # The current node is a helper and received a request from a coordinator for a piece it has already downloaded + # Should send a Have message to the coordinator + self.connection.send_have(interest) + break + if self.helper and self.downloader.storage.inactive_requests[interest] == []: + # The current node is a helper and received a request from a coordinator for a piece that is downloading + # (all blocks are requested to the swarm, and have not arrived yet) + break + +# _2fastbt + if interest is None: + d.send_not_interested() + else: + d.example_interest = interest + + # Arno: LIVEWRAP: no endgame + if not self.downloader.endgamemode and \ + self.downloader.storage.is_endgame() and \ + not (self.downloader.picker.videostatus and self.downloader.picker.videostatus.live_streaming): + self.downloader.start_endgame() + + + def fix_download_endgame(self, new_unchoke = False): +# 2fastbt_ + # do not download from coordinator + if self.downloader.paused or self.connection.connection.is_coordinator_con(): + if DEBUG: print >>sys.stderr, "Downloader: fix_download_endgame: paused", self.downloader.paused, "or is_coordinator_con", self.connection.connection.is_coordinator_con() + return +# _2fastbt + + if len(self.active_requests) >= self._backlog(new_unchoke): + if not (self.active_requests or self.backlog) and not self.choked: + self.downloader.queued_out[self] = 1 + if DEBUG: print >>sys.stderr, "Downloader: fix_download_endgame: returned" + return +# 2fastbt_ + want = [a for a in self.downloader.all_requests if self.have[a[0]] and a not in self.active_requests and (self.helper is None or self.connection.connection.is_helper_con() or not self.helper.is_ignored(a[0]))] +# _2fastbt + if not (self.active_requests or want): + self.send_not_interested() + if DEBUG: print >>sys.stderr, "Downloader: fix_download_endgame: not interested" + return + if want: + self.send_interested() + if self.choked: + if DEBUG: print >>sys.stderr, "Downloader: fix_download_endgame: choked" + return + shuffle(want) + del want[self.backlog - len(self.active_requests):] + self.active_requests.extend(want) + for piece, begin, length in want: +# 2fastbt_ + if self.helper is None or self.connection.connection.is_helper_con() or self.helper.reserve_piece(piece,self): + self.connection.send_request(piece, begin, length) + self.downloader.chunk_requested(length) +# _2fastbt + + def got_have(self, index): +# print >>sys.stderr,"Downloader: got_have",index + if DEBUG: + print >>sys.stderr,"Downloader: got_have",index + if index == self.downloader.numpieces-1: + self.downloader.totalmeasure.update_rate(self.downloader.storage.total_length-(self.downloader.numpieces-1)*self.downloader.storage.piece_length) + self.peermeasure.update_rate(self.downloader.storage.total_length-(self.downloader.numpieces-1)*self.downloader.storage.piece_length) + else: + self.downloader.totalmeasure.update_rate(self.downloader.storage.piece_length) + self.peermeasure.update_rate(self.downloader.storage.piece_length) + + # Arno: LIVEWRAP + if not self.downloader.picker.is_valid_piece(index): + if DEBUG: + print >>sys.stderr,"Downloader: got_have",index,"is invalid piece" + return # TODO: should we request_more()? + + if self.have[index]: + return + + self.have[index] = True + self.downloader.picker.got_have(index,self.connection) + # ProxyService_ + # + # Aggregate the haves bitfields and send them to the coordinator + # If I am a coordinator, i will exit shortly + self.downloader.aggregate_and_send_haves() + # + # _ProxyService + + if self.have.complete(): + self.downloader.picker.became_seed() + if self.downloader.picker.am_I_complete(): + self.downloader.add_disconnected_seed(self.connection.get_readable_id()) + self.connection.close() + return + if self.downloader.endgamemode: + self.fix_download_endgame() + elif ( not self.downloader.paused + and not self.downloader.picker.is_blocked(index) + and self.downloader.storage.do_I_have_requests(index) ): + if not self.choked: + self._request_more() + else: + self.send_interested() + + def _check_interests(self): + if self.interested or self.downloader.paused: + return + for i in xrange(len(self.have)): + if ( self.have[i] and not self.downloader.picker.is_blocked(i) + and ( self.downloader.endgamemode + or self.downloader.storage.do_I_have_requests(i) ) ): + self.send_interested() + return + + def got_have_bitfield(self, have): + if self.downloader.picker.am_I_complete() and have.complete(): + # Arno: If we're both seeds + if self.downloader.super_seeding: + self.connection.send_bitfield(have.tostring()) # be nice, show you're a seed too + self.connection.close() + self.downloader.add_disconnected_seed(self.connection.get_readable_id()) + return + + if DEBUGBF: + st = time.time() + + if have.complete(): + # Arno: He is seed + self.downloader.picker.got_seed() + else: + # Arno: pass on HAVE knowledge to PiecePicker and if LIVEWRAP: + # filter out valid pieces + + # STBSPEED: if we haven't hooked in yet, don't iterate over whole range + # just over the active ranges in the received Bitfield + activerangeiterators = [] + if self.downloader.picker.videostatus and self.downloader.picker.videostatus.live_streaming and self.downloader.picker.videostatus.get_live_startpos() is None: + # Not hooked in + activeranges = have.get_active_ranges() + + if len(activeranges) == 0: + # Bug, fallback to whole range + activerangeiterators = [self.downloader.picker.get_valid_range_iterator()] + else: + # Create iterators for the active ranges + for (s,e) in activeranges: + activerangeiterators.append(xrange(s,e+1)) + else: + # Hooked in, use own valid range as active range + + # Arno, 2010-04-20: Not correct for VOD with seeking, then we + # should store the HAVE info for things before playback too. + + activerangeiterators = [self.downloader.picker.get_valid_range_iterator()] + + if DEBUGBF: + print >>sys.stderr,"Downloader: got_have_field: live: Filtering bitfield",activerangeiterators + + if not self.downloader.picker.videostatus or self.downloader.picker.videostatus.live_streaming: + if DEBUGBF: + print >>sys.stderr,"Downloader: got_have_field: live or normal filter" + # Transfer HAVE knowledge to PiecePicker and filter pieces if live + validhave = Bitfield(self.downloader.numpieces) + for iterator in activerangeiterators: + for i in iterator: + if have[i]: + validhave[i] = True + self.downloader.picker.got_have(i,self.connection) + else: # VOD + if DEBUGBF: + print >>sys.stderr,"Downloader: got_have_field: VOD filter" + validhave = Bitfield(self.downloader.numpieces) + (first,last) = self.downloader.picker.videostatus.download_range() + for i in xrange(first,last): + if have[i]: + validhave[i] = True + self.downloader.picker.got_have(i,self.connection) + # ProxyService_ + # + # Aggregate the haves bitfields and send them to the coordinator + # ARNOPS: Shouldn't this be done after have = validhave? + self.downloader.aggregate_and_send_haves() + # + # _ProxyService + + """ + # SANITY CHECK + checkhave = Bitfield(self.downloader.numpieces) + for i in self.downloader.picker.get_valid_range_iterator(): + if have[i]: + checkhave[i] = True + + assert validhave.tostring() == checkhave.tostring() + """ + + # Store filtered bitfield instead of received one + have = validhave + + if DEBUGBF: + et = time.time() + diff = et - st + print >>sys.stderr,"Download: got_have_field: took",diff + + + self.have = have + + #print >>sys.stderr,"Downloader: got_have_bitfield: valid",`have.toboollist()` + + if self.downloader.endgamemode and not self.downloader.paused: + for piece, begin, length in self.downloader.all_requests: + if self.have[piece]: + self.send_interested() + break + return + self._check_interests() + + def get_rate(self): + return self.measure.get_rate() + + def get_short_term_rate(self): + return self.short_term_measure.get_rate() + + def is_snubbed(self): +# 2fastbt_ + if not self.choked and clock() - self.last2 > self.downloader.snub_time and \ + not self.connection.connection.is_helper_con() and \ + not self.connection.connection.is_coordinator_con(): +# _2fastbt + for index, begin, length in self.active_requests: + self.connection.send_cancel(index, begin, length) + self.got_choke() # treat it just like a choke + return clock() - self.last > self.downloader.snub_time + + def peer_is_complete(self): + return self.have.complete() + +class Downloader: + def __init__(self, infohash, storage, picker, backlog, max_rate_period, + numpieces, chunksize, measurefunc, snub_time, + kickbans_ok, kickfunc, banfunc, scheduler = None): + self.infohash = infohash + self.b64_infohash = b64encode(infohash) + self.storage = storage + self.picker = picker + self.backlog = backlog + self.max_rate_period = max_rate_period + self.measurefunc = measurefunc + self.totalmeasure = Measure(max_rate_period*storage.piece_length/storage.request_size) + self.numpieces = numpieces + self.chunksize = chunksize + self.snub_time = snub_time + self.kickfunc = kickfunc + self.banfunc = banfunc + self.disconnectedseeds = {} + self.downloads = [] + self.perip = {} + self.gotbaddata = {} + self.kicked = {} + self.banned = {} + self.kickbans_ok = kickbans_ok + self.kickbans_halted = False + self.super_seeding = False + self.endgamemode = False + self.endgame_queued_pieces = [] + self.all_requests = [] + self.discarded = 0L + self.download_rate = 0 +# self.download_rate = 25000 # 25K/s test rate + self.bytes_requested = 0 + self.last_time = clock() + self.queued_out = {} + self.requeueing = False + self.paused = False + self.scheduler = scheduler + + # hack: we should not import this since it is not part of the + # core nor should we import here, but otherwise we will get + # import errors + # + # _event_reporter stores events that are logged somewhere... + # from BaseLib.Core.Statistics.StatusReporter import get_reporter_instance + # self._event_reporter = get_reporter_instance() + self._event_reporter = get_status_holder("LivingLab") + + # check periodicaly + self.scheduler(self.dlr_periodic_check, 1) + + def dlr_periodic_check(self): + self.picker.check_outstanding_requests(self.downloads) + + ds = [d for d in self.downloads if not d.choked] + shuffle(ds) + for d in ds: + d._request_more() + + self.scheduler(self.dlr_periodic_check, 1) + + def set_download_rate(self, rate): + self.download_rate = rate * 1000 + self.bytes_requested = 0 + + def queue_limit(self): + if not self.download_rate: + return 10e10 # that's a big queue! + t = clock() + self.bytes_requested -= (t - self.last_time) * self.download_rate + self.last_time = t + if not self.requeueing and self.queued_out and self.bytes_requested < 0: + self.requeueing = True + q = self.queued_out.keys() + shuffle(q) + self.queued_out = {} + for d in q: + d._request_more() + self.requeueing = False + if -self.bytes_requested > 5*self.download_rate: + self.bytes_requested = -5*self.download_rate + ql = max(int(-self.bytes_requested/self.chunksize), 0) + # if DEBUG: + # print >> sys.stderr, 'Downloader: download_rate: %s, bytes_requested: %s, chunk: %s -> queue limit: %d' % \ + # (self.download_rate, self.bytes_requested, self.chunksize, ql) + return ql + + def chunk_requested(self, size): + self.bytes_requested += size + + external_data_received = chunk_requested + + def make_download(self, connection): + ip = connection.get_ip() + if self.perip.has_key(ip): + perip = self.perip[ip] + else: + perip = self.perip.setdefault(ip, PerIPStats(ip)) + perip.peerid = connection.get_readable_id() + perip.numconnections += 1 + d = SingleDownload(self, connection) + perip.lastdownload = d + self.downloads.append(d) + self._event_reporter.create_and_add_event("connection-established", [self.b64_infohash, str(ip)]) + return d + + def piece_flunked(self, index): + if self.paused: + return + if self.endgamemode: + if self.downloads: + while self.storage.do_I_have_requests(index): + nb, nl = self.storage.new_request(index) + self.all_requests.append((index, nb, nl)) + for d in self.downloads: + d.fix_download_endgame() + return + self._reset_endgame() + return + ds = [d for d in self.downloads if not d.choked] + shuffle(ds) + for d in ds: + d._request_more() + ds = [d for d in self.downloads if not d.interested and d.have[index]] + for d in ds: + d.example_interest = index + d.send_interested() + + def has_downloaders(self): + return len(self.downloads) + + def lost_peer(self, download): + ip = download.ip + self.perip[ip].numconnections -= 1 + if self.perip[ip].lastdownload == download: + self.perip[ip].lastdownload = None + self.downloads.remove(download) + if self.endgamemode and not self.downloads: # all peers gone + self._reset_endgame() + + self._event_reporter.create_and_add_event("connection-upload", [self.b64_infohash, ip, download.connection.total_uploaded]) + self._event_reporter.create_and_add_event("connection-download", [self.b64_infohash, ip, download.connection.total_downloaded]) + self._event_reporter.create_and_add_event("connection-lost", [self.b64_infohash, ip]) + + def _reset_endgame(self): + if DEBUG: print >>sys.stderr, "Downloader: _reset_endgame" + self.storage.reset_endgame(self.all_requests) + self.endgamemode = False + self.all_requests = [] + self.endgame_queued_pieces = [] + + def add_disconnected_seed(self, id): +# if not self.disconnectedseeds.has_key(id): +# self.picker.seed_seen_recently() + self.disconnectedseeds[id]=clock() + +# def expire_disconnected_seeds(self): + + def num_disconnected_seeds(self): + # first expire old ones + expired = [] + for id, t in self.disconnectedseeds.items(): + if clock() - t > EXPIRE_TIME: #Expire old seeds after so long + expired.append(id) + for id in expired: +# self.picker.seed_disappeared() + del self.disconnectedseeds[id] + return len(self.disconnectedseeds) + # if this isn't called by a stats-gathering function + # it should be scheduled to run every minute or two. + + def _check_kicks_ok(self): + if len(self.gotbaddata) > 10: + self.kickbans_ok = False + self.kickbans_halted = True + return self.kickbans_ok and len(self.downloads) > 2 + + def try_kick(self, download): + if self._check_kicks_ok(): + download.guard.download = None + ip = download.ip + id = download.connection.get_readable_id() + self.kicked[ip] = id + self.perip[ip].peerid = id + self.kickfunc(download.connection) + + def try_ban(self, ip): + if self._check_kicks_ok(): + self.banfunc(ip) + self.banned[ip] = self.perip[ip].peerid + if self.kicked.has_key(ip): + del self.kicked[ip] + + def set_super_seed(self): + self.super_seeding = True + + def check_complete(self, index): + if self.endgamemode and not self.all_requests: + self.endgamemode = False + if self.endgame_queued_pieces and not self.endgamemode: + self.requeue_piece_download() + if self.picker.am_I_complete(): + assert not self.all_requests + assert not self.endgamemode + + for download in self.downloads: + if download.have.complete(): + download.connection.send_have(index) # be nice, tell the other seed you completed + self.add_disconnected_seed(download.connection.get_readable_id()) + download.connection.close() + + self._event_reporter.create_and_add_event("connection-seed", [self.b64_infohash, download.ip, download.connection.total_uploaded]) + else: + self._event_reporter.create_and_add_event("connection-upload", [self.b64_infohash, download.ip, download.connection.total_uploaded]) + self._event_reporter.create_and_add_event("connection-download", [self.b64_infohash, download.ip, download.connection.total_downloaded]) + + self._event_reporter.create_and_add_event("complete", [self.b64_infohash]) + # self._event_reporter.flush() + + return True + return False + + def too_many_partials(self): + return len(self.storage.dirty) > (len(self.downloads)/2) + + def cancel_requests(self, requests, allowrerequest=True): + + # todo: remove duplicates + slowpieces = [piece_id for piece_id, _, _ in requests] + + if self.endgamemode: + if self.endgame_queued_pieces: + for piece_id, _, _ in requests: + if not self.storage.do_I_have(piece_id): + try: + self.endgame_queued_pieces.remove(piece_id) + except: + pass + + # remove the items in requests from self.all_requests + if not allowrerequest: + self.all_requests = [request for request in self.all_requests if not request in requests] + if DEBUG: print >>sys.stderr, "Downloader: cancel_requests: all_requests", len(self.all_requests), "remaining" + + for download in self.downloads: + hit = False + for request in download.active_requests: + if request in requests: + hit = True + if DEBUG: print >>sys.stderr, "Downloader:cancel_requests: canceling", request, "on", download.ip + download.connection.send_cancel(*request) + if not self.endgamemode: + self.storage.request_lost(*request) + if hit: + download.active_requests = [request for request in download.active_requests if not request in requests] + # Arno: VOD: all these peers were slow for their individually + # assigned pieces. These pieces have high priority, so don't + # retrieve any of theses pieces from these slow peers, just + # give them something further in the future. + if allowrerequest: + download._request_more() + else: + # Arno: ALT is to just kick peer. Good option if we have lots (See Encryper.to_connect() queue + #print >>sys.stderr,"Downloader: Kicking slow peer",d.ip + #d.connection.close() # bye bye, zwaai zwaai + download._request_more(slowpieces=slowpieces) + + if not self.endgamemode and download.choked: + download._check_interests() + + def cancel_piece_download(self, pieces, allowrerequest=True): + if self.endgamemode: + if self.endgame_queued_pieces: + for piece in pieces: + try: + self.endgame_queued_pieces.remove(piece) + except: + pass + + if allowrerequest: + for index, nb, nl in self.all_requests: + if index in pieces: + self.storage.request_lost(index, nb, nl) + + else: + new_all_requests = [] + for index, nb, nl in self.all_requests: + if index in pieces: + self.storage.request_lost(index, nb, nl) + else: + new_all_requests.append((index, nb, nl)) + self.all_requests = new_all_requests + if DEBUG: print >>sys.stderr, "Downloader: cancel_piece_download: all_requests", len(self.all_requests), "remaining" + + for d in self.downloads: + hit = False + for index, nb, nl in d.active_requests: + if index in pieces: + hit = True + d.connection.send_cancel(index, nb, nl) + if not self.endgamemode: + self.storage.request_lost(index, nb, nl) + if hit: + d.active_requests = [ r for r in d.active_requests + if r[0] not in pieces ] + # Arno: VOD: all these peers were slow for their individually + # assigned pieces. These pieces have high priority, so don't + # retrieve any of theses pieces from these slow peers, just + # give them something further in the future. + if not allowrerequest: + # Arno: ALT is to just kick peer. Good option if we have lots (See Encryper.to_connect() queue + #print >>sys.stderr,"Downloader: Kicking slow peer",d.ip + #d.connection.close() # bye bye, zwaai zwaai + d._request_more(slowpieces=pieces) + else: + d._request_more() + if not self.endgamemode and d.choked: + d._check_interests() + + def requeue_piece_download(self, pieces = []): + if self.endgame_queued_pieces: + for piece in pieces: + if not piece in self.endgame_queued_pieces: + self.endgame_queued_pieces.append(piece) + pieces = self.endgame_queued_pieces + if self.endgamemode: + if self.all_requests: + self.endgame_queued_pieces = pieces + return + self.endgamemode = False + self.endgame_queued_pieces = None + + ds = [d for d in self.downloads] + shuffle(ds) + for d in ds: + if d.choked: + d._check_interests() + else: + d._request_more() + + def start_endgame(self): + assert not self.endgamemode + self.endgamemode = True + assert not self.all_requests + for d in self.downloads: + if d.active_requests: + assert d.interested and not d.choked + for request in d.active_requests: + assert not request in self.all_requests + self.all_requests.append(request) + for d in self.downloads: + d.fix_download_endgame() + if DEBUG: print >>sys.stderr, "Downloader: start_endgame: we have", len(self.all_requests), "requests remaining" + + def pause(self, flag): + self.paused = flag + if flag: + for d in self.downloads: + for index, begin, length in d.active_requests: + d.connection.send_cancel(index, begin, length) + d._letgo() + d.send_not_interested() + if self.endgamemode: + self._reset_endgame() + else: + shuffle(self.downloads) + for d in self.downloads: + d._check_interests() + if d.interested and not d.choked: + d._request_more() + + def live_invalidate(self,piece,mevirgin=False): # Arno: LIVEWRAP + #print >>sys.stderr,"Downloader: live_invalidate",piece + for d in self.downloads: + d.have[piece] = False + # STBSPEED: If I have no pieces yet, no need to loop to invalidate them. + if not mevirgin: + self.storage.live_invalidate(piece) + + def live_invalidate_ranges(self,toinvalidateranges,toinvalidateset): + """ STBPEED: Faster version of live_invalidate that copies have arrays + rather than iterate over them for clearing + """ + if len(toinvalidateranges) == 1: + (s,e) = toinvalidateranges[0] + emptyrange = [False for piece in xrange(s,e+1)] + assert len(emptyrange) == e+1-s + + for d in self.downloads: + newhave = d.have[0:s] + emptyrange + d.have[e+1:] + + #oldhave = d.have + d.have = Bitfield(length=len(newhave),fromarray=newhave) + #assert oldhave.tostring() == d.have.tostring() + """ + for piece in toinvalidateset: + d.have[piece] = False + print >>sys.stderr,"d len",len(d.have) + print >>sys.stderr,"new len",len(newhave) + + for i in xrange(0,len(newhave)): + if d.have[i] != newhave[i]: + print >>sys.stderr,"newhave diff",i + assert False + """ + + else: + (s1,e1) = toinvalidateranges[0] + (s2,e2) = toinvalidateranges[1] + emptyrange1 = [False for piece in xrange(s1,e1+1)] + emptyrange2 = [False for piece in xrange(s2,e2+1)] + + assert len(emptyrange1) == e1+1-s1 + assert len(emptyrange2) == e2+1-s2 + + for d in self.downloads: + newhave = emptyrange1 + d.have[e1+1:s2] + emptyrange2 + + #oldhave = d.have + d.have = Bitfield(length=len(newhave),fromarray=newhave) + #assert oldhave.tostring() == d.have.tostring() + """ + for piece in toinvalidateset: + d.have[piece] = False + print >>sys.stderr,"d len",len(d.have) + print >>sys.stderr,"new len",len(newhave) + for i in xrange(0,len(newhave)): + if d.have[i] != newhave[i]: + print >>sys.stderr,"newhave diff",i + assert False + """ + + # ProxyService_ + # + def aggregate_and_send_haves(self): + """ Aggregates the information from the haves bitfields for all the active connections, + then calls the helper class to send the aggregated information as a PROXY_HAVE message + """ + if self.picker.helper: + # The current node is a coordinator + if DEBUG: + print >> sys.stderr,"Downloader: aggregate_and_send_haves: helper None or helper conn" + + # haves_vector is a matrix, having on each line a Bitfield + haves_vector = [None] * len(self.downloads) + for i in range(0, len(self.downloads)): + haves_vector[i] = self.downloads[i].have + + #Calculate the aggregated haves + aggregated_haves = Bitfield(self.numpieces) + for piece in range (0, self.numpieces): + aggregated_value = False + # For every column in the haves_vector matrix + for d in range(0, len(self.downloads)): + # For every active connection + aggregated_value = aggregated_value or haves_vector[d][piece] # Logical OR operation + aggregated_haves[piece] = aggregated_value + + self.picker.helper.send_proxy_have(aggregated_haves) + # + # _ProxyService \ No newline at end of file diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/DownloaderFeedback.py b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/DownloaderFeedback.py new file mode 100644 index 0000000..b5804b6 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/DownloaderFeedback.py @@ -0,0 +1,224 @@ +# Written by Bram Cohen +# see LICENSE.txt for license information + +from threading import Event + +try: + True +except: + True = 1 + False = 0 + +class DownloaderFeedback: + def __init__(self, choker, ghttpdl, hhttpdl, add_task, upfunc, downfunc, + ratemeasure, leftfunc, file_length, finflag, sp, statistics, + statusfunc = None, interval = None, infohash = None, voddownload=None): + self.choker = choker + self.ghttpdl = ghttpdl + self.hhttpdl = hhttpdl + self.add_task = add_task + self.upfunc = upfunc + self.downfunc = downfunc + self.ratemeasure = ratemeasure + self.leftfunc = leftfunc + self.file_length = file_length + self.finflag = finflag + self.sp = sp + self.statistics = statistics + self.lastids = [] + self.spewdata = None + self.infohash = infohash + self.voddownload = voddownload + self.doneprocessing = Event() + self.doneprocessing.set() + if statusfunc: + self.autodisplay(statusfunc, interval) + + + def _rotate(self): + cs = self.choker.connections + for id in self.lastids: + for i in xrange(len(cs)): + if cs[i].get_id() == id: + return cs[i:] + cs[:i] + return cs + + def spews(self): + l = [] + cs = self._rotate() + self.lastids = [c.get_id() for c in cs] + for c in cs: # c: Connecter.Connection + a = {} + a['id'] = c.get_readable_id() + a['ip'] = c.get_ip() + if c.is_locally_initiated(): + a['port'] = c.get_port() + else: + a['port'] = 0 + try: + a['optimistic'] = (c is self.choker.connections[0]) + except: + a['optimistic'] = False + if c.is_locally_initiated(): + a['direction'] = 'L' + else: + a['direction'] = 'R' + ##a['unauth_permid'] = c.get_unauth_permid() + u = c.get_upload() + a['uprate'] = int(u.measure.get_rate()) + a['uinterested'] = u.is_interested() + a['uchoked'] = u.is_choked() + d = c.get_download() + a['downrate'] = int(d.measure.get_rate()) + a['dinterested'] = d.is_interested() + a['dchoked'] = d.is_choked() + a['snubbed'] = d.is_snubbed() + a['utotal'] = d.connection.upload.measure.get_total() + a['dtotal'] = d.connection.download.measure.get_total() + if d.connection.download.have: + a['completed'] = float(len(d.connection.download.have)-d.connection.download.have.numfalse)/float(len(d.connection.download.have)) + else: + a['completed'] = 1.0 + # The total download speed of the peer as measured from its + # HAVE messages. + a['speed'] = d.connection.download.peermeasure.get_rate() + a['g2g'] = c.use_g2g + a['g2g_score'] = c.g2g_score() + + # RePEX: include number of pex messages in the stats + a['pex_received'] = c.pex_received + + l.append(a) + + for dl in self.ghttpdl.get_downloads(): + if dl.goodseed: + a = {} + a['id'] = 'url list' + a['ip'] = dl.baseurl + a['optimistic'] = False + a['direction'] = 'L' + a['uprate'] = 0 + a['uinterested'] = False + a['uchoked'] = False + a['downrate'] = int(dl.measure.get_rate()) + a['dinterested'] = True + a['dchoked'] = not dl.active + a['snubbed'] = not dl.active + a['utotal'] = None + a['dtotal'] = dl.measure.get_total() + a['completed'] = 1.0 + a['speed'] = None + + l.append(a) + for dl in self.hhttpdl.get_downloads(): + if dl.goodseed: + a = {} + a['id'] = 'http seed' + a['ip'] = dl.baseurl + a['optimistic'] = False + a['direction'] = 'L' + a['uprate'] = 0 + a['uinterested'] = False + a['uchoked'] = False + a['downrate'] = int(dl.measure.get_rate()) + a['dinterested'] = True + a['dchoked'] = not dl.active + a['snubbed'] = not dl.active + a['utotal'] = None + a['dtotal'] = dl.measure.get_total() + a['completed'] = 1.0 + a['speed'] = None + + l.append(a) + return l + + + def gather(self, displayfunc = None, getpeerlist=False): + """ Called by SingleDownload to obtain download statistics to become the + DownloadStates for each Download """ + s = {'stats': self.statistics.update()} + if getpeerlist: + s['spew'] = self.spews() + else: + s['spew'] = None + s['up'] = self.upfunc() + if self.finflag.isSet(): + s['done'] = self.file_length + s['down'] = 0.0 + s['frac'] = 1.0 + s['wanted'] = 0 + s['time'] = 0 + s['vod'] = False + s['vod_prebuf_frac'] = 1.0 + s['vod_playable'] = True + s['vod_playable_after'] = 0.0 + s['vod_stats'] = {'harry':1} + if self.voddownload is not None: + #s['vod'] = True + s['vod_stats'] = self.voddownload.get_stats() + +# if self.voddownload: +# s['vod_duration'] = self.voddownload.get_duration() +# else: +# s['vod_duration'] = None + return s + s['down'] = self.downfunc() + obtained, desired, have = self.leftfunc() + s['done'] = obtained + s['wanted'] = desired + if desired > 0: + s['frac'] = float(obtained)/desired + else: + s['frac'] = 1.0 + if desired == obtained: + s['time'] = 0 + else: + s['time'] = self.ratemeasure.get_time_left(desired-obtained) + + if self.voddownload is not None: + s['vod_prebuf_frac'] = self.voddownload.get_prebuffering_progress() + s['vod_playable'] = self.voddownload.is_playable() + s['vod_playable_after'] = self.voddownload.get_playable_after() + s['vod'] = True + s['vod_stats'] = self.voddownload.get_stats() +# s['vod_duration'] = self.voddownload.get_duration() + else: + s['vod_prebuf_frac'] = 0.0 + s['vod_playable'] = False + s['vod_playable_after'] = float(2 ** 31) + s['vod'] = False + s['vod_stats'] = {} +# s['vod_duration'] = None + return s + + + def display(self, displayfunc): + if not self.doneprocessing.isSet(): + return + self.doneprocessing.clear() + stats = self.gather() + if self.finflag.isSet(): + displayfunc(dpflag = self.doneprocessing, + upRate = stats['up'], + statistics = stats['stats'], spew = stats['spew']) + elif stats['time'] is not None: + displayfunc(dpflag = self.doneprocessing, + fractionDone = stats['frac'], sizeDone = stats['done'], + downRate = stats['down'], upRate = stats['up'], + statistics = stats['stats'], spew = stats['spew'], + timeEst = stats['time']) + else: + displayfunc(dpflag = self.doneprocessing, + fractionDone = stats['frac'], sizeDone = stats['done'], + downRate = stats['down'], upRate = stats['up'], + statistics = stats['stats'], spew = stats['spew']) + + + def autodisplay(self, displayfunc, interval): + self.displayfunc = displayfunc + self.interval = interval + self._autodisplay() + + def _autodisplay(self): + self.add_task(self._autodisplay, self.interval) + self.display(self.displayfunc) diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/Encrypter.py b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/Encrypter.py new file mode 100644 index 0000000..396e59f --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/Encrypter.py @@ -0,0 +1,771 @@ +# Written by Bram Cohen, Pawel Garbacki, George Milescu +# see LICENSE.txt for license information + +import sys +from base64 import b64encode +from cStringIO import StringIO +from binascii import b2a_hex +from socket import error as socketerror +from urllib import quote +from struct import unpack +from time import time +from sets import Set +from traceback import print_exc + +from BaseLib.Core.BitTornado.BT1.MessageID import protocol_name,option_pattern +from BaseLib.Core.BitTornado.BT1.convert import toint +from BaseLib.Core.Statistics.Status.Status import get_status_holder +from BaseLib.Core.ProxyService.ProxyServiceUtil import * + +try: + True +except: + True = 1 + False = 0 + +DEBUG = False + +if sys.platform == 'win32': + # Arno: On windows XP SP2 there is a limit on "the number of concurrent, + # incomplete outbound TCP connection attempts. When the limit is reached, + # subsequent connection attempts are put in a queue and resolved at a fixed + # rate so that there are only a limited number of connections in the + # incomplete state. During normal operation, when programs are connecting + # to available hosts at valid IP addresses, no limit is imposed on the + # number of connections in the incomplete state. When the number of + # incomplete connections exceeds the limit, for example, as a result of + # programs connecting to IP addresses that are not valid, connection-rate + # limitations are invoked, and this event is logged." + # Source: http://go.microsoft.com/fwlink/events.asp and fill in + # Product: "Windos Operating System" + # Event: 4226 + # Which directs to: + # http://www.microsoft.com/technet/support/ee/transform.aspx?ProdName=Windows%20Operating%20System&ProdVer=5.2&EvtID=4226&EvtSrc=Tcpip&LCID=1033 + # + # The ABC/BitTornado people felt the need to therefore impose a rate limit + # themselves. Normally, I would be against this, because the kernel usually + # does a better job at this than some app programmers. But here it makes + # somewhat sense because it appears that when the Win32 "connection-rate + # limitations" are triggered, this causes socket timeout + # errors. For ABC/BitTornado this should not be a big problem, as none of + # the TCP connections it initiates are really vital that they proceed + # quickly. + # + # For Tribler, we have one very important TCP connection at the moment, + # that is when the VideoPlayer/VLC tries to connect to our HTTP-based + # VideoServer on 127.0.0.1 to play the video. We have actually seen these + # connections timeout when we set MAX_INCOMPLETE to > 10. + # + # So we keep this app-level rate limit mechanism FOR NOW and add a security + # margin. To support our SwarmPlayer that wants quick startup of many + # connections we decrease the autoclosing timeout, such that bad conns + # get removed from this rate-limit admin faster. + # + # Windows die die die. + # + # Update, 2009-10-21: + # This limiting has been disabled starting Vista SP2 and beyond: + # http://support.microsoft.com/kb/969710 + # + # Go Vista?! + # + + # [E1101] Module 'sys' has no 'getwindowsversion' member + # pylint: disable-msg=E1101 + winvertuple = sys.getwindowsversion() + # pylint: enable-msg=E1101 + spstr = winvertuple[4] + if winvertuple[0] == 5 or winvertuple[0] == 6 and spstr < "Service Pack 2": + MAX_INCOMPLETE = 8 # safety margin. Even 9 gives video socket timeout, 10 is official limit + else: + MAX_INCOMPLETE = 1024 # inf +else: + MAX_INCOMPLETE = 32 + +AUTOCLOSE_TIMEOUT = 15 # secs. Setting this to e.g. 7 causes Video HTTP timeouts + +def make_readable(s): + if not s: + return '' + if quote(s).find('%') >= 0: + return b2a_hex(s).upper() + return '"'+s+'"' + +def show(s): + return b2a_hex(s) + +class IncompleteCounter: + def __init__(self): + self.c = 0 + def increment(self): + self.c += 1 + def decrement(self): + #print_stack() + self.c -= 1 + def toomany(self): + #print >>sys.stderr,"IncompleteCounter: c",self.c + return self.c >= MAX_INCOMPLETE + +# Arno: This is a global counter!!!! +incompletecounter = IncompleteCounter() + + +# header, reserved, download id, my id, [length, message] + +class Connection: +# 2fastbt_ + def __init__(self, Encoder, connection, id, ext_handshake = False, + locally_initiated = None, dns = None, coord_con = False, challenge = None): +# _2fastbt + self.Encoder = Encoder + self.connection = connection # SocketHandler.SingleSocket + self.connecter = Encoder.connecter + self.id = id + self.readable_id = make_readable(id) + self.coord_con = coord_con + # Challenge sent by the coordinator to identify the Helper + self.challenge = challenge + if locally_initiated is not None: + self.locally_initiated = locally_initiated + elif coord_con: + self.locally_initiated = True + else: + self.locally_initiated = (id != None) +# _2fastbt + self.complete = False + self.keepalive = lambda: None + self.closed = False + self.buffer = StringIO() +# overlay + self.dns = dns + self.support_extend_messages = False + self.connecter_conn = None +# _overlay + self.support_merklehash= False + self.na_want_internal_conn_from = None + self.na_address_distance = None + + if self.locally_initiated: + incompletecounter.increment() +# 2fastbt_ + self.create_time = time() +# _2fastbt + + if self.locally_initiated or ext_handshake: + if DEBUG: + print >>sys.stderr,"Encoder.Connection: writing protname + options + infohash" + self.connection.write(chr(len(protocol_name)) + protocol_name + + option_pattern + self.Encoder.download_id) + if ext_handshake: + if DEBUG: + print >>sys.stderr,"Encoder.Connection: writing my peer-ID" + if coord_con: + # on the helper-coordinator BT communication a special peer id is used + proxy_peer_id = encode_challenge_in_peerid(self.Encoder.my_id, self.challenge) + self.connection.write(proxy_peer_id) + else: + self.connection.write(self.Encoder.my_id) + self.next_len, self.next_func = 20, self.read_peer_id + else: + self.next_len, self.next_func = 1, self.read_header_len + self.Encoder.raw_server.add_task(self._auto_close, AUTOCLOSE_TIMEOUT) + + def get_ip(self, real=False): + return self.connection.get_ip(real) + + def get_port(self, real=False): + return self.connection.get_port(real) + + def get_myip(self, real=False): + return self.connection.get_myip(real) + + def get_myport(self, real=False): + return self.connection.get_myport(real) + + def get_id(self): + return self.id + + def get_readable_id(self): + return self.readable_id + + def is_locally_initiated(self): + return self.locally_initiated + + def is_flushed(self): + return self.connection.is_flushed() + + def supports_merklehash(self): + return self.support_merklehash + + def supports_extend_messages(self): + return self.support_extend_messages + + def set_options(self, s): +# overlay_ + r = unpack("B", s[5]) + if r[0] & 0x10: # left + 43 bit + self.support_extend_messages = True + if DEBUG: + print >>sys.stderr,"encoder: Peer supports EXTEND" + if r[0] & 0x20: # left + 42 bit + self.support_merklehash= True + if DEBUG: + print >>sys.stderr,"encoder: Peer supports Merkle hashes" +# _overlay + + def read_header_len(self, s): + if ord(s) != len(protocol_name): + return None + return len(protocol_name), self.read_header + + def read_header(self, s): + if s != protocol_name: + return None + return 8, self.read_reserved + + def read_reserved(self, s): + if DEBUG: + print >>sys.stderr,"encoder: Reserved bits:", show(s) + print >>sys.stderr,"encoder: Reserved bits=", show(option_pattern) + self.set_options(s) + return 20, self.read_download_id + + def read_download_id(self, s): + if s != self.Encoder.download_id: + return None + if not self.locally_initiated: + self.Encoder.connecter.external_connection_made += 1 + if self.coord_con: + # on the helper-coordinator BT communication a special peer id is used + proxy_peer_id = encode_challenge_in_peerid(self.Encoder.my_id, self.challenge) + self.connection.write(chr(len(protocol_name)) + protocol_name + option_pattern + self.Encoder.download_id + proxy_peer_id) + else: + self.connection.write(chr(len(protocol_name)) + protocol_name + option_pattern + self.Encoder.download_id + self.Encoder.my_id) + + return 20, self.read_peer_id + + def read_peer_id(self, s): +# 2fastbt_ + """ In the scenario of locally initiating: + - I may or may not (normally not) get the remote peerid from a tracker before connecting. + - If I've gotten the remote peerid, set it as self.id, otherwise set self.id as 0. + - I send handshake message without my peerid. + - After I received peer's handshake message, if self.id isn't 0 (i.e., I had the remote peerid), + check the remote peerid, otherwise set self.id as the remote id. If the check is failed, drop the connection. + - Then I send self.Encoder.my_id to the remote peer. + - The remote peer will record self.Encoder.id as my peerid. + - Anyway, self.id should be the same with the remote id if handshake is ok. + + Note self.Encoder.id is a unique id to each swarm I have. + Normally self.id isn't equal to self.Encoder.my_id. + + In the scenario of remotely initiating: + - I don't have remote id + - I received the handshake message to join a swarm. + - Before I read the remote id, I send my handshake with self.Encoder.my_id, my unique id of the swarm. + - I read the remote id and set it as my.id + + before read_peer_id(), self.id = 0 if locally init without remote id + self.id = remote id if locally init with remote id + self.id = None if remotely init + after read_peer_id(), self.id = remote id if locally init + self.id = remote id if remotely init + """ +# _2fastbt + if DEBUG: + print >>sys.stderr,"Encoder.Connection: read_peer_id" + + if not self.id: # remote init or local init without remote peer's id or remote init + self.id = s + self.readable_id = make_readable(s) + else: # local init with remote id + if s != self.id: + if DEBUG: + print >>sys.stderr,"Encoder.Connection: read_peer_id: s != self.id, returning None" + return None + self.complete = self.Encoder.got_id(self) + + if DEBUG: + print >>sys.stderr,"Encoder.Connection: read_peer_id: complete is",self.complete + + + if not self.complete: + if DEBUG: + print >>sys.stderr,"Encoder.Connection: read_peer_id: self not complete!!!, returning None" + return None + if self.locally_initiated: + if self.coord_con: + # on the helper-coordinator BT communication a special peer id is used + proxy_peer_id = encode_challenge_in_peerid(self.Encoder.my_id, self.challenge) + self.connection.write(proxy_peer_id) + else: + self.connection.write(self.Encoder.my_id) + incompletecounter.decrement() + # Arno: open new conn from queue if at limit. Faster than RawServer task + self.Encoder._start_connection_from_queue(sched=False) + + c = self.Encoder.connecter.connection_made(self) + self.keepalive = c.send_keepalive + return 4, self.read_len + + def read_len(self, s): + l = toint(s) + if l > self.Encoder.max_len: + return None + return l, self.read_message + + def read_message(self, s): + if s != '': + self.connecter.got_message(self, s) + #else: + # print >>sys.stderr,"encoder: got keepalive from",s.getpeername() + return 4, self.read_len + + def read_dead(self, s): + return None + + def _auto_close(self): + if not self.complete and not self.is_coordinator_con(): + if DEBUG: + print >>sys.stderr,"encoder: autoclosing ",self.get_myip(),self.get_myport(),"to",self.get_ip(),self.get_port() + + self.Encoder._event_reporter.create_and_add_event("connection-timeout", [b64encode(self.Encoder.connecter.infohash), self.get_ip(), self.get_port()]) + + # RePEX: inform repexer of timeout + repexer = self.Encoder.repexer + if repexer and not self.closed: + try: + repexer.connection_timeout(self) + except: + print_exc() + self.close() + + def close(self,closeall=False): + if DEBUG: + print >>sys.stderr,"encoder: closing connection",self.get_ip() + #print_stack() + + if not self.closed: + self.connection.close() + self.sever(closeall=closeall) + + + def sever(self,closeall=False): + self.closed = True + if self.Encoder.connections.has_key(self.connection): + self.Encoder.admin_close(self.connection) + + # RePEX: inform repexer of closed connection + repexer = self.Encoder.repexer + if repexer and not self.complete: + try: + repexer.connection_closed(self) + except: + print_exc() + + if self.complete: + self.connecter.connection_lost(self) + elif self.locally_initiated: + incompletecounter.decrement() + # Arno: open new conn from queue if at limit. Faster than RawServer task + if not closeall: + self.Encoder._start_connection_from_queue(sched=False) + + def send_message_raw(self, message): + if not self.closed: + self.connection.write(message) # SingleSocket + + def data_came_in(self, connection, s): + self.Encoder.measurefunc(len(s)) + while 1: + if self.closed: + return + i = self.next_len - self.buffer.tell() + if i > len(s): + self.buffer.write(s) + return + self.buffer.write(s[:i]) + s = s[i:] + m = self.buffer.getvalue() + self.buffer.reset() + self.buffer.truncate() + try: + x = self.next_func(m) + except: + print_exc() + self.next_len, self.next_func = 1, self.read_dead + raise + if x is None: + if DEBUG: + print >>sys.stderr,"encoder: function failed",self.next_func + self.close() + return + self.next_len, self.next_func = x + + def connection_flushed(self, connection): + if self.complete: + self.connecter.connection_flushed(self) + + def connection_lost(self, connection): + if self.Encoder.connections.has_key(connection): + self.sever() +# 2fastbt_ + def is_coordinator_con(self): + #if DEBUG: + # print >>sys.stderr,"encoder: is_coordinator_con: coordinator is ",self.Encoder.coordinator_ip + if self.coord_con: + return True + elif self.get_ip() == self.Encoder.coordinator_ip and self.get_ip() != '127.0.0.1': # Arno: for testing + return True + else: + return False + + def is_helper_con(self): + coordinator = self.connecter.coordinator + if coordinator is None: + return False + return coordinator.is_helper_ip(self.get_ip()) +# _2fastbt + + # NETWORK AWARE + def na_set_address_distance(self): + """ Calc address distance. Currently simple: if same /24 then 0 + else 1. TODO: IPv6 + """ + hisip = self.get_ip(real=True) + myip = self.get_myip(real=True) + + a = hisip.split(".") + b = myip.split(".") + if a[0] == b[0] and a[1] == b[1] and a[2] == b[2]: + if DEBUG: + print >>sys.stderr,"encoder.connection: na: Found peer on local LAN",self.get_ip() + self.na_address_distance = 0 + else: + self.na_address_distance = 1 + + def na_get_address_distance(self): + return self.na_address_distance + + + + + +class Encoder: + def __init__(self, connecter, raw_server, my_id, max_len, + schedulefunc, keepalive_delay, download_id, + measurefunc, config): + self.raw_server = raw_server + self.connecter = connecter + self.my_id = my_id + self.max_len = max_len + self.schedulefunc = schedulefunc + self.keepalive_delay = keepalive_delay + self.download_id = download_id + self.measurefunc = measurefunc + self.config = config + self.connections = {} + self.banned = {} + self.to_connect = Set() + self.trackertime = 0 + self.paused = False + if self.config['max_connections'] == 0: + self.max_connections = 2 ** 30 + else: + self.max_connections = self.config['max_connections'] + """ + In r529 there was a problem when a single Windows client + would connect to our text-based seeder (i.e. btlaunchmany) + with no other clients present. Apparently both the seeder + and client would connect to eachother simultaneously, but + not end up with a good connection, halting the client. + + Arno, 2006-03-10: Reappears in ~r890, fixed in r892. It + appears to be a problem of writing to a nonblocking socket + before it signalled it is ready for writing, although the + evidence is inconclusive. + + Arno: 2006-12-15: Reappears in r2319. There is some weird + socket problem here. Using Python 2.4.4 doesn't solve it. + The problem I see here is that as soon as we register + at the tracker, the single seeder tries to connect to + us. He succeeds, but after a short while the connection + appears to be closed by him. We then wind up with no + connection at all and have to wait until we recontact + the tracker. + + My workaround is to refuse these initial connections from + the seeder and wait until I've started connecting to peers + based on the info I got from the tracker before accepting + remote connections. + + Arno: 2007-02-16: I think I finally found it. The Tribler + tracker (BitTornado/BT1/track.py) will do a NAT check + (BitTornado/BT1/NATCheck) by default, which consists of + initiating a connection and then closing it after a good + BT handshake was received. + + The solution now is to make sure we check IP and port to + identify existing connections. I already added that 2006-12-15, + so I just removed the restriction on initial connections, + which are superfluous. + """ + self.rerequest = None +# 2fastbt_ + self.toofast_banned = {} + self.coordinator_ip = None +# _2fastbt + + # hack: we should not import this since it is not part of the + # core nor should we import here, but otherwise we will get + # import errors + # + # _event_reporter stores events that are logged somewhere... + # from BaseLib.Core.Statistics.StatusReporter import get_reporter_instance + self._event_reporter = get_status_holder("LivingLab") + + # the addresses that have already been reported + self._known_addresses = {} + + schedulefunc(self.send_keepalives, keepalive_delay) + + # RePEX: added repexer field. + # Note: perhaps call it observer in the future and make the + # download engine more observable? + self.repexer = None + + def send_keepalives(self): + self.schedulefunc(self.send_keepalives, self.keepalive_delay) + if self.paused: + return + for c in self.connections.values(): + c.keepalive() + + def start_connections(self, dnsidlist): + """ Arno: dnsidlist is a list of tuples (dns,id) where dns is a (ip,port) tuple + and id is apparently always 0. It must be unequal to None at least, + because Encrypter.Connection used the id to see if a connection is + locally initiated?! """ + + if DEBUG: + print >>sys.stderr,"encoder: adding",len(dnsidlist),"peers to queue, current len",len(self.to_connect) + if not self.to_connect: + self.raw_server.add_task(self._start_connection_from_queue) + + # all reported addresses are stored in self._known_addresses + # to prevent duplicated addresses being send + new_addresses = [] + known_addresses = self._known_addresses + for dns, _ in dnsidlist: + address = "%s:%s" % dns + if not address in known_addresses: + known_addresses[address] = True + new_addresses.append(address) + + if new_addresses: + self._event_reporter.create_and_add_event("known-hosts", [b64encode(self.connecter.infohash), ";".join(new_addresses)]) + + # prevent 'to much' memory usage + if len(known_addresses) > 2500: + known_addresses.clear() + + self.to_connect.update(dnsidlist) + # make sure addrs from various sources, like tracker, ut_pex and DHT are mixed + # TODO: or not? For Tribler Supported we may want the tracker to + # be more authoritative, such that official seeders found fast. Nah. + + #random.shuffle(self.to_connect) + #Jelle: Since objects are already placed in the Set in pseudo random order, they don't have to + # be shuffled (and a Set cannot be shuffled). + + self.trackertime = int(time()) + + def _start_connection_from_queue(self,sched=True): + try: + if not self.to_connect: + return + + if self.connecter.external_connection_made: + max_initiate = self.config['max_initiate'] + else: + max_initiate = int(self.config['max_initiate']*1.5) + cons = len(self.connections) + + if DEBUG: + print >>sys.stderr,"encoder: conns",cons,"max conns",self.max_connections,"max init",max_initiate + + if cons >= self.max_connections or cons >= max_initiate: + delay = 60.0 + elif self.paused or incompletecounter.toomany(): + delay = 1.0 + else: + delay = 0.0 + dns, id = self.to_connect.pop() + self.start_connection(dns, id) + if self.to_connect and sched: + if DEBUG: + print >>sys.stderr,"encoder: start_from_queue delay",delay + self.raw_server.add_task(self._start_connection_from_queue, delay) + except: + print_exc() + raise + + def start_connection(self, dns, id, coord_con = False, forcenew = False, challenge = None): + """ Locally initiated connection """ + if DEBUG: + print >>sys.stderr,"encoder: start_connection:",dns + print >>sys.stderr,"encoder: start_connection: qlen",len(self.to_connect),"nconns",len(self.connections),"maxi",self.config['max_initiate'],"maxc",self.config['max_connections'] + + if ( self.paused + or len(self.connections) >= self.max_connections + or id == self.my_id + or self.banned.has_key(dns[0]) ) and not forcenew: + if DEBUG: + print >>sys.stderr,"encoder: start_connection: we're paused or too busy" + return True + for v in self.connections.values(): # avoid duplicated connection from a single ip + if v is None: + continue + if id and v.id == id and not forcenew: + if DEBUG: + print >>sys.stderr,"encoder: start_connection: already connected to peer",`id` + return True + ip = v.get_ip(True) + port = v.get_port(False) + + if DEBUG: + print >>sys.stderr,"encoder: start_connection: candidate",ip,port,"want",dns[0],dns[1] + + if self.config['security'] and ip != 'unknown' and ip == dns[0] and port == dns[1] and not forcenew: + if DEBUG: + print >>sys.stderr,"encoder: start_connection: using existing",ip,"want port",dns[1],"existing port",port,"id",`id` + return True + try: + if DEBUG: + print >>sys.stderr,"encoder: start_connection: Setting up new to peer", dns,"id",`id` + c = self.raw_server.start_connection(dns) + con = Connection(self, c, id, dns = dns, coord_con = coord_con, challenge = challenge) + self.connections[c] = con + c.set_handler(con) + except socketerror: + if DEBUG: + print >>sys.stderr,"Encoder.connection failed" + return False + return True + + def _start_connection(self, dns, id): + def foo(self=self, dns=dns, id=id): + self.start_connection(dns, id) + + self.schedulefunc(foo, 0) + + def got_id(self, connection): + """ check if the connection can be accepted """ + + if connection.id == self.my_id: + # NETWORK AWARE + ret = self.connecter.na_got_loopback(connection) + if DEBUG: + print >>sys.stderr,"encoder: got_id: connection to myself? keep",ret + if ret == False: + self.connecter.external_connection_made -= 1 + return ret + + ip = connection.get_ip(True) + port = connection.get_port(False) + + # NETWORK AWARE + connection.na_set_address_distance() + + if self.config['security'] and self.banned.has_key(ip): + if DEBUG: + print >>sys.stderr,"encoder: got_id: security ban on IP" + return False + for v in self.connections.values(): + if connection is not v: + # NETWORK AWARE + if DEBUG: + print >>sys.stderr,"encoder: got_id: new internal conn from peer? ids",connection.id,v.id + if connection.id == v.id: + if DEBUG: + print >>sys.stderr,"encoder: got_id: new internal conn from peer? addrs",v.na_want_internal_conn_from,ip + if v.na_want_internal_conn_from == ip: + # We were expecting a connection from this peer that shares + # a NAT with us via the internal network. This is it. + self.connecter.na_got_internal_connection(v,connection) + return True + elif v.create_time < connection.create_time: + if DEBUG: + print >>sys.stderr,"encoder: got_id: create time bad?!" + return False + # don't allow multiple connections from the same ip if security is set. + if self.config['security'] and ip != 'unknown' and ip == v.get_ip(True) and port == v.get_port(False): + print >>sys.stderr,"encoder: got_id: closing duplicate connection" + v.close() + return True + + def external_connection_made(self, connection): + """ Remotely initiated connection """ + if DEBUG: + print >>sys.stderr,"encoder: external_conn_made",connection.get_ip() + if self.paused or len(self.connections) >= self.max_connections: + print >>sys.stderr,"encoder: external_conn_made: paused or too many" + connection.close() + return False + con = Connection(self, connection, None) + self.connections[connection] = con + connection.set_handler(con) + return True + + def externally_handshaked_connection_made(self, connection, options, msg_remainder): + if DEBUG: + print >>sys.stderr,"encoder: external_handshaked_conn_made",connection.get_ip() + # 2fastbt_ + if self.paused or len(self.connections) >= self.max_connections: + connection.close() + return False + + con = Connection(self, connection, None, True) + con.set_options(options) + # before: connection.handler = Encoder + # Don't forget to count the external conns! + self.connections[connection] = con + connection.set_handler(con) + # after: connection.handler = Encrypter.Connecter + + if msg_remainder: + con.data_came_in(con, msg_remainder) + return True + + def close_all(self): + if DEBUG: + print >>sys.stderr,"encoder: closing all connections" + copy = self.connections.values()[:] + for c in copy: + c.close(closeall=True) + self.connections = {} + + def ban(self, ip): + self.banned[ip] = 1 + + def pause(self, flag): + self.paused = flag + +# 2fastbt_ + def set_coordinator_ip(self,ip): + self.coordinator_ip = ip +# _2fastbt + + def set_rerequester(self,rerequest): + self.rerequest = rerequest + + def admin_close(self,conn): + del self.connections[conn] + now = int(time()) + if DEBUG: + print >>sys.stderr,"encoder: admin_close: now-tt is",now-self.trackertime + if len(self.connections) == 0 and (now-self.trackertime) < 20: + #if DEBUG: + # print >>sys.stderr,"encoder: admin_close: Recontacting tracker, last request got just dead peers: TEMP DISABLED, ARNO WORKING ON IT" + ###self.rerequest.encoder_wants_new_peers() + pass diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/FileSelector.py b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/FileSelector.py new file mode 100644 index 0000000..e9bbfa0 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/FileSelector.py @@ -0,0 +1,243 @@ +# Written by John Hoffman +# see LICENSE.txt for license information + +from random import shuffle +try: + True +except: + True = 1 + False = 0 + + +class FileSelector: + def __init__(self, files, piece_length, bufferdir, + storage, storagewrapper, sched, failfunc): + self.files = files + + # JD: Store piece length + self.piece_length = piece_length + + self.storage = storage + self.storagewrapper = storagewrapper + self.sched = sched + self.failfunc = failfunc + self.downloader = None + self.picker = None + + storage.set_bufferdir(bufferdir) + + self.numfiles = len(files) + self.priority = [1] * self.numfiles + self.new_priority = None + self.new_partials = None + self.filepieces = [] + total = 0L + for file, length in files: + if not length: + self.filepieces.append(()) + else: + pieces = range( int(total/piece_length), + int((total+length-1)/piece_length)+1 ) + self.filepieces.append(tuple(pieces)) + total += length + self.numpieces = int((total+piece_length-1)/piece_length) + self.piece_priority = [1] * self.numpieces + + + + def init_priority(self, new_priority): + try: + assert len(new_priority) == self.numfiles + for v in new_priority: + assert type(v) in (type(0), type(0L)) + assert v >= -1 + assert v <= 2 + except: +# print_exc() + return False + try: + for f in xrange(self.numfiles): + if new_priority[f] < 0: + self.storage.disable_file(f) + self.new_priority = new_priority + except (IOError, OSError), e: + self.failfunc("can't open partial file for " + + self.files[f][0] + ': ' + str(e)) + return False + return True + + ''' + d['priority'] = [file #1 priority [,file #2 priority...] ] + a list of download priorities for each file. + Priority may be -1, 0, 1, 2. -1 = download disabled, + 0 = highest, 1 = normal, 2 = lowest. + Also see Storage.pickle and StorageWrapper.pickle for additional keys. + ''' + def unpickle(self, d): + if d.has_key('priority'): + if not self.init_priority(d['priority']): + return + pieces = self.storage.unpickle(d) + if not pieces: # don't bother, nothing restoreable + return + new_piece_priority = self._get_piece_priority_list(self.new_priority) + self.storagewrapper.reblock([i == -1 for i in new_piece_priority]) + self.new_partials = self.storagewrapper.unpickle(d, pieces) + + + def tie_in(self, picker, cancelfunc, requestmorefunc): + self.picker = picker + self.cancelfunc = cancelfunc + self.requestmorefunc = requestmorefunc + + if self.new_priority: + self.priority = self.new_priority + self.new_priority = None + self.new_piece_priority = self._set_piece_priority(self.priority) + + if self.new_partials: + shuffle(self.new_partials) + for p in self.new_partials: + self.picker.requested(p) + self.new_partials = None + + + def _set_files_disabled(self, old_priority, new_priority): + old_disabled = [p == -1 for p in old_priority] + new_disabled = [p == -1 for p in new_priority] + data_to_update = [] + for f in xrange(self.numfiles): + if new_disabled[f] != old_disabled[f]: + data_to_update.extend(self.storage.get_piece_update_list(f)) + buffer = [] + for piece, start, length in data_to_update: + if self.storagewrapper.has_data(piece): + data = self.storagewrapper.read_raw(piece, start, length) + if data is None: + return False + buffer.append((piece, start, data)) + + files_updated = False + try: + for f in xrange(self.numfiles): + if new_disabled[f] and not old_disabled[f]: + self.storage.disable_file(f) + files_updated = True + if old_disabled[f] and not new_disabled[f]: + self.storage.enable_file(f) + files_updated = True + except (IOError, OSError), e: + if new_disabled[f]: + msg = "can't open partial file for " + else: + msg = 'unable to open ' + self.failfunc(msg + self.files[f][0] + ': ' + str(e)) + return False + if files_updated: + self.storage.reset_file_status() + + changed_pieces = {} + for piece, start, data in buffer: + if not self.storagewrapper.write_raw(piece, start, data): + return False + data.release() + changed_pieces[piece] = 1 + if not self.storagewrapper.doublecheck_data(changed_pieces): + return False + + return True + + + def _get_piece_priority_list(self, file_priority_list): + l = [-1] * self.numpieces + for f in xrange(self.numfiles): + if file_priority_list[f] == -1: + continue + for i in self.filepieces[f]: + if l[i] == -1: + l[i] = file_priority_list[f] + continue + l[i] = min(l[i], file_priority_list[f]) + return l + + + def _set_piece_priority(self, new_priority): + new_piece_priority = self._get_piece_priority_list(new_priority) + pieces = range(self.numpieces) + shuffle(pieces) + new_blocked = [] + new_unblocked = [] + for piece in pieces: + self.picker.set_priority(piece, new_piece_priority[piece]) + o = self.piece_priority[piece] == -1 + n = new_piece_priority[piece] == -1 + if n and not o: + new_blocked.append(piece) + if o and not n: + new_unblocked.append(piece) + if new_blocked: + self.cancelfunc(new_blocked) + self.storagewrapper.reblock([i == -1 for i in new_piece_priority]) + if new_unblocked: + self.requestmorefunc(new_unblocked) + + return new_piece_priority + + + def set_priorities_now(self, new_priority = None): + if not new_priority: + new_priority = self.new_priority + self.new_priority = None # potential race condition + if not new_priority: + return + old_priority = self.priority + self.priority = new_priority + if not self._set_files_disabled(old_priority, new_priority): + return + self.piece_priority = self._set_piece_priority(new_priority) + + def set_priorities(self, new_priority): + self.new_priority = new_priority + def s(self=self): + self.set_priorities_now() + self.sched(s) + + def set_priority(self, f, p): + new_priority = self.get_priorities() + new_priority[f] = p + self.set_priorities(new_priority) + + def get_priorities(self): + priority = self.new_priority + if not priority: + priority = self.priority # potential race condition + return [i for i in priority] + + def __setitem__(self, index, val): + self.set_priority(index, val) + + def __getitem__(self, index): + try: + return self.new_priority[index] + except: + return self.priority[index] + + + def finish(self): + pass +# for f in xrange(self.numfiles): +# if self.priority[f] == -1: +# self.storage.delete_file(f) + + def pickle(self): + d = {'priority': self.priority} + try: + s = self.storage.pickle() + sw = self.storagewrapper.pickle() + for k in s.keys(): + d[k] = s[k] + for k in sw.keys(): + d[k] = sw[k] + except (IOError, OSError): + pass + return d diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/Filter.py b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/Filter.py new file mode 100644 index 0000000..a564efb --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/Filter.py @@ -0,0 +1,15 @@ +# Written by Bram Cohen +# see LICENSE.txt for license information + +class Filter: + def __init__(self, callback): + self.callback = callback + + def check(self, ip, paramslist, headers): + + def params(key, default = None, l = paramslist): + if l.has_key(key): + return l[key][0] + return default + + return None diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/GetRightHTTPDownloader.py b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/GetRightHTTPDownloader.py new file mode 100644 index 0000000..6d1e4ea --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/GetRightHTTPDownloader.py @@ -0,0 +1,415 @@ +# Written by John Hoffman +# see LICENSE.txt for license information + +# Patched by Diego Andres Rabaioli. +# This is the HTTPDownloader class that implements the GetRight +# style WebSeeding technique. Compared to the John Hoffman's style it +# doesn't require any web server support.However the biggest gap (see +# http://www.bittorrent.org/beps/bep_0019.html) is not taken into +# account when requesting pieces. + +import sys +from random import randint +from urlparse import urlparse +from httplib import HTTPConnection +import urllib +from threading import Thread,currentThread,Lock +from traceback import print_exc, print_stack + +from BaseLib.Core.BitTornado.__init__ import product_name,version_short +from BaseLib.Core.BitTornado.CurrentRateMeasure import Measure +from BaseLib.Core.Utilities.timeouturlopen import find_proxy + +# ProxyService_ +# +try: + from BaseLib.Core.ProxyService.Helper import SingleDownloadHelperInterface +except ImportError: + class SingleDownloadHelperInterface: + + def __init__(self): + pass +# +# _ProxyService + +DEBUG = False + +EXPIRE_TIME = 60 * 60 + +VERSION = product_name+'/'+version_short + +class haveComplete: + def complete(self): + return True + def __getitem__(self, x): + return True +haveall = haveComplete() + +class SingleDownload(SingleDownloadHelperInterface): + + def __init__(self, downloader, url): + SingleDownloadHelperInterface.__init__(self) + self.downloader = downloader + self.baseurl = url + + try: + (self.scheme, self.netloc, path, pars, query, fragment) = urlparse(url) + except: + self.downloader.errorfunc('cannot parse http seed address: '+url) + return + if self.scheme != 'http': + self.downloader.errorfunc('http seed url not http: '+url) + return + + # Arno, 2010-03-08: Make proxy aware + self.proxyhost = find_proxy(url) + try: + if self.proxyhost is None: + self.connection = HTTPConnection(self.netloc) + else: + self.connection = HTTPConnection(self.proxyhost) + except: + self.downloader.errorfunc('cannot connect to http seed: '+url) + return + + self.seedurl = path + self.measure = Measure(downloader.max_rate_period) + self.index = None + self.piece_size = self.downloader.storage._piecelen( 0 ) + self.total_len = self.downloader.storage.total_length + self.url = '' + self.requests = [] + self.request_size = 0 + self.endflag = False + self.error = None + self.retry_period = 30 + self._retry_period = None + self.errorcount = 0 + self.goodseed = False + self.active = False + self.cancelled = False + # HTTP Video Support + self.request_lock = Lock() + self.video_support_policy = True # TODO : get from constructor parameters + self.video_support_enabled = False # Don't start immediately with support + self.video_support_speed = 0.0 # Start with the faster rescheduling speed + self.video_support_slow_start = False # If enabled delay the first request (give chance to peers to give bandwidth) + # Arno, 2010-04-07: Wait 1 second before using HTTP seed. TODO good policy + # If Video Support policy is not eneabled then use Http seed normaly + if not self.video_support_policy: + self.resched(1) + + + def resched(self, len = None): + if self.video_support_policy: + if ( not self.video_support_enabled ) or self.video_support_slow_start: + return + if len is None: + len = self.retry_period + if self.errorcount > 3: + len = min(1.0,len) * (self.errorcount - 2) + + # Arno, 2010-04-07: If immediately, don't go via queue. Actual work is + # done by other thread, so no worries of hogging NetworkThread. + if len > 0: + self.downloader.rawserver.add_task(self.download, len) + else: + self.download() + + def _want(self, index): + if self.endflag: + return self.downloader.storage.do_I_have_requests(index) + else: + return self.downloader.storage.is_unstarted(index) + + def download(self): + from BaseLib.Core.Session import Session + session = Session.get_instance() + session.uch.perform_usercallback(self._download) + + def _download(self): +# 2fastbt_ + self.request_lock.acquire() + if DEBUG: + print "http-sdownload: download()" + if self.is_frozen_by_helper(): + if DEBUG: + print "http-sdownload: blocked, rescheduling" + self.resched(1) + return +# _2fastbt + self.cancelled = False + if self.downloader.picker.am_I_complete(): + self.downloader.downloads.remove(self) + return + self.index = self.downloader.picker.next(haveall, self._want, self) +# 2fastbt_ + if self.index is None and self.frozen_by_helper: + self.resched(0.01) + return +# _2fastbt + if ( self.index is None and not self.endflag + and not self.downloader.peerdownloader.has_downloaders() ): + self.endflag = True + self.index = self.downloader.picker.next(haveall, self._want, self) + if self.index is None: + self.endflag = True + self.resched() + else: + self.url = self.seedurl + start = self.piece_size * self.index + end = start + self.downloader.storage._piecelen( self.index ) - 1 + self.request_range = '%d-%d' % ( start, end ) + self._get_requests() + # Just overwrite other blocks and don't ask for ranges. + self._request() + # Diego : 2010-05-19 : Moving thread creation on _download and not on + # _request anymore. One Lock handles sync problems between threads performing + # new requests before the previous response is read. + """ + # Arno, 2010-04-07: Use threads from pool to Download, more efficient + # than creating a new one for every piece. + from BaseLib.Core.Session import Session + session = Session.get_instance() + session.uch.perform_usercallback(self._request) + # Diego + rq = Thread(target = self._request) + rq.setName( "GetRightHTTPDownloader"+rq.getName() ) + rq.setDaemon(True) + rq.start() + """ + self.active = True + + def _request(self): + import encodings.ascii + import encodings.punycode + import encodings.idna + + self.error = None + self.received_data = None + try: + #print >>sys.stderr, 'HTTP piece ', self.index + if self.proxyhost is None: + realurl = self.url + else: + realurl = self.scheme+'://'+self.netloc+self.url + + self.connection.request( 'GET', realurl, None, + {'Host': self.netloc, 'User-Agent': VERSION, 'Range' : 'bytes=%s' % self.request_range } ) + + r = self.connection.getresponse() + self.connection_status = r.status + self.received_data = r.read() + + except Exception, e: + print_exc() + + self.error = 'error accessing http seed: '+str(e) + try: + self.connection.close() + except: + pass + try: + self.connection = HTTPConnection(self.netloc) + except: + self.connection = None # will cause an exception and retry next cycle + self.downloader.rawserver.add_task(self.request_finished) + + def request_finished(self): + self.active = False + if self.error is not None: + if self.goodseed: + self.downloader.errorfunc(self.error) + self.errorcount += 1 + if self.received_data: + self.errorcount = 0 + if not self._got_data(): + self.received_data = None + if not self.received_data: + self._release_requests() + self.downloader.peerdownloader.piece_flunked(self.index) + self.request_lock.release() + if self._retry_period is not None: + self.resched(self._retry_period) + self._retry_period = None + return + self.resched() + + def _got_data(self): + if self.connection_status == 503: # seed is busy + try: + self.retry_period = max(int(self.received_data), 5) + except: + pass + return False + + if self.connection_status != 200 and self.connection_status != 206: # 206 = partial download OK + self.errorcount += 1 + return False + # Arno, 2010-04-07: retry_period set to 0 for faster DL speeds + # Diego, 2010-04-16: retry_period set depending on the level of support asked by the MovieOnDemandTransporter + self._retry_period = self.video_support_speed + + if len(self.received_data) != self.request_size: + if self.goodseed: + self.downloader.errorfunc('corrupt data from http seed - redownloading') + return False + self.measure.update_rate(len(self.received_data)) + self.downloader.measurefunc(len(self.received_data)) + if self.cancelled: + return False + if not self._fulfill_requests(): + return False + if not self.goodseed: + self.goodseed = True + self.downloader.seedsfound += 1 + if self.downloader.storage.do_I_have(self.index): + self.downloader.picker.complete(self.index) + self.downloader.peerdownloader.check_complete(self.index) + self.downloader.gotpiecefunc(self.index) + return True + + def _get_requests(self): + self.requests = [] + self.request_size = 0L + while self.downloader.storage.do_I_have_requests(self.index): + r = self.downloader.storage.new_request(self.index) + self.requests.append(r) + self.request_size += r[1] + self.requests.sort() + + def _fulfill_requests(self): + start = 0L + success = True + while self.requests: + begin, length = self.requests.pop(0) +# 2fastbt_ + if not self.downloader.storage.piece_came_in(self.index, begin, [], + self.received_data[start:start+length], length): +# _2fastbt + success = False + break + start += length + return success + + def _release_requests(self): + for begin, length in self.requests: + self.downloader.storage.request_lost(self.index, begin, length) + self.requests = [] + + def _request_ranges(self): + s = '' + begin, length = self.requests[0] + for begin1, length1 in self.requests[1:]: + if begin + length == begin1: + length += length1 + continue + else: + if s: + s += ',' + s += str(begin)+'-'+str(begin+length-1) + begin, length = begin1, length1 + if s: + s += ',' + s += str(begin)+'-'+str(begin+length-1) + return s + +# 2fastbt_ + def helper_forces_unchoke(self): + pass + + def helper_set_freezing(self,val): + self.frozen_by_helper = val +# _2fastbt + + def slow_start_wake_up( self ): + self.video_support_slow_start = False + self.resched(0) + + def is_slow_start( self ): + return self.video_support_slow_start + + def start_video_support( self, level = 0.0, sleep_time = None ): + ''' + Level indicates how fast a new request is scheduled and therefore the level of support required. + 0 = maximum support. (immediate rescheduling) + 1 ~= 0.01 seconds between each request + 2 ~= 0.1 seconds between each request + and so on... at the moment just level 0 is asked. To be noted that level is a float! + ''' + + if DEBUG: + print >>sys.stderr,"GetRightHTTPDownloader: START" + self.video_support_speed = 0.001 * ( ( 10 ** level ) - 1 ) + if not self.video_support_enabled: + self.video_support_enabled = True + if sleep_time: + if not self.video_support_slow_start: + self.video_support_slow_start = True + self.downloader.rawserver.add_task( self.slow_start_wake_up, sleep_time ) + else: + self.resched( self.video_support_speed ) + + def stop_video_support( self ): + if DEBUG: + print >>sys.stderr,"GetRightHTTPDownloader: STOP" + if not self.video_support_enabled: + return + self.video_support_enabled = False + + def is_video_support_enabled( self ): + return self.video_support_enabled + + +class GetRightHTTPDownloader: + def __init__(self, storage, picker, rawserver, + finflag, errorfunc, peerdownloader, + max_rate_period, infohash, measurefunc, gotpiecefunc): + self.storage = storage + self.picker = picker + self.rawserver = rawserver + self.finflag = finflag + self.errorfunc = errorfunc + self.peerdownloader = peerdownloader + self.infohash = infohash + self.max_rate_period = max_rate_period + self.gotpiecefunc = gotpiecefunc + self.measurefunc = measurefunc + self.downloads = [] + self.seedsfound = 0 + self.video_support_enabled = False + + def make_download(self, url): + self.downloads.append(SingleDownload(self, url)) + return self.downloads[-1] + + def get_downloads(self): + if self.finflag.isSet(): + return [] + return self.downloads + + def cancel_piece_download(self, pieces): + for d in self.downloads: + if d.active and d.index in pieces: + d.cancelled = True + + # Diego : wrap each single http download + def start_video_support( self, level = 0.0, sleep_time = None ): + for d in self.downloads: + d.start_video_support( level, sleep_time ) + self.video_support_enabled = True + + def stop_video_support( self ): + for d in self.downloads: + d.stop_video_support() + self.video_support_enabled = False + + def is_video_support_enabled( self ): + return self.video_support_enabled + + def is_slow_start( self ): + for d in self.downloads: + if d.is_slow_start(): + return True + return False + diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/HTTPDownloader.py b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/HTTPDownloader.py new file mode 100644 index 0000000..edcc917 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/HTTPDownloader.py @@ -0,0 +1,299 @@ +# Written by John Hoffman, George Milescu +# see LICENSE.txt for license information + +from BaseLib.Core.BitTornado.CurrentRateMeasure import Measure +from random import randint +from urlparse import urlparse +from httplib import HTTPConnection +from urllib import quote +from threading import Thread +from BaseLib.Core.BitTornado.__init__ import product_name,version_short + +# ProxyService_ +# +try: + from BaseLib.Core.ProxyService.Helper import SingleDownloadHelperInterface +except ImportError: + class SingleDownloadHelperInterface: + + def __init__(self): + pass +# +# ProxyService + +try: + True +except: + True = 1 + False = 0 + +# 2fastbt_ +DEBUG = False +# _2fastbt + +EXPIRE_TIME = 60 * 60 + +VERSION = product_name+'/'+version_short + +class haveComplete: + def complete(self): + return True + def __getitem__(self, x): + return True +haveall = haveComplete() + +# 2fastbt_ +class SingleDownload(SingleDownloadHelperInterface): +# _2fastbt + def __init__(self, downloader, url): +# 2fastbt_ + SingleDownloadHelperInterface.__init__(self) +# _2fastbt + self.downloader = downloader + self.baseurl = url + try: + (scheme, self.netloc, path, pars, query, fragment) = urlparse(url) + except: + self.downloader.errorfunc('cannot parse http seed address: '+url) + return + if scheme != 'http': + self.downloader.errorfunc('http seed url not http: '+url) + return + try: + self.connection = HTTPConnection(self.netloc) + except: + self.downloader.errorfunc('cannot connect to http seed: '+url) + return + self.seedurl = path + if pars: + self.seedurl += ';'+pars + self.seedurl += '?' + if query: + self.seedurl += query+'&' + self.seedurl += 'info_hash='+quote(self.downloader.infohash) + + self.measure = Measure(downloader.max_rate_period) + self.index = None + self.url = '' + self.requests = [] + self.request_size = 0 + self.endflag = False + self.error = None + self.retry_period = 30 + self._retry_period = None + self.errorcount = 0 + self.goodseed = False + self.active = False + self.cancelled = False + self.resched(randint(2, 10)) + + def resched(self, len = None): + if len is None: + len = self.retry_period + if self.errorcount > 3: + len = len * (self.errorcount - 2) + self.downloader.rawserver.add_task(self.download, len) + + def _want(self, index): + if self.endflag: + return self.downloader.storage.do_I_have_requests(index) + else: + return self.downloader.storage.is_unstarted(index) + + def download(self): +# 2fastbt_ + if DEBUG: + print "http-sdownload: download()" + if self.is_frozen_by_helper(): + if DEBUG: + print "http-sdownload: blocked, rescheduling" + self.resched(1) + return +# _2fastbt + self.cancelled = False + if self.downloader.picker.am_I_complete(): + self.downloader.downloads.remove(self) + return + self.index = self.downloader.picker.next(haveall, self._want, self) +# 2fastbt_ + if self.index is None and self.frozen_by_helper: + self.resched(0.01) + return +# _2fastbt + if ( self.index is None and not self.endflag + and not self.downloader.peerdownloader.has_downloaders() ): + self.endflag = True + self.index = self.downloader.picker.next(haveall, self._want, self) + if self.index is None: + self.endflag = True + self.resched() + else: + self.url = ( self.seedurl+'&piece='+str(self.index) ) + self._get_requests() + if self.request_size < self.downloader.storage._piecelen(self.index): + self.url += '&ranges='+self._request_ranges() + rq = Thread(target = self._request) + rq.setName( "HTTPDownloader"+rq.getName() ) + rq.setDaemon(True) + rq.start() + self.active = True + + def _request(self): + import encodings.ascii + import encodings.punycode + import encodings.idna + + self.error = None + self.received_data = None + try: + self.connection.request('GET', self.url, None, + {'User-Agent': VERSION}) + r = self.connection.getresponse() + self.connection_status = r.status + self.received_data = r.read() + except Exception, e: + self.error = 'error accessing http seed: '+str(e) + try: + self.connection.close() + except: + pass + try: + self.connection = HTTPConnection(self.netloc) + except: + self.connection = None # will cause an exception and retry next cycle + self.downloader.rawserver.add_task(self.request_finished) + + def request_finished(self): + self.active = False + if self.error is not None: + if self.goodseed: + self.downloader.errorfunc(self.error) + self.errorcount += 1 + if self.received_data: + self.errorcount = 0 + if not self._got_data(): + self.received_data = None + if not self.received_data: + self._release_requests() + self.downloader.peerdownloader.piece_flunked(self.index) + if self._retry_period: + self.resched(self._retry_period) + self._retry_period = None + return + self.resched() + + def _got_data(self): + if self.connection_status == 503: # seed is busy + try: + self.retry_period = max(int(self.received_data), 5) + except: + pass + return False + if self.connection_status != 200: + self.errorcount += 1 + return False + self._retry_period = 1 + if len(self.received_data) != self.request_size: + if self.goodseed: + self.downloader.errorfunc('corrupt data from http seed - redownloading') + return False + self.measure.update_rate(len(self.received_data)) + self.downloader.measurefunc(len(self.received_data)) + if self.cancelled: + return False + if not self._fulfill_requests(): + return False + if not self.goodseed: + self.goodseed = True + self.downloader.seedsfound += 1 + if self.downloader.storage.do_I_have(self.index): + self.downloader.picker.complete(self.index) + self.downloader.peerdownloader.check_complete(self.index) + self.downloader.gotpiecefunc(self.index) + return True + + def _get_requests(self): + self.requests = [] + self.request_size = 0L + while self.downloader.storage.do_I_have_requests(self.index): + r = self.downloader.storage.new_request(self.index) + self.requests.append(r) + self.request_size += r[1] + self.requests.sort() + + def _fulfill_requests(self): + start = 0L + success = True + while self.requests: + begin, length = self.requests.pop(0) +# 2fastbt_ + if not self.downloader.storage.piece_came_in(self.index, begin, [], + self.received_data[start:start+length], length): +# _2fastbt + success = False + break + start += length + return success + + def _release_requests(self): + for begin, length in self.requests: + self.downloader.storage.request_lost(self.index, begin, length) + self.requests = [] + + def _request_ranges(self): + s = '' + begin, length = self.requests[0] + for begin1, length1 in self.requests[1:]: + if begin + length == begin1: + length += length1 + continue + else: + if s: + s += ',' + s += str(begin)+'-'+str(begin+length-1) + begin, length = begin1, length1 + if s: + s += ',' + s += str(begin)+'-'+str(begin+length-1) + return s + +# 2fastbt_ + def helper_forces_unchoke(self): + pass + + def helper_set_freezing(self,val): + self.frozen_by_helper = val +# _2fastbt + + + +class HTTPDownloader: + def __init__(self, storage, picker, rawserver, + finflag, errorfunc, peerdownloader, + max_rate_period, infohash, measurefunc, gotpiecefunc): + self.storage = storage + self.picker = picker + self.rawserver = rawserver + self.finflag = finflag + self.errorfunc = errorfunc + self.peerdownloader = peerdownloader + self.infohash = infohash + self.max_rate_period = max_rate_period + self.gotpiecefunc = gotpiecefunc + self.measurefunc = measurefunc + self.downloads = [] + self.seedsfound = 0 + + def make_download(self, url): + self.downloads.append(SingleDownload(self, url)) + return self.downloads[-1] + + def get_downloads(self): + if self.finflag.isSet(): + return [] + return self.downloads + + def cancel_piece_download(self, pieces): + for d in self.downloads: + if d.active and d.index in pieces: + d.cancelled = True diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/HoffmanHTTPDownloader.py b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/HoffmanHTTPDownloader.py new file mode 100644 index 0000000..0ecce37 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/HoffmanHTTPDownloader.py @@ -0,0 +1,311 @@ +# Written by John Hoffman +# see LICENSE.txt for license information + +import sys +from random import randint +from urlparse import urlparse +from httplib import HTTPConnection +import urllib +from threading import Thread +from traceback import print_exc + +from BaseLib.Core.BitTornado.__init__ import product_name,version_short +from BaseLib.Core.BitTornado.CurrentRateMeasure import Measure +from BaseLib.Core.Utilities.timeouturlopen import find_proxy + +# ProxyService_ +# +try: + from BaseLib.Core.ProxyService.Helper import SingleDownloadHelperInterface +except ImportError: + class SingleDownloadHelperInterface: + + def __init__(self): + pass +# +# _ProxyService + +try: + True +except: + True = 1 + False = 0 + +# 2fastbt_ +DEBUG = False +# _2fastbt + +EXPIRE_TIME = 60 * 60 + +VERSION = product_name+'/'+version_short + +class haveComplete: + def complete(self): + return True + def __getitem__(self, x): + return True +haveall = haveComplete() + +# 2fastbt_ +class SingleDownload(SingleDownloadHelperInterface): +# _2fastbt + def __init__(self, downloader, url): +# 2fastbt_ + SingleDownloadHelperInterface.__init__(self) +# _2fastbt + self.downloader = downloader + self.baseurl = url + try: + (self.scheme, self.netloc, path, pars, query, fragment) = urlparse(url) + except: + self.downloader.errorfunc('cannot parse http seed address: '+url) + return + if self.scheme != 'http': + self.downloader.errorfunc('http seed url not http: '+url) + return + + # Arno, 2010-03-08: Make proxy aware + self.proxyhost = find_proxy(url) + try: + if self.proxyhost is None: + self.connection = HTTPConnection(self.netloc) + else: + self.connection = HTTPConnection(self.proxyhost) + except: + self.downloader.errorfunc('cannot connect to http seed: '+url) + return + self.seedurl = path + if pars: + self.seedurl += ';'+pars + self.seedurl += '?' + if query: + self.seedurl += query+'&' + self.seedurl += 'info_hash='+urllib.quote(self.downloader.infohash) + + self.measure = Measure(downloader.max_rate_period) + self.index = None + self.url = '' + self.requests = [] + self.request_size = 0 + self.endflag = False + self.error = None + self.retry_period = 30 + self._retry_period = None + self.errorcount = 0 + self.goodseed = False + self.active = False + self.cancelled = False + self.resched(randint(2, 10)) + + def resched(self, len = None): + if len is None: + len = self.retry_period + if self.errorcount > 3: + len = len * (self.errorcount - 2) + self.downloader.rawserver.add_task(self.download, len) + + def _want(self, index): + if self.endflag: + return self.downloader.storage.do_I_have_requests(index) + else: + return self.downloader.storage.is_unstarted(index) + + def download(self): +# 2fastbt_ + if DEBUG: + print "http-sdownload: download()" + if self.is_frozen_by_helper(): + if DEBUG: + print "http-sdownload: blocked, rescheduling" + self.resched(1) + return +# _2fastbt + self.cancelled = False + if self.downloader.picker.am_I_complete(): + self.downloader.downloads.remove(self) + return + self.index = self.downloader.picker.next(haveall, self._want, self) +# 2fastbt_ + if self.index is None and self.frozen_by_helper: + self.resched(0.01) + return +# _2fastbt + if ( self.index is None and not self.endflag + and not self.downloader.peerdownloader.has_downloaders() ): + self.endflag = True + self.index = self.downloader.picker.next(haveall, self._want, self) + if self.index is None: + self.endflag = True + self.resched() + else: + self.url = ( self.seedurl+'&piece='+str(self.index) ) + self._get_requests() + if self.request_size < self.downloader.storage._piecelen(self.index): + self.url += '&ranges='+self._request_ranges() + rq = Thread(target = self._request) + rq.setName( "HoffmanHTTPDownloader"+rq.getName() ) + rq.setDaemon(True) + rq.start() + self.active = True + + def _request(self): + import encodings.ascii + import encodings.punycode + import encodings.idna + + self.error = None + self.received_data = None + try: + self.connection.request('GET', self.url, None, + {'User-Agent': VERSION}) + r = self.connection.getresponse() + self.connection_status = r.status + self.received_data = r.read() + except Exception, e: + print_exc() + + self.error = 'error accessing http seed: '+str(e) + try: + self.connection.close() + except: + pass + try: + self.connection = HTTPConnection(self.netloc) + except: + self.connection = None # will cause an exception and retry next cycle + self.downloader.rawserver.add_task(self.request_finished) + + def request_finished(self): + self.active = False + if self.error is not None: + if self.goodseed: + self.downloader.errorfunc(self.error) + self.errorcount += 1 + if self.received_data: + self.errorcount = 0 + if not self._got_data(): + self.received_data = None + if not self.received_data: + self._release_requests() + self.downloader.peerdownloader.piece_flunked(self.index) + if self._retry_period: + self.resched(self._retry_period) + self._retry_period = None + return + self.resched() + + def _got_data(self): + if self.connection_status == 503: # seed is busy + try: + self.retry_period = max(int(self.received_data), 5) + except: + pass + return False + if self.connection_status != 200: + self.errorcount += 1 + return False + self._retry_period = 1 + if len(self.received_data) != self.request_size: + if self.goodseed: + self.downloader.errorfunc('corrupt data from http seed - redownloading') + return False + self.measure.update_rate(len(self.received_data)) + self.downloader.measurefunc(len(self.received_data)) + if self.cancelled: + return False + if not self._fulfill_requests(): + return False + if not self.goodseed: + self.goodseed = True + self.downloader.seedsfound += 1 + if self.downloader.storage.do_I_have(self.index): + self.downloader.picker.complete(self.index) + self.downloader.peerdownloader.check_complete(self.index) + self.downloader.gotpiecefunc(self.index) + return True + + def _get_requests(self): + self.requests = [] + self.request_size = 0L + while self.downloader.storage.do_I_have_requests(self.index): + r = self.downloader.storage.new_request(self.index) + self.requests.append(r) + self.request_size += r[1] + self.requests.sort() + + def _fulfill_requests(self): + start = 0L + success = True + while self.requests: + begin, length = self.requests.pop(0) +# 2fastbt_ + if not self.downloader.storage.piece_came_in(self.index, begin, [], + self.received_data[start:start+length], length): +# _2fastbt + success = False + break + start += length + return success + + def _release_requests(self): + for begin, length in self.requests: + self.downloader.storage.request_lost(self.index, begin, length) + self.requests = [] + + def _request_ranges(self): + s = '' + begin, length = self.requests[0] + for begin1, length1 in self.requests[1:]: + if begin + length == begin1: + length += length1 + continue + else: + if s: + s += ',' + s += str(begin)+'-'+str(begin+length-1) + begin, length = begin1, length1 + if s: + s += ',' + s += str(begin)+'-'+str(begin+length-1) + return s + +# 2fastbt_ + def helper_forces_unchoke(self): + pass + + def helper_set_freezing(self,val): + self.frozen_by_helper = val +# _2fastbt + + + +class HoffmanHTTPDownloader: + def __init__(self, storage, picker, rawserver, + finflag, errorfunc, peerdownloader, + max_rate_period, infohash, measurefunc, gotpiecefunc): + self.storage = storage + self.picker = picker + self.rawserver = rawserver + self.finflag = finflag + self.errorfunc = errorfunc + self.peerdownloader = peerdownloader + self.infohash = infohash + self.max_rate_period = max_rate_period + self.gotpiecefunc = gotpiecefunc + self.measurefunc = measurefunc + self.downloads = [] + self.seedsfound = 0 + + def make_download(self, url): + self.downloads.append(SingleDownload(self, url)) + return self.downloads[-1] + + def get_downloads(self): + if self.finflag.isSet(): + return [] + return self.downloads + + def cancel_piece_download(self, pieces): + for d in self.downloads: + if d.active and d.index in pieces: + d.cancelled = True diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/MessageID.py b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/MessageID.py new file mode 100644 index 0000000..8d37030 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/MessageID.py @@ -0,0 +1,249 @@ +# Written by Jie Yang, Arno Bakker, George Milescu +# see LICENSE.txt for license information +# +# All message IDs in BitTorrent Protocol and our extensions +# +# Arno: please don't define stuff until the spec is ready +# + +protocol_name = 'BitTorrent protocol' +# Enable Tribler extensions: +# Left-most bit = Azureus Enhanced Messaging Protocol (AEMP) +# Left+42 bit = Tribler Simple Merkle Hashes extension v0. Outdated, but still sent for compatibility. +# Left+43 bit = Tribler Overlay swarm extension +# AND uTorrent extended protocol, conflicting. See EXTEND message +# Right-most bit = BitTorrent DHT extension +#option_pattern = chr(0)*8 +option_pattern = '\x00\x00\x00\x00\x00\x30\x00\x00' + + +CHOKE = chr(0) +UNCHOKE = chr(1) +INTERESTED = chr(2) +NOT_INTERESTED = chr(3) + +# index +HAVE = chr(4) +# index, bitfield +BITFIELD = chr(5) +# index, begin, length +REQUEST = chr(6) +# index, begin, piece +PIECE = chr(7) +# index, begin, piece +CANCEL = chr(8) +# 2-byte port +PORT = chr(9) + +# uTorrent and Bram's BitTorrent now support an extended protocol +EXTEND = chr(20) + + +# +# Tribler specific message IDs +# + +# IDs 255 and 254 are reserved. Tribler extensions number downwards + +## PermID /Overlay Swarm Extension +# ctxt +CHALLENGE = chr(253) +# rdata1 +RESPONSE1 = chr(252) +# rdata2 +RESPONSE2 = chr(251) + +## Merkle Hash Extension +# Merkle: PIECE message with hashes +HASHPIECE = chr(250) + +## Buddycast Extension +# payload is beencoded dict +BUDDYCAST = chr(249) + +# bencoded torrent_hash (Arno,2007-08-14: shouldn't be bencoded, but is) +GET_METADATA = chr(248) +# {'torrent_hash', 'metadata', ... } +METADATA = chr(247) + +## ProxyService extension, reused from Cooperative Download (2fast) +# For connectability test +DIALBACK_REQUEST = chr(244) +DIALBACK_REPLY = chr(243) +# torrent_hash +ASK_FOR_HELP = chr(246) +# torrent_hash +STOP_HELPING = chr(245) +# torrent_hash + bencode([piece num,...]) +REQUEST_PIECES = chr(242) +# torrent_hash + bencode([piece num,...]) +CANCEL_PIECE = chr(241) +# torrent_hash +JOIN_HELPERS = chr(224) +# torrent_hash +RESIGN_AS_HELPER = chr(223) +# torrent_hash + bencode([piece num,...]) +DROPPED_PIECE = chr(222) +PROXY_HAVE = chr(221) +PROXY_UNHAVE = chr(220) + + +# SecureOverlay empty payload +KEEP_ALIVE = chr(240) + +## Social-Network feature +SOCIAL_OVERLAP = chr(239) + +# Remote query extension +QUERY = chr(238) +QUERY_REPLY = chr(237) + +# Bartercast, payload is bencoded dict +BARTERCAST = chr(236) + +# g2g info (uplink statistics, etc) +G2G_PIECE_XFER = chr(235) + +# Friendship messages +FRIENDSHIP = chr(234) + +# Generic Crawler messages +CRAWLER_REQUEST = chr(232) +CRAWLER_REPLY = chr(231) + +VOTECAST = chr(226) +CHANNELCAST = chr(225) + +GET_SUBS = chr(230) +SUBS = chr(229) + +####### FREE ID = 227/228 + < 220 + + +# +# EXTEND_MSG_CS sub-messages +# +# Closed swarms +# CS : removed, unused. Using CS_CHALLENGE_A message ID in extend handshake +CS_CHALLENGE_A = chr(227) +CS_CHALLENGE_B = chr(228) +CS_POA_EXCHANGE_A = chr(229) +CS_POA_EXCHANGE_B = chr(230) + +# +# Crawler sub-messages +# +CRAWLER_DATABASE_QUERY = chr(1) +CRAWLER_SEEDINGSTATS_QUERY = chr(2) +CRAWLER_NATCHECK = chr(3) +CRAWLER_FRIENDSHIP_STATS = chr(4) +CRAWLER_NATTRAVERSAL = chr(5) +CRAWLER_VIDEOPLAYBACK_INFO_QUERY = chr(6) +CRAWLER_VIDEOPLAYBACK_EVENT_QUERY = chr(7) +CRAWLER_REPEX_QUERY = chr(8) # RePEX: query a peer's SwarmCache history +CRAWLER_PUNCTURE_QUERY = chr(9) +CRAWLER_CHANNEL_QUERY = chr(10) + + +# +# Summaries +# + +PermIDMessages = [CHALLENGE, RESPONSE1, RESPONSE2] +BuddyCastMessages = [CHANNELCAST, VOTECAST, BARTERCAST, BUDDYCAST, KEEP_ALIVE] +MetadataMessages = [GET_METADATA, METADATA] +DialbackMessages = [DIALBACK_REQUEST,DIALBACK_REPLY] +HelpCoordinatorMessages = [ASK_FOR_HELP,STOP_HELPING,REQUEST_PIECES,CANCEL_PIECE] +HelpHelperMessages = [JOIN_HELPERS,RESIGN_AS_HELPER,DROPPED_PIECE,PROXY_HAVE,PROXY_UNHAVE] +SocialNetworkMessages = [SOCIAL_OVERLAP] +RemoteQueryMessages = [QUERY,QUERY_REPLY] +VoDMessages = [G2G_PIECE_XFER] +FriendshipMessages = [FRIENDSHIP] +CrawlerMessages = [CRAWLER_REQUEST, CRAWLER_REPLY] +SubtitleMessages = [GET_SUBS, SUBS] + +# All overlay-swarm messages +OverlaySwarmMessages = PermIDMessages + BuddyCastMessages + MetadataMessages + HelpCoordinatorMessages + HelpHelperMessages + SocialNetworkMessages + RemoteQueryMessages + CrawlerMessages + + +# +# Printing +# + +message_map = { + CHOKE:"CHOKE", + UNCHOKE:"UNCHOKE", + INTERESTED:"INTEREST", + NOT_INTERESTED:"NOT_INTEREST", + HAVE:"HAVE", + BITFIELD:"BITFIELD", + REQUEST:"REQUEST", + CANCEL:"CANCEL", + PIECE:"PIECE", + PORT:"PORT", + EXTEND:"EXTEND", + + CHALLENGE:"CHALLENGE", + RESPONSE1:"RESPONSE1", + RESPONSE2:"RESPONSE2", + HASHPIECE:"HASHPIECE", + BUDDYCAST:"BUDDYCAST", + GET_METADATA:"GET_METADATA", + METADATA:"METADATA", + ASK_FOR_HELP:"ASK_FOR_HELP", + STOP_HELPING:"STOP_HELPING", + REQUEST_PIECES:"REQUEST_PIECES", + CANCEL_PIECE:"CANCEL_PIECE", + JOIN_HELPERS:"JOIN_HELPERS", + RESIGN_AS_HELPER:"RESIGN_AS_HELPER", + DROPPED_PIECE:"DROPPED_PIECE", + PROXY_HAVE:"PROXY_HAVE", + PROXY_UNHAVE:"PROXY_UNHAVE", + DIALBACK_REQUEST:"DIALBACK_REQUEST", + DIALBACK_REPLY:"DIALBACK_REPLY", + KEEP_ALIVE:"KEEP_ALIVE", + SOCIAL_OVERLAP:"SOCIAL_OVERLAP", + QUERY:"QUERY", + QUERY_REPLY:"QUERY_REPLY", + VOTECAST:"VOTECAST", + BARTERCAST:"BARTERCAST", + G2G_PIECE_XFER: "G2G_PIECE_XFER", + FRIENDSHIP:"FRIENDSHIP", + VOTECAST:"VOTECAST", + CHANNELCAST:"CHANNELCAST", + + CRAWLER_REQUEST:"CRAWLER_REQUEST", + CRAWLER_REQUEST+CRAWLER_DATABASE_QUERY:"CRAWLER_DATABASE_QUERY_REQUEST", + CRAWLER_REQUEST+CRAWLER_SEEDINGSTATS_QUERY:"CRAWLER_SEEDINGSTATS_QUERY_REQUEST", + CRAWLER_REQUEST+CRAWLER_NATCHECK:"CRAWLER_NATCHECK_QUERY_REQUEST", + CRAWLER_REQUEST+CRAWLER_NATTRAVERSAL:"CRAWLER_NATTRAVERSAL_QUERY_REQUEST", + CRAWLER_REQUEST+CRAWLER_FRIENDSHIP_STATS:"CRAWLER_FRIENDSHIP_STATS_REQUEST", + CRAWLER_REQUEST+CRAWLER_VIDEOPLAYBACK_INFO_QUERY:"CRAWLER_VIDEOPLAYBACK_INFO_QUERY_REQUEST", + CRAWLER_REQUEST+CRAWLER_VIDEOPLAYBACK_EVENT_QUERY:"CRAWLER_VIDEOPLAYBACK_EVENT_QUERY_REQUEST", + CRAWLER_REQUEST+CRAWLER_REPEX_QUERY:"CRAWLER_REPEX_QUERY_REQUEST", # RePEX: query a peer's SwarmCache history + CRAWLER_REQUEST+CRAWLER_PUNCTURE_QUERY:"CRAWLER_PUNCTURE_QUERY_REQUEST", + CRAWLER_REQUEST+CRAWLER_CHANNEL_QUERY:"CRAWLER_CHANNEL_QUERY_REQUEST", + + CRAWLER_REPLY:"CRAWLER_REPLY", + CRAWLER_REPLY+CRAWLER_DATABASE_QUERY:"CRAWLER_DATABASE_QUERY_REPLY", + CRAWLER_REPLY+CRAWLER_SEEDINGSTATS_QUERY:"CRAWLER_SEEDINGSTATS_QUERY_REPLY", + CRAWLER_REPLY+CRAWLER_NATCHECK:"CRAWLER_NATCHECK_QUERY_REPLY", + CRAWLER_REPLY+CRAWLER_NATTRAVERSAL:"CRAWLER_NATTRAVERSAL_QUERY_REPLY", + CRAWLER_REPLY+CRAWLER_FRIENDSHIP_STATS:"CRAWLER_FRIENDSHIP_STATS", + CRAWLER_REPLY+CRAWLER_FRIENDSHIP_STATS:"CRAWLER_FRIENDSHIP_STATS_REPLY", + CRAWLER_REPLY+CRAWLER_VIDEOPLAYBACK_INFO_QUERY:"CRAWLER_VIDEOPLAYBACK_INFO_QUERY_REPLY", + CRAWLER_REPLY+CRAWLER_VIDEOPLAYBACK_EVENT_QUERY:"CRAWLER_VIDEOPLAYBACK_EVENT_QUERY_REPLY", + CRAWLER_REPLY+CRAWLER_REPEX_QUERY:"CRAWLER_REPEX_QUERY_REPLY", # RePEX: query a peer's SwarmCache history + CRAWLER_REPLY+CRAWLER_PUNCTURE_QUERY:"CRAWLER_PUNCTURE_QUERY_REPLY", + CRAWLER_REPLY+CRAWLER_CHANNEL_QUERY:"CRAWLER_CHANNEL_QUERY_REPLY" +} + +def getMessageName(s): + """ + Return the message name for message id s. This may be either a one + or a two byte sting + """ + if s in message_map: + return message_map[s] + else: + return "Unknown_MessageID_" + "_".join([str(ord(c)) for c in s]) diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/NatCheck.py b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/NatCheck.py new file mode 100644 index 0000000..a44379a --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/NatCheck.py @@ -0,0 +1,94 @@ +# Written by Bram Cohen +# see LICENSE.txt for license information + +from cStringIO import StringIO +from socket import error as socketerror +try: + True +except: + True = 1 + False = 0 + +protocol_name = 'BitTorrent protocol' + +# header, reserved, download id, my id, [length, message] + +class NatCheck: + def __init__(self, resultfunc, downloadid, peerid, ip, port, rawserver): + self.resultfunc = resultfunc + self.downloadid = downloadid + self.peerid = peerid + self.ip = ip + self.port = port + self.closed = False + self.buffer = StringIO() + self.next_len = 1 + self.next_func = self.read_header_len + try: + self.connection = rawserver.start_connection((ip, port), self) + self.connection.write(chr(len(protocol_name)) + protocol_name + + (chr(0) * 8) + downloadid) + except socketerror: + self.answer(False) + except IOError: + self.answer(False) + + def answer(self, result): + self.closed = True + try: + self.connection.close() + except AttributeError: + pass + self.resultfunc(result, self.downloadid, self.peerid, self.ip, self.port) + + def read_header_len(self, s): + if ord(s) != len(protocol_name): + return None + return len(protocol_name), self.read_header + + def read_header(self, s): + if s != protocol_name: + return None + return 8, self.read_reserved + + def read_reserved(self, s): + return 20, self.read_download_id + + def read_download_id(self, s): + if s != self.downloadid: + return None + return 20, self.read_peer_id + + def read_peer_id(self, s): + if s != self.peerid: + return None + self.answer(True) + return None + + def data_came_in(self, connection, s): + while 1: + if self.closed: + return + i = self.next_len - self.buffer.tell() + if i > len(s): + self.buffer.write(s) + return + self.buffer.write(s[:i]) + s = s[i:] + m = self.buffer.getvalue() + self.buffer.reset() + self.buffer.truncate() + x = self.next_func(m) + if x is None: + if not self.closed: + self.answer(False) + return + self.next_len, self.next_func = x + + def connection_lost(self, connection): + if not self.closed: + self.closed = True + self.resultfunc(False, self.downloadid, self.peerid, self.ip, self.port) + + def connection_flushed(self, connection): + pass diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/PiecePicker.py b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/PiecePicker.py new file mode 100644 index 0000000..fa4d9d2 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/PiecePicker.py @@ -0,0 +1,799 @@ +# Written by Bram Cohen and Pawel Garbacki, George Milescu +# see LICENSE.txt for license information + +from random import randrange, shuffle +from BaseLib.Core.BitTornado.clock import clock +# 2fastbt_ +from traceback import extract_tb,print_stack +from BaseLib.Core.BitTornado.bitfield import Bitfield +import sys +import time +# _2fastbt + +try: + True +except: + True = 1 + False = 0 + +DEBUG = False + +""" + rarest_first_cutoff = number of downloaded pieces at which to switch from random to rarest first. + rarest_first_priority_cutoff = number of peers which need to have a piece before other partials + take priority over rarest first. +""" + +class PiecePicker: +# 2fastbt_ + def __init__(self, numpieces, + rarest_first_cutoff = 1, rarest_first_priority_cutoff = 3, + priority_step = 20, helper = None, coordinator = None, rate_predictor = None): +# TODO: fix PiecePickerSVC and PiecePickerVOD __init calls +# _2fastbt + # If we have less than the cutoff pieces, choose pieces at random. Otherwise, + # go for rarest first. + self.rarest_first_cutoff = rarest_first_cutoff + + self.priority_step = priority_step + + # cutoff = number of non-seeds which need to have a piece before other + # partials take priority over rarest first. In effect, equal to: + # rarest_first_priority_cutoff + priority_step - #seeds + # before a seed is discovered, it is equal to (as set here): + # rarest_first_priority_cutoff + # + # This cutoff is used as an interest level (see below). When in random piece + # mode, asking for really rare pieces is disfavoured. + self.rarest_first_priority_cutoff = rarest_first_priority_cutoff + priority_step + self.cutoff = rarest_first_priority_cutoff + + # total number of pieces + self.numpieces = numpieces + + # pieces we have started to download (in transit) + self.started = [] + + # !!! the following statistics involve peers, and exclude seeds !!! + + # total number of pieces owned by peers + self.totalcount = 0 + + # how many peers (non-seeder peers) have a certain piece + self.numhaves = [0] * numpieces + + # priority of each peace; -1 to avoid downloading it + self.priority = [1] * numpieces + + self.removed_partials = {} + + # self.crosscount[x] = the number of pieces owned by x peers + # (inverse of self.numhaves) + self.crosscount = [numpieces] + + # self.crosscount2[x] = the number of pieces owned by x peers and me + # (inverse of self.numhaves[x]+self.has[x]) + self.crosscount2 = [numpieces] + + # whether we have a certain piece + self.has = [0] * numpieces + + # number of (complete) pieces we got + self.numgot = 0 + + # whether we're done downloading + self.done = False + + # peer information + self.peer_connections = {} + + # seeding information + self.seed_connections = {} + self.seed_time = None + self.superseed = False + self.seeds_connected = 0 + +# 2fastbt_ + self.helper = helper + self.coordinator = coordinator + self.rate_predictor = rate_predictor + self.videostatus = None +# _2fastbt + # Arno, 2010-08-11: STBSPEED, moved to fast_initialize() + # self._init_interests() + + def _init_interests(self): + """ + Interests are sets of pieces ordered by priority (0 = high). The + priority to the outside world is coarse-grained and is fine-tuned + by the number of peers owning a piece. + + The interest level of a piece is self.level_in_interests[piece], + which is equal to: + + self.priority[piece] * self.priority_step + self.numhaves[piece]. + + Every level is a subset of pieces. The placement in the subset + with self.pos_in_interests[piece], so + + piece == self.interests + [self.level_in_interests[piece]] + [self.pos_in_interests[piece]] + + holds. Pieces within the same subset are kept shuffled. + """ + + self.interests = [[] for x in xrange(self.priority_step)] + self.level_in_interests = [self.priority_step] * self.numpieces + interests = range(self.numpieces) + shuffle(interests) + self.pos_in_interests = [0] * self.numpieces + for i in xrange(self.numpieces): + self.pos_in_interests[interests[i]] = i + self.interests.append(interests) + + def got_piece(self, piece, begin, length): + """ + Used by the streaming piece picker for additional information. + """ + pass + + def check_outstanding_requests(self, downloads): + """ + Used by the streaming piece picker to cancel slow requests. + """ + pass + + def got_have(self, piece, connection = None): + """ A peer reports to have the given piece. """ + + self.totalcount+=1 + numint = self.numhaves[piece] + self.numhaves[piece] += 1 + self.crosscount[numint] -= 1 + if numint+1==len(self.crosscount): + self.crosscount.append(0) + self.crosscount[numint+1] += 1 + if not self.done: + numintplus = numint+self.has[piece] + self.crosscount2[numintplus] -= 1 + if numintplus+1 == len(self.crosscount2): + self.crosscount2.append(0) + self.crosscount2[numintplus+1] += 1 + numint = self.level_in_interests[piece] + self.level_in_interests[piece] += 1 + if self.superseed: + self.seed_got_haves[piece] += 1 + numint = self.level_in_interests[piece] + self.level_in_interests[piece] += 1 + elif self.has[piece]: + return True + elif self.priority[piece] == -1: + return False + if numint == len(self.interests) - 1: + self.interests.append([]) + self._shift_over(piece, self.interests[numint], self.interests[numint + 1]) + return False + + # ProxyService_ + # + def redirect_haves_to_coordinator(self, connection = None, helper_con = False, piece = None): + """ The method is called by the Downloader when a HAVE message is received. + + If the current node is a helper, it will send the HAVE information to the coordinator. + + @param connection: the connection for which the HAVE message was received + @param helper_con: True if it is a connection to a helper + @param piece: the received piece + """ + + if self.helper : + # The current node is a coordinator + if DEBUG: + print >> sys.stderr,"PiecePicker: proxy_got_have: sending haves to coordinator" + + # Create the piece list - a copy of numhaves for simplicity + piece_list = self.numhaves + print "sending piece_list=", piece_list + + # Send the bitfield + self.helper.send_proxy_have(piece_list) + else: + # if the node is a helper or a non-proxy node, do nothing + return + # + # _ProxyService + + + def lost_have(self, piece): + """ We lost a peer owning the given piece. """ + self.totalcount-=1 + numint = self.numhaves[piece] + self.numhaves[piece] -= 1 + self.crosscount[numint] -= 1 + self.crosscount[numint-1] += 1 + if not self.done: + numintplus = numint+self.has[piece] + self.crosscount2[numintplus] -= 1 + self.crosscount2[numintplus-1] += 1 + numint = self.level_in_interests[piece] + self.level_in_interests[piece] -= 1 + if self.superseed: + numint = self.level_in_interests[piece] + self.level_in_interests[piece] -= 1 + elif self.has[piece] or self.priority[piece] == -1: + return + self._shift_over(piece, self.interests[numint], self.interests[numint - 1]) + + + # Arno: LIVEWRAP + def is_valid_piece(self, piece): + return True + + def get_valid_range_iterator(self): + return xrange(0,len(self.has)) + + def invalidate_piece(self,piece): + """ A piece ceases to exist at the neighbours. Required for LIVEWRAP. """ + + if self.has[piece]: + self.has[piece] = 0 + #print >>sys.stderr,"PiecePicker: Clearing piece",piece + self.numgot -= 1 + + # undo self._remove_from_interests(piece); ripped from set_priority + + # reinsert into interests + p = self.priority[piece] + level = self.numhaves[piece] + (self.priority_step * p) + self.level_in_interests[piece] = level + while len(self.interests) < level+1: + self.interests.append([]) + + # insert at a random spot in the list at the current level + l2 = self.interests[level] + parray = self.pos_in_interests + newp = randrange(len(l2)+1) + if newp == len(l2): + parray[piece] = len(l2) + l2.append(piece) + else: + old = l2[newp] + parray[old] = len(l2) + l2.append(old) + l2[newp] = piece + parray[piece] = newp + + # modelled after lost_have + + #assert not self.done + #assert not self.seeds_connected + + numint = self.numhaves[piece] + if numint == 0: + return + + # set numhaves to 0 + self.totalcount -= numint + self.numhaves[piece] = 0 + self.crosscount[numint] -= 1 + self.crosscount[0] += 1 + numintplus = numint+0 + self.crosscount2[numintplus] -= 1 + self.crosscount2[0] += 1 + numint = self.level_in_interests[piece] + self.level_in_interests[piece] = 0 + self._shift_over(piece, self.interests[numint], self.interests[0]) + + def set_downloader(self,dl): + self.downloader = dl + + def _shift_over(self, piece, l1, l2): + """ Moves 'piece' from interests list l1 to l2. """ + + assert self.superseed or (not self.has[piece] and self.priority[piece] >= 0) + parray = self.pos_in_interests + + # remove piece from l1 + p = parray[piece] + assert l1[p] == piece + q = l1[-1] + l1[p] = q + parray[q] = p + del l1[-1] + + # add piece to a random place in l2 + newp = randrange(len(l2)+1) + if newp == len(l2): + parray[piece] = len(l2) + l2.append(piece) + else: + old = l2[newp] + parray[old] = len(l2) + l2.append(old) + l2[newp] = piece + parray[piece] = newp + + def got_seed(self): + self.seeds_connected += 1 + self.cutoff = max(self.rarest_first_priority_cutoff-self.seeds_connected, 0) + + def became_seed(self): + """ A peer just became a seed. """ + + self.got_seed() + self.totalcount -= self.numpieces + self.numhaves = [i-1 for i in self.numhaves] + if self.superseed or not self.done: + self.level_in_interests = [i-1 for i in self.level_in_interests] + del self.interests[0] + del self.crosscount[0] + if not self.done: + del self.crosscount2[0] + + def lost_seed(self): + self.seeds_connected -= 1 + self.cutoff = max(self.rarest_first_priority_cutoff-self.seeds_connected, 0) + + # boudewijn: for VOD we need additional information. added BEGIN + # and LENGTH parameter + def requested(self, piece, begin=None, length=None): + """ Given piece has been requested or a partial of it is on disk. """ + if piece not in self.started: + self.started.append(piece) + + def _remove_from_interests(self, piece, keep_partial = False): + l = self.interests[self.level_in_interests[piece]] + p = self.pos_in_interests[piece] + assert l[p] == piece + q = l[-1] + l[p] = q + self.pos_in_interests[q] = p + del l[-1] + try: + self.started.remove(piece) + if keep_partial: + self.removed_partials[piece] = 1 + except ValueError: + pass + + def complete(self, piece): + """ Succesfully received the given piece. """ + assert not self.has[piece] + self.has[piece] = 1 + self.numgot += 1 + + if self.numgot == self.numpieces: + self.done = True + self.crosscount2 = self.crosscount + else: + numhaves = self.numhaves[piece] + self.crosscount2[numhaves] -= 1 + if numhaves+1 == len(self.crosscount2): + self.crosscount2.append(0) + self.crosscount2[numhaves+1] += 1 + self._remove_from_interests(piece) + + # ProxyService_ + # + def _proxynext(self, haves, wantfunc, complete_first, helper_con, willrequest=True, connection=None, proxyhave=None, lookatstarted=False, onlystarted=False): + """ Determine which piece to download next from a peer. _proxynext has three extra arguments compared to _next + + @param haves: set of pieces owned by that peer + @param wantfunc: custom piece filter + @param complete_first: whether to complete partial pieces first + @param helper_con: True for Coordinator, False for Helper + @param willrequest: + @param connection: + @param proxyhave: a bitfield with the pieces that the helper "sees" in the swarm + @param lookatstarted: if True, the picker will search in the already started pieces first, and then in the available pieces + @param onlystarted: if True, the picker will only search in the already started pieces + @return: a piece number or None + """ + + # First few (rarest_first_cutoff) pieces are selected at random + # and completed. Subsequent pieces are downloaded rarest-first. + + # cutoff = True: random mode + # False: rarest-first mode + cutoff = self.numgot < self.rarest_first_cutoff + + # whether to complete existing partials first -- do so before the + # cutoff, or if forced by complete_first, but not for seeds. + complete_first = (complete_first or cutoff) and not haves.complete() + + # most interesting piece + best = None + + # interest level of best piece + bestnum = 2 ** 30 + + # select piece we started to download with best interest index. + if lookatstarted: + # No active requested (started) pieces will be rerequested + for i in self.started: + if proxyhave == None: + proxyhave_i = False + else: + proxyhave_i = proxyhave[i] + if (haves[i] or proxyhave_i) and wantfunc(i) and (self.helper is None or helper_con or not self.helper.is_ignored(i)): + if self.level_in_interests[i] < bestnum: + best = i + bestnum = self.level_in_interests[i] + + if best is not None: + # found a piece -- return it if we are completing partials first + # or if there is a cutoff + if complete_first or (cutoff and len(self.interests) > self.cutoff): + return best + + if onlystarted: + # Only look at started downloads - used by the helper + return best + + if haves.complete(): + # peer has all pieces - look for any more interesting piece + r = [ (0, min(bestnum, len(self.interests))) ] + elif cutoff and len(self.interests) > self.cutoff: + # no best piece - start looking for low-priority pieces first + r = [ (self.cutoff, min(bestnum, len(self.interests))), + (0, self.cutoff) ] + else: + # look for the most interesting piece + r = [ (0, min(bestnum, len(self.interests))) ] +# print "piecepicker: r=", r + + # select first acceptable piece, best interest index first. + # r is an interest-range + for lo, hi in r: + for i in xrange(lo, hi): + # Randomize the list of pieces in the interest level i + random_interests = [] + random_interests.extend(self.interests[i]) + shuffle(random_interests) + for j in random_interests: + if proxyhave == None: + proxyhave_j = False + else: + proxyhave_j = proxyhave[j] + if (haves[j] or proxyhave_j) and wantfunc(j) and (self.helper is None or helper_con or not self.helper.is_ignored(j)): + return j + + if best is not None: + return best + return None + # + # _ProxyService + +# 2fastbt_ + def _next(self, haves, wantfunc, complete_first, helper_con, willrequest=True, connection=None): +# _2fastbt + """ Determine which piece to download next from a peer. + + @param haves: set of pieces owned by that peer + @param wantfunc: custom piece filter + @param complete_first: whether to complete partial pieces first + @param helper_con: True for Coordinator, False for Helper + @param willrequest: + @param connection: the connection object on which the returned piece will be requested + @return: a piece number or None + """ + + # First few (rarest_first_cutoff) pieces are selected at random + # and completed. Subsequent pieces are downloaded rarest-first. + + # cutoff = True: random mode + # False: rarest-first mode + cutoff = self.numgot < self.rarest_first_cutoff + + # whether to complete existing partials first -- do so before the + # cutoff, or if forced by complete_first, but not for seeds. + complete_first = (complete_first or cutoff) and not haves.complete() + + # most interesting piece + best = None + + # interest level of best piece + bestnum = 2 ** 30 + + # select piece we started to download with best interest index. + for i in self.started: +# 2fastbt_ + if haves[i] and wantfunc(i) and (self.helper is None or helper_con or not self.helper.is_ignored(i)): +# _2fastbt + if self.level_in_interests[i] < bestnum: + best = i + bestnum = self.level_in_interests[i] + + if best is not None: + # found a piece -- return it if we are completing partials first + # or if there is a cutoff + if complete_first or (cutoff and len(self.interests) > self.cutoff): + return best + + if haves.complete(): + # peer has all pieces - look for any more interesting piece + r = [ (0, min(bestnum, len(self.interests))) ] + elif cutoff and len(self.interests) > self.cutoff: + # no best piece - start looking for low-priority pieces first + r = [ (self.cutoff, min(bestnum, len(self.interests))), + (0, self.cutoff) ] + else: + # look for the most interesting piece + r = [ (0, min(bestnum, len(self.interests))) ] + + # select first acceptable piece, best interest index first. + # r is an interest-range + for lo, hi in r: + for i in xrange(lo, hi): + for j in self.interests[i]: +# 2fastbt_ + if haves[j] and wantfunc(j) and (self.helper is None or helper_con or not self.helper.is_ignored(j)): +# _2fastbt + return j + + if best is not None: + return best + return None + +# 2fastbt_ + def next(self, haves, wantfunc, sdownload, complete_first = False, helper_con = False, slowpieces= [], willrequest = True, connection = None, proxyhave = None): + """ Return the next piece number to be downloaded + + @param haves: set of pieces owned by that peer + @param wantfunc: custom piece filter + @param sdownload: + @param complete_first: whether to complete partial pieces first + @param helper_con: True for Coordinator, False for Helper + @param slowpieces: + @param willrequest: + @param connection: the connection object on which the returned piece will be requested + @param proxyhave: a bitfield with the pieces that the helper "sees" in the swarm + @return: a piece number or None + """ +# try: + # Helper connection (helper_con) is true for coordinator + # Helper connection (helper_con) is false for helpers + # self.helper is None for Coordinator and is notNone for Helper + while True: +# print "started =", self.started + if helper_con : + # The current node is a coordinator + + # First try to request a piece that the peer advertised via a HAVE message + piece = self._proxynext(haves, wantfunc, complete_first, helper_con, willrequest = willrequest, connection = connection, proxyhave = None, lookatstarted=False) + + # If no piece could be requested, try to find a piece that the node advertised via a PROXY_HAVE message + if piece is None: + piece = self._proxynext(haves, wantfunc, complete_first, helper_con, willrequest = willrequest, connection = connection, proxyhave = proxyhave, lookatstarted=False) + + if piece is None: + # The piece picker failed to return a piece + if DEBUG: + print >> sys.stderr,"PiecePicker: next: _next returned no pieces for proxyhave!", + break + + if DEBUG: + print >> sys.stderr,"PiecePicker: next: helper None or helper conn, returning", piece + print >> sys.stderr,"PiecePicker: next: haves[", piece, "]=", haves[piece] + print >> sys.stderr,"PiecePicker: next: proxyhave[", piece, "]=", proxyhave[piece] + if not haves[piece]: + # If the piece was not advertised with a BT HAVE message, send a proxy request for it + # Reserve the piece to one of the helpers + self.coordinator.send_request_pieces(piece, connection.get_id()) + return None + else: + # The piece was advertised with a BT HAVE message + # Return the selected piece + return piece + + if self.helper is not None: + # The current node is a helper + + # Look into the pieces that are already downloading + piece = self._proxynext(haves, wantfunc, complete_first, helper_con, willrequest = willrequest, connection = connection, proxyhave = None, lookatstarted=True, onlystarted=True) + if piece is not None: + if DEBUG: + print >> sys.stderr,"PiecePicker: next: helper: continuing already started download for", requested_piece + return piece + + # If no already started downloads, look at new coordinator requests + requested_piece = self.helper.next_request() + if requested_piece is not None: + if DEBUG: + print >> sys.stderr,"PiecePicker: next: helper: got request from coordinator for", requested_piece + return requested_piece + else: + # There is no pending requested piece from the coordinator + if DEBUG: + print >> sys.stderr,"PiecePicker: next: helper: no piece pending" + return None + + # The current node not a helper, neither a coordinator + # First try to request a piece that the peer advertised via a HAVE message + piece = self._next(haves, wantfunc, complete_first, helper_con, willrequest = willrequest, connection = connection) + + if piece is None: + # The piece picker failed to return a piece + if DEBUG: + print >> sys.stderr,"PiecePicker: next: _next returned no pieces!", + break + + # We should never get here + if DEBUG: + print >> sys.stderr,"PiecePicker: next: helper: an error occurred. Returning piece",piece + return piece + + # Arno, 2008-05-20: 2fast code: if we got capacity to DL something, + # ask coordinator what new pieces to dl for it. + if self.rate_predictor and self.rate_predictor.has_capacity(): + return self._next(haves, wantfunc, complete_first, True, willrequest = willrequest, connection = connection) + else: + return None + + def set_rate_predictor(self, rate_predictor): + self.rate_predictor = rate_predictor +# _2fastbt + + def am_I_complete(self): + return self.done + + def bump(self, piece): + """ Piece was received but contained bad data? """ + + l = self.interests[self.level_in_interests[piece]] + pos = self.pos_in_interests[piece] + del l[pos] + l.append(piece) + for i in range(pos, len(l)): + self.pos_in_interests[l[i]] = i + try: + self.started.remove(piece) + except: + pass + + def set_priority(self, piece, p): + """ Define the priority with which a piece needs to be downloaded. + A priority of -1 means 'do not download'. """ + + if self.superseed: + return False # don't muck with this if you're a superseed + oldp = self.priority[piece] + if oldp == p: + return False + self.priority[piece] = p + if p == -1: + # when setting priority -1, + # make sure to cancel any downloads for this piece + if not self.has[piece]: + self._remove_from_interests(piece, True) + return True + if oldp == -1: + level = self.numhaves[piece] + (self.priority_step * p) + self.level_in_interests[piece] = level + if self.has[piece]: + return True + while len(self.interests) < level+1: + self.interests.append([]) + l2 = self.interests[level] + parray = self.pos_in_interests + newp = randrange(len(l2)+1) + if newp == len(l2): + parray[piece] = len(l2) + l2.append(piece) + else: + old = l2[newp] + parray[old] = len(l2) + l2.append(old) + l2[newp] = piece + parray[piece] = newp + if self.removed_partials.has_key(piece): + del self.removed_partials[piece] + self.started.append(piece) + # now go to downloader and try requesting more + return True + numint = self.level_in_interests[piece] + newint = numint + ((p - oldp) * self.priority_step) + self.level_in_interests[piece] = newint + if self.has[piece]: + return False + while len(self.interests) < newint+1: + self.interests.append([]) + self._shift_over(piece, self.interests[numint], self.interests[newint]) + return False + + def is_blocked(self, piece): + return self.priority[piece] < 0 + + + def set_superseed(self): + assert self.done + self.superseed = True + self.seed_got_haves = [0] * self.numpieces + self._init_interests() # assume everyone is disconnected + + def next_have(self, connection, looser_upload): + if self.seed_time is None: + self.seed_time = clock() + return None + if clock() < self.seed_time+10: # wait 10 seconds after seeing the first peers + return None # to give time to grab have lists + if not connection.upload.super_seeding: + return None + if connection in self.seed_connections: + if looser_upload: + num = 1 # send a new have even if it hasn't spread that piece elsewhere + else: + num = 2 + if self.seed_got_haves[self.seed_connections[connection]] < num: + return None + if not connection.upload.was_ever_interested: # it never downloaded it? + connection.upload.skipped_count += 1 + if connection.upload.skipped_count >= 3: # probably another stealthed seed + return -1 # signal to close it + for tier in self.interests: + for piece in tier: + if not connection.download.have[piece]: + seedint = self.level_in_interests[piece] + self.level_in_interests[piece] += 1 # tweak it up one, so you don't duplicate effort + if seedint == len(self.interests) - 1: + self.interests.append([]) + self._shift_over(piece, + self.interests[seedint], self.interests[seedint + 1]) + self.seed_got_haves[piece] = 0 # reset this + self.seed_connections[connection] = piece + connection.upload.seed_have_list.append(piece) + return piece + return -1 # something screwy; terminate connection + + def got_peer(self, connection): + self.peer_connections[connection] = { "connection": connection } + + def lost_peer(self, connection): + if connection.download.have.complete(): + self.lost_seed() + else: + has = connection.download.have + for i in xrange(0, self.numpieces): + if has[i]: + self.lost_have(i) + + if connection in self.seed_connections: + del self.seed_connections[connection] + del self.peer_connections[connection] + + + def fast_initialize(self,completeondisk): + if completeondisk: + self.has = [1] * self.numpieces + self.numgot = self.numpieces + self.done = True + self.interests = [[] for x in xrange(self.priority_step)] + self.interests.append([]) + self.level_in_interests = [self.priority_step] * self.numpieces + self.pos_in_interests = [0] * self.numpieces # Incorrect, but shouldn't matter + else: + self._init_interests() + + def print_complete(self): + print >>sys.stderr,"pp: self.numpieces",`self.numpieces` + print >>sys.stderr,"pp: self.started",`self.started` + print >>sys.stderr,"pp: self.totalcount",`self.totalcount` + print >>sys.stderr,"pp: self.numhaves",`self.numhaves` + print >>sys.stderr,"pp: self.priority",`self.priority` + print >>sys.stderr,"pp: self.removed_partials",`self.removed_partials` + print >>sys.stderr,"pp: self.crosscount",`self.crosscount` + print >>sys.stderr,"pp: self.crosscount2",`self.crosscount2` + print >>sys.stderr,"pp: self.has",`self.has` + print >>sys.stderr,"pp: self.numgot",`self.numgot` + print >>sys.stderr,"pp: self.done",`self.done` + print >>sys.stderr,"pp: self.peer_connections",`self.peer_connections` + print >>sys.stderr,"pp: self.seed_connections",`self.seed_connections` + print >>sys.stderr,"pp: self.seed_time",`self.seed_time` + print >>sys.stderr,"pp: self.superseed",`self.superseed` + print >>sys.stderr,"pp: self.seeds_connected",`self.seeds_connected` + print >>sys.stderr,"pp: self.interests",`self.interests` + print >>sys.stderr,"pp: self.level_in_interests",`self.level_in_interests` + print >>sys.stderr,"pp: self.pos_in_interests",`self.pos_in_interests` + + diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/Rerequester.py b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/Rerequester.py new file mode 100644 index 0000000..0ed1d9b --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/Rerequester.py @@ -0,0 +1,606 @@ +# Written by Bram Cohen +# modified for multitracker operation by John Hoffman +# modified for mainline DHT support by Fabian van der Werf +# Modified by Raul Jimenez to integrate KTH DHT +# see LICENSE.txt for license information + +import sys +from BaseLib.Core.BitTornado.zurllib import urlopen +from urllib import quote +from btformats import check_peers +from BaseLib.Core.BitTornado.bencode import bdecode +from threading import Thread, Lock, currentThread +from cStringIO import StringIO +from traceback import print_exc,print_stack +from socket import error, gethostbyname, inet_aton, inet_ntoa +from random import shuffle +from BaseLib.Core.Utilities.Crypto import sha +from time import time +from struct import pack, unpack +import binascii +from BaseLib.Core.simpledefs import * + + +import BaseLib.Core.DecentralizedTracking.mainlineDHT as mainlineDHT +if mainlineDHT.dht_imported: + from BaseLib.Core.DecentralizedTracking.kadtracker.identifier import Id, IdError + + +try: + from os import getpid +except ImportError: + def getpid(): + return 1 + +try: + True +except: + True = 1 + False = 0 + +DEBUG = False +DEBUG_DHT = False + +mapbase64 = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz.-' +keys = {} +basekeydata = str(getpid()) + repr(time()) + 'tracker' + +def add_key(tracker): + key = '' + for i in sha(basekeydata+tracker).digest()[-6:]: + key += mapbase64[ord(i) & 0x3F] + keys[tracker] = key + +def get_key(tracker): + try: + return "&key="+keys[tracker] + except: + add_key(tracker) + return "&key="+keys[tracker] + +class fakeflag: + def __init__(self, state=False): + self.state = state + def wait(self): + pass + def isSet(self): + return self.state + +class Rerequester: + def __init__(self, trackerlist, interval, sched, howmany, minpeers, + connect, externalsched, amount_left, up, down, + port, ip, myid, infohash, timeout, errorfunc, excfunc, + maxpeers, doneflag, upratefunc, downratefunc, + unpauseflag = fakeflag(True), config=None): + + self.excfunc = excfunc + newtrackerlist = [] + for tier in trackerlist: + if len(tier) > 1: + shuffle(tier) + newtrackerlist += [tier] + self.trackerlist = newtrackerlist + self.lastsuccessful = '' + self.rejectedmessage = 'rejected by tracker - ' + self.port = port + + if DEBUG: + print >>sys.stderr,"Rerequest tracker: infohash is",`infohash`,"port is",self.port,"myid",`myid`,"quoted id",quote(myid) + + self.url = ('?info_hash=%s&peer_id=%s&port=%s' % + (quote(infohash), quote(myid), str(port))) + self.ip = ip + self.interval = interval + self.last = None + self.trackerid = None + self.announce_interval = 1 * 60 + self.sched = sched + self.howmany = howmany + self.minpeers = minpeers + self.connect = connect + self.externalsched = externalsched + self.amount_left = amount_left + self.up = up + self.down = down + self.timeout = timeout + self.errorfunc = errorfunc + self.maxpeers = maxpeers + self.doneflag = doneflag + self.upratefunc = upratefunc + self.downratefunc = downratefunc + self.unpauseflag = unpauseflag + self.last_failed = True + self.never_succeeded = True + self.errorcodes = {} + self.lock = SuccessLock() + self.special = None + self.started = False + self.stopped = False + self.schedid = 'arno481' + self.infohash = infohash + self.dht = mainlineDHT.dht + self.config = config + self.notifiers = [] # Diego : warn who is interested about peers returned (only) by tracker + + + def start(self): + if not self.started: + self.started = True + self.sched(self.c, self.interval/2) + self.d(0) + + def c(self): + if self.stopped: + return + if not self.unpauseflag.isSet() and self.howmany() < self.minpeers: + self.announce(3, self._c) + else: + self._c() + + def _c(self): + self.sched(self.c, self.interval) + + def d(self, event = 3): + if self.stopped: + return + if not self.unpauseflag.isSet(): + self._d() + return + self.announce(event, self._d) + + def _d(self): + if self.never_succeeded: + self.sched(self.d, 60) # retry in 60 seconds + else: + self.sched(self.d, self.announce_interval) + + def encoder_wants_new_peers(self): + """ The list of peers we gave to the encoder via self.connect() + did not give any live connections, reconnect to get some more. + Officially we should cancel the outstanding + self.sched(self.d,self.announce_interval) + """ + self.d(0) + + def announce(self, event = 3, callback = lambda: None, specialurl = None): + # IPVSIX: Azureus 3.1.1.0 used as Ubuntu IPv6 tracker doesn't support BEP 7 + if ':' in self.ip: + compact = 0 + else: + compact = 1 + + if specialurl is not None: + s = self.url+'&uploaded=0&downloaded=0&left=1' # don't add to statistics + if self.howmany() >= self.maxpeers: + s += '&numwant=0' + else: + s += '&no_peer_id=1' + if compact: + s+= '&compact=1' + self.last_failed = True # force true, so will display an error + self.special = specialurl + self.rerequest(s, callback) + return + + else: + s = ('%s&uploaded=%s&downloaded=%s&left=%s' % + (self.url, str(self.up()), str(self.down()), + str(self.amount_left()))) + if self.last is not None: + s += '&last=' + quote(str(self.last)) + if self.trackerid is not None: + s += '&trackerid=' + quote(str(self.trackerid)) + if self.howmany() >= self.maxpeers: + s += '&numwant=0' + else: + s += '&no_peer_id=1' + if compact: + s+= '&compact=1' + if event != 3: + s += '&event=' + ['started', 'completed', 'stopped'][event] + if event == 2: + self.stopped = True + self.rerequest(s, callback) + + + def snoop(self, peers, callback = lambda: None): # tracker call support + self.rerequest(self.url + +'&event=stopped&port=0&uploaded=0&downloaded=0&left=1&tracker=1&numwant=' + +str(peers), callback) + + + def rerequest(self, s, callback): + # ProxyService_ + # + proxy_mode = self.config.get('proxy_mode',0) + if DEBUG: + if proxy_mode == PROXY_MODE_PRIVATE: + if True: + print "_rerequest exited."# + str(proxy_mode) + return + else: + if True: + print "_rerequest did not exit"# + str(proxy_mode) + # + # _ProxyService + + if not self.lock.isfinished(): # still waiting for prior cycle to complete?? + def retry(self = self, s = s, callback = callback): + self.rerequest(s, callback) + self.sched(retry, 5) # retry in 5 seconds + return + self.lock.reset() + rq = Thread(target = self._rerequest, args = [s, callback]) + rq.setName( "TrackerRerequestA"+rq.getName() ) + # Arno: make this a daemon thread so the client closes sooner. + rq.setDaemon(True) + rq.start() + + def _rerequest(self, s, callback): + try: + def fail(self = self, callback = callback): + self._fail(callback) + if self.ip: + try: + # IPVSIX + if ':' in self.ip: + # TODO: support for ipv4= field + urlip = "["+self.ip+"]" # URL encoding for IPv6, see RFC3986 + field = "ipv6" + else: + urlip = self.ip + field = "ip" + + s += '&' + field + '=' + urlip + except: + self.errorcodes['troublecode'] = 'unable to resolve: '+self.ip + self.externalsched(fail) + self.errorcodes = {} + if self.special is None: + + #Do dht request + if self.dht: + self._dht_rerequest() + elif DEBUG_DHT: + print >>sys.stderr,"Rerequester: No DHT support loaded" + + for t in range(len(self.trackerlist)): + for tr in range(len(self.trackerlist[t])): + tracker = self.trackerlist[t][tr] + # Arno: no udp support yet + if tracker.startswith( 'udp:' ): + if DEBUG: + print >>sys.stderr,"Rerequester: Ignoring tracker",tracker + continue + #elif DEBUG: + # print >>sys.stderr,"Rerequester: Trying tracker",tracker + if self.rerequest_single(tracker, s, callback): + if not self.last_failed and tr != 0: + del self.trackerlist[t][tr] + self.trackerlist[t] = [tracker] + self.trackerlist[t] + return + else: + tracker = self.special + self.special = None + if self.rerequest_single(tracker, s, callback): + return + # no success from any tracker + self.externalsched(fail) + except: + self.exception(callback) + + + def _fail(self, callback): + if ( (self.upratefunc() < 100 and self.downratefunc() < 100) + or not self.amount_left() ): + for f in ['rejected', 'bad_data', 'troublecode']: + if self.errorcodes.has_key(f): + r = self.errorcodes[f] + break + else: + r = 'Problem connecting to tracker - unspecified error:'+`self.errorcodes` + self.errorfunc(r) + + self.last_failed = True + self.lock.give_up() + self.externalsched(callback) + + + def rerequest_single(self, t, s, callback): + l = self.lock.set() + rq = Thread(target = self._rerequest_single, args = [t, s+get_key(t), l, callback]) + rq.setName( "TrackerRerequestB"+rq.getName() ) + # Arno: make this a daemon thread so the client closes sooner. + rq.setDaemon(True) + rq.start() + self.lock.wait() + if self.lock.success: + self.lastsuccessful = t + self.last_failed = False + self.never_succeeded = False + return True + if not self.last_failed and self.lastsuccessful == t: + # if the last tracker hit was successful, and you've just tried the tracker + # you'd contacted before, don't go any further, just fail silently. + self.last_failed = True + self.externalsched(callback) + self.lock.give_up() + return True + return False # returns true if it wants rerequest() to exit + + + def _rerequest_single(self, t, s, l, callback): + try: + closer = [None] + def timedout(self = self, l = l, closer = closer): + if self.lock.trip(l): + self.errorcodes['troublecode'] = 'Problem connecting to tracker - timeout exceeded' + self.lock.unwait(l) + try: + closer[0]() + except: + pass + + self.externalsched(timedout, self.timeout) + + err = None + try: + if DEBUG: + print >>sys.stderr,"Rerequest tracker:" + print >>sys.stderr,t+s + h = urlopen(t+s) + closer[0] = h.close + data = h.read() + except (IOError, error), e: + err = 'Problem connecting to tracker - ' + str(e) + if DEBUG: + print_exc() + except: + err = 'Problem connecting to tracker' + if DEBUG: + print_exc() + + + #if DEBUG: + # print >>sys.stderr,"rerequest: Got data",data + + try: + h.close() + except: + pass + if err: + if self.lock.trip(l): + self.errorcodes['troublecode'] = err + self.lock.unwait(l) + return + + if not data: + if self.lock.trip(l): + self.errorcodes['troublecode'] = 'no data from tracker' + self.lock.unwait(l) + return + + try: + r = bdecode(data, sloppy=1) + if DEBUG: + print >>sys.stderr,"Rerequester: Tracker returns:", r + check_peers(r) + + #print >>sys.stderr,"Rerequester: Tracker returns, post check done" + + except ValueError, e: + if DEBUG: + print_exc() + if self.lock.trip(l): + self.errorcodes['bad_data'] = 'bad data from tracker - ' + str(e) + self.lock.unwait(l) + return + + if r.has_key('failure reason'): + if self.lock.trip(l): + self.errorcodes['rejected'] = self.rejectedmessage + r['failure reason'] + self.lock.unwait(l) + return + + if self.lock.trip(l, True): # success! + self.lock.unwait(l) + else: + callback = lambda: None # attempt timed out, don't do a callback + + # even if the attempt timed out, go ahead and process data + def add(self = self, r = r, callback = callback): + #print >>sys.stderr,"Rerequester: add: postprocessing",r + self.postrequest(r, callback, self.notifiers) + + #print >>sys.stderr,"Rerequester: _request_single: scheduling processing of returned",r + self.externalsched(add) + except: + + print_exc() + + self.exception(callback) + + def _dht_rerequest(self): + if DEBUG_DHT: + print >>sys.stderr,"Rerequester: _dht_rerequest",`self.infohash` + try: + info_hash_id = Id(self.infohash) + except (IdError): + print >>sys.stderr,"Rerequester: _dht_rerequest: self.info_hash is not a valid identifier" + return + + if 'dialback' in self.config and self.config['dialback']: + from BaseLib.Core.NATFirewall.DialbackMsgHandler import DialbackMsgHandler + + if DialbackMsgHandler.getInstance().isConnectable(): + if DEBUG_DHT: + print >>sys.stderr,"Rerequester: _dht_rerequest: get_peers AND announce" + self.dht.get_peers(info_hash_id, self._dht_got_peers, self.port) + return + #raul: I added this return so when the peer is NOT connectable + # it does a get_peers lookup but it does not announce + if DEBUG_DHT: + print >>sys.stderr,"Rerequester: _dht_rerequest: JUST get_peers, DO NOT announce" + self.dht.get_peers(info_hash_id, self._dht_got_peers) + + + def _dht_got_peers(self, peers): + if DEBUG_DHT: + print >>sys.stderr,"Rerequester: DHT: Received",len(peers),"peers",currentThread().getName() + """ + raul: This is quite weird but I leave as it is. + """ + p = [{'ip': peer[0],'port': peer[1]} for peer in peers] + if p: + r = {'peers':p} + def add(self = self, r = r): + self.postrequest(r, lambda : None) + self.externalsched(add) + + + def add_notifier( self, cb ): + self.notifiers.append( cb ) + + def postrequest(self, r, callback, notifiers = []): + try: + if r.has_key('warning message'): + self.errorfunc('warning from tracker - ' + r['warning message']) + self.announce_interval = r.get('interval', self.announce_interval) + self.interval = r.get('min interval', self.interval) + + if DEBUG: + print >> sys.stderr,"Rerequester: announce min is",self.announce_interval,self.interval + + self.trackerid = r.get('tracker id', self.trackerid) + self.last = r.get('last', self.last) + # ps = len(r['peers']) + self.howmany() + peers = [] + p = r.get('peers') + if p is not None: + if type(p) == type(''): + for x in xrange(0, len(p), 6): + ip = '.'.join([str(ord(i)) for i in p[x:x+4]]) + port = (ord(p[x+4]) << 8) | ord(p[x+5]) + peers.append(((ip, port), 0)) # Arno: note: not just (ip,port)!!! + else: + # IPVSIX: Azureus 3.1.1.0 used as Ubuntu IPv6 tracker + # doesn't support BEP 7. Hence these may be IPv6. + # + for x in p: + peers.append(((x['ip'].strip(), x['port']), x.get('peer id', 0))) + else: + # IPv6 Tracker Extension, http://www.bittorrent.org/beps/bep_0007.html + p = r.get('peers6') + if type(p) == type(''): + for x in xrange(0, len(p), 18): + #ip = '.'.join([str(ord(i)) for i in p[x:x+16]]) + hexip = binascii.b2a_hex(p[x:x+16]) + ip = '' + for i in xrange(0,len(hexip),4): + ip += hexip[i:i+4] + if i+4 != len(hexip): + ip += ':' + port = (ord(p[x+16]) << 8) | ord(p[x+17]) + peers.append(((ip, port), 0)) # Arno: note: not just (ip,port)!!! + else: + for x in p: + peers.append(((x['ip'].strip(), x['port']), x.get('peer id', 0))) + + + # Arno, 2009-04-06: Need more effort to support IPv6, e.g. + # see SocketHandler.SingleSocket.get_ip(). The getsockname() + # + getpeername() calls should be make to accept IPv6 returns. + # Plus use inet_ntop() instead of inet_ntoa(), but former only + # supported on UNIX :-( See new ipaddr module in Python 2.7 + # + print >>sys.stderr,"Rerequester: Got IPv6 peer addresses, not yet supported, ignoring." + peers = [] + + if DEBUG: + print >>sys.stderr,"Rerequester: postrequest: Got peers",peers + ps = len(peers) + self.howmany() + if ps < self.maxpeers: + if self.doneflag.isSet(): + if r.get('num peers', 1000) - r.get('done peers', 0) > ps * 1.2: + self.last = None + else: + if r.get('num peers', 1000) > ps * 1.2: + self.last = None + + + if peers: + shuffle(peers) + self.connect(peers) # Encoder.start_connections(peers) + for notifier in notifiers: + notifier( peers ) + + callback() + except: + print >>sys.stderr,"Rerequester: Error in postrequest" + import traceback + traceback.print_exc() + + def exception(self, callback): + data = StringIO() + print_exc(file = data) + def r(s = data.getvalue(), callback = callback): + if self.excfunc: + self.excfunc(s) + else: + print s + callback() + self.externalsched(r) + + +class SuccessLock: + def __init__(self): + self.lock = Lock() + self.pause = Lock() + self.code = 0L + self.success = False + self.finished = True + + def reset(self): + self.success = False + self.finished = False + + def set(self): + self.lock.acquire() + if not self.pause.locked(): + self.pause.acquire() + self.first = True + self.code += 1L + self.lock.release() + return self.code + + def trip(self, code, s = False): + self.lock.acquire() + try: + if code == self.code and not self.finished: + r = self.first + self.first = False + if s: + self.finished = True + self.success = True + return r + finally: + self.lock.release() + + def give_up(self): + self.lock.acquire() + self.success = False + self.finished = True + self.lock.release() + + def wait(self): + self.pause.acquire() + + def unwait(self, code): + if code == self.code and self.pause.locked(): + self.pause.release() + + def isfinished(self): + self.lock.acquire() + x = self.finished + self.lock.release() + return x diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/Statistics.py b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/Statistics.py new file mode 100644 index 0000000..0ee040e --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/Statistics.py @@ -0,0 +1,182 @@ +# Written by Edward Keyes +# see LICENSE.txt for license information + +from threading import Event +try: + True +except: + True = 1 + False = 0 + +class Statistics_Response: + pass # empty class + + +class Statistics: + def __init__(self, upmeasure, downmeasure, connecter, ghttpdl, hhttpdl, + ratelimiter, rerequest_lastfailed, fdatflag): + self.upmeasure = upmeasure + self.downmeasure = downmeasure + self.connecter = connecter + self.ghttpdl = ghttpdl + self.hhttpdl = hhttpdl + self.ratelimiter = ratelimiter + self.downloader = connecter.downloader + self.picker = connecter.downloader.picker + self.storage = connecter.downloader.storage + self.torrentmeasure = connecter.downloader.totalmeasure + self.rerequest_lastfailed = rerequest_lastfailed + self.fdatflag = fdatflag + self.fdatactive = False + self.piecescomplete = None + self.placesopen = None + self.storage_totalpieces = len(self.storage.hashes) + + + def set_dirstats(self, files, piece_length): + self.piecescomplete = 0 + self.placesopen = 0 + self.filelistupdated = Event() + self.filelistupdated.set() + frange = xrange(len(files)) + self.filepieces = [[] for x in frange] + self.filepieces2 = [[] for x in frange] + self.fileamtdone = [0.0 for x in frange] + self.filecomplete = [False for x in frange] + self.fileinplace = [False for x in frange] + start = 0L + for i in frange: + l = files[i][1] + if l == 0: + self.fileamtdone[i] = 1.0 + self.filecomplete[i] = True + self.fileinplace[i] = True + else: + fp = self.filepieces[i] + fp2 = self.filepieces2[i] + for piece in range(int(start/piece_length), + int((start+l-1)/piece_length)+1): + fp.append(piece) + fp2.append(piece) + start += l + + + def update(self): + s = Statistics_Response() + s.upTotal = self.upmeasure.get_total() + s.downTotal = self.downmeasure.get_total() + s.last_failed = self.rerequest_lastfailed() + s.external_connection_made = self.connecter.external_connection_made + if s.downTotal > 0: + s.shareRating = float(s.upTotal)/s.downTotal + elif s.upTotal == 0: + s.shareRating = 0.0 + else: + s.shareRating = -1.0 + s.torrentRate = self.torrentmeasure.get_rate() + s.torrentTotal = self.torrentmeasure.get_total() + s.numSeeds = self.picker.seeds_connected + s.numOldSeeds = self.downloader.num_disconnected_seeds() + s.numPeers = len(self.downloader.downloads)-s.numSeeds + s.numCopies = 0.0 + for i in self.picker.crosscount: + if i==0: + s.numCopies+=1 + else: + s.numCopies+=1-float(i)/self.picker.numpieces + break + if self.picker.done: + s.numCopies2 = s.numCopies + 1 + else: + s.numCopies2 = 0.0 + for i in self.picker.crosscount2: + if i==0: + s.numCopies2+=1 + else: + s.numCopies2+=1-float(i)/self.picker.numpieces + break + s.discarded = self.downloader.discarded + s.numSeeds += self.ghttpdl.seedsfound + s.numSeeds += self.hhttpdl.seedsfound + s.numOldSeeds += self.ghttpdl.seedsfound + s.numOldSeeds += self.hhttpdl.seedsfound + if s.numPeers == 0 or self.picker.numpieces == 0: + s.percentDone = 0.0 + else: + s.percentDone = 100.0*(float(self.picker.totalcount)/self.picker.numpieces)/s.numPeers + + s.backgroundallocating = self.storage.bgalloc_active + s.storage_totalpieces = len(self.storage.hashes) + s.storage_active = len(self.storage.stat_active) + s.storage_new = len(self.storage.stat_new) + s.storage_dirty = len(self.storage.dirty) + numdownloaded = self.storage.stat_numdownloaded + s.storage_justdownloaded = numdownloaded + s.storage_numcomplete = self.storage.stat_numfound + numdownloaded + s.storage_numflunked = self.storage.stat_numflunked + s.storage_isendgame = self.downloader.endgamemode + + s.peers_kicked = self.downloader.kicked.items() + s.peers_banned = self.downloader.banned.items() + + try: + s.upRate = int(self.ratelimiter.upload_rate/1000) + assert s.upRate < 5000 + except: + s.upRate = 0 + s.upSlots = self.ratelimiter.slots + + s.have = self.storage.get_have_copy() + + if self.piecescomplete is None: # not a multi-file torrent + return s + + if self.fdatflag.isSet(): + if not self.fdatactive: + self.fdatactive = True + else: + self.fdatactive = False + + if self.piecescomplete != self.picker.numgot: + for i in xrange(len(self.filecomplete)): + if self.filecomplete[i]: + continue + oldlist = self.filepieces[i] + newlist = [ piece + for piece in oldlist + if not self.storage.have[piece] ] + if len(newlist) != len(oldlist): + self.filepieces[i] = newlist + self.fileamtdone[i] = ( + (len(self.filepieces2[i])-len(newlist)) + /float(len(self.filepieces2[i])) ) + if not newlist: + self.filecomplete[i] = True + self.filelistupdated.set() + + self.piecescomplete = self.picker.numgot + + if ( self.filelistupdated.isSet() + or self.placesopen != len(self.storage.places) ): + for i in xrange(len(self.filecomplete)): + if not self.filecomplete[i] or self.fileinplace[i]: + continue + while self.filepieces2[i]: + piece = self.filepieces2[i][-1] + if self.storage.places[piece] != piece: + break + del self.filepieces2[i][-1] + if not self.filepieces2[i]: + self.fileinplace[i] = True + self.storage.set_file_readonly(i) + self.filelistupdated.set() + + self.placesopen = len(self.storage.places) + + s.fileamtdone = self.fileamtdone + s.filecomplete = self.filecomplete + s.fileinplace = self.fileinplace + s.filelistupdated = self.filelistupdated + + return s + diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/Storage.py b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/Storage.py new file mode 100644 index 0000000..7a0f971 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/Storage.py @@ -0,0 +1,597 @@ +# Written by Bram Cohen +# see LICENSE.txt for license information + +from BaseLib.Core.BitTornado.piecebuffer import BufferPool +from threading import Lock +from time import strftime, localtime +import os +from os.path import exists, getsize, getmtime as getmtime_, basename +from traceback import print_exc +try: + from os import fsync +except ImportError: + fsync = lambda x: None +from bisect import bisect +import sys + +try: + True +except: + True = 1 + False = 0 + +DEBUG = False + +#MAXREADSIZE = 32768 +MAXREADSIZE = 2 ** 16 # Arno: speed opt +MAXLOCKSIZE = 1000000000L +MAXLOCKRANGE = 3999999999L # only lock first 4 gig of file + +_pool = BufferPool() +PieceBuffer = _pool.new + +def getmtime(path): + # On some OS's, getmtime returns a float + return int(getmtime_(path)) + +def dummy_status(fractionDone = None, activity = None): + pass + +class Storage: + def __init__(self, files, piece_length, doneflag, config, + disabled_files = None): + # can raise IOError and ValueError + self.files = files + self.piece_length = piece_length + self.doneflag = doneflag + self.disabled = [False] * len(files) + self.file_ranges = [] + self.disabled_ranges = [] + self.working_ranges = [] + numfiles = 0 + total = 0L + self.so_far = 0L + self.handles = {} + self.whandles = {} + self.tops = {} + self.sizes = {} + self.mtimes = {} + if config.get('lock_files', True): + self.lock_file, self.unlock_file = self._lock_file, self._unlock_file + else: + self.lock_file, self.unlock_file = lambda x1, x2: None, lambda x1, x2: None + self.lock_while_reading = config.get('lock_while_reading', False) + self.lock = Lock() + + if not disabled_files: + disabled_files = [False] * len(files) + + for i in xrange(len(files)): + file, length = files[i] + if doneflag.isSet(): # bail out if doneflag is set + return + self.disabled_ranges.append(None) + if length == 0: + self.file_ranges.append(None) + self.working_ranges.append([]) + else: + range = (total, total + length, 0, file) + self.file_ranges.append(range) + self.working_ranges.append([range]) + numfiles += 1 + total += length + if disabled_files[i]: + l = 0 + else: + if exists(file): + l = getsize(file) + if l > length: + h = open(file, 'rb+') + h.truncate(length) + h.flush() + h.close() + l = length + else: + l = 0 + h = open(file, 'wb+') + h.flush() + h.close() + self.mtimes[file] = getmtime(file) + self.tops[file] = l + self.sizes[file] = length + self.so_far += l + + self.total_length = total + self._reset_ranges() + + self.max_files_open = config['max_files_open'] + if self.max_files_open > 0 and numfiles > self.max_files_open: + self.handlebuffer = [] + else: + self.handlebuffer = None + + + if os.name == 'nt': + def _lock_file(self, name, f): + import msvcrt + for p in range(0, min(self.sizes[name], MAXLOCKRANGE), MAXLOCKSIZE): + f.seek(p) + msvcrt.locking(f.fileno(), msvcrt.LK_LOCK, + min(MAXLOCKSIZE, self.sizes[name]-p)) + + def _unlock_file(self, name, f): + import msvcrt + for p in range(0, min(self.sizes[name], MAXLOCKRANGE), MAXLOCKSIZE): + f.seek(p) + msvcrt.locking(f.fileno(), msvcrt.LK_UNLCK, + min(MAXLOCKSIZE, self.sizes[name]-p)) + + elif os.name == 'posix': + def _lock_file(self, name, f): + import fcntl + fcntl.flock(f.fileno(), fcntl.LOCK_EX) + + def _unlock_file(self, name, f): + import fcntl + fcntl.flock(f.fileno(), fcntl.LOCK_UN) + + else: + def _lock_file(self, name, f): + pass + def _unlock_file(self, name, f): + pass + + # Arno, 2010-04-16: STBSPEED + def get_length_initial_content(self): + return self.so_far + + def was_preallocated(self, pos, length): + for file, begin, end in self._intervals(pos, length): + if self.tops.get(file, 0) < end: + return False + return True + + + def _sync(self, file): + self._close(file) + if self.handlebuffer: + self.handlebuffer.remove(file) + + def sync(self): + # may raise IOError or OSError + for file in self.whandles.keys(): + self._sync(file) + + + def set_readonly(self, f=None): + if f is None: + self.sync() + return + file = self.files[f][0] + if self.whandles.has_key(file): + self._sync(file) + + + def get_total_length(self): + return self.total_length + + + def _open(self, file, mode): + if self.mtimes.has_key(file): + try: + if self.handlebuffer is not None: + assert getsize(file) == self.tops[file] + newmtime = getmtime(file) + oldmtime = self.mtimes[file] + assert newmtime <= oldmtime+1 + assert newmtime >= oldmtime-1 + except: + if DEBUG: + print( file+' modified: ' + +strftime('(%x %X)', localtime(self.mtimes[file])) + +strftime(' != (%x %X) ?', localtime(getmtime(file))) ) + raise IOError('modified during download') + try: + return open(file, mode) + except: + if DEBUG: + print_exc() + raise + + + def _close(self, file): + f = self.handles[file] + del self.handles[file] + if self.whandles.has_key(file): + del self.whandles[file] + f.flush() + self.unlock_file(file, f) + f.close() + self.tops[file] = getsize(file) + self.mtimes[file] = getmtime(file) + else: + if self.lock_while_reading: + self.unlock_file(file, f) + f.close() + + + def _close_file(self, file): + if not self.handles.has_key(file): + return + self._close(file) + if self.handlebuffer: + self.handlebuffer.remove(file) + + + def _get_file_handle(self, file, for_write): + if self.handles.has_key(file): + if for_write and not self.whandles.has_key(file): + self._close(file) + try: + f = self._open(file, 'rb+') + self.handles[file] = f + self.whandles[file] = 1 + self.lock_file(file, f) + except (IOError, OSError), e: + if DEBUG: + print_exc() + raise IOError('unable to reopen '+file+': '+str(e)) + + if self.handlebuffer: + if self.handlebuffer[-1] != file: + self.handlebuffer.remove(file) + self.handlebuffer.append(file) + elif self.handlebuffer is not None: + self.handlebuffer.append(file) + else: + try: + if for_write: + f = self._open(file, 'rb+') + self.handles[file] = f + self.whandles[file] = 1 + self.lock_file(file, f) + else: + f = self._open(file, 'rb') + self.handles[file] = f + if self.lock_while_reading: + self.lock_file(file, f) + except (IOError, OSError), e: + if DEBUG: + print_exc() + raise IOError('unable to open '+file+': '+str(e)) + + if self.handlebuffer is not None: + self.handlebuffer.append(file) + if len(self.handlebuffer) > self.max_files_open: + self._close(self.handlebuffer.pop(0)) + + return self.handles[file] + + + def _reset_ranges(self): + self.ranges = [] + for l in self.working_ranges: + self.ranges.extend(l) + self.begins = [i[0] for i in self.ranges] + + def _intervals(self, pos, amount): + r = [] + stop = pos + amount + p = bisect(self.begins, pos) - 1 + while p < len(self.ranges): + begin, end, offset, file = self.ranges[p] + if begin >= stop: + break + r.append(( file, + offset + max(pos, begin) - begin, + offset + min(end, stop) - begin )) + p += 1 + return r + + + def read(self, pos, amount, flush_first = False): + r = PieceBuffer() + for file, pos, end in self._intervals(pos, amount): + if DEBUG: + print >>sys.stderr,'reading '+file+' from '+str(pos)+' to '+str(end)+' amount '+str(amount) + try: + self.lock.acquire() + h = self._get_file_handle(file, False) + if flush_first and self.whandles.has_key(file): + h.flush() + fsync(h) + h.seek(pos) + while pos < end: + length = min(end-pos, MAXREADSIZE) + data = h.read(length) + if len(data) != length: + raise IOError('error reading data from '+ file) + r.append(data) + pos += length + self.lock.release() + except: + self.lock.release() + raise IOError('error reading data from '+ file) + return r + + def write(self, pos, s): + # might raise an IOError + total = 0 + for file, begin, end in self._intervals(pos, len(s)): + if DEBUG: + print 'writing '+file+' from '+str(pos)+' to '+str(end) + self.lock.acquire() + h = self._get_file_handle(file, True) + h.seek(begin) + h.write(s[total: total + end - begin]) + self.lock.release() + total += end - begin + + def top_off(self): + for begin, end, offset, file in self.ranges: + l = offset + end - begin + if l > self.tops.get(file, 0): + self.lock.acquire() + h = self._get_file_handle(file, True) + h.seek(l-1) + h.write(chr(0xFF)) + self.lock.release() + + def flush(self): + # may raise IOError or OSError + for file in self.whandles.keys(): + self.lock.acquire() + self.handles[file].flush() + self.lock.release() + + def close(self): + for file, f in self.handles.items(): + try: + self.unlock_file(file, f) + except: + pass + try: + f.close() + except: + pass + self.handles = {} + self.whandles = {} + self.handlebuffer = None + + + def _get_disabled_ranges(self, f): + if not self.file_ranges[f]: + return ((), (), ()) + r = self.disabled_ranges[f] + if r: + return r + start, end, offset, file = self.file_ranges[f] + if DEBUG: + print 'calculating disabled range for '+self.files[f][0] + print 'bytes: '+str(start)+'-'+str(end) + print 'file spans pieces '+str(int(start/self.piece_length))+'-'+str(int((end-1)/self.piece_length)+1) + pieces = range(int(start/self.piece_length), + int((end-1)/self.piece_length)+1) + offset = 0 + disabled_files = [] + if len(pieces) == 1: + if ( start % self.piece_length == 0 + and end % self.piece_length == 0 ): # happens to be a single, + # perfect piece + working_range = [(start, end, offset, file)] + update_pieces = [] + else: + midfile = os.path.join(self.bufferdir, str(f)) + working_range = [(start, end, 0, midfile)] + disabled_files.append((midfile, start, end)) + length = end - start + self.sizes[midfile] = length + piece = pieces[0] + update_pieces = [(piece, start-(piece*self.piece_length), length)] + else: + update_pieces = [] + if start % self.piece_length != 0: # doesn't begin on an even piece boundary + end_b = pieces[1]*self.piece_length + startfile = os.path.join(self.bufferdir, str(f)+'b') + working_range_b = [ ( start, end_b, 0, startfile ) ] + disabled_files.append((startfile, start, end_b)) + length = end_b - start + self.sizes[startfile] = length + offset = length + piece = pieces.pop(0) + update_pieces.append((piece, start-(piece*self.piece_length), length)) + else: + working_range_b = [] + if f != len(self.files)-1 and end % self.piece_length != 0: + # doesn't end on an even piece boundary + start_e = pieces[-1] * self.piece_length + endfile = os.path.join(self.bufferdir, str(f)+'e') + working_range_e = [ ( start_e, end, 0, endfile ) ] + disabled_files.append((endfile, start_e, end)) + length = end - start_e + self.sizes[endfile] = length + piece = pieces.pop(-1) + update_pieces.append((piece, 0, length)) + else: + working_range_e = [] + if pieces: + working_range_m = [ ( pieces[0]*self.piece_length, + (pieces[-1]+1)*self.piece_length, + offset, file ) ] + else: + working_range_m = [] + working_range = working_range_b + working_range_m + working_range_e + + if DEBUG: + print str(working_range) + print str(update_pieces) + r = (tuple(working_range), tuple(update_pieces), tuple(disabled_files)) + self.disabled_ranges[f] = r + return r + + + def set_bufferdir(self, dir): + self.bufferdir = dir + + def enable_file(self, f): + if not self.disabled[f]: + return + self.disabled[f] = False + r = self.file_ranges[f] + if not r: + return + file = r[3] + if not exists(file): + h = open(file, 'wb+') + h.flush() + h.close() + if not self.tops.has_key(file): + self.tops[file] = getsize(file) + if not self.mtimes.has_key(file): + self.mtimes[file] = getmtime(file) + self.working_ranges[f] = [r] + + def disable_file(self, f): + if self.disabled[f]: + return + self.disabled[f] = True + r = self._get_disabled_ranges(f) + if not r: + return + for file, begin, end in r[2]: + if not os.path.isdir(self.bufferdir): + os.makedirs(self.bufferdir) + if not exists(file): + h = open(file, 'wb+') + h.flush() + h.close() + if not self.tops.has_key(file): + self.tops[file] = getsize(file) + if not self.mtimes.has_key(file): + self.mtimes[file] = getmtime(file) + self.working_ranges[f] = r[0] + + reset_file_status = _reset_ranges + + + def get_piece_update_list(self, f): + return self._get_disabled_ranges(f)[1] + + + def delete_file(self, f): + try: + os.remove(self.files[f][0]) + except: + pass + + + ''' + Pickled data format: + + d['files'] = [ file #, size, mtime {, file #, size, mtime...} ] + file # in torrent, and the size and last modification + time for those files. Missing files are either empty + or disabled. + d['partial files'] = [ name, size, mtime... ] + Names, sizes and last modification times of files containing + partial piece data. Filenames go by the following convention: + {file #, 0-based}{nothing, "b" or "e"} + eg: "0e" "3" "4b" "4e" + Where "b" specifies the partial data for the first piece in + the file, "e" the last piece, and no letter signifying that + the file is disabled but is smaller than one piece, and that + all the data is cached inside so adjacent files may be + verified. + ''' + def pickle(self): + files = [] + pfiles = [] + for i in xrange(len(self.files)): + if not self.files[i][1]: # length == 0 + continue + if self.disabled[i]: + for file, start, end in self._get_disabled_ranges(i)[2]: + pfiles.extend([basename(file), getsize(file), getmtime(file)]) + continue + file = self.files[i][0] + files.extend([i, getsize(file), getmtime(file)]) + return {'files': files, 'partial files': pfiles} + + + def unpickle(self, data): + # assume all previously-disabled files have already been disabled + try: + files = {} + pfiles = {} + l = data['files'] + assert len(l) % 3 == 0 + l = [l[x:x+3] for x in xrange(0, len(l), 3)] + for f, size, mtime in l: + files[f] = (size, mtime) + l = data.get('partial files', []) + assert len(l) % 3 == 0 + l = [l[x:x+3] for x in xrange(0, len(l), 3)] + for file, size, mtime in l: + pfiles[file] = (size, mtime) + + valid_pieces = {} + for i in xrange(len(self.files)): + if self.disabled[i]: + continue + r = self.file_ranges[i] + if not r: + continue + start, end, offset, file = r + if DEBUG: + print 'adding '+file + for p in xrange( int(start/self.piece_length), + int((end-1)/self.piece_length)+1 ): + valid_pieces[p] = 1 + + if DEBUG: + print valid_pieces.keys() + + def test(old, size, mtime): + oldsize, oldmtime = old + if size != oldsize: + return False + if mtime > oldmtime+1: + return False + if mtime < oldmtime-1: + return False + return True + + for i in xrange(len(self.files)): + if self.disabled[i]: + for file, start, end in self._get_disabled_ranges(i)[2]: + f1 = basename(file) + if ( not pfiles.has_key(f1) + or not test(pfiles[f1],getsize(file),getmtime(file)) ): + if DEBUG: + print 'removing '+file + for p in xrange( int(start/self.piece_length), + int((end-1)/self.piece_length)+1 ): + if valid_pieces.has_key(p): + del valid_pieces[p] + continue + file, size = self.files[i] + if not size: + continue + if ( not files.has_key(i) + or not test(files[i], getsize(file), getmtime(file)) ): + start, end, offset, file = self.file_ranges[i] + if DEBUG: + print 'removing '+file + for p in xrange( int(start/self.piece_length), + int((end-1)/self.piece_length)+1 ): + if valid_pieces.has_key(p): + del valid_pieces[p] + except: + if DEBUG: + print_exc() + return [] + + if DEBUG: + print valid_pieces.keys() + return valid_pieces.keys() + diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/StorageWrapper.py b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/StorageWrapper.py new file mode 100644 index 0000000..e33f746 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/StorageWrapper.py @@ -0,0 +1,1357 @@ +# Written by Bram Cohen, Arno Bakker, George Milescu +# see LICENSE.txt for license information + +import sys +from traceback import print_exc +from random import randrange +from copy import deepcopy +import pickle +import traceback +import time + +from BaseLib.Core.Merkle.merkle import MerkleTree +from BaseLib.Core.Utilities.Crypto import sha +from BaseLib.Core.BitTornado.bitfield import Bitfield +from BaseLib.Core.BitTornado.clock import clock +from BaseLib.Core.BitTornado.bencode import bencode + +try: + True +except: + True = 1 + False = 0 +from bisect import insort + +DEBUG = False + +STATS_INTERVAL = 0.2 +RARE_RAWSERVER_TASKID = -481 # This must be a rawserver task ID that is never valid. + + +def dummy_status(fractionDone = None, activity = None): + pass + +class Olist: + def __init__(self, l = []): + self.d = {} + for i in l: + self.d[i] = 1 + def __len__(self): + return len(self.d) + def includes(self, i): + return self.d.has_key(i) + def add(self, i): + self.d[i] = 1 + def extend(self, l): + for i in l: + self.d[i] = 1 + def pop(self, n=0): + # assert self.d + k = self.d.keys() + if n == 0: + i = min(k) + elif n == -1: + i = max(k) + else: + k.sort() + i = k[n] + del self.d[i] + return i + def remove(self, i): + if self.d.has_key(i): + del self.d[i] + +class fakeflag: + def __init__(self, state=False): + self.state = state + def wait(self): + pass + def isSet(self): + return self.state + + +class StorageWrapper: + def __init__(self, videoinfo, storage, request_size, hashes, + piece_size, root_hash, finished, failed, + statusfunc = dummy_status, flag = fakeflag(), check_hashes = True, + data_flunked = lambda x: None, + piece_from_live_source_func = lambda i,d: None, + backfunc = None, + config = {}, unpauseflag = fakeflag(True)): + + if DEBUG: + print >>sys.stderr, "StorageWrapper: __init__: wrapped around", storage.files + self.videoinfo = videoinfo + self.storage = storage + self.request_size = long(request_size) + self.hashes = hashes + self.piece_size = long(piece_size) + self.piece_length = long(piece_size) + self.finished = finished + self.report_failure = failed + self.statusfunc = statusfunc + self.flag = flag + self.check_hashes = check_hashes + self.data_flunked = data_flunked + self.piece_from_live_source_func = piece_from_live_source_func + self.backfunc = backfunc + self.config = config + self.unpauseflag = unpauseflag + + self.live_streaming = self.videoinfo['live'] + + self.alloc_type = config.get('alloc_type', 'normal') + self.double_check = config.get('double_check', 0) + self.triple_check = config.get('triple_check', 0) + if self.triple_check: + self.double_check = True + self.bgalloc_enabled = False + self.bgalloc_active = False + self.total_length = storage.get_total_length() + self.amount_left = self.total_length + if self.total_length <= self.piece_size * (len(hashes) - 1): + raise ValueError, 'bad data in responsefile - total too small' + if self.total_length > self.piece_size * len(hashes): + raise ValueError, 'bad data in responsefile - total too big' + self.numactive = [0] * len(hashes) + self.inactive_requests = [1] * len(hashes) + self.amount_inactive = self.total_length + self.amount_obtained = 0 + self.amount_desired = self.total_length + self.have = Bitfield(len(hashes)) + self.have_cloaked_data = None + self.blocked = [False] * len(hashes) + self.blocked_holes = [] + self.blocked_movein = Olist() + self.blocked_moveout = Olist() + self.waschecked = [False] * len(hashes) + self.places = {} # Arno, maps piece to actual position in files + self.holes = [] + self.stat_active = {} + self.stat_new = {} + self.dirty = {} + self.stat_numflunked = 0 + self.stat_numdownloaded = 0 + self.stat_numfound = 0 + self.download_history = {} + self.failed_pieces = {} + self.out_of_place = 0 + self.write_buf_max = config['write_buffer_size']*1048576L + self.write_buf_size = 0L + self.write_buf = {} # structure: piece: [(start, data), ...] + self.write_buf_list = [] + # Arno, 2010-04-23: STBSPEED: the piece that were correct on disk at start + self.pieces_on_disk_at_startup = [] + + # Merkle: + self.merkle_torrent = (root_hash is not None) + self.root_hash = root_hash + # STBSPEED: no hashchecking for live, so no need for this expensive op. + if self.live_streaming: + self.initial_hashes = None + else: + self.initial_hashes = deepcopy(self.hashes) + + if self.merkle_torrent: + self.hashes_unpickled = False + # Must see if we're initial seeder + self.check_hashes = True + # Fallback for if we're not an initial seeder or don't have a + # Merkle tree on disk. + self.merkletree = MerkleTree(self.piece_size,self.total_length,self.root_hash,None) + else: + # Normal BT + self.hashes_unpickled = True + + self.initialize_tasks = [ + ['checking existing data', 0, self.init_hashcheck, self.hashcheckfunc], + ['moving data', 1, self.init_movedata, self.movedatafunc], + ['allocating disk space', 1, self.init_alloc, self.allocfunc] ] + self.initialize_done = None + + # Arno: move starting of periodic _bgalloc to init_alloc + self.backfunc(self._bgsync, max(self.config['auto_flush']*60, 60)) + + + def _bgsync(self): + if self.config['auto_flush']: + self.sync() + self.backfunc(self._bgsync, max(self.config['auto_flush']*60, 60)) + + + def old_style_init(self): + while self.initialize_tasks: + msg, done, init, next = self.initialize_tasks.pop(0) + if init(): + self.statusfunc(activity = msg, fractionDone = done) + t = clock() + STATS_INTERVAL + x = 0 + while x is not None: + if t < clock(): + t = clock() + STATS_INTERVAL + self.statusfunc(fractionDone = x) + self.unpauseflag.wait() + if self.flag.isSet(): + return False + x = next() + + self.statusfunc(fractionDone = 0) + return True + + + def initialize(self, donefunc, statusfunc = None): + if DEBUG: + print >>sys.stderr,"StorageWrapper: initialize: enter, backfunc is",self.backfunc + + self.initialize_done = donefunc + if statusfunc is None: + statusfunc = self.statusfunc + self.initialize_status = statusfunc + self.initialize_next = None + + """ + Arno: 2007-01-02: + This next line used to read: + self.backfunc(self._initialize) + So without the task ID. I've changed this to accomodate the + following situation. In video-on-demand, it may occur that + a torrent is stopped and then immediately after it is + restarted. In particular, we use this when a user selects + a torrent from the mainwin to be played (again). Because the + torrent does not necessarily use a VOD-piecepicker we have + to stop the current DL process and start a new one. + + When stopping and starting a torrent quickly a problem occurs. + When a torrent is stopped, its infohash is registered in kill list + of the (real) RawServer class. The next time the rawserver looks + for tasks to execute it will first check the kill list. If it's not + empty it will remove all tasks that have the given infohash as taskID. + This mechanism ensures that when a torrent is stopped, any outstanding + tasks belonging to the torrent are removed from the rawserver task queue. + + It can occur that we've stopped the torrent and the + infohash is on the kill list, but the queue has not yet been cleared of + old entries because the thread that runs the rawserver did not get to + executing new tasks yet. This causes a problem right here, because + we now want to schedule a new task on behalf of the new download process. + If it is enqueued now, it will be removed the next time the rawserver + checks its task list and because the infohash is on the kill list be + deleted. + + My fix is to schedule this first task of the new torrent under a + different task ID. Hence, when the rawserver checks its queue it + will not delete it, thinking it belonged to the old download + process. The really clean solution is to stop using infohash as + taskid, and use a unique ID for a download process. This will + take a bit of work to ensure it works correctly, so in the mean + time we'll use this fix. + """ + + # Arno, STBSPEED: potentially we can just call + # self.initialize_done(success=True) + # here for live, but we need to check if all variables are set correctly + # (also if different disk allocation policies are used, etc. + # + self.backfunc(self._initialize, id = RARE_RAWSERVER_TASKID) + + def _initialize(self): + + if DEBUG: + print >>sys.stderr,"StorageWrapper: _initialize: enter" + if not self.unpauseflag.isSet(): + self.backfunc(self._initialize, 1) + return + + if DEBUG: + print >>sys.stderr,"StorageWrapper: _initialize: next is",self.initialize_next + + if self.initialize_next: + x = self.initialize_next() + if x is None: + self.initialize_next = None + else: + self.initialize_status(fractionDone = x) + else: + if not self.initialize_tasks: + self.initialize_done(success=True) + self.initialize_done = None + return + msg, done, init, next = self.initialize_tasks.pop(0) + if DEBUG: + print >>sys.stderr,"StorageWrapper: _initialize performing task",msg + if DEBUG: + st = time.time() + if init(): + self.initialize_status(activity = msg, fractionDone = done) + self.initialize_next = next + if DEBUG: + et = time.time() + diff = et - st + print >>sys.stderr,"StorageWrapper: _initialize: task took",diff + + self.backfunc(self._initialize) + + + def init_hashcheck(self): + + if DEBUG: + print >>sys.stderr,"StorageWrapper: init_hashcheck: #hashes",len(self.hashes),"amountleft",self.amount_left + + + if self.live_streaming: + # STBSPEED by Milton + self.set_nohashcheck() + return False + + # Non-live streaming + if self.flag.isSet(): + if DEBUG: + print >>sys.stderr,"StorageWrapper: init_hashcheck: FLAG IS SET" + return False + self.check_list = [] + if not self.hashes or self.amount_left == 0: + self.check_total = 0 + self.finished() + if DEBUG: + print >>sys.stderr,"StorageWrapper: init_hashcheck: Download finished" + return False + + if DEBUG: + print >>sys.stderr,"StorageWrapper: init_hashcheck: self.places",`self.places` + + self.check_targets = {} + got = {} + for p, v in self.places.iteritems(): + assert not got.has_key(v) + got[v] = 1 + + # Arno, 2010-04-16: STBSPEED: Avoid costly calculations if new VOD + if len(self.places) == 0 and self.storage.get_length_initial_content() == 0L: + self.set_nohashcheck() + return False + + # STBSPEED: TODO: optimize + for i in xrange(len(self.hashes)): + # Arno, 2010-04-16: STBSPEED: Only execute if there is persistent + # state (=self.places) on already hashchecked pieces. + if len(self.places) > 0: + if self.places.has_key(i): # restored from pickled + self.check_targets[self.hashes[i]] = [] + if self.places[i] == i: + continue + else: + assert not got.has_key(i) + self.out_of_place += 1 + if got.has_key(i): + continue + + if self._waspre(i): + # Arno: If there is data on disk, check it + if self.blocked[i]: + self.places[i] = i + else: + self.check_list.append(i) + continue + if not self.check_hashes: + self.failed('file supposed to be complete on start-up, but data is missing') + return False + self.holes.append(i) + if self.blocked[i] or self.check_targets.has_key(self.hashes[i]): + self.check_targets[self.hashes[i]] = [] # in case of a hash collision, discard + else: + self.check_targets[self.hashes[i]] = [i] + self.check_total = len(self.check_list) + self.check_numchecked = 0.0 + self.lastlen = self._piecelen(len(self.hashes) - 1) + self.numchecked = 0.0 + if DEBUG: + print "StorageWrapper: init_hashcheck: checking",self.check_list + print "StorageWrapper: init_hashcheck: return self.check_total > 0 is ",(self.check_total > 0) + return self.check_total > 0 + + + def set_nohashcheck(self): + if DEBUG: + print "StorageWrapper: init_hashcheck: live or empty files, skipping" + self.places = {} + self.check_targets = {} + self.check_list = [] + self.check_total = len(self.check_list) + self.check_numchecked = 0.0 + self.lastlen = self._piecelen(len(self.hashes) - 1) + self.numchecked = 0.0 + self.check_targets[self.hashes[0]] = [0] + self.holes = range(len(self.hashes)) + + + # Arno, 2010-04-20: STBSPEED + def get_pieces_on_disk_at_startup(self): + """ Returns list of pieces currently on disk that were succesfully + hashchecked. If the file was complete on disk, this list is empty. + See download_bt1.py::BT1Download::startEngine for how this is dealt with. + """ + if DEBUG: + print >>sys.stderr,"StorageWrapper: get_pieces_on_disk_at_startup: self.places len",len(self.places),"on disk",len(self.pieces_on_disk_at_startup) + + return self.pieces_on_disk_at_startup + + + def _markgot(self, piece, pos): + if DEBUG: + print str(piece)+' at '+str(pos) + self.places[piece] = pos + self.have[piece] = True + self.pieces_on_disk_at_startup.append(piece) + len = self._piecelen(piece) + self.amount_obtained += len + self.amount_left -= len + self.amount_inactive -= len + self.inactive_requests[piece] = None + self.waschecked[piece] = self.check_hashes + self.stat_numfound += 1 + + def hashcheckfunc(self): + try: + if self.live_streaming: + return None + if self.flag.isSet(): + return None + if not self.check_list: + return None + + i = self.check_list.pop(0) + if not self.check_hashes: + self._markgot(i, i) + else: + d1 = self.read_raw(i, 0, self.lastlen) + if d1 is None: + return None + sh = sha(d1[:]) + d1.release() + sp = sh.digest() + d2 = self.read_raw(i, self.lastlen, self._piecelen(i)-self.lastlen) + if d2 is None: + return None + sh.update(d2[:]) + d2.release() + s = sh.digest() + + + if DEBUG: + if s != self.hashes[i]: + print >>sys.stderr,"StorageWrapper: hashcheckfunc: piece corrupt",i + + # Merkle: If we didn't read the hashes from persistent storage then + # we can't check anything. Exception is the case where we are the + # initial seeder. In that case we first calculate all hashes, + # and then compute the hash tree. If the root hash equals the + # root hash in the .torrent we're a seeder. Otherwise, we are + # client with messed up data and no (local) way of checking it. + # + if not self.hashes_unpickled: + if DEBUG: + print "StorageWrapper: Merkle torrent, saving calculated hash",i + self.initial_hashes[i] = s + self._markgot(i, i) + elif s == self.hashes[i]: + self._markgot(i, i) + elif (self.check_targets.get(s) + and self._piecelen(i) == self._piecelen(self.check_targets[s][-1])): + self._markgot(self.check_targets[s].pop(), i) + self.out_of_place += 1 + elif (not self.have[-1] and sp == self.hashes[-1] + and (i == len(self.hashes) - 1 + or not self._waspre(len(self.hashes) - 1))): + self._markgot(len(self.hashes) - 1, i) + self.out_of_place += 1 + else: + self.places[i] = i + self.numchecked += 1 + if self.amount_left == 0: + if not self.hashes_unpickled: + # Merkle: The moment of truth. Are we an initial seeder? + self.merkletree = MerkleTree(self.piece_size,self.total_length,None,self.initial_hashes) + if self.merkletree.compare_root_hashes(self.root_hash): + if DEBUG: + print "StorageWrapper: Merkle torrent, initial seeder!" + self.hashes = self.initial_hashes + else: + # Bad luck + if DEBUG: + print "StorageWrapper: Merkle torrent, NOT a seeder!" + self.failed('download corrupted, hash tree does not compute; please delete and restart') + return 1 + self.finished() + + return (self.numchecked / self.check_total) + + except Exception, e: + print_exc() + self.failed('download corrupted: '+str(e)+'; please delete and restart') + + + def init_movedata(self): + if self.flag.isSet(): + return False + if self.alloc_type != 'sparse': + return False + self.storage.top_off() # sets file lengths to their final size + self.movelist = [] + if self.out_of_place == 0: + for i in self.holes: + self.places[i] = i + self.holes = [] + return False + self.tomove = float(self.out_of_place) + for i in xrange(len(self.hashes)): + if not self.places.has_key(i): + self.places[i] = i + elif self.places[i] != i: + self.movelist.append(i) + self.holes = [] + return True + + def movedatafunc(self): + if self.flag.isSet(): + return None + if not self.movelist: + return None + i = self.movelist.pop(0) + old = self.read_raw(self.places[i], 0, self._piecelen(i)) + if old is None: + return None + if not self.write_raw(i, 0, old): + return None + if self.double_check and self.have[i]: + if self.triple_check: + old.release() + old = self.read_raw(i, 0, self._piecelen(i), + flush_first = True) + if old is None: + return None + if sha(old[:]).digest() != self.hashes[i]: + self.failed('download corrupted, piece on disk failed triple check; please delete and restart') + return None + old.release() + + self.places[i] = i + self.tomove -= 1 + return (self.tomove / self.out_of_place) + + + def init_alloc(self): + if self.flag.isSet(): + return False + if not self.holes: + return False + self.numholes = float(len(self.holes)) + self.alloc_buf = chr(0xFF) * self.piece_size + ret = False + if self.alloc_type == 'pre-allocate': + self.bgalloc_enabled = True + ret = True + if self.alloc_type == 'background': + self.bgalloc_enabled = True + # Arno: only enable this here, eats CPU otherwise + if self.bgalloc_enabled: + self.backfunc(self._bgalloc, 0.1) + if ret: + return ret + if self.blocked_moveout: + return True + return False + + + def _allocfunc(self): + while self.holes: + n = self.holes.pop(0) + if self.blocked[n]: # assume not self.blocked[index] + if not self.blocked_movein: + self.blocked_holes.append(n) + continue + if not self.places.has_key(n): + b = self.blocked_movein.pop(0) + oldpos = self._move_piece(b, n) + self.places[oldpos] = oldpos + return None + if self.places.has_key(n): + oldpos = self._move_piece(n, n) + self.places[oldpos] = oldpos + return None + return n + return None + + def allocfunc(self): + if self.flag.isSet(): + return None + + if self.blocked_moveout: + self.bgalloc_active = True + n = self._allocfunc() + if n is not None: + if self.blocked_moveout.includes(n): + self.blocked_moveout.remove(n) + b = n + else: + b = self.blocked_moveout.pop(0) + oldpos = self._move_piece(b, n) + self.places[oldpos] = oldpos + return len(self.holes) / self.numholes + + if self.holes and self.bgalloc_enabled: + self.bgalloc_active = True + n = self._allocfunc() + if n is not None: + self.write_raw(n, 0, self.alloc_buf[:self._piecelen(n)]) + self.places[n] = n + return len(self.holes) / self.numholes + + self.bgalloc_active = False + return None + + def bgalloc(self): + if self.bgalloc_enabled: + if not self.holes and not self.blocked_moveout and self.backfunc: + self.backfunc(self.storage.flush) + # force a flush whenever the "finish allocation" button is hit + self.bgalloc_enabled = True + return False + + def _bgalloc(self): + self.allocfunc() + if self.config.get('alloc_rate', 0) < 0.1: + self.config['alloc_rate'] = 0.1 + self.backfunc(self._bgalloc, + float(self.piece_size)/(self.config['alloc_rate']*1048576)) + + def _waspre(self, piece): + return self.storage.was_preallocated(piece * self.piece_size, self._piecelen(piece)) + + def _piecelen(self, piece): + if piece < len(self.hashes) - 1: + return self.piece_size + else: + return self.total_length - (piece * self.piece_size) + + def get_amount_left(self): + return self.amount_left + + def do_I_have_anything(self): + return self.amount_left < self.total_length + + def _make_inactive(self, index): + """ Mark the blocks that form a piece and save that information to inactive_requests. Each block is marked with a (begin, length) pair. + + @param index: the index of the piece for which blocks are being calculated + """ + length = self._piecelen(index) + l = [] + x = 0 + while x + self.request_size < length: + l.append((x, self.request_size)) + x += self.request_size + l.append((x, length - x)) + self.inactive_requests[index] = l # Note: letter L not number 1 + + def is_endgame(self): + return not self.amount_inactive + + def reset_endgame(self, requestlist): + for index, begin, length in requestlist: + self.request_lost(index, begin, length) + + def get_have_list(self): + return self.have.tostring() + + def get_have_copy(self): + return self.have.copy() + + def get_have_list_cloaked(self): + if self.have_cloaked_data is None: + newhave = Bitfield(copyfrom = self.have) + unhaves = [] + n = min(randrange(2, 5), len(self.hashes)) # between 2-4 unless torrent is small + while len(unhaves) < n: + unhave = randrange(min(32, len(self.hashes))) # all in first 4 bytes + if not unhave in unhaves: + unhaves.append(unhave) + newhave[unhave] = False + self.have_cloaked_data = (newhave.tostring(), unhaves) + return self.have_cloaked_data + + def do_I_have(self, index): + return self.have[index] + + def do_I_have_requests(self, index): + return not not self.inactive_requests[index] + + def is_unstarted(self, index): + return (not self.have[index] and not self.numactive[index] + and not self.dirty.has_key(index)) + + def get_hash(self, index): + return self.hashes[index] + + def get_stats(self): + return self.amount_obtained, self.amount_desired, self.have + + def new_request(self, index): + """ Return a block mark to be downloaded from a piece + + @param index: the index of the piece for which a block will be downloaded + @return: a (begin, length) pair + """ + + if DEBUG: + print >>sys.stderr,"StorageWrapper: new_request",index,"#" + + # returns (begin, length) + if self.inactive_requests[index] == 1: # number 1, not letter L + self._make_inactive(index) + self.numactive[index] += 1 + self.stat_active[index] = 1 + if not self.dirty.has_key(index): + self.stat_new[index] = 1 + rs = self.inactive_requests[index] +# r = min(rs) +# rs.remove(r) + r = rs.pop(0) + self.amount_inactive -= r[1] + return r + + + def request_too_slow(self,index): + """ Arno's addition to get pieces we requested from slow peers to be + back in the PiecePicker's list of candidates """ + if self.amount_inactive == 0: + # all has been requested, endgame about to start, don't mess around + return + + self.inactive_requests[index] = 1 # number 1, not letter L + self.amount_inactive += self._piecelen(index) + + + def write_raw(self, index, begin, data): + try: + self.storage.write(self.piece_size * index + begin, data) + return True + except IOError, e: + traceback.print_exc() + self.failed('IO Error: ' + str(e)) + return False + + + def _write_to_buffer(self, piece, start, data): + if not self.write_buf_max: + return self.write_raw(self.places[piece], start, data) + self.write_buf_size += len(data) + while self.write_buf_size > self.write_buf_max: + old = self.write_buf_list.pop(0) + if not self._flush_buffer(old, True): + return False + if self.write_buf.has_key(piece): + self.write_buf_list.remove(piece) + else: + self.write_buf[piece] = [] + self.write_buf_list.append(piece) + self.write_buf[piece].append((start, data)) + return True + + def _flush_buffer(self, piece, popped = False): + if not self.write_buf.has_key(piece): + return True + if not popped: + self.write_buf_list.remove(piece) + l = self.write_buf[piece] + del self.write_buf[piece] + l.sort() + for start, data in l: + self.write_buf_size -= len(data) + if not self.write_raw(self.places[piece], start, data): + return False + return True + + def sync(self): + spots = {} + for p in self.write_buf_list: + spots[self.places[p]] = p + l = spots.keys() + l.sort() + for i in l: + try: + self._flush_buffer(spots[i]) + except: + pass + try: + self.storage.sync() + except IOError, e: + self.failed('IO Error: ' + str(e)) + except OSError, e: + self.failed('OS Error: ' + str(e)) + + + def _move_piece(self, index, newpos): + oldpos = self.places[index] + if DEBUG: + print 'moving '+str(index)+' from '+str(oldpos)+' to '+str(newpos) + assert oldpos != index + assert oldpos != newpos + assert index == newpos or not self.places.has_key(newpos) + old = self.read_raw(oldpos, 0, self._piecelen(index)) + if old is None: + return -1 + if not self.write_raw(newpos, 0, old): + return -1 + self.places[index] = newpos + if self.have[index] and ( + self.triple_check or (self.double_check and index == newpos)): + if self.triple_check: + old.release() + old = self.read_raw(newpos, 0, self._piecelen(index), + flush_first = True) + if old is None: + return -1 + if sha(old[:]).digest() != self.hashes[index]: + self.failed('download corrupted, piece on disk failed triple check; please delete and restart') + return -1 + old.release() + + if self.blocked[index]: + self.blocked_moveout.remove(index) + if self.blocked[newpos]: + self.blocked_movein.remove(index) + else: + self.blocked_movein.add(index) + else: + self.blocked_movein.remove(index) + if self.blocked[newpos]: + self.blocked_moveout.add(index) + else: + self.blocked_moveout.remove(index) + + return oldpos + + def _clear_space(self, index): + h = self.holes.pop(0) + n = h + if self.blocked[n]: # assume not self.blocked[index] + if not self.blocked_movein: + self.blocked_holes.append(n) + return True # repeat + if not self.places.has_key(n): + b = self.blocked_movein.pop(0) + oldpos = self._move_piece(b, n) + if oldpos < 0: + return False + n = oldpos + if self.places.has_key(n): + oldpos = self._move_piece(n, n) + if oldpos < 0: + return False + n = oldpos + if index == n or index in self.holes: + if n == h: + self.write_raw(n, 0, self.alloc_buf[:self._piecelen(n)]) + self.places[index] = n + if self.blocked[n]: + # because n may be a spot cleared 10 lines above, it's possible + # for it to be blocked. While that spot could be left cleared + # and a new spot allocated, this condition might occur several + # times in a row, resulting in a significant amount of disk I/O, + # delaying the operation of the engine. Rather than do this, + # queue the piece to be moved out again, which will be performed + # by the background allocator, with which data movement is + # automatically limited. + self.blocked_moveout.add(index) + return False + for p, v in self.places.iteritems(): + if v == index: + break + else: + self.failed('download corrupted; please delete and restart') + return False + self._move_piece(p, n) + self.places[index] = index + return False + + ## Arno: don't think we need length here, FIXME + def piece_came_in(self, index, begin, hashlist, piece, baddataguard, source = None): + assert not self.have[index] + # Merkle: Check that the hashes are valid using the known root_hash + # If so, put them in the hash tree and the normal list of hashes to + # allow (1) us to send this piece to others using the right hashes + # and (2) us to check the validity of the piece when it has been + # received completely. + # + if self.merkle_torrent and len(hashlist) > 0: + if self.merkletree.check_hashes(hashlist): + self.merkletree.update_hash_admin(hashlist,self.hashes) + else: + raise ValueError("bad list of hashes") + + if not self.places.has_key(index): + while self._clear_space(index): + pass + if DEBUG: + print 'new place for '+str(index)+' at '+str(self.places[index]) + if self.flag.isSet(): + return False + + if self.failed_pieces.has_key(index): + old = self.read_raw(self.places[index], begin, len(piece)) + if old is None: + return True + if old[:].tostring() != piece: + try: + self.failed_pieces[index][self.download_history[index][begin]] = 1 + except: + self.failed_pieces[index][None] = 1 + old.release() + self.download_history.setdefault(index, {})[begin] = source + + if not self._write_to_buffer(index, begin, piece): + return True + + self.amount_obtained += len(piece) + self.dirty.setdefault(index, []).append((begin, len(piece))) + self.numactive[index] -= 1 + assert self.numactive[index] >= 0 + if not self.numactive[index]: + del self.stat_active[index] + if self.stat_new.has_key(index): + del self.stat_new[index] + + if self.inactive_requests[index] or self.numactive[index]: + return True + + del self.dirty[index] + if not self._flush_buffer(index): + return True + + length = self._piecelen(index) + # Check hash + data = self.read_raw(self.places[index], 0, length, + flush_first = self.triple_check) + if data is None: + return True + + pieceok = False + if self.live_streaming: + # LIVESOURCEAUTH + if self.piece_from_live_source_func(index,data[:]): + pieceok = True + else: + hash = sha(data[:]).digest() + data.release() + if hash == self.hashes[index]: + pieceok = True + + if not pieceok: + self.amount_obtained -= length + self.data_flunked(length, index) + self.inactive_requests[index] = 1 # number 1, not letter L + self.amount_inactive += length + self.stat_numflunked += 1 + + self.failed_pieces[index] = {} + allsenders = {} + for d in self.download_history[index].values(): + allsenders[d] = 1 + if len(allsenders) == 1: + culprit = allsenders.keys()[0] + if culprit is not None: + culprit.failed(index, bump = True) + del self.failed_pieces[index] # found the culprit already + + if self.live_streaming: + # TODO: figure out how to use the Download.BadDataGuard + # cf. the culprit business above. + print >>sys.stderr,"////////////////////////////////////////////////////////////// kicking peer" + raise ValueError("Arno quick fix: Unauth data unacceptable") + + return False + + self.have[index] = True + self.inactive_requests[index] = None + self.waschecked[index] = True + + self.amount_left -= length + self.stat_numdownloaded += 1 + + for d in self.download_history[index].values(): + if d is not None: + d.good(index) + del self.download_history[index] + if self.failed_pieces.has_key(index): + for d in self.failed_pieces[index].keys(): + if d is not None: + d.failed(index) + del self.failed_pieces[index] + + if self.amount_left == 0: + self.finished() + return True + + + def request_lost(self, index, begin, length): + + if DEBUG: + print >>sys.stderr,"StorageWrapper: request_lost",index,"#" + + assert not (begin, length) in self.inactive_requests[index] + insort(self.inactive_requests[index], (begin, length)) + self.amount_inactive += length + self.numactive[index] -= 1 + if not self.numactive[index]: + del self.stat_active[index] + if self.stat_new.has_key(index): + del self.stat_new[index] + + + def get_piece(self, index, begin, length): + # Merkle: Get (sub)piece from disk and its associated hashes + # do_get_piece() returns PieceBuffer + pb = self.do_get_piece(index,begin,length) + if self.merkle_torrent and pb is not None and begin == 0: + hashlist = self.merkletree.get_hashes_for_piece(index) + else: + hashlist = [] + return [pb,hashlist] + + def do_get_piece(self, index, begin, length): + if not self.have[index]: + return None + data = None + if not self.waschecked[index]: + data = self.read_raw(self.places[index], 0, self._piecelen(index)) + if data is None: + return None + if not self.live_streaming and sha(data[:]).digest() != self.hashes[index]: + self.failed('file supposed to be complete on start-up, but piece failed hash check') + return None + self.waschecked[index] = True + if length == -1 and begin == 0: + return data # optimization + if length == -1: + if begin > self._piecelen(index): + return None + length = self._piecelen(index)-begin + if begin == 0: + return self.read_raw(self.places[index], 0, length) + elif begin + length > self._piecelen(index): + return None + if data is not None: + s = data[begin:begin+length] + data.release() + return s + data = self.read_raw(self.places[index], begin, length) + if data is None: + return None + s = data.getarray() + data.release() + return s + + def read_raw(self, piece, begin, length, flush_first = False): + try: + return self.storage.read(self.piece_size * piece + begin, + length, flush_first) + except IOError, e: + self.failed('IO Error: ' + str(e)) + return None + + + def set_file_readonly(self, n): + try: + self.storage.set_readonly(n) + except IOError, e: + self.failed('IO Error: ' + str(e)) + except OSError, e: + self.failed('OS Error: ' + str(e)) + + + def has_data(self, index): + return index not in self.holes and index not in self.blocked_holes + + def doublecheck_data(self, pieces_to_check): + if not self.double_check: + return + sources = [] + for p, v in self.places.iteritems(): + if pieces_to_check.has_key(v): + sources.append(p) + assert len(sources) == len(pieces_to_check) + sources.sort() + for index in sources: + if self.have[index]: + piece = self.read_raw(self.places[index], 0, self._piecelen(index), + flush_first = True) + if piece is None: + return False + if sha(piece[:]).digest() != self.hashes[index]: + self.failed('download corrupted, piece on disk failed double check; please delete and restart') + return False + piece.release() + return True + + + def reblock(self, new_blocked): + # assume downloads have already been canceled and chunks made inactive + for i in xrange(len(new_blocked)): + if new_blocked[i] and not self.blocked[i]: + length = self._piecelen(i) + self.amount_desired -= length + if self.have[i]: + self.amount_obtained -= length + continue + if self.inactive_requests[i] == 1: # number 1, not letter L + self.amount_inactive -= length + continue + inactive = 0 + for nb, nl in self.inactive_requests[i]: + inactive += nl + self.amount_inactive -= inactive + self.amount_obtained -= length - inactive + + if self.blocked[i] and not new_blocked[i]: + length = self._piecelen(i) + self.amount_desired += length + if self.have[i]: + self.amount_obtained += length + continue + if self.inactive_requests[i] == 1: + self.amount_inactive += length + continue + inactive = 0 + for nb, nl in self.inactive_requests[i]: + inactive += nl + self.amount_inactive += inactive + self.amount_obtained += length - inactive + + self.blocked = new_blocked + + self.blocked_movein = Olist() + self.blocked_moveout = Olist() + for p, v in self.places.iteritems(): + if p != v: + if self.blocked[p] and not self.blocked[v]: + self.blocked_movein.add(p) + elif self.blocked[v] and not self.blocked[p]: + self.blocked_moveout.add(p) + + self.holes.extend(self.blocked_holes) # reset holes list + self.holes.sort() + self.blocked_holes = [] + + + """ + Pickled data format: + + d['pieces'] = either a string containing a bitfield of complete pieces, + or the numeric value "1" signifying a seed. If it is + a seed, d['places'] and d['partials'] should be empty + and needn't even exist. d['merkletree'] must exist + if it's a seed and a Merkle torrent. + d['partials'] = [ piece, [ offset, length... ]... ] + a list of partial data that had been previously + downloaded, plus the given offsets. Adjacent partials + are merged so as to save space, and so that if the + request size changes then new requests can be + calculated more efficiently. + d['places'] = [ piece, place, {,piece, place ...} ] + the piece index, and the place it's stored. + If d['pieces'] specifies a complete piece or d['partials'] + specifies a set of partials for a piece which has no + entry in d['places'], it can be assumed that + place[index] = index. A place specified with no + corresponding data in d['pieces'] or d['partials'] + indicates allocated space with no valid data, and is + reserved so it doesn't need to be hash-checked. + d['merkletree'] = pickle.dumps(self.merkletree) + if we're using a Merkle torrent the Merkle tree, otherwise + there is no 'merkletree' in the dictionary. + """ + def pickle(self): + if self.have.complete(): + if self.merkle_torrent: + return {'pieces': 1, 'merkletree': pickle.dumps(self.merkletree) } + else: + return {'pieces': 1 } + pieces = Bitfield(len(self.hashes)) + places = [] + partials = [] + for p in xrange(len(self.hashes)): + if self.blocked[p] or not self.places.has_key(p): + continue + h = self.have[p] + pieces[p] = h + pp = self.dirty.get(p) + if not h and not pp: # no data + places.extend([self.places[p], self.places[p]]) + elif self.places[p] != p: + places.extend([p, self.places[p]]) + if h or not pp: + continue + pp.sort() + r = [] + while len(pp) > 1: + if pp[0][0]+pp[0][1] == pp[1][0]: + pp[0] = list(pp[0]) + pp[0][1] += pp[1][1] + del pp[1] + else: + r.extend(pp[0]) + del pp[0] + r.extend(pp[0]) + partials.extend([p, r]) + if self.merkle_torrent: + return {'pieces': pieces.tostring(), 'places': places, 'partials': partials, 'merkletree': pickle.dumps(self.merkletree) } + else: + return {'pieces': pieces.tostring(), 'places': places, 'partials': partials } + + + def unpickle(self, data, valid_places): + got = {} + places = {} + dirty = {} + download_history = {} + stat_active = {} + stat_numfound = self.stat_numfound + amount_obtained = self.amount_obtained + amount_inactive = self.amount_inactive + amount_left = self.amount_left + inactive_requests = [x for x in self.inactive_requests] + restored_partials = [] + + try: + if data.has_key('merkletree'): + try: + if DEBUG: + print "StorageWrapper: Unpickling Merkle tree!" + self.merkletree = pickle.loads(data['merkletree']) + self.hashes = self.merkletree.get_piece_hashes() + self.hashes_unpickled = True + except Exception, e: + print "StorageWrapper: Exception while unpickling Merkle tree",str(e) + print_exc() + if data['pieces'] == 1: # a seed + assert not data.get('places', None) + assert not data.get('partials', None) + # Merkle: restore Merkle tree + have = Bitfield(len(self.hashes)) + for i in xrange(len(self.hashes)): + have[i] = True + assert have.complete() + _places = [] + _partials = [] + else: + have = Bitfield(len(self.hashes), data['pieces']) + _places = data['places'] + assert len(_places) % 2 == 0 + _places = [_places[x:x+2] for x in xrange(0, len(_places), 2)] + _partials = data['partials'] + assert len(_partials) % 2 == 0 + _partials = [_partials[x:x+2] for x in xrange(0, len(_partials), 2)] + + for index, place in _places: + if place not in valid_places: + continue + assert not got.has_key(index) + assert not got.has_key(place) + places[index] = place + got[index] = 1 + got[place] = 1 + + for index in xrange(len(self.hashes)): + if DEBUG: + print "StorageWrapper: Unpickle: Checking if we have piece",index + if have[index]: + if not places.has_key(index): + if index not in valid_places: + have[index] = False + continue + assert not got.has_key(index) + places[index] = index + got[index] = 1 + length = self._piecelen(index) + amount_obtained += length + stat_numfound += 1 + amount_inactive -= length + amount_left -= length + inactive_requests[index] = None + + for index, plist in _partials: + assert not dirty.has_key(index) + assert not have[index] + if not places.has_key(index): + if index not in valid_places: + continue + assert not got.has_key(index) + places[index] = index + got[index] = 1 + assert len(plist) % 2 == 0 + plist = [plist[x:x+2] for x in xrange(0, len(plist), 2)] + dirty[index] = plist + stat_active[index] = 1 + download_history[index] = {} + # invert given partials + length = self._piecelen(index) + l = [] + if plist[0][0] > 0: + l.append((0, plist[0][0])) + for i in xrange(len(plist)-1): + end = plist[i][0]+plist[i][1] + assert not end > plist[i+1][0] + l.append((end, plist[i+1][0]-end)) + end = plist[-1][0]+plist[-1][1] + assert not end > length + if end < length: + l.append((end, length-end)) + # split them to request_size + ll = [] + amount_obtained += length + amount_inactive -= length + for nb, nl in l: + while nl > 0: + r = min(nl, self.request_size) + ll.append((nb, r)) + amount_inactive += r + amount_obtained -= r + nb += self.request_size + nl -= self.request_size + inactive_requests[index] = ll + restored_partials.append(index) + + assert amount_obtained + amount_inactive == self.amount_desired + except: +# print_exc() + return [] # invalid data, discard everything + + self.have = have + self.places = places + self.dirty = dirty + self.download_history = download_history + self.stat_active = stat_active + self.stat_numfound = stat_numfound + self.amount_obtained = amount_obtained + self.amount_inactive = amount_inactive + self.amount_left = amount_left + self.inactive_requests = inactive_requests + + return restored_partials + + def failed(self,s): + # Arno: report failure of hash check + self.report_failure(s) + if self.initialize_done is not None: + self.initialize_done(success=False) + + def live_invalidate(self,piece): # Arno: LIVEWRAP + # Assumption: not outstanding requests + length = self._piecelen(piece) + oldhave = self.have[piece] + self.have[piece] = False + #self.waschecked[piece] = False + self.inactive_requests[piece] = 1 + if oldhave: + self.amount_left += length + self.amount_obtained -= length diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/T2T.py b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/T2T.py new file mode 100644 index 0000000..2b2f857 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/T2T.py @@ -0,0 +1,191 @@ +# Written by John Hoffman +# see LICENSE.txt for license information + +from Rerequester import Rerequester +from urllib import quote +from threading import Event +from random import randrange +import __init__ +try: + True +except: + True = 1 + False = 0 + +DEBUG = False + + +def excfunc(x): + print x + +class T2TConnection: + def __init__(self, myid, tracker, hash, interval, peers, timeout, + rawserver, disallow, isdisallowed): + self.tracker = tracker + self.interval = interval + self.hash = hash + self.operatinginterval = interval + self.peers = peers + self.rawserver = rawserver + self.disallow = disallow + self.isdisallowed = isdisallowed + self.active = True + self.busy = False + self.errors = 0 + self.rejected = 0 + self.trackererror = False + self.peerlists = [] + + self.rerequester = Rerequester([[tracker]], interval, + rawserver.add_task, lambda: 0, peers, self.addtolist, + rawserver.add_task, lambda: 1, 0, 0, 0, '', + myid, hash, timeout, self.errorfunc, excfunc, peers, Event(), + lambda: 0, lambda: 0) + + if self.isactive(): + rawserver.add_task(self.refresh, randrange(int(self.interval/10), self.interval)) + # stagger announces + + def isactive(self): + if self.isdisallowed(self.tracker): # whoops! + self.deactivate() + return self.active + + def deactivate(self): + self.active = False + + def refresh(self): + if not self.isactive(): + return + self.lastsuccessful = True + self.newpeerdata = [] + if DEBUG: + print 'contacting %s for info_hash=%s' % (self.tracker, quote(self.hash)) + self.rerequester.snoop(self.peers, self.callback) + + def callback(self): + self.busy = False + if self.lastsuccessful: + self.errors = 0 + self.rejected = 0 + if self.rerequester.announce_interval > (3*self.interval): + # I think I'm stripping from a regular tracker; boost the number of peers requested + self.peers = int(self.peers * (self.rerequester.announce_interval / self.interval)) + self.operatinginterval = self.rerequester.announce_interval + if DEBUG: + print ("%s with info_hash=%s returned %d peers" % + (self.tracker, quote(self.hash), len(self.newpeerdata))) + self.peerlists.append(self.newpeerdata) + self.peerlists = self.peerlists[-10:] # keep up to the last 10 announces + if self.isactive(): + self.rawserver.add_task(self.refresh, self.operatinginterval) + + def addtolist(self, peers): + for peer in peers: + self.newpeerdata.append((peer[1],peer[0][0],peer[0][1])) + + def errorfunc(self, r): + self.lastsuccessful = False + if DEBUG: + print "%s with info_hash=%s gives error: '%s'" % (self.tracker, quote(self.hash), r) + if r == self.rerequester.rejectedmessage + 'disallowed': # whoops! + if DEBUG: + print ' -- disallowed - deactivating' + self.deactivate() + self.disallow(self.tracker) # signal other torrents on this tracker + return + if r[:8].lower() == 'rejected': # tracker rejected this particular torrent + self.rejected += 1 + if self.rejected == 3: # rejected 3 times + if DEBUG: + print ' -- rejected 3 times - deactivating' + self.deactivate() + return + self.errors += 1 + if self.errors >= 3: # three or more errors in a row + self.operatinginterval += self.interval # lengthen the interval + if DEBUG: + print ' -- lengthening interval to '+str(self.operatinginterval)+' seconds' + + def harvest(self): + x = [] + for list in self.peerlists: + x += list + self.peerlists = [] + return x + + +class T2TList: + def __init__(self, enabled, trackerid, interval, maxpeers, timeout, rawserver): + self.enabled = enabled + self.trackerid = trackerid + self.interval = interval + self.maxpeers = maxpeers + self.timeout = timeout + self.rawserver = rawserver + self.list = {} + self.torrents = {} + self.disallowed = {} + self.oldtorrents = [] + + def parse(self, allowed_list): + if not self.enabled: + return + + # step 1: Create a new list with all tracker/torrent combinations in allowed_dir + newlist = {} + for hash, data in allowed_list.items(): + if data.has_key('announce-list'): + for tier in data['announce-list']: + for tracker in tier: + self.disallowed.setdefault(tracker, False) + newlist.setdefault(tracker, {}) + newlist[tracker][hash] = None # placeholder + + # step 2: Go through and copy old data to the new list. + # if the new list has no place for it, then it's old, so deactivate it + for tracker, hashdata in self.list.items(): + for hash, t2t in hashdata.items(): + if not newlist.has_key(tracker) or not newlist[tracker].has_key(hash): + t2t.deactivate() # this connection is no longer current + self.oldtorrents += [t2t] + # keep it referenced in case a thread comes along and tries to access. + else: + newlist[tracker][hash] = t2t + if not newlist.has_key(tracker): + self.disallowed[tracker] = False # reset when no torrents on it left + + self.list = newlist + newtorrents = {} + + # step 3: If there are any entries that haven't been initialized yet, do so. + # At the same time, copy all entries onto the by-torrent list. + for tracker, hashdata in newlist.items(): + for hash, t2t in hashdata.items(): + if t2t is None: + hashdata[hash] = T2TConnection(self.trackerid, tracker, hash, + self.interval, self.maxpeers, self.timeout, + self.rawserver, self._disallow, self._isdisallowed) + newtorrents.setdefault(hash,[]) + newtorrents[hash] += [hashdata[hash]] + + self.torrents = newtorrents + + # structures: + # list = {tracker: {hash: T2TConnection, ...}, ...} + # torrents = {hash: [T2TConnection, ...]} + # disallowed = {tracker: flag, ...} + # oldtorrents = [T2TConnection, ...] + + def _disallow(self,tracker): + self.disallowed[tracker] = True + + def _isdisallowed(self,tracker): + return self.disallowed[tracker] + + def harvest(self,hash): + harvest = [] + if self.enabled: + for t2t in self.torrents[hash]: + harvest += t2t.harvest() + return harvest diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/Uploader.py b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/Uploader.py new file mode 100644 index 0000000..cfadae7 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/Uploader.py @@ -0,0 +1,195 @@ +# Written by Bram Cohen +# see LICENSE.txt for license information + +from BaseLib.Core.BitTornado.CurrentRateMeasure import Measure +from BaseLib.Core.Statistics.Status.Status import get_status_holder + +import sys + +try: + True +except: + True = 1 + False = 0 + +class Upload: + def __init__(self, connection, ratelimiter, totalup, choker, storage, + picker, config): + self.connection = connection + self.ratelimiter = ratelimiter + self.totalup = totalup + self.choker = choker + self.storage = storage + self.picker = picker + self.config = config + self.max_slice_length = config['max_slice_length'] + self.choked = True + self.cleared = True + self.interested = False + self.super_seeding = False + self.buffer = [] + self.measure = Measure(config['max_rate_period'], config['upload_rate_fudge']) + self.was_ever_interested = False + if storage.get_amount_left() == 0: + if choker.super_seed: + self.super_seeding = True # flag, and don't send bitfield + self.seed_have_list = [] # set from piecepicker + self.skipped_count = 0 + else: + if config['breakup_seed_bitfield']: + bitfield, msgs = storage.get_have_list_cloaked() + connection.send_bitfield(bitfield) + for have in msgs: + connection.send_have(have) + else: + connection.send_bitfield(storage.get_have_list()) + else: + if storage.do_I_have_anything(): + connection.send_bitfield(storage.get_have_list()) + + self.piecedl = None + self.piecebuf = None + # Merkle + self.hashlist = [] + + def send_haves(self, connection): + """ + Send all pieces I have a series of HAVEs - this is done + by closed swarms after successfully connecting (will send blank + bitfields until remote node is authorized) + """ + have_list = self.storage.get_have_list() + + print >>sys.stderr, "Have list:",have_list + + def send_bitfield(self, connection): + """ + Send the bitfield (again) + """ + if self.storage.get_amount_left() == 0: + if not self.super_seeding: + if self.config['breakup_seed_bitfield']: + bitfield, msgs = self.storage.get_have_list_cloaked() + connection.send_bitfield(bitfield) + for have in msgs: + connection.send_have(have) + else: + connection.send_bitfield(self.storage.get_have_list()) + else: + if self.storage.do_I_have_anything(): + connection.send_bitfield(self.storage.get_have_list()) + + + def got_not_interested(self): + if self.interested: + self.interested = False + del self.buffer[:] + self.piecedl = None + if self.piecebuf: + self.piecebuf.release() + self.piecebuf = None + self.choker.not_interested(self.connection) + + def got_interested(self): + if not self.interested: + self.interested = True + self.was_ever_interested = True + self.choker.interested(self.connection) + + def get_upload_chunk(self): + if self.choked or not self.buffer: + return None + index, begin, length = self.buffer.pop(0) + if self.config['buffer_reads']: + if index != self.piecedl: + if self.piecebuf: + self.piecebuf.release() + self.piecedl = index + # Merkle + [ self.piecebuf, self.hashlist ] = self.storage.get_piece(index, 0, -1) + try: + piece = self.piecebuf[begin:begin+length] + assert len(piece) == length + except: # fails if storage.get_piece returns None or if out of range + self.connection.close() + return None + if begin == 0: + hashlist = self.hashlist + else: + hashlist = [] + else: + if self.piecebuf: + self.piecebuf.release() + self.piecedl = None + [piece, hashlist] = self.storage.get_piece(index, begin, length) + if piece is None: + self.connection.close() + return None + self.measure.update_rate(len(piece)) + self.totalup.update_rate(len(piece)) + + status = get_status_holder("LivingLab") + s_upload = status.get_or_create_status_element("uploaded",0) + s_upload.inc(len(piece)) + + # BarterCast counter + self.connection.total_uploaded += length + + return (index, begin, hashlist, piece) + + def got_request(self, index, begin, length): + if ((self.super_seeding and not index in self.seed_have_list) + or (not self.connection.connection.is_coordinator_con() and not self.interested) + or length > self.max_slice_length): + self.connection.close() + return + if not self.cleared: + self.buffer.append((index, begin, length)) + if not self.choked and self.connection.next_upload is None: + self.ratelimiter.queue(self.connection) + + def got_cancel(self, index, begin, length): + try: + self.buffer.remove((index, begin, length)) + except ValueError: + pass + + def choke(self): + if not self.choked: + self.choked = True + self.connection.send_choke() + self.piecedl = None + if self.piecebuf: + self.piecebuf.release() + self.piecebuf = None + + def choke_sent(self): + del self.buffer[:] + self.cleared = True + + def unchoke(self): + if self.choked: + try: + if self.connection.send_unchoke(): + self.choked = False + self.cleared = False + except: + pass + + def disconnected(self): + if self.piecebuf: + self.piecebuf.release() + self.piecebuf = None + + def is_choked(self): + return self.choked + + def is_interested(self): + return self.interested + + def has_queries(self): + return not self.choked and self.buffer + + def get_rate(self): + return self.measure.get_rate() + diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/__init__.py b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/__init__.py new file mode 100644 index 0000000..1902f5a --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/__init__.py @@ -0,0 +1,2 @@ +# Written by Bram Cohen +# see LICENSE.txt for license information diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/btformats.py b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/btformats.py new file mode 100644 index 0000000..7478791 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/btformats.py @@ -0,0 +1,130 @@ +# Written by Bram Cohen +# see LICENSE.txt for license information + +import sys +from types import UnicodeType, StringType, LongType, IntType, ListType, DictType +from re import compile + +#reg = compile(r'^[^/\\.~][^/\\]*$') +#reg = compile(r'^[^/\\]*$') + +ints = (LongType, IntType) + +def check_info(info): + if type(info) != DictType: + raise ValueError, 'bad metainfo - not a dictionary' + + if info.has_key('pieces'): + pieces = info.get('pieces') + if type(pieces) != StringType or len(pieces) % 20 != 0: + raise ValueError, 'bad metainfo - bad pieces key' + elif info.has_key('root hash'): + # Merkle + root_hash = info.get('root hash') + if type(root_hash) != StringType or len(root_hash) != 20: + raise ValueError, 'bad metainfo - bad root hash key' + piecelength = info.get('piece length') + if type(piecelength) not in ints or piecelength <= 0: + raise ValueError, 'bad metainfo - illegal piece length' + name = info.get('name') + if StringType != type(name) != UnicodeType: + raise ValueError, 'bad metainfo - bad name' + #if not reg.match(name): + # raise ValueError, 'name %s disallowed for security reasons' % name + if info.has_key('files') == info.has_key('length'): + raise ValueError, 'single/multiple file mix' + if info.has_key('length'): + length = info.get('length') + if type(length) not in ints or length < 0: + raise ValueError, 'bad metainfo - bad length' + else: + files = info.get('files') + if type(files) != ListType: + raise ValueError + for f in files: + if type(f) != DictType: + raise ValueError, 'bad metainfo - bad file value' + length = f.get('length') + if type(length) not in ints or length < 0: + raise ValueError, 'bad metainfo - bad length' + path = f.get('path') + if type(path) != ListType or path == []: + raise ValueError, 'bad metainfo - bad path' + for p in path: + if StringType != type(p) != UnicodeType: + raise ValueError, 'bad metainfo - bad path dir' + #if not reg.match(p): + # raise ValueError, 'path %s disallowed for security reasons' % p + for i in xrange(len(files)): + for j in xrange(i): + if files[i]['path'] == files[j]['path']: + raise ValueError, 'bad metainfo - duplicate path' + +def check_message(message): + if type(message) != DictType: + raise ValueError + check_info(message.get('info')) + if StringType != type(message.get('announce')) != UnicodeType: + raise ValueError + +def check_peers(message): + if type(message) != DictType: + raise ValueError + if message.has_key('failure reason'): + if type(message['failure reason']) != StringType: + raise ValueError + return + peers = message.get('peers') + if peers is not None: + if type(peers) == ListType: + for p in peers: + if type(p) != DictType: + raise ValueError + if type(p.get('ip')) != StringType: + raise ValueError + port = p.get('port') + if type(port) not in ints or p <= 0: + raise ValueError + if p.has_key('peer id'): + id = p['peer id'] + if type(id) != StringType or len(id) != 20: + raise ValueError + elif type(peers) != StringType or len(peers) % 6 != 0: + raise ValueError + + # IPv6 Tracker extension. http://www.bittorrent.org/beps/bep_0007.html + peers6 = message.get('peers6') + if peers6 is not None: + if type(peers6) == ListType: + for p in peers6: + if type(p) != DictType: + raise ValueError + if type(p.get('ip')) != StringType: + raise ValueError + port = p.get('port') + if type(port) not in ints or p <= 0: + raise ValueError + if p.has_key('peer id'): + id = p['peer id'] + if type(id) != StringType or len(id) != 20: + raise ValueError + elif type(peers6) != StringType or len(peers6) % 18 != 0: + raise ValueError + + interval = message.get('interval', 1) + if type(interval) not in ints or interval <= 0: + raise ValueError + minint = message.get('min interval', 1) + if type(minint) not in ints or minint <= 0: + raise ValueError + if type(message.get('tracker id', '')) != StringType: + raise ValueError + npeers = message.get('num peers', 0) + if type(npeers) not in ints or npeers < 0: + raise ValueError + dpeers = message.get('done peers', 0) + if type(dpeers) not in ints or dpeers < 0: + raise ValueError + last = message.get('last', 0) + if type(last) not in ints or last < 0: + raise ValueError diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/convert.py b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/convert.py new file mode 100644 index 0000000..0f9a832 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/convert.py @@ -0,0 +1,12 @@ +# Written by Bram Cohen and Arno Bakker +# see LICENSE.txt for license information + +from binascii import b2a_hex + +def toint(s): + return long(b2a_hex(s), 16) + +def tobinary(i): + return (chr(i >> 24) + chr((i >> 16) & 0xFF) + + chr((i >> 8) & 0xFF) + chr(i & 0xFF)) + diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/fakeopen.py b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/fakeopen.py new file mode 100644 index 0000000..659566a --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/fakeopen.py @@ -0,0 +1,87 @@ +# Written by Bram Cohen +# see LICENSE.txt for license information + +class FakeHandle: + def __init__(self, name, fakeopen): + self.name = name + self.fakeopen = fakeopen + self.pos = 0 + + def flush(self): + pass + + def close(self): + pass + + def seek(self, pos): + self.pos = pos + + def read(self, amount = None): + old = self.pos + f = self.fakeopen.files[self.name] + if self.pos >= len(f): + return '' + if amount is None: + self.pos = len(f) + return ''.join(f[old:]) + else: + self.pos = min(len(f), old + amount) + return ''.join(f[old:self.pos]) + + def write(self, s): + f = self.fakeopen.files[self.name] + while len(f) < self.pos: + f.append(chr(0)) + self.fakeopen.files[self.name][self.pos : self.pos + len(s)] = list(s) + self.pos += len(s) + +class FakeOpen: + def __init__(self, initial = {}): + self.files = {} + for key, value in initial.items(): + self.files[key] = list(value) + + def open(self, filename, mode): + """currently treats everything as rw - doesn't support append""" + self.files.setdefault(filename, []) + return FakeHandle(filename, self) + + def exists(self, file): + return self.files.has_key(file) + + def getsize(self, file): + return len(self.files[file]) + +def test_normal(): + f = FakeOpen({'f1': 'abcde'}) + assert f.exists('f1') + assert not f.exists('f2') + assert f.getsize('f1') == 5 + h = f.open('f1', 'rw') + assert h.read(3) == 'abc' + assert h.read(1) == 'd' + assert h.read() == 'e' + assert h.read(2) == '' + h.write('fpq') + h.seek(4) + assert h.read(2) == 'ef' + h.write('ghij') + h.seek(0) + assert h.read() == 'abcdefghij' + h.seek(2) + h.write('p') + h.write('q') + assert h.read(1) == 'e' + h.seek(1) + assert h.read(5) == 'bpqef' + + h2 = f.open('f2', 'rw') + assert h2.read() == '' + h2.write('mnop') + h2.seek(1) + assert h2.read() == 'nop' + + assert f.exists('f1') + assert f.exists('f2') + assert f.getsize('f1') == 10 + assert f.getsize('f2') == 4 diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/track.py b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/track.py new file mode 100644 index 0000000..2ffd823 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/BT1/track.py @@ -0,0 +1,1092 @@ +# Written by Bram Cohen, Arno Bakker +# see LICENSE.txt for license information +import sys, os +import signal +import re +import pickle +from threading import Event, Thread +from urllib import quote, unquote +from urlparse import urlparse +from os.path import exists +from cStringIO import StringIO +from traceback import print_exc +from time import time, gmtime, strftime, localtime +from random import shuffle, seed +from types import StringType, IntType, LongType, DictType +from binascii import b2a_hex + +from BaseLib.Core.simpledefs import * +from BaseLib.Core.BitTornado.parseargs import parseargs, formatDefinitions +from BaseLib.Core.BitTornado.RawServer import RawServer +from BaseLib.Core.BitTornado.HTTPHandler import HTTPHandler, months +from BaseLib.Core.BitTornado.parsedir import parsedir +from NatCheck import NatCheck +from T2T import T2TList +from Filter import Filter +from BaseLib.Core.BitTornado.subnetparse import IP_List, ipv6_to_ipv4, to_ipv4, is_valid_ip, is_ipv4 +from BaseLib.Core.BitTornado.iprangeparse import IP_List as IP_Range_List +from BaseLib.Core.BitTornado.torrentlistparse import parsetorrentlist +from BaseLib.Core.BitTornado.bencode import bencode, bdecode, Bencached +from BaseLib.Core.BitTornado.zurllib import urlopen +from BaseLib.Core.Utilities.Crypto import sha +from BaseLib.Core.BitTornado.clock import clock +from BaseLib.Core.BitTornado.__init__ import version_short, createPeerID +from BaseLib.Core.simpledefs import TRIBLER_TORRENT_EXT + +try: + True +except: + True = 1 + False = 0 + +DEBUG=False + +from BaseLib.Core.defaults import trackerdefaults + +defaults = [] +for k,v in trackerdefaults.iteritems(): + defaults.append((k,v,"See triblerAPI")) + + +def statefiletemplate(x): + if type(x) != DictType: + raise ValueError + for cname, cinfo in x.items(): + if cname == 'peers': + for y in cinfo.values(): # The 'peers' key is a dictionary of SHA hashes (torrent ids) + if type(y) != DictType: # ... for the active torrents, and each is a dictionary + raise ValueError + for id, info in y.items(): # ... of client ids interested in that torrent + if (len(id) != 20): + raise ValueError + if type(info) != DictType: # ... each of which is also a dictionary + raise ValueError # ... which has an IP, a Port, and a Bytes Left count for that client for that torrent + if type(info.get('ip', '')) != StringType: + raise ValueError + port = info.get('port') + if type(port) not in (IntType,LongType) or port < 0: + raise ValueError + left = info.get('left') + if type(left) not in (IntType,LongType) or left < 0: + raise ValueError + elif cname == 'completed': + if (type(cinfo) != DictType): # The 'completed' key is a dictionary of SHA hashes (torrent ids) + raise ValueError # ... for keeping track of the total completions per torrent + for y in cinfo.values(): # ... each torrent has an integer value + if type(y) not in (IntType,LongType): + raise ValueError # ... for the number of reported completions for that torrent + elif cname == 'allowed': + if (type(cinfo) != DictType): # a list of info_hashes and included data + raise ValueError + if x.has_key('allowed_dir_files'): + adlist = [z[1] for z in x['allowed_dir_files'].values()] + for y in cinfo.keys(): # and each should have a corresponding key here + if not y in adlist: + raise ValueError + elif cname == 'allowed_dir_files': + if (type(cinfo) != DictType): # a list of files, their attributes and info hashes + raise ValueError + dirkeys = {} + for y in cinfo.values(): # each entry should have a corresponding info_hash + if not y[1]: + continue + if not x['allowed'].has_key(y[1]): + raise ValueError + if dirkeys.has_key(y[1]): # and each should have a unique info_hash + raise ValueError + dirkeys[y[1]] = 1 + + +alas = 'your file may exist elsewhere in the universe\nbut alas, not here\n' + +local_IPs = IP_List() +local_IPs.set_intranet_addresses() + + +def isotime(secs = None): + if secs == None: + secs = time() + return strftime('%Y-%m-%d %H:%M UTC', gmtime(secs)) + +http_via_filter = re.compile(' for ([0-9.]+)\Z') + +def _get_forwarded_ip(headers): + if headers.has_key('http_x_forwarded_for'): + header = headers['http_x_forwarded_for'] + try: + x,y = header.split(',') + except: + return header + if not local_IPs.includes(x): + return x + return y + if headers.has_key('http_client_ip'): + return headers['http_client_ip'] + if headers.has_key('http_via'): + x = http_via_filter.search(headers['http_via']) + try: + return x.group(1) + except: + pass + if headers.has_key('http_from'): + return headers['http_from'] + return None + +def get_forwarded_ip(headers): + x = _get_forwarded_ip(headers) + if not is_valid_ip(x) or local_IPs.includes(x): + return None + return x + +def compact_peer_info(ip, port): + try: + s = ( ''.join([chr(int(i)) for i in ip.split('.')]) + + chr((port & 0xFF00) >> 8) + chr(port & 0xFF) ) + if len(s) != 6: + raise ValueError + except: + s = '' # not a valid IP, must be a domain name + return s + +def compact_ip(ip): + return ''.join([chr(int(i)) for i in ip.split('.')]) + +def decompact_ip(cip): + return '.'.join([str(ord(i)) for i in cip]) + + +class Tracker: + def __init__(self, config, rawserver): + self.config = config + self.response_size = config['tracker_response_size'] + self.dfile = config['tracker_dfile'] + self.natcheck = config['tracker_nat_check'] + favicon = config['tracker_favicon'] + self.parse_dir_interval = config['tracker_parse_dir_interval'] + self.favicon = None + if favicon: + try: + h = open(favicon,'rb') + self.favicon = h.read() + h.close() + except: + print "**warning** specified favicon file -- %s -- does not exist." % favicon + self.rawserver = rawserver + self.cached = {} # format: infohash: [[time1, l1, s1], [time2, l2, s2], [time3, l3, s3]] + self.cached_t = {} # format: infohash: [time, cache] + self.times = {} + self.state = {} + self.seedcount = {} + + self.allowed_IPs = None + self.banned_IPs = None + if config['tracker_allowed_ips'] or config['tracker_banned_ips']: + self.allowed_ip_mtime = 0 + self.banned_ip_mtime = 0 + self.read_ip_lists() + + self.only_local_override_ip = config['tracker_only_local_override_ip'] + if self.only_local_override_ip == 2: + self.only_local_override_ip = not config['tracker_nat_check'] + + if exists(self.dfile): + try: + h = open(self.dfile, 'rb') + if self.config['tracker_dfile_format'] == ITRACKDBFORMAT_BENCODE: + ds = h.read() + tempstate = bdecode(ds) + else: + tempstate = pickle.load(h) + h.close() + if not tempstate.has_key('peers'): + tempstate = {'peers': tempstate} + statefiletemplate(tempstate) + self.state = tempstate + except: + print '**warning** statefile '+self.dfile+' corrupt; resetting' + self.downloads = self.state.setdefault('peers', {}) + self.completed = self.state.setdefault('completed', {}) + + self.becache = {} # format: infohash: [[l1, s1], [l2, s2], [l3, s3]] + for infohash, ds in self.downloads.items(): + self.seedcount[infohash] = 0 + for x,y in ds.items(): + ip = y['ip'] + if ( (self.allowed_IPs and not self.allowed_IPs.includes(ip)) + or (self.banned_IPs and self.banned_IPs.includes(ip)) ): + del ds[x] + continue + if not y['left']: + self.seedcount[infohash] += 1 + if y.get('nat',-1): + continue + gip = y.get('given_ip') + if is_valid_ip(gip) and ( + not self.only_local_override_ip or local_IPs.includes(ip) ): + ip = gip + self.natcheckOK(infohash,x,ip,y['port'],y['left']) + + for x in self.downloads.keys(): + self.times[x] = {} + for y in self.downloads[x].keys(): + self.times[x][y] = 0 + + self.trackerid = createPeerID('-T-') + seed(self.trackerid) + + self.reannounce_interval = config['tracker_reannounce_interval'] + self.save_dfile_interval = config['tracker_save_dfile_interval'] + self.show_names = config['tracker_show_names'] + rawserver.add_task(self.save_state, self.save_dfile_interval) + self.prevtime = clock() + self.timeout_downloaders_interval = config['tracker_timeout_downloaders_interval'] + rawserver.add_task(self.expire_downloaders, self.timeout_downloaders_interval) + self.logfile = None + self.log = None + if (config['tracker_logfile']) and (config['tracker_logfile'] != '-'): + try: + self.logfile = config['tracker_logfile'] + self.log = open(self.logfile,'a') + sys.stdout = self.log + print "# Log Started: ", isotime() + except: + print "**warning** could not redirect stdout to log file: ", sys.exc_info()[0] + + if config['tracker_hupmonitor']: + def huphandler(signum, frame, self = self): + try: + self.log.close() + self.log = open(self.logfile,'a') + sys.stdout = self.log + print "# Log reopened: ", isotime() + except: + print "**warning** could not reopen logfile" + + signal.signal(signal.SIGHUP, huphandler) + + self.allow_get = config['tracker_allow_get'] + + self.t2tlist = T2TList(config['tracker_multitracker_enabled'], self.trackerid, + config['tracker_multitracker_reannounce_interval'], + config['tracker_multitracker_maxpeers'], config['tracker_multitracker_http_timeout'], + self.rawserver) + + if config['tracker_allowed_list']: + if config['tracker_allowed_dir']: + print '**warning** allowed_dir and allowed_list options cannot be used together' + print '**warning** disregarding allowed_dir' + config['tracker_allowed_dir'] = '' + self.allowed = self.state.setdefault('allowed_list',{}) + self.allowed_list_mtime = 0 + self.parse_allowed() + self.remove_from_state('allowed','allowed_dir_files') + if config['tracker_multitracker_allowed'] == ITRACKMULTI_ALLOW_AUTODETECT: + config['tracker_multitracker_allowed'] = ITRACKMULTI_ALLOW_NONE + config['tracker_allowed_controls'] = 0 + + elif config['tracker_allowed_dir']: + self.allowed = self.state.setdefault('allowed',{}) + self.allowed_dir_files = self.state.setdefault('allowed_dir_files',{}) + self.allowed_dir_blocked = {} + self.parse_allowed() + self.remove_from_state('allowed_list') + + else: + self.allowed = None + self.remove_from_state('allowed','allowed_dir_files', 'allowed_list') + if config['tracker_multitracker_allowed'] == ITRACKMULTI_ALLOW_AUTODETECT: + config['tracker_multitracker_allowed'] = ITRACKMULTI_ALLOW_NONE + config['tracker_allowed_controls'] = 0 + + self.uq_broken = unquote('+') != ' ' + self.keep_dead = config['tracker_keep_dead'] + self.Filter = Filter(rawserver.add_task) + + aggregator = config['tracker_aggregator'] + if aggregator == 0: + self.is_aggregator = False + self.aggregator_key = None + else: + self.is_aggregator = True + if aggregator == 1: + self.aggregator_key = None + else: + self.aggregator_key = aggregator + self.natcheck = False + + send = config['tracker_aggregate_forward'] + if not send: + self.aggregate_forward = None + else: + try: + self.aggregate_forward, self.aggregate_password = send + except: + self.aggregate_forward = send + self.aggregate_password = None + + self.cachetime = 0 + self.track_cachetimeupdate() + + def track_cachetimeupdate(self): + self.cachetime += 1 # raw clock, but more efficient for cache + self.rawserver.add_task(self.track_cachetimeupdate,1) + + def aggregate_senddata(self, query): + url = self.aggregate_forward+'?'+query + if self.aggregate_password is not None: + url += '&password='+self.aggregate_password + rq = Thread(target = self._aggregate_senddata, args = [url]) + rq.setName( "AggregateSendData"+rq.getName() ) + rq.setDaemon(True) + rq.start() + + def _aggregate_senddata(self, url): # just send, don't attempt to error check, + try: # discard any returned data + h = urlopen(url) + h.read() + h.close() + except: + return + + + def get_infopage(self): + try: + if not self.config['tracker_show_infopage']: + return (404, 'Not Found', {'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, alas) + red = self.config['tracker_infopage_redirect'] + if red: + return (302, 'Found', {'Content-Type': 'text/html', 'Location': red}, + 'Click Here') + + s = StringIO() + s.write('\n' \ + 'Tribler Tracker Statistics\n') + if self.favicon is not None: + s.write('\n') + s.write('\n\n' \ + '

Tribler Tracker Statistics

\n') + if self.config['tracker_allowed_dir']: + if self.show_names: + names = [ (self.allowed[hash]['name'],hash) + for hash in self.allowed.keys() ] + else: + names = [ (None,hash) + for hash in self.allowed.keys() ] + else: + names = [ (None,hash) for hash in self.downloads.keys() ] + if not names: + s.write('

Not tracking any files yet...

\n') + else: + names.sort() + tn = 0 + tc = 0 + td = 0 + tt = 0 # Total transferred + ts = 0 # Total size + nf = 0 # Number of files displayed + if self.config['tracker_allowed_dir'] and self.show_names: + s.write('\n' \ + '\n') + else: + s.write('
info hashtorrent namesizecompletedownloadingdownloadedtransferred
\n' \ + '\n') + for name,hash in names: + l = self.downloads[hash] + n = self.completed.get(hash, 0) + tn = tn + n + c = self.seedcount[hash] + tc = tc + c + d = len(l) - c + td = td + d + if self.config['tracker_allowed_dir'] and self.show_names: + if self.allowed.has_key(hash): + nf = nf + 1 + sz = self.allowed[hash]['length'] # size + ts = ts + sz + szt = sz * n # Transferred for this torrent + tt = tt + szt + if self.allow_get == 1: + # P2PURL + url = self.allowed[hash].get('url') + if url: + linkname = '' + name + '' + else: + #linkname = '' + name + '' + linkname = '' + name + '' + else: + linkname = name + s.write('\n' \ + % (b2a_hex(hash), linkname, size_format(sz), c, d, n, size_format(szt))) + else: + s.write('\n' \ + % (b2a_hex(hash), c, d, n)) + ttn = 0 + for i in self.completed.values(): + ttn = ttn + i + if self.config['tracker_allowed_dir'] and self.show_names: + s.write('\n' + % (nf, size_format(ts), tc, td, tn, ttn, size_format(tt))) + else: + s.write('\n' + % (nf, tc, td, tn, ttn)) + s.write('
info hashcompletedownloadingdownloaded
%s%s%s%i%i%i%s
%s%i%i%i
%i files%s%i%i%i/%i%s
%i files%i%i%i/%i
\n' \ + '
    \n' \ + '
  • info hash: SHA1 hash of the "info" section of the metainfo (*.torrent)
    • \n' \ + '
  • complete: number of connected clients with the complete file
    • \n' \ + '
  • downloading: number of connected clients still downloading
    • \n' \ + '
  • downloaded: reported complete downloads (total: current/all)
    • \n' \ + '
  • transferred: torrent size * total downloaded (does not include partial transfers)
    • \n' \ + '
\n' \ + '
\n' + '
%s (%s)
\n' % (version_short, isotime())) + + + s.write('\n' \ + '\n') + return (200, 'OK', {'Content-Type': 'text/html; charset=iso-8859-1'}, s.getvalue()) + except: + print_exc() + return (500, 'Internal Server Error', {'Content-Type': 'text/html; charset=iso-8859-1'}, 'Server Error') + + + def scrapedata(self, hash, return_name = True): + l = self.downloads[hash] + n = self.completed.get(hash, 0) + c = self.seedcount[hash] + d = len(l) - c + f = {'complete': c, 'incomplete': d, 'downloaded': n} + if return_name and self.show_names and self.config['tracker_allowed_dir']: + f['name'] = self.allowed[hash]['name'] + return (f) + + def get_scrape(self, paramslist): + fs = {} + if paramslist.has_key('info_hash'): + if self.config['tracker_scrape_allowed'] not in [ITRACKSCRAPE_ALLOW_SPECIFIC,ITRACKSCRAPE_ALLOW_FULL]: + return (400, 'Not Authorized', {'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, + bencode({'failure reason': + 'specific scrape function is not available with this tracker.'})) + for hash in paramslist['info_hash']: + if self.allowed is not None: + if self.allowed.has_key(hash): + fs[hash] = self.scrapedata(hash) + else: + if self.downloads.has_key(hash): + fs[hash] = self.scrapedata(hash) + else: + if self.config['tracker_scrape_allowed'] != ITRACKSCRAPE_ALLOW_FULL: + return (400, 'Not Authorized', {'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, + bencode({'failure reason': + 'full scrape function is not available with this tracker.'})) + if self.allowed is not None: + keys = self.allowed.keys() + else: + keys = self.downloads.keys() + for hash in keys: + fs[hash] = self.scrapedata(hash) + + return (200, 'OK', {'Content-Type': 'text/plain'}, bencode({'files': fs})) + + + def get_file_by_name(self,name): + # Assumption: name is in UTF-8, as is the names in self.allowed + for hash,rec in self.allowed.iteritems(): + if 'name' in rec and rec['name'] == name: + return self.get_file(hash) + return (404, 'Not Found', {'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, alas) + + def get_file(self, hash): + if not self.allow_get: + return (400, 'Not Authorized', {'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, + 'get function is not available with this tracker.') + if not self.allowed.has_key(hash): + return (404, 'Not Found', {'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, alas) + fname = self.allowed[hash]['file'] + fpath = self.allowed[hash]['path'] + return (200, 'OK', {'Content-Type': 'application/x-bittorrent', + 'Content-Disposition': 'attachment; filename=' + fname}, + open(fpath, 'rb').read()) + + + def get_tstream_from_httpseed(self, httpseedurl): + if not self.allow_get: + return (400, 'Not Authorized', {'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, + 'get function is not available with this tracker.') + + # TODO: normalize? + wanturlhash = sha(httpseedurl).digest() + # TODO: reverse table? + found = False + for infohash,a in self.allowed.iteritems(): + for goturlhash in a['url-hash-list']: + if goturlhash == wanturlhash: + found = True + break + if found: + break + + if not found or not self.allowed.has_key(infohash): + return (404, 'Not Found', {'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, alas) + + fname = self.allowed[infohash]['file'] + fpath = self.allowed[infohash]['path'] + print >>sys.stderr,"tracker: get_stream: Sending",fname + return (200, 'OK', {'Content-Type': 'application/x-bittorrent', + 'Content-Disposition': 'attachment; filename=' + fname}, + open(fpath, 'rb').read()) + + + def check_allowed(self, infohash, paramslist): + if ( self.aggregator_key is not None + and not ( paramslist.has_key('password') + and paramslist['password'][0] == self.aggregator_key ) ): + return (200, 'Not Authorized', {'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, + bencode({'failure reason': + 'Requested download is not authorized for use with this tracker.'})) + + if self.allowed is not None: + if not self.allowed.has_key(infohash): + return (200, 'Not Authorized', {'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, + bencode({'failure reason': + 'Requested download is not authorized for use with this tracker.'})) + if self.config['tracker_allowed_controls']: + if self.allowed[infohash].has_key('failure reason'): + return (200, 'Not Authorized', {'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, + bencode({'failure reason': self.allowed[infohash]['failure reason']})) + + if paramslist.has_key('tracker'): + if ( self.config['tracker_multitracker_allowed'] == ITRACKMULTI_ALLOW_NONE or # turned off + paramslist['peer_id'][0] == self.trackerid ): # oops! contacted myself + return (200, 'Not Authorized', {'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, + bencode({'failure reason': 'disallowed'})) + + if ( self.config['tracker_multitracker_allowed'] == ITRACKMULTI_ALLOW_AUTODETECT + and not self.allowed[infohash].has_key('announce-list') ): + return (200, 'Not Authorized', {'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, + bencode({'failure reason': + 'Requested download is not authorized for multitracker use.'})) + + return None + + + def add_data(self, infohash, event, ip, paramslist): + peers = self.downloads.setdefault(infohash, {}) + ts = self.times.setdefault(infohash, {}) + self.completed.setdefault(infohash, 0) + self.seedcount.setdefault(infohash, 0) + + def params(key, default = None, l = paramslist): + if l.has_key(key): + return l[key][0] + return default + + myid = params('peer_id','') + if len(myid) != 20: + raise ValueError, 'id not of length 20' + if event not in ['started', 'completed', 'stopped', 'snooped', None]: + raise ValueError, 'invalid event' + port = long(params('port','')) + if port < 0 or port > 65535: + raise ValueError, 'invalid port' + left = long(params('left','')) + if left < 0: + raise ValueError, 'invalid amount left' + uploaded = long(params('uploaded','')) + downloaded = long(params('downloaded','')) + + peer = peers.get(myid) + islocal = local_IPs.includes(ip) + mykey = params('key') + if peer: + auth = peer.get('key',-1) == mykey or peer.get('ip') == ip + + gip = params('ip') + if is_valid_ip(gip) and (islocal or not self.only_local_override_ip): + ip1 = gip + else: + ip1 = ip + + if params('numwant') is not None: + rsize = min(int(params('numwant')),self.response_size) + else: + rsize = self.response_size + + if event == 'stopped': + if peer: + if auth: + self.delete_peer(infohash,myid) + + elif not peer: + ts[myid] = clock() + peer = {'ip': ip, 'port': port, 'left': left} + if mykey: + peer['key'] = mykey + if gip: + peer['given ip'] = gip + if port: + if not self.natcheck or islocal: + peer['nat'] = 0 + self.natcheckOK(infohash,myid,ip1,port,left) + else: + NatCheck(self.connectback_result,infohash,myid,ip1,port,self.rawserver) + else: + peer['nat'] = 2**30 + if event == 'completed': + self.completed[infohash] += 1 + if not left: + self.seedcount[infohash] += 1 + + peers[myid] = peer + + else: + if not auth: + return rsize # return w/o changing stats + + ts[myid] = clock() + if not left and peer['left']: + self.completed[infohash] += 1 + self.seedcount[infohash] += 1 + if not peer.get('nat', -1): + for bc in self.becache[infohash]: + bc[1][myid] = bc[0][myid] + del bc[0][myid] + if peer['left']: + peer['left'] = left + + if port: + recheck = False + if ip != peer['ip']: + peer['ip'] = ip + recheck = True + if gip != peer.get('given ip'): + if gip: + peer['given ip'] = gip + elif peer.has_key('given ip'): + del peer['given ip'] + recheck = True + + natted = peer.get('nat', -1) + if recheck: + if natted == 0: + l = self.becache[infohash] + y = not peer['left'] + for x in l: + del x[y][myid] + if not self.natcheck or islocal: + del peer['nat'] # restart NAT testing + if natted and natted < self.natcheck: + recheck = True + + if recheck: + if not self.natcheck or islocal: + peer['nat'] = 0 + self.natcheckOK(infohash,myid,ip1,port,left) + else: + NatCheck(self.connectback_result,infohash,myid,ip1,port,self.rawserver) + + return rsize + + + def peerlist(self, infohash, stopped, tracker, is_seed, return_type, rsize): + data = {} # return data + seeds = self.seedcount[infohash] + data['complete'] = seeds + data['incomplete'] = len(self.downloads[infohash]) - seeds + + if ( self.config['tracker_allowed_controls'] + and self.allowed[infohash].has_key('warning message') ): + data['warning message'] = self.allowed[infohash]['warning message'] + + if tracker: + data['interval'] = self.config['tracker_multitracker_reannounce_interval'] + if not rsize: + return data + cache = self.cached_t.setdefault(infohash, None) + if ( not cache or len(cache[1]) < rsize + or cache[0] + self.config['tracker_min_time_between_cache_refreshes'] < clock() ): + bc = self.becache.setdefault(infohash,[[{}, {}], [{}, {}], [{}, {}]]) + cache = [ clock(), bc[0][0].values() + bc[0][1].values() ] + self.cached_t[infohash] = cache + shuffle(cache[1]) + cache = cache[1] + + data['peers'] = cache[-rsize:] + del cache[-rsize:] + return data + + data['interval'] = self.reannounce_interval + if stopped or not rsize: # save some bandwidth + data['peers'] = [] + return data + + bc = self.becache.setdefault(infohash,[[{}, {}], [{}, {}], [{}, {}]]) + len_l = len(bc[0][0]) + len_s = len(bc[0][1]) + if not (len_l+len_s): # caches are empty! + data['peers'] = [] + return data + l_get_size = int(float(rsize)*(len_l)/(len_l+len_s)) + cache = self.cached.setdefault(infohash,[None,None,None])[return_type] + if cache and ( not cache[1] + or (is_seed and len(cache[1]) < rsize) + or len(cache[1]) < l_get_size + or cache[0]+self.config['tracker_min_time_between_cache_refreshes'] < self.cachetime ): + cache = None + if not cache: + peers = self.downloads[infohash] + vv = [[],[],[]] + for key, ip, port in self.t2tlist.harvest(infohash): # empty if disabled + if not peers.has_key(key): + vv[0].append({'ip': ip, 'port': port, 'peer id': key}) + vv[1].append({'ip': ip, 'port': port}) + vv[2].append(compact_peer_info(ip, port)) + cache = [ self.cachetime, + bc[return_type][0].values()+vv[return_type], + bc[return_type][1].values() ] + shuffle(cache[1]) + shuffle(cache[2]) + self.cached[infohash][return_type] = cache + for rr in xrange(len(self.cached[infohash])): + if rr != return_type: + try: + self.cached[infohash][rr][1].extend(vv[rr]) + except: + pass + if len(cache[1]) < l_get_size: + peerdata = cache[1] + if not is_seed: + peerdata.extend(cache[2]) + cache[1] = [] + cache[2] = [] + else: + if not is_seed: + peerdata = cache[2][l_get_size-rsize:] + del cache[2][l_get_size-rsize:] + rsize -= len(peerdata) + else: + peerdata = [] + if rsize: + peerdata.extend(cache[1][-rsize:]) + del cache[1][-rsize:] + if return_type == 2: + peerdata = ''.join(peerdata) + data['peers'] = peerdata + return data + + + def get(self, connection, path, headers): + real_ip = connection.get_ip() + ip = real_ip + if is_ipv4(ip): + ipv4 = True + else: + try: + ip = ipv6_to_ipv4(ip) + ipv4 = True + except ValueError: + ipv4 = False + + # Arno: log received GET + if self.config['tracker_logfile']: + self.getlog(ip, path, headers) + + if ( (self.allowed_IPs and not self.allowed_IPs.includes(ip)) + or (self.banned_IPs and self.banned_IPs.includes(ip)) ): + return (400, 'Not Authorized', {'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, + bencode({'failure reason': + 'your IP is not allowed on this tracker'})) + + nip = get_forwarded_ip(headers) + if nip and not self.only_local_override_ip: + ip = nip + try: + ip = to_ipv4(ip) + ipv4 = True + except ValueError: + ipv4 = False + + paramslist = {} + def params(key, default = None, l = paramslist): + if l.has_key(key): + return l[key][0] + return default + + try: + (scheme, netloc, path, pars, query, fragment) = urlparse(path) + if self.uq_broken == 1: + path = path.replace('+',' ') + query = query.replace('+',' ') + path = unquote(path)[1:] + for s in query.split('&'): + if s: + i = s.find('=') + if i == -1: + break + kw = unquote(s[:i]) + paramslist.setdefault(kw, []) + paramslist[kw] += [unquote(s[i+1:])] + + if DEBUG: + print >>sys.stderr,"tracker: Got request /"+path+'?'+query + + if path == '' or path == 'index.html': + return self.get_infopage() + if (path == 'file'): + # Arno: 2010-02-26: name based retrieval + if paramslist.has_key('name'): + return self.get_file_by_name(params('name')) + else: + return self.get_file(params('info_hash')) + + if path == 'tlookup': + return self.get_tstream_from_httpseed(unquote(query)) + if path == 'favicon.ico' and self.favicon is not None: + return (200, 'OK', {'Content-Type' : 'image/x-icon'}, self.favicon) + + # automated access from here on + + if path == 'scrape': + return self.get_scrape(paramslist) + + if path != 'announce': + return (404, 'Not Found', {'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, alas) + + # main tracker function + + filtered = self.Filter.check(real_ip, paramslist, headers) + if filtered: + return (400, 'Not Authorized', {'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, + bencode({'failure reason': filtered})) + + infohash = params('info_hash') + if not infohash: + raise ValueError, 'no info hash' + + notallowed = self.check_allowed(infohash, paramslist) + if notallowed: + return notallowed + + event = params('event') + + rsize = self.add_data(infohash, event, ip, paramslist) + + except ValueError, e: + print_exc() + return (400, 'Bad Request', {'Content-Type': 'text/plain'}, + 'you sent me garbage - ' + str(e)) + + if self.aggregate_forward and not paramslist.has_key('tracker'): + self.aggregate_senddata(query) + + if self.is_aggregator: # don't return peer data here + return (200, 'OK', {'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, + bencode({'response': 'OK'})) + + if params('compact') and ipv4: + return_type = 2 + elif params('no_peer_id'): + return_type = 1 + else: + return_type = 0 + + data = self.peerlist(infohash, event=='stopped', + params('tracker'), not params('left'), + return_type, rsize) + + if paramslist.has_key('scrape'): + data['scrape'] = self.scrapedata(infohash, False) + + return (200, 'OK', {'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, bencode(data)) + + + def natcheckOK(self, infohash, peerid, ip, port, not_seed): + if DEBUG: + print >>sys.stderr,"tracker: natcheck: Recorded succes" + bc = self.becache.setdefault(infohash,[[{}, {}], [{}, {}], [{}, {}]]) + bc[0][not not_seed][peerid] = Bencached(bencode({'ip': ip, 'port': port, + 'peer id': peerid})) + bc[1][not not_seed][peerid] = Bencached(bencode({'ip': ip, 'port': port})) + bc[2][not not_seed][peerid] = compact_peer_info(ip, port) + + + def natchecklog(self, peerid, ip, port, result): + year, month, day, hour, minute, second, a, b, c = localtime(time()) + print '%s - %s [%02d/%3s/%04d:%02d:%02d:%02d] "!natcheck-%s:%i" %i 0 - -' % ( + ip, quote(peerid), day, months[month], year, hour, minute, second, + ip, port, result) + + def getlog(self, ip, path, headers): + year, month, day, hour, minute, second, a, b, c = localtime(time()) + print '%s - %s [%02d/%3s/%04d:%02d:%02d:%02d] "GET %s HTTP/1.1" 100 0 - -' % ( + ip, ip, day, months[month], year, hour, minute, second, path) + + def connectback_result(self, result, downloadid, peerid, ip, port): + record = self.downloads.get(downloadid, {}).get(peerid) + if ( record is None + or (record['ip'] != ip and record.get('given ip') != ip) + or record['port'] != port ): + if self.config['tracker_log_nat_checks']: + self.natchecklog(peerid, ip, port, 404) + if DEBUG: + print >>sys.stderr,"tracker: natcheck: No record found for tested peer" + return + if self.config['tracker_log_nat_checks']: + if result: + x = 200 + else: + x = 503 + self.natchecklog(peerid, ip, port, x) + if not record.has_key('nat'): + record['nat'] = int(not result) + if result: + self.natcheckOK(downloadid,peerid,ip,port,record['left']) + elif result and record['nat']: + record['nat'] = 0 + self.natcheckOK(downloadid,peerid,ip,port,record['left']) + elif not result: + record['nat'] += 1 + if DEBUG: + print >>sys.stderr,"tracker: natcheck: Recorded failed attempt" + + + def remove_from_state(self, *l): + for s in l: + try: + del self.state[s] + except: + pass + + def save_state(self): + self.rawserver.add_task(self.save_state, self.save_dfile_interval) + h = open(self.dfile, 'wb') + if self.config['tracker_dfile_format'] == ITRACKDBFORMAT_BENCODE: + h.write(bencode(self.state)) + else: + pickle.dump(self.state,h,-1) + h.close() + + + def parse_allowed(self,source=None): + if DEBUG: + print >>sys.stderr,"tracker: parse_allowed: Source is",source,"alloweddir",self.config['tracker_allowed_dir'] + + if source is None: + self.rawserver.add_task(self.parse_allowed, self.parse_dir_interval) + + if self.config['tracker_allowed_dir']: + r = parsedir( self.config['tracker_allowed_dir'], self.allowed, + self.allowed_dir_files, self.allowed_dir_blocked, + [".torrent",TRIBLER_TORRENT_EXT] ) + ( self.allowed, self.allowed_dir_files, self.allowed_dir_blocked, + added, garbage2 ) = r + + if DEBUG: + print >>sys.stderr,"tracker: parse_allowed: Found new",`added` + + self.state['allowed'] = self.allowed + self.state['allowed_dir_files'] = self.allowed_dir_files + + self.t2tlist.parse(self.allowed) + + else: + f = self.config['tracker_allowed_list'] + if self.allowed_list_mtime == os.path.getmtime(f): + return + try: + r = parsetorrentlist(f, self.allowed) + (self.allowed, added, garbage2) = r + self.state['allowed_list'] = self.allowed + except (IOError, OSError): + print '**warning** unable to read allowed torrent list' + return + self.allowed_list_mtime = os.path.getmtime(f) + + for infohash in added.keys(): + self.downloads.setdefault(infohash, {}) + self.completed.setdefault(infohash, 0) + self.seedcount.setdefault(infohash, 0) + + + def read_ip_lists(self): + self.rawserver.add_task(self.read_ip_lists,self.parse_dir_interval) + + f = self.config['tracker_allowed_ips'] + if f and self.allowed_ip_mtime != os.path.getmtime(f): + self.allowed_IPs = IP_List() + try: + self.allowed_IPs.read_fieldlist(f) + self.allowed_ip_mtime = os.path.getmtime(f) + except (IOError, OSError): + print '**warning** unable to read allowed_IP list' + + f = self.config['tracker_banned_ips'] + if f and self.banned_ip_mtime != os.path.getmtime(f): + self.banned_IPs = IP_Range_List() + try: + self.banned_IPs.read_rangelist(f) + self.banned_ip_mtime = os.path.getmtime(f) + except (IOError, OSError): + print '**warning** unable to read banned_IP list' + + + def delete_peer(self, infohash, peerid): + dls = self.downloads[infohash] + peer = dls[peerid] + if not peer['left']: + self.seedcount[infohash] -= 1 + if not peer.get('nat',-1): + l = self.becache[infohash] + y = not peer['left'] + for x in l: + del x[y][peerid] + del self.times[infohash][peerid] + del dls[peerid] + + def expire_downloaders(self): + for x in self.times.keys(): + for myid, t in self.times[x].items(): + if t < self.prevtime: + self.delete_peer(x,myid) + self.prevtime = clock() + if (self.keep_dead != 1): + for key, value in self.downloads.items(): + if len(value) == 0 and ( + self.allowed is None or not self.allowed.has_key(key) ): + del self.times[key] + del self.downloads[key] + del self.seedcount[key] + self.rawserver.add_task(self.expire_downloaders, self.timeout_downloaders_interval) + + +def track(args): + if not args: + print formatDefinitions(defaults, 80) + return + try: + config, files = parseargs(args, defaults, 0, 0) + except ValueError, e: + print 'error: ' + str(e) + print 'run with no arguments for parameter explanations' + return + r = RawServer(Event(), config['tracker_timeout_check_interval'], + config['tracker_socket_timeout'], ipv6_enable = config['ipv6_enabled']) + t = Tracker(config, r) + r.bind(config['minport'], config['bind'], + reuse = True, ipv6_socket_style = config['ipv6_binds_v4']) + r.listen_forever(HTTPHandler(t.get, config['min_time_between_log_flushes'])) + t.save_state() + print '# Shutting down: ' + isotime() + +def size_format(s): + if (s < 1024): + r = str(s) + 'B' + elif (s < 1048576): + r = str(int(s/1024)) + 'KiB' + elif (s < 1073741824L): + r = str(int(s/1048576)) + 'MiB' + elif (s < 1099511627776L): + r = str(int((s/1073741824.0)*100.0)/100.0) + 'GiB' + else: + r = str(int((s/1099511627776.0)*100.0)/100.0) + 'TiB' + return(r) + diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/CurrentRateMeasure.py b/instrumentation/next-share/BaseLib/Core/BitTornado/CurrentRateMeasure.py new file mode 100644 index 0000000..43f8ce2 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/CurrentRateMeasure.py @@ -0,0 +1,38 @@ +# Written by Bram Cohen +# see LICENSE.txt for license information + +from clock import clock + +class Measure: + def __init__(self, max_rate_period, fudge = 1): + self.max_rate_period = max_rate_period + self.ratesince = clock() - fudge + self.last = self.ratesince + self.rate = 0.0 + self.total = 0L + + def update_rate(self, amount): + self.total += amount + t = clock() + self.rate = (self.rate * (self.last - self.ratesince) + + amount) / (t - self.ratesince + 0.0001) + self.last = t + if self.ratesince < t - self.max_rate_period: + self.ratesince = t - self.max_rate_period + + def get_rate(self): + self.update_rate(0) + #print 'Rate: %f (%d bytes)' % (self.rate, self.total) + return self.rate + + def get_rate_noupdate(self): + return self.rate + + def time_until_rate(self, newrate): + if self.rate <= newrate: + return 0 + t = clock() - self.ratesince + return ((self.rate * t) / newrate) - t + + def get_total(self): + return self.total diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/HTTPHandler.py b/instrumentation/next-share/BaseLib/Core/BitTornado/HTTPHandler.py new file mode 100644 index 0000000..06c3211 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/HTTPHandler.py @@ -0,0 +1,194 @@ +# Written by Bram Cohen +# see LICENSE.txt for license information + +from cStringIO import StringIO +import sys +import time +from clock import clock +from gzip import GzipFile +try: + True +except: + True = 1 + False = 0 + +DEBUG = False + +weekdays = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'] + +months = [None, 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', + 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] + +class HTTPConnection: + def __init__(self, handler, connection): + self.handler = handler + self.connection = connection + self.buf = '' + self.closed = False + self.done = False + self.donereading = False + self.next_func = self.read_type + + def get_ip(self): + return self.connection.get_ip() + + def data_came_in(self, data): + if self.donereading or self.next_func is None: + return True + self.buf += data + while 1: + try: + i = self.buf.index('\n') + except ValueError: + return True + val = self.buf[:i] + self.buf = self.buf[i+1:] + self.next_func = self.next_func(val) + if self.donereading: + return True + if self.next_func is None or self.closed: + return False + + def read_type(self, data): + self.header = data.strip() + words = data.split() + if len(words) == 3: + self.command, self.path, garbage = words + self.pre1 = False + elif len(words) == 2: + self.command, self.path = words + self.pre1 = True + if self.command != 'GET': + return None + else: + return None + if self.command not in ('HEAD', 'GET'): + return None + self.headers = {} + return self.read_header + + def read_header(self, data): + data = data.strip() + if data == '': + self.donereading = True + if self.headers.get('accept-encoding', '').find('gzip') > -1: + self.encoding = 'gzip' + else: + self.encoding = 'identity' + r = self.handler.getfunc(self, self.path, self.headers) + if r is not None: + self.answer(r) + return None + try: + i = data.index(':') + except ValueError: + return None + self.headers[data[:i].strip().lower()] = data[i+1:].strip() + if DEBUG: + print data[:i].strip() + ": " + data[i+1:].strip() + return self.read_header + + def answer(self, (responsecode, responsestring, headers, data)): + if self.closed: + return + if self.encoding == 'gzip': + compressed = StringIO() + gz = GzipFile(fileobj = compressed, mode = 'wb', compresslevel = 9) + gz.write(data) + gz.close() + cdata = compressed.getvalue() + if len(cdata) >= len(data): + self.encoding = 'identity' + else: + if DEBUG: + print "Compressed: %i Uncompressed: %i\n" % (len(cdata), len(data)) + data = cdata + headers['Content-Encoding'] = 'gzip' + + # i'm abusing the identd field here, but this should be ok + if self.encoding == 'identity': + ident = '-' + else: + ident = self.encoding + self.handler.log( self.connection.get_ip(), ident, '-', + self.header, responsecode, len(data), + self.headers.get('referer','-'), + self.headers.get('user-agent','-') ) + self.done = True + r = StringIO() + r.write('HTTP/1.0 ' + str(responsecode) + ' ' + + responsestring + '\r\n') + if not self.pre1: + headers['Content-Length'] = len(data) + for key, value in headers.items(): + r.write(key + ': ' + str(value) + '\r\n') + r.write('\r\n') + if self.command != 'HEAD': + r.write(data) + self.connection.write(r.getvalue()) + if self.connection.is_flushed(): + self.connection.shutdown(1) + +class HTTPHandler: + def __init__(self, getfunc, minflush): + self.connections = {} + self.getfunc = getfunc + self.minflush = minflush + self.lastflush = clock() + + def external_connection_made(self, connection): + self.connections[connection] = HTTPConnection(self, connection) + + def connection_flushed(self, connection): + if self.connections[connection].done: + connection.shutdown(1) + + def connection_lost(self, connection): + ec = self.connections[connection] + ec.closed = True + del ec.connection + del ec.next_func + del self.connections[connection] + + def data_came_in(self, connection, data): + c = self.connections[connection] + if not c.data_came_in(data) and not c.closed: + c.connection.shutdown(1) + + def log(self, ip, ident, username, header, + responsecode, length, referrer, useragent): + year, month, day, hour, minute, second, a, b, c = time.localtime(time.time()) + if DEBUG: + print >>sys.stderr,'HTTPHandler: %s %s %s [%02d/%3s/%04d:%02d:%02d:%02d] "%s" %i %i "%s" "%s"' % ( + ip, ident, username, day, months[month], year, hour, + minute, second, header, responsecode, length, referrer, useragent) + t = clock() + if t - self.lastflush > self.minflush: + self.lastflush = t + sys.stdout.flush() + + +class DummyHTTPHandler: + def __init__(self): + pass + + def external_connection_made(self, connection): + print >> sys.stderr,"DummyHTTPHandler: ext_conn_made" + reply = 'HTTP/1.1 404 Not Found\r\nContent-Type: text/plain\r\n\r\nTribler Internal Tracker not activated.\r\n' + connection.write(reply) + connection.close() + + def connection_flushed(self, connection): + pass + + def connection_lost(self, connection): + pass + + def data_came_in(self, connection, data): + print >> sys.stderr,"DummyHTTPHandler: data_came_in",len(data) + pass + + def log(self, ip, ident, username, header, + responsecode, length, referrer, useragent): + year, month, day, hour, minute, second, a, b, c = time.localtime(time.time()) + pass diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/PSYCO.py b/instrumentation/next-share/BaseLib/Core/BitTornado/PSYCO.py new file mode 100644 index 0000000..58fd571 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/PSYCO.py @@ -0,0 +1,8 @@ +# Written by BitTornado authors +# see LICENSE.txt for license information + +# edit this file to enable/disable Psyco +# psyco = 1 -- enabled +# psyco = 0 -- disabled + +psyco = 0 diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/RateLimiter.py b/instrumentation/next-share/BaseLib/Core/BitTornado/RateLimiter.py new file mode 100644 index 0000000..f6a7d73 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/RateLimiter.py @@ -0,0 +1,178 @@ +# Written by Bram Cohen and Pawel Garbacki +# see LICENSE.txt for license information + +from clock import clock +from CurrentRateMeasure import Measure +from math import sqrt +import sys + +try: + True +except: + True = 1 + False = 0 +try: + sum([1]) +except: + sum = lambda a: reduce(lambda x, y: x+y, a, 0) + +DEBUG = False + +MAX_RATE_PERIOD = 20.0 +MAX_RATE = 10e10 +PING_BOUNDARY = 1.2 +PING_SAMPLES = 7 +PING_DISCARDS = 1 +PING_THRESHHOLD = 5 +PING_DELAY = 5 # cycles 'til first upward adjustment +PING_DELAY_NEXT = 2 # 'til next +ADJUST_UP = 1.05 +ADJUST_DOWN = 0.95 +UP_DELAY_FIRST = 5 +UP_DELAY_NEXT = 2 +SLOTS_STARTING = 6 +SLOTS_FACTOR = 1.66/1000 + +class RateLimiter: + def __init__(self, sched, unitsize, slotsfunc = lambda x: None): + self.sched = sched + self.last = None + self.unitsize = unitsize + self.slotsfunc = slotsfunc + self.measure = Measure(MAX_RATE_PERIOD) + self.autoadjust = False + self.upload_rate = MAX_RATE * 1000 + self.slots = SLOTS_STARTING # garbage if not automatic + + def set_upload_rate(self, rate): + if DEBUG: + print >>sys.stderr, "RateLimiter: set_upload_rate", rate + + # rate = -1 # test automatic + if rate < 0: + if self.autoadjust: + return + self.autoadjust = True + self.autoadjustup = 0 + self.pings = [] + rate = MAX_RATE + self.slots = SLOTS_STARTING + self.slotsfunc(self.slots) + else: + self.autoadjust = False + if not rate: + rate = MAX_RATE + self.upload_rate = rate * 1000 + self.lasttime = clock() + self.bytes_sent = 0 + + def queue(self, conn): + if DEBUG: print >>sys.stderr, "RateLimiter: queue", conn + assert conn.next_upload is None + if self.last is None: + self.last = conn + conn.next_upload = conn + self.try_send(True) + else: + conn.next_upload = self.last.next_upload + self.last.next_upload = conn +# 2fastbt_ + if not conn.connection.is_coordinator_con(): + self.last = conn +# _2fastbt + + def try_send(self, check_time = False): + if DEBUG: print >>sys.stderr, "RateLimiter: try_send" + t = clock() + self.bytes_sent -= (t - self.lasttime) * self.upload_rate + #print 'try_send: bytes_sent: %s' % self.bytes_sent + self.lasttime = t + if check_time: + self.bytes_sent = max(self.bytes_sent, 0) + cur = self.last.next_upload + while self.bytes_sent <= 0: + bytes = cur.send_partial(self.unitsize) + self.bytes_sent += bytes + self.measure.update_rate(bytes) + if bytes == 0 or cur.backlogged(): + if self.last is cur: + self.last = None + cur.next_upload = None + break + else: + self.last.next_upload = cur.next_upload + cur.next_upload = None + cur = self.last.next_upload + else: +# 2fastbt_ + if not cur.connection.is_coordinator_con() or not cur.upload.buffer: +# _2fastbt + self.last = cur + cur = cur.next_upload +# 2fastbt_ + else: + pass +# _2fastbt + else: + # 01/04/10 Boudewijn: because we use a -very- small value + # to indicate a 0bps rate, we will schedule the call to be + # made in a very long time. This results in no upload for + # a very long time. + # + # the try_send method has protection again calling to + # soon, so we can simply schedule the call to be made + # sooner. + delay = min(5.0, self.bytes_sent / self.upload_rate) + self.sched(self.try_send, delay) + + def adjust_sent(self, bytes): + # if DEBUG: print >>sys.stderr, "RateLimiter: adjust_sent", bytes + self.bytes_sent = min(self.bytes_sent+bytes, self.upload_rate*3) + self.measure.update_rate(bytes) + + + def ping(self, delay): + ##raise Exception('Is this called?') + if DEBUG: + print >>sys.stderr, delay + if not self.autoadjust: + return + self.pings.append(delay > PING_BOUNDARY) + if len(self.pings) < PING_SAMPLES+PING_DISCARDS: + return + if DEBUG: + print >>sys.stderr, 'RateLimiter: cycle' + pings = sum(self.pings[PING_DISCARDS:]) + del self.pings[:] + if pings >= PING_THRESHHOLD: # assume flooded + if self.upload_rate == MAX_RATE: + self.upload_rate = self.measure.get_rate()*ADJUST_DOWN + else: + self.upload_rate = min(self.upload_rate, + self.measure.get_rate()*1.1) + self.upload_rate = max(int(self.upload_rate*ADJUST_DOWN), 2) + self.slots = int(sqrt(self.upload_rate*SLOTS_FACTOR)) + self.slotsfunc(self.slots) + if DEBUG: + print >>sys.stderr, 'RateLimiter: adjust down to '+str(self.upload_rate) + self.lasttime = clock() + self.bytes_sent = 0 + self.autoadjustup = UP_DELAY_FIRST + else: # not flooded + if self.upload_rate == MAX_RATE: + return + self.autoadjustup -= 1 + if self.autoadjustup: + return + self.upload_rate = int(self.upload_rate*ADJUST_UP) + self.slots = int(sqrt(self.upload_rate*SLOTS_FACTOR)) + self.slotsfunc(self.slots) + if DEBUG: + print >>sys.stderr, 'RateLimiter: adjust up to '+str(self.upload_rate) + self.lasttime = clock() + self.bytes_sent = 0 + self.autoadjustup = UP_DELAY_NEXT + + + + diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/RateMeasure.py b/instrumentation/next-share/BaseLib/Core/BitTornado/RateMeasure.py new file mode 100644 index 0000000..1c42a2f --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/RateMeasure.py @@ -0,0 +1,70 @@ +# Written by Bram Cohen +# see LICENSE.txt for license information + +from clock import clock +try: + True +except: + True = 1 + False = 0 + +FACTOR = 0.999 + +class RateMeasure: + def __init__(self): + self.last = None + self.time = 1.0 + self.got = 0.0 + self.remaining = None + self.broke = False + self.got_anything = False + self.last_checked = None + self.rate = 0 + + def data_came_in(self, amount): + if not self.got_anything: + self.got_anything = True + self.last = clock() + return + self.update(amount) + + def data_rejected(self, amount): + pass + + def get_time_left(self, left): + t = clock() + if not self.got_anything: + return None + if t - self.last > 15: + self.update(0) + try: + remaining = left/self.rate + delta = max(remaining/20, 2) + if self.remaining is None: + self.remaining = remaining + elif abs(self.remaining-remaining) > delta: + self.remaining = remaining + else: + self.remaining -= t - self.last_checked + except ZeroDivisionError: + self.remaining = None + if self.remaining is not None and self.remaining < 0.1: + self.remaining = 0.1 + self.last_checked = t + return self.remaining + + def update(self, amount): + t = clock() + t1 = int(t) + l1 = int(self.last) + for i in xrange(l1, t1): + self.time *= FACTOR + self.got *= FACTOR + self.got += amount + if t - self.last < 20: + self.time += t - self.last + self.last = t + try: + self.rate = self.got / self.time + except ZeroDivisionError: + pass diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/RawServer.py b/instrumentation/next-share/BaseLib/Core/BitTornado/RawServer.py new file mode 100644 index 0000000..0ebaf35 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/RawServer.py @@ -0,0 +1,274 @@ +# Written by Bram Cohen and Pawel Garbacki +# see LICENSE.txt for license information + +from bisect import insort +from SocketHandler import SocketHandler +import socket +from cStringIO import StringIO +from traceback import print_exc +from select import error +from threading import Event, RLock +from thread import get_ident +from clock import clock +import sys +import time + +try: + True +except: + True = 1 + False = 0 + +DEBUG = False + +def autodetect_ipv6(): + try: + assert sys.version_info >= (2, 3) + assert socket.has_ipv6 + socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + except: + return 0 + return 1 + +def autodetect_socket_style(): + if sys.platform.find('linux') < 0: + return 1 + else: + try: + f = open('/proc/sys/net/ipv6/bindv6only', 'r') + dual_socket_style = int(f.read()) + f.close() + return int(not dual_socket_style) + except: + return 0 + + +READSIZE = 100000 + +class RawServer: + def __init__(self, doneflag, timeout_check_interval, timeout, noisy = True, + ipv6_enable = True, failfunc = lambda x: None, errorfunc = None, + sockethandler = None, excflag = Event()): + self.timeout_check_interval = timeout_check_interval + self.timeout = timeout + self.servers = {} + self.single_sockets = {} + self.dead_from_write = [] + self.doneflag = doneflag + self.noisy = noisy + self.failfunc = failfunc + self.errorfunc = errorfunc + self.exccount = 0 + self.funcs = [] + self.externally_added = [] + self.finished = Event() + self.tasks_to_kill = [] + self.excflag = excflag + self.lock = RLock() + + if sockethandler is None: + sockethandler = SocketHandler(timeout, ipv6_enable, READSIZE) + self.sockethandler = sockethandler + + self.thread_ident = None + self.interrupt_socket = sockethandler.get_interrupt_socket() + + self.add_task(self.scan_for_timeouts, timeout_check_interval) + + def get_exception_flag(self): + return self.excflag + + def _add_task(self, func, delay, id = None): + if delay < 0: + delay = 0 + insort(self.funcs, (clock() + delay, func, id)) + + def add_task(self, func, delay = 0, id = None): + #if DEBUG: + # print >>sys.stderr,"rawserver: add_task(",func,delay,")" + if delay < 0: + delay = 0 + self.lock.acquire() + self.externally_added.append((func, delay, id)) + if self.thread_ident != get_ident(): + self.interrupt_socket.interrupt() + self.lock.release() + + def scan_for_timeouts(self): + self.add_task(self.scan_for_timeouts, self.timeout_check_interval) + self.sockethandler.scan_for_timeouts() + + def bind(self, port, bind = '', reuse = False, + ipv6_socket_style = 1): + self.sockethandler.bind(port, bind, reuse, ipv6_socket_style) + + def find_and_bind(self, first_try, minport, maxport, bind = '', reuse = False, + ipv6_socket_style = 1, randomizer = False): +# 2fastbt_ + result = self.sockethandler.find_and_bind(first_try, minport, maxport, bind, reuse, + ipv6_socket_style, randomizer) +# _2fastbt + return result + + def start_connection_raw(self, dns, socktype, handler = None): + return self.sockethandler.start_connection_raw(dns, socktype, handler) + + def start_connection(self, dns, handler = None, randomize = False): + return self.sockethandler.start_connection(dns, handler, randomize) + + def get_stats(self): + return self.sockethandler.get_stats() + + def pop_external(self): + self.lock.acquire() + while self.externally_added: + (a, b, c) = self.externally_added.pop(0) + self._add_task(a, b, c) + self.lock.release() + + def listen_forever(self, handler): + if DEBUG: + print >>sys.stderr,"rawserver: listen forever()" + # handler=btlanuchmany: MultiHandler, btdownloadheadless: Encoder + self.thread_ident = get_ident() + self.sockethandler.set_handler(handler) + try: + while not self.doneflag.isSet(): + try: + self.pop_external() + self._kill_tasks() + if self.funcs: + period = self.funcs[0][0] + 0.001 - clock() + else: + period = 2 ** 30 + if period < 0: + period = 0 + + #if DEBUG: + # print >>sys.stderr,"rawserver: do_poll",period + events = self.sockethandler.do_poll(period) + + if self.doneflag.isSet(): + if DEBUG: + print >> sys.stderr,"rawserver: stopping because done flag set" + return + + #print >>sys.stderr,"RawServer: funcs is",`self.funcs` + + + while self.funcs and self.funcs[0][0] <= clock(): + garbage1, func, id = self.funcs.pop(0) + if id in self.tasks_to_kill: + pass + try: +# print func.func_name + if DEBUG: + if func.func_name != "_bgalloc": + print >> sys.stderr,"RawServer:f",func.func_name + #st = time.time() + func() + #et = time.time() + #diff = et - st + #print >>sys.stderr,func,"took %.5f" % (diff) + + except (SystemError, MemoryError), e: + self.failfunc(e) + return + except KeyboardInterrupt,e: +# self.exception(e) + return + except error: + if DEBUG: + print >> sys.stderr,"rawserver: func: ERROR exception" + print_exc() + pass + except Exception,e: + if DEBUG: + print >> sys.stderr,"rawserver: func: any exception" + print_exc() + if self.noisy: + self.exception(e) + self.sockethandler.close_dead() + self.sockethandler.handle_events(events) + if self.doneflag.isSet(): + if DEBUG: + print >> sys.stderr,"rawserver: stopping because done flag set2" + return + self.sockethandler.close_dead() + except (SystemError, MemoryError), e: + if DEBUG: + print >> sys.stderr,"rawserver: SYS/MEM exception",e + self.failfunc(e) + return + except error: + if DEBUG: + print >> sys.stderr,"rawserver: ERROR exception" + print_exc() + if self.doneflag.isSet(): + return + except KeyboardInterrupt,e: + self.failfunc(e) + return + except Exception,e: + if DEBUG: + print >> sys.stderr,"rawserver: other exception" + print_exc() + self.exception(e) + ## Arno: Don't stop till we drop + ##if self.exccount > 10: + ## print >> sys.stderr,"rawserver: stopping because exccount > 10" + ## return + finally: +# self.sockethandler.shutdown() + self.finished.set() + + def is_finished(self): + return self.finished.isSet() + + def wait_until_finished(self): + self.finished.wait() + + def _kill_tasks(self): + if self.tasks_to_kill: + new_funcs = [] + for (t, func, id) in self.funcs: + if id not in self.tasks_to_kill: + new_funcs.append((t, func, id)) + self.funcs = new_funcs + self.tasks_to_kill = [] + + def kill_tasks(self, id): + self.tasks_to_kill.append(id) + + def exception(self,e,kbint=False): + if not kbint: + self.excflag.set() + self.exccount += 1 + if self.errorfunc is None: + print_exc() + else: + if not kbint: # don't report here if it's a keyboard interrupt + self.errorfunc(e) + + def shutdown(self): + self.sockethandler.shutdown() + + + # + # Interface for Khashmir + # + def create_udpsocket(self,port,host): + if DEBUG: + print >>sys.stderr,"rawudp: create_udp_socket",host,port + return self.sockethandler.create_udpsocket(port,host) + + def start_listening_udp(self,serversocket,handler): + if DEBUG: + print >>sys.stderr,"rawudp: start_listen:",serversocket,handler + self.sockethandler.start_listening_udp(serversocket,handler) + + def stop_listening_udp(self,serversocket): + if DEBUG: + print >>sys.stderr,"rawudp: stop_listen:",serversocket + self.sockethandler.stop_listening_udp(serversocket) + diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/ServerPortHandler.py b/instrumentation/next-share/BaseLib/Core/BitTornado/ServerPortHandler.py new file mode 100644 index 0000000..b3eceeb --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/ServerPortHandler.py @@ -0,0 +1,238 @@ +# Written by John Hoffman +# see LICENSE.txt for license information + +import sys +from cStringIO import StringIO +from binascii import b2a_hex +#from RawServer import RawServer + +try: + True +except: + True = 1 + False = 0 + +from BT1.Encrypter import protocol_name + + +def toint(s): + return long(b2a_hex(s), 16) + +default_task_id = [] + +DEBUG = False + +def show(s): + for i in xrange(len(s)): + print ord(s[i]), + print + +class SingleRawServer: + def __init__(self, info_hash, multihandler, doneflag, protocol): + self.info_hash = info_hash + self.doneflag = doneflag + self.protocol = protocol + self.multihandler = multihandler + self.rawserver = multihandler.rawserver + self.finished = False + self.running = False + self.handler = None + self.taskqueue = [] + + def shutdown(self): + if not self.finished: + self.multihandler.shutdown_torrent(self.info_hash) + + def _shutdown(self): + if DEBUG: + print >>sys.stderr,"SingleRawServer: _shutdown" + if not self.finished: + self.finished = True + self.running = False + self.rawserver.kill_tasks(self.info_hash) + if self.handler: + self.handler.close_all() + + def _external_connection_made(self, c, options, msg_remainder): + if DEBUG: + print >> sys.stderr,"SingleRawServer: _external_conn_made, running?",self.running + if self.running: + c.set_handler(self.handler) + self.handler.externally_handshaked_connection_made( + c, options, msg_remainder) + + ### RawServer functions ### + + def add_task(self, func, delay=0, id = default_task_id): + if id is default_task_id: + id = self.info_hash + if not self.finished: + self.rawserver.add_task(func, delay, id) + +# def bind(self, port, bind = '', reuse = False): +# pass # not handled here + + def start_connection(self, dns, handler = None): + if not handler: + handler = self.handler + c = self.rawserver.start_connection(dns, handler) + return c + +# def listen_forever(self, handler): +# pass # don't call with this + + def start_listening(self, handler): + self.handler = handler # Encoder + self.running = True + return self.shutdown # obviously, doesn't listen forever + + def is_finished(self): + return self.finished + + def get_exception_flag(self): + return self.rawserver.get_exception_flag() + +class NewSocketHandler: # hand a new socket off where it belongs + def __init__(self, multihandler, connection): # connection: SingleSocket + self.multihandler = multihandler + self.connection = connection + connection.set_handler(self) + self.closed = False + self.buffer = StringIO() + self.complete = False + self.next_len, self.next_func = 1, self.read_header_len + self.multihandler.rawserver.add_task(self._auto_close, 15) + + def _auto_close(self): + if not self.complete: + self.close() + + def close(self): + if not self.closed: + self.connection.close() + self.closed = True + +# header format: +# connection.write(chr(len(protocol_name)) + protocol_name + +# (chr(0) * 8) + self.encrypter.download_id + self.encrypter.my_id) + + # copied from Encrypter and modified + + def read_header_len(self, s): + if s == 'G': + self.protocol = 'HTTP' + self.firstbyte = s + if DEBUG: + print >>sys.stderr,"NewSocketHandler: Got HTTP connection" + return True + else: + l = ord(s) + return l, self.read_header + + def read_header(self, s): + self.protocol = s + return 8, self.read_reserved + + def read_reserved(self, s): + self.options = s + return 20, self.read_download_id + + def read_download_id(self, s): + if DEBUG: + print >>sys.stderr,"NewSocketHandler: Swarm id is",`s`,self.connection.socket.getpeername() + if self.multihandler.singlerawservers.has_key(s): + if self.multihandler.singlerawservers[s].protocol == self.protocol: + if DEBUG: + print >>sys.stderr,"NewSocketHandler: Found rawserver for swarm id" + return True + if DEBUG: + print >>sys.stderr,"NewSocketHandler: No rawserver found for swarm id",`s` + return None + + def read_dead(self, s): + return None + + def data_came_in(self, garbage, s): +# if DEBUG: +# print "NewSocketHandler data came in", sha(s).hexdigest() + while 1: + if self.closed: + return + i = self.next_len - self.buffer.tell() + if i > len(s): + self.buffer.write(s) + return + self.buffer.write(s[:i]) + s = s[i:] + m = self.buffer.getvalue() + self.buffer.reset() + self.buffer.truncate() + try: + x = self.next_func(m) + except: + self.next_len, self.next_func = 1, self.read_dead + raise + if x is None: + if DEBUG: + print >> sys.stderr,"NewSocketHandler:",self.next_func,"returned None" + self.close() + return + if x == True: # ready to process + if self.protocol == 'HTTP': + if DEBUG: + print >> sys.stderr,"NewSocketHandler: Reporting HTTP connection" + self.multihandler.httphandler.external_connection_made(self.connection) + self.multihandler.httphandler.data_came_in(self.connection,self.firstbyte) + self.multihandler.httphandler.data_came_in(self.connection,s) + else: + if DEBUG: + print >> sys.stderr,"NewSocketHandler: Reporting connection via",self.multihandler.singlerawservers[m]._external_connection_made + self.multihandler.singlerawservers[m]._external_connection_made(self.connection, self.options, s) + self.complete = True + return + self.next_len, self.next_func = x + + def connection_flushed(self, ss): + pass + + def connection_lost(self, ss): + self.closed = True + +class MultiHandler: + def __init__(self, rawserver, doneflag): + self.rawserver = rawserver + self.masterdoneflag = doneflag + self.singlerawservers = {} + self.connections = {} + self.taskqueues = {} + self.httphandler = None + + def newRawServer(self, info_hash, doneflag, protocol=protocol_name): + new = SingleRawServer(info_hash, self, doneflag, protocol) + self.singlerawservers[info_hash] = new + return new + + def shutdown_torrent(self, info_hash): + if DEBUG: + print >>sys.stderr,"MultiHandler: shutdown_torrent",`info_hash` + self.singlerawservers[info_hash]._shutdown() + del self.singlerawservers[info_hash] + + def listen_forever(self): + if DEBUG: + print >>sys.stderr,"MultiHandler: listen_forever()" + self.rawserver.listen_forever(self) + for srs in self.singlerawservers.values(): + srs.finished = True + srs.running = False + srs.doneflag.set() + + def set_httphandler(self,httphandler): + self.httphandler = httphandler + + ### RawServer handler functions ### + # be wary of name collisions + + def external_connection_made(self, ss): + # ss: SingleSocket + NewSocketHandler(self, ss) diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/SocketHandler.py b/instrumentation/next-share/BaseLib/Core/BitTornado/SocketHandler.py new file mode 100644 index 0000000..6c1b360 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/SocketHandler.py @@ -0,0 +1,629 @@ +# Written by Bram Cohen +# see LICENSE.txt for license information + +import socket +import errno +try: + from select import poll, POLLIN, POLLOUT, POLLERR, POLLHUP + timemult = 1000 +except ImportError: + from selectpoll import poll, POLLIN, POLLOUT, POLLERR, POLLHUP + timemult = 1 +from time import sleep +from clock import clock +import sys +from random import shuffle, randrange +from traceback import print_exc + +try: + True +except: + True = 1 + False = 0 + +DEBUG = False + +all = POLLIN | POLLOUT + +if sys.platform == 'win32': + SOCKET_BLOCK_ERRORCODE=10035 # WSAEWOULDBLOCK +else: + SOCKET_BLOCK_ERRORCODE=errno.EWOULDBLOCK + +class InterruptSocketHandler: + @staticmethod + def data_came_in(interrupt_socket, data): + pass + +class InterruptSocket: + """ + When we need the poll to return before the timeout expires, we + will send some data to the InterruptSocket and discard the data. + """ + def __init__(self, socket_handler): + self.socket_handler = socket_handler + self.handler = InterruptSocketHandler + + self.ip = "127.0.0.1" + self.port = None + self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.interrupt_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + + # we assume that one port in the range below is free + for self.port in xrange(10000, 12345): + try: + if DEBUG: print >>sys.stderr, "InterruptSocket: Trying to start InterruptSocket on port", self.port + self.socket.bind((self.ip, self.port)) + break + except: + pass + + # start listening to the InterruptSocket + self.socket_handler.single_sockets[self.socket.fileno()] = self + self.socket_handler.poll.register(self.socket, POLLIN) + + def interrupt(self): + self.interrupt_socket.sendto("+", (self.ip, self.port)) + + def get_ip(self): + return self.ip + + def get_port(self): + return self.port + +class UdpSocket: + """ Class to hold socket and handler for a UDP socket. """ + def __init__(self, socket, handler): + self.socket = socket + self.handler = handler + +class SingleSocket: + """ + There are two places to create SingleSocket: + incoming connection -- SocketHandler.handle_events + outgoing connection -- SocketHandler.start_connection_raw + """ + + def __init__(self, socket_handler, sock, handler, ip = None): + self.socket_handler = socket_handler + self.socket = sock + self.handler = handler + self.buffer = [] + self.last_hit = clock() + self.fileno = sock.fileno() + self.connected = False + self.skipped = 0 +# self.check = StreamCheck() + self.myip = None + self.myport = -1 + self.ip = None + self.port = -1 + try: + myname = self.socket.getsockname() + self.myip = myname[0] + self.myport = myname[1] + peername = self.socket.getpeername() + self.ip = peername[0] + self.port = peername[1] + except: + #print_exc() + if ip is None: + self.ip = 'unknown' + else: + self.ip = ip + # RePEX: Measurement TODO: Remove when measurement test has been done + self.data_sent = 0 + self.data_received = 0 + + def get_ip(self, real=False): + if real: + try: + peername = self.socket.getpeername() + self.ip = peername[0] + self.port = peername[1] + except: + ## print_exc() + pass + return self.ip + + def get_port(self, real=False): + if real: + self.get_ip(True) + return self.port + + def get_myip(self, real=False): + if real: + try: + myname = self.socket.getsockname() + self.myip = myname[0] + self.myport = myname[1] + except: + print_exc() + pass + return self.myip + + def get_myport(self, real=False): + if real: + self.get_myip(True) + return self.myport + + def close(self): + ''' + for x in xrange(5,0,-1): + try: + f = inspect.currentframe(x).f_code + print (f.co_filename,f.co_firstlineno,f.co_name) + del f + except: + pass + print '' + ''' + assert self.socket + self.connected = False + sock = self.socket + self.socket = None + self.buffer = [] + del self.socket_handler.single_sockets[self.fileno] + self.socket_handler.poll.unregister(sock) + sock.close() + + def shutdown(self, val): + self.socket.shutdown(val) + + def is_flushed(self): + return not self.buffer + + def write(self, s): +# self.check.write(s) + # Arno: fishy concurrency problem, sometimes self.socket is None + if self.socket is None: + return + #assert self.socket is not None + self.buffer.append(s) + if len(self.buffer) == 1: + self.try_write() + + def try_write(self): + + if self.connected: + dead = False + try: + while self.buffer: + buf = self.buffer[0] + amount = self.socket.send(buf) + self.data_sent += amount # RePEX: Measurement TODO: Remove when measurement test has been done + if amount == 0: + self.skipped += 1 + break + self.skipped = 0 + if amount != len(buf): + self.buffer[0] = buf[amount:] + break + del self.buffer[0] + except socket.error, e: + #if DEBUG: + # print_exc(file=sys.stderr) + blocked=False + try: + blocked = (e[0] == SOCKET_BLOCK_ERRORCODE) + dead = not blocked + except: + dead = True + if not blocked: + self.skipped += 1 + if self.skipped >= 5: + dead = True + if dead: + self.socket_handler.dead_from_write.append(self) + return + if self.buffer: + self.socket_handler.poll.register(self.socket, all) + else: + self.socket_handler.poll.register(self.socket, POLLIN) + + def set_handler(self, handler): # can be: NewSocketHandler, Encoder, En_Connection + self.handler = handler + + +class SocketHandler: + def __init__(self, timeout, ipv6_enable, readsize = 100000): + self.timeout = timeout + self.ipv6_enable = ipv6_enable + self.readsize = readsize + self.poll = poll() + # {socket: SingleSocket} + self.single_sockets = {} + self.dead_from_write = [] + self.max_connects = 1000 + self.servers = {} + self.btengine_said_reachable = False + self.interrupt_socket = None + self.udp_sockets = {} + + def scan_for_timeouts(self): + t = clock() - self.timeout + tokill = [] + for s in self.single_sockets.values(): + # Only SingleSockets can be closed because of timeouts + if type(s) is SingleSocket and s.last_hit < t: + tokill.append(s) + for k in tokill: + if k.socket is not None: + if DEBUG: + print >> sys.stderr,"SocketHandler: scan_timeout closing connection",k.get_ip() + self._close_socket(k) + + def bind(self, port, bind = [], reuse = False, ipv6_socket_style = 1): + port = int(port) + addrinfos = [] + self.servers = {} + self.interfaces = [] + # if bind != [] bind to all specified addresses (can be IPs or hostnames) + # else bind to default ipv6 and ipv4 address + if bind: + if self.ipv6_enable: + socktype = socket.AF_UNSPEC + else: + socktype = socket.AF_INET + for addr in bind: + if sys.version_info < (2, 2): + addrinfos.append((socket.AF_INET, None, None, None, (addr, port))) + else: + addrinfos.extend(socket.getaddrinfo(addr, port, + socktype, socket.SOCK_STREAM)) + else: + if self.ipv6_enable: + addrinfos.append([socket.AF_INET6, None, None, None, ('', port)]) + if not addrinfos or ipv6_socket_style != 0: + addrinfos.append([socket.AF_INET, None, None, None, ('', port)]) + for addrinfo in addrinfos: + try: + server = socket.socket(addrinfo[0], socket.SOCK_STREAM) + if reuse: + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.setblocking(0) + if DEBUG: + print >> sys.stderr,"SocketHandler: Try to bind socket on", addrinfo[4], "..." + server.bind(addrinfo[4]) + self.servers[server.fileno()] = server + if bind: + self.interfaces.append(server.getsockname()[0]) + if DEBUG: + print >> sys.stderr,"SocketHandler: OK" + server.listen(64) + self.poll.register(server, POLLIN) + except socket.error, e: + for server in self.servers.values(): + try: + server.close() + except: + pass + if self.ipv6_enable and ipv6_socket_style == 0 and self.servers: + raise socket.error('blocked port (may require ipv6_binds_v4 to be set)') + raise socket.error(str(e)) + if not self.servers: + raise socket.error('unable to open server port') + self.port = port + + def find_and_bind(self, first_try, minport, maxport, bind = '', reuse = False, + ipv6_socket_style = 1, randomizer = False): + e = 'maxport less than minport - no ports to check' + if maxport-minport < 50 or not randomizer: + portrange = range(minport, maxport+1) + if randomizer: + shuffle(portrange) + portrange = portrange[:20] # check a maximum of 20 ports + else: + portrange = [] + while len(portrange) < 20: + listen_port = randrange(minport, maxport+1) + if not listen_port in portrange: + portrange.append(listen_port) + if first_try != 0: # try 22 first, because TU only opens port 22 for SSH... + try: + self.bind(first_try, bind, reuse = reuse, + ipv6_socket_style = ipv6_socket_style) + return first_try + except socket.error, e: + pass + for listen_port in portrange: + try: + #print >> sys.stderr, listen_port, bind, reuse + self.bind(listen_port, bind, reuse = reuse, + ipv6_socket_style = ipv6_socket_style) + return listen_port + except socket.error, e: + raise + raise socket.error(str(e)) + + + def set_handler(self, handler): + self.handler = handler + + + def start_connection_raw(self, dns, socktype = socket.AF_INET, handler = None): + # handler = Encoder, self.handler = Multihandler + if handler is None: + handler = self.handler + sock = socket.socket(socktype, socket.SOCK_STREAM) + sock.setblocking(0) + try: + if DEBUG: + print >>sys.stderr,"SocketHandler: Initiate connection to",dns,"with socket #",sock.fileno() + # Arno,2007-01-23: http://docs.python.org/lib/socket-objects.html + # says that connect_ex returns an error code (and can still throw + # exceptions). The original code never checked the return code. + # + err = sock.connect_ex(dns) + if DEBUG: + if err == 0: + msg = 'No error' + else: + msg = errno.errorcode[err] + print >>sys.stderr,"SocketHandler: connect_ex on socket #",sock.fileno(),"returned",err,msg + if err != 0: + if sys.platform == 'win32' and err == 10035: + # Arno, 2007-02-23: win32 always returns WSAEWOULDBLOCK, whether + # the connect is to a live peer or not. Win32's version + # of EINPROGRESS + pass + elif err == errno.EINPROGRESS: # or err == errno.EALREADY or err == errno.EWOULDBLOCK: + # [Stevens98] says that UNICES return EINPROGRESS when the connect + # does not immediately succeed, which is almost always the case. + pass + else: + raise socket.error((err,errno.errorcode[err])) + except socket.error, e: + if DEBUG: + print >> sys.stderr,"SocketHandler: SocketError in connect_ex",str(e) + raise + except Exception, e: + if DEBUG: + print >> sys.stderr,"SocketHandler: Exception in connect_ex",str(e) + raise socket.error(str(e)) + self.poll.register(sock, POLLIN) + s = SingleSocket(self, sock, handler, dns[0]) # create socket to connect the peers obtained from tracker + self.single_sockets[sock.fileno()] = s + #if DEBUG: + # print >> sys.stderr,"SocketHandler: Created Socket" + return s + + + def start_connection(self, dns, handler = None, randomize = False): + if handler is None: + handler = self.handler + if sys.version_info < (2, 2): + s = self.start_connection_raw(dns, socket.AF_INET, handler) + else: +# if self.ipv6_enable: +# socktype = socket.AF_UNSPEC +# else: +# socktype = socket.AF_INET + try: + try: + """ + Arno: When opening a new connection, the network thread calls the + getaddrinfo() function (=DNS resolve), as apparently the input + sometimes is a hostname. At the same time the tracker thread uses + this same function to resolve the tracker name to an IP address. + However, on Python for Windows this method has concurrency control + protection that allows only 1 request at a time. + + In some cases resolving the tracker name takes a very long time, + meanwhile blocking the network thread!!!! And that only wanted to + resolve some IP address to some IP address, i.e., do nothing!!! + + Sol: don't call getaddrinfo() is the input is an IP address, and + submit a bug to python that it shouldn't lock when the op is + a null op + """ + socket.inet_aton(dns[0]) # IPVSIX: change to inet_pton() + #print >>sys.stderr,"SockHand: start_conn: after inet_aton",dns[0],"<",dns,">" + addrinfos=[(socket.AF_INET, None, None, None, (dns[0], dns[1]))] + except: + #print_exc() + try: + # Jie: we attempt to use this socktype to connect ipv6 addresses. + socktype = socket.AF_UNSPEC + addrinfos = socket.getaddrinfo(dns[0], int(dns[1]), + socktype, socket.SOCK_STREAM) + except: + socktype = socket.AF_INET + addrinfos = socket.getaddrinfo(dns[0], int(dns[1]), + socktype, socket.SOCK_STREAM) + except socket.error, e: + raise + except Exception, e: + raise socket.error(str(e)) + if randomize: + shuffle(addrinfos) + for addrinfo in addrinfos: + try: + s = self.start_connection_raw(addrinfo[4], addrinfo[0], handler) + break + except Exception,e: + print_exc() + pass # FIXME Arno: ???? raise e + else: + raise socket.error('unable to connect') + return s + + + def _sleep(self): + sleep(1) + + def handle_events(self, events): + for sock, event in events: + #print >>sys.stderr,"SocketHandler: event on sock#",sock + s = self.servers.get(sock) # socket.socket + if s: + if event & (POLLHUP | POLLERR) != 0: + if DEBUG: + print >> sys.stderr,"SocketHandler: Got event, close server socket" + self.poll.unregister(s) + del self.servers[sock] + else: + try: + newsock, addr = s.accept() + if DEBUG: + print >> sys.stderr,"SocketHandler: Got connection from",newsock.getpeername() + if not self.btengine_said_reachable: + try: + from BaseLib.Core.NATFirewall.DialbackMsgHandler import DialbackMsgHandler + dmh = DialbackMsgHandler.getInstance() + dmh.network_btengine_reachable_callback() + except ImportError: + if DEBUG: + print_exc() + pass + self.btengine_said_reachable = True + + # Only use the new socket if we can spare the + # connections. Otherwise we will silently drop + # the connection. + if len(self.single_sockets) < self.max_connects: + newsock.setblocking(0) + nss = SingleSocket(self, newsock, self.handler) # create socket for incoming peers and tracker + self.single_sockets[newsock.fileno()] = nss + self.poll.register(newsock, POLLIN) + self.handler.external_connection_made(nss) + else: + print >> sys.stderr,"SocketHandler: too many connects" + newsock.close() + + except socket.error,e: + if DEBUG: + print >> sys.stderr,"SocketHandler: SocketError while accepting new connection",str(e) + self._sleep() + continue + + s = self.udp_sockets.get(sock) + if s: + try: + (data, addr) = s.socket.recvfrom(65535) + if not data: + if DEBUG: + print >> sys.stderr, "SocketHandler: UDP no-data", addr + else: + if DEBUG: + print >> sys.stderr,"SocketHandler: Got UDP data",addr,"len",len(data) + s.handler.data_came_in(addr, data) + + except socket.error, e: + if DEBUG: + print >> sys.stderr,"SocketHandler: UDP Socket error",str(e) + continue + + s = self.single_sockets.get(sock) + if s: + if (event & (POLLHUP | POLLERR)): + if DEBUG: + print >> sys.stderr,"SocketHandler: Got event, connect socket got error" + print >> sys.stderr,"SocketHandler: Got event, connect socket got error",s.ip,s.port + self._close_socket(s) + continue + if (event & POLLIN): + try: + s.last_hit = clock() + data = s.socket.recv(100000) + if not data: + if DEBUG: + print >> sys.stderr,"SocketHandler: no-data closing connection",s.get_ip(),s.get_port() + self._close_socket(s) + else: + #if DEBUG: + # print >> sys.stderr,"SocketHandler: Got data",s.get_ip(),s.get_port(),"len",len(data) + + # btlaunchmany: NewSocketHandler, btdownloadheadless: Encrypter.Connection + if hasattr(s, 'data_received'): s.data_received += len(data) # RePEX: Measurement TODO: Remove when measurement test has been done + s.handler.data_came_in(s, data) + except socket.error, e: + if DEBUG: + print >> sys.stderr,"SocketHandler: Socket error",str(e) + code, msg = e + if code != SOCKET_BLOCK_ERRORCODE: + if DEBUG: + print >> sys.stderr,"SocketHandler: closing connection because not WOULDBLOCK",s.get_ip(),"error",code + self._close_socket(s) + continue + if (event & POLLOUT) and s.socket and not s.is_flushed(): + s.connected = True + s.try_write() + if s.is_flushed(): + s.handler.connection_flushed(s) + + def close_dead(self): + while self.dead_from_write: + old = self.dead_from_write + self.dead_from_write = [] + for s in old: + if s.socket: + if DEBUG: + print >> sys.stderr,"SocketHandler: close_dead closing connection",s.get_ip() + self._close_socket(s) + + def _close_socket(self, s): + if DEBUG: + print >> sys.stderr,"SocketHandler: closing connection to ",s.get_ip() + s.close() + s.handler.connection_lost(s) + + def do_poll(self, t): + r = self.poll.poll(t*timemult) + if r is None: + connects = len(self.single_sockets) + to_close = int(connects*0.05)+1 # close 5% of sockets + self.max_connects = connects-to_close + closelist = self.single_sockets.values() + shuffle(closelist) + closelist = closelist[:to_close] + for sock in closelist: + if DEBUG: + print >> sys.stderr,"SocketHandler: do_poll closing connection",sock.get_ip() + self._close_socket(sock) + return [] + return r + + def get_stats(self): + return { 'interfaces': self.interfaces, + 'port': self.port } + + + def shutdown(self): + for ss in self.single_sockets.values(): + try: + ss.close() + except: + pass + for server in self.servers.values(): + try: + server.close() + except: + pass + + # + # Interface for Khasmir, called from RawServer + # + # + def create_udpsocket(self,port,host): + server = socket.socket(socket.AF_INET,socket.SOCK_DGRAM) + server.bind((host,port)) + server.setblocking(0) + return server + + def start_listening_udp(self,serversocket,handler): + self.udp_sockets[serversocket.fileno()] = UdpSocket(serversocket, handler) + self.poll.register(serversocket, POLLIN) + + def stop_listening_udp(self,serversocket): + self.poll.unregister(serversocket) + del self.udp_sockets[serversocket.fileno()] + + def get_interrupt_socket(self): + """ + Create a socket to interrupt the poll when the thread needs to + continue without waiting for the timeout + """ + if not self.interrupt_socket: + self.interrupt_socket = InterruptSocket(self) + return self.interrupt_socket diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/__init__.py b/instrumentation/next-share/BaseLib/Core/BitTornado/__init__.py new file mode 100644 index 0000000..c16bf91 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/__init__.py @@ -0,0 +1,109 @@ +# Written by BitTornado authors and Arno Bakker +# see LICENSE.txt for license information + +## Arno: FIXME _idprefix is also defined in BitTornado.__init__ and that's the one +## actually used in connections, so make sure they are defined in one place +## (here) and correct. +## + +from BaseLib.__init__ import LIBRARYNAME + +if LIBRARYNAME == "Tribler": + version_id = '5.2.1' + product_name = 'Tribler' + version_short = 'Tribler-' + version_id + report_email = 'tribler@tribler.org' + # Arno: looking at Azureus BTPeerIDByteDecoder this letter is free + # 'T' is BitTornado, 'A' is ABC, 'TR' is Transmission + TRIBLER_PEERID_LETTER='R' +else: + version_id = '3.2.0' # aka M32 + product_name = 'NextShare' + version_short = 'NextShare-' + version_id + report_email = 'support@p2p-next.org' + # Arno: looking at Azureus BTPeerIDByteDecoder this letter is free + # 'T' is BitTornado, 'A' is ABC, 'TR' is Transmission + TRIBLER_PEERID_LETTER='N' + + +version = version_short + ' (' + product_name + ')' +_idprefix = TRIBLER_PEERID_LETTER + + +from types import StringType +from time import time, clock +from string import strip +import socket +import random +try: + from os import getpid +except ImportError: + def getpid(): + return 1 +from base64 import decodestring +import sys +from traceback import print_exc + +mapbase64 = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz.-' + +#for subver in version_short[2:].split('.'): +for subver in version_short.split('-')[1].split('.'): + try: + subver = int(subver) + except: + subver = 0 + _idprefix += mapbase64[subver] +_idprefix += ('-' * (6-len(_idprefix))) +_idrandom = [None] + + + + +def resetPeerIDs(): + try: + f = open('/dev/urandom', 'rb') + x = f.read(20) + f.close() + except: + # Arno: make sure there is some randomization when on win32 + random.seed() + x = '' + while len(x) < 20: + #r = random.randint(0,sys.maxint) + r = random.randint(0,255) + x += chr(r) + x = x[:20] + + s = '' + for i in x: + s += mapbase64[ord(i) & 0x3F] + _idrandom[0] = s[:11] # peer id = iprefix (6) + ins (3) + random + +def createPeerID(ins = '---'): + assert type(ins) is StringType + assert len(ins) == 3 + resetPeerIDs() + return _idprefix + ins + _idrandom[0] + +def decodePeerID(id): + client = None + version = None + try: + if id[0] == '-': + # Azureus type ID: + client = id[1:3] + encversion = id[3:7] + else: + # Shadow type ID: + client = id[0] + encversion = id[1:4] + version = '' + for i in range(len(encversion)): + for j in range(len(mapbase64)): + if mapbase64[j] == encversion[i]: + if len(version) > 0: + version += '.' + version += str(j) + except: + print_exc(file=sys.stderr) + return [client,version] diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/bencode.py b/instrumentation/next-share/BaseLib/Core/BitTornado/bencode.py new file mode 100644 index 0000000..c8ff912 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/bencode.py @@ -0,0 +1,351 @@ +# Written by Petru Paler, Uoti Urpala, Ross Cohen and John Hoffman +# see LICENSE.txt for license information + +from types import IntType, LongType, StringType, ListType, TupleType, DictType +try: + from types import BooleanType +except ImportError: + BooleanType = None +try: + from types import UnicodeType +except ImportError: + UnicodeType = None + +from traceback import print_exc,print_stack +import sys + +DEBUG = False + +def decode_int(x, f): + f += 1 + newf = x.index('e', f) + try: + n = int(x[f:newf]) + except: + n = long(x[f:newf]) + if x[f] == '-': + if x[f + 1] == '0': + raise ValueError + elif x[f] == '0' and newf != f+1: + raise ValueError + return (n, newf+1) + +def decode_string(x, f): + colon = x.index(':', f) + try: + n = int(x[f:colon]) + except (OverflowError, ValueError): + n = long(x[f:colon]) + if x[f] == '0' and colon != f+1: + raise ValueError + colon += 1 + return (x[colon:colon+n], colon+n) + +def decode_unicode(x, f): + s, f = decode_string(x, f+1) + return (s.decode('UTF-8'), f) + +def decode_list(x, f): + r, f = [], f+1 + while x[f] != 'e': + v, f = decode_func[x[f]](x, f) + r.append(v) + return (r, f + 1) + +def decode_dict(x, f): + r, f = {}, f+1 + lastkey = None + while x[f] != 'e': + k, f = decode_string(x, f) + # Arno, 2008-09-12: uTorrent 1.8 violates the bencoding spec, its keys + # in an EXTEND handshake message are not sorted. Be liberal in what we + # receive: + ##if lastkey >= k: + ## raise ValueError + lastkey = k + r[k], f = decode_func[x[f]](x, f) + return (r, f + 1) + +decode_func = {} +decode_func['l'] = decode_list +decode_func['d'] = decode_dict +decode_func['i'] = decode_int +decode_func['0'] = decode_string +decode_func['1'] = decode_string +decode_func['2'] = decode_string +decode_func['3'] = decode_string +decode_func['4'] = decode_string +decode_func['5'] = decode_string +decode_func['6'] = decode_string +decode_func['7'] = decode_string +decode_func['8'] = decode_string +decode_func['9'] = decode_string +#decode_func['u'] = decode_unicode + +def bdecode(x, sloppy = 0): + try: + r, l = decode_func[x[0]](x, 0) +# except (IndexError, KeyError): + except (IndexError, KeyError, ValueError): + if DEBUG: + print_exc() + raise ValueError, "bad bencoded data" + if not sloppy and l != len(x): + raise ValueError, "bad bencoded data" + return r + +def test_bdecode(): + try: + bdecode('0:0:') + assert 0 + except ValueError: + pass + try: + bdecode('ie') + assert 0 + except ValueError: + pass + try: + bdecode('i341foo382e') + assert 0 + except ValueError: + pass + assert bdecode('i4e') == 4L + assert bdecode('i0e') == 0L + assert bdecode('i123456789e') == 123456789L + assert bdecode('i-10e') == -10L + try: + bdecode('i-0e') + assert 0 + except ValueError: + pass + try: + bdecode('i123') + assert 0 + except ValueError: + pass + try: + bdecode('') + assert 0 + except ValueError: + pass + try: + bdecode('i6easd') + assert 0 + except ValueError: + pass + try: + bdecode('35208734823ljdahflajhdf') + assert 0 + except ValueError: + pass + try: + bdecode('2:abfdjslhfld') + assert 0 + except ValueError: + pass + assert bdecode('0:') == '' + assert bdecode('3:abc') == 'abc' + assert bdecode('10:1234567890') == '1234567890' + try: + bdecode('02:xy') + assert 0 + except ValueError: + pass + try: + bdecode('l') + assert 0 + except ValueError: + pass + assert bdecode('le') == [] + try: + bdecode('leanfdldjfh') + assert 0 + except ValueError: + pass + assert bdecode('l0:0:0:e') == ['', '', ''] + try: + bdecode('relwjhrlewjh') + assert 0 + except ValueError: + pass + assert bdecode('li1ei2ei3ee') == [1, 2, 3] + assert bdecode('l3:asd2:xye') == ['asd', 'xy'] + assert bdecode('ll5:Alice3:Bobeli2ei3eee') == [['Alice', 'Bob'], [2, 3]] + try: + bdecode('d') + assert 0 + except ValueError: + pass + try: + bdecode('defoobar') + assert 0 + except ValueError: + pass + assert bdecode('de') == {} + assert bdecode('d3:agei25e4:eyes4:bluee') == {'age': 25, 'eyes': 'blue'} + assert bdecode('d8:spam.mp3d6:author5:Alice6:lengthi100000eee') == {'spam.mp3': {'author': 'Alice', 'length': 100000}} + try: + bdecode('d3:fooe') + assert 0 + except ValueError: + pass + try: + bdecode('di1e0:e') + assert 0 + except ValueError: + pass + try: + bdecode('d1:b0:1:a0:e') + assert 0 + except ValueError: + pass + try: + bdecode('d1:a0:1:a0:e') + assert 0 + except ValueError: + pass + try: + bdecode('i03e') + assert 0 + except ValueError: + pass + try: + bdecode('l01:ae') + assert 0 + except ValueError: + pass + try: + bdecode('9999:x') + assert 0 + except ValueError: + pass + try: + bdecode('l0:') + assert 0 + except ValueError: + pass + try: + bdecode('d0:0:') + assert 0 + except ValueError: + pass + try: + bdecode('d0:') + assert 0 + except ValueError: + pass + +bencached_marker = [] + +class Bencached: + def __init__(self, s): + self.marker = bencached_marker + self.bencoded = s + +BencachedType = type(Bencached('')) # insufficient, but good as a filter + +def encode_bencached(x, r): + assert x.marker == bencached_marker + r.append(x.bencoded) + +def encode_int(x, r): + r.extend(('i', str(x), 'e')) + +def encode_bool(x, r): + encode_int(int(x), r) + +def encode_string(x, r): + r.extend((str(len(x)), ':', x)) + +def encode_unicode(x, r): + #r.append('u') + encode_string(x.encode('UTF-8'), r) + +def encode_list(x, r): + r.append('l') + for e in x: + encode_func[type(e)](e, r) + r.append('e') + +def encode_dict(x, r): + r.append('d') + ilist = x.items() + ilist.sort() + for k, v in ilist: + + if DEBUG: + print >>sys.stderr,"bencode: Encoding",`k`,`v` + + try: + r.extend((str(len(k)), ':', k)) + except: + print >> sys.stderr, "k: %s" % k + raise + + encode_func[type(v)](v, r) + r.append('e') + +encode_func = {} +encode_func[BencachedType] = encode_bencached +encode_func[IntType] = encode_int +encode_func[LongType] = encode_int +encode_func[StringType] = encode_string +encode_func[ListType] = encode_list +encode_func[TupleType] = encode_list +encode_func[DictType] = encode_dict +if BooleanType: + encode_func[BooleanType] = encode_bool +# Arno, 2010-01-27: No more implicit Unicode support. +# We should disable this now and then to see if the higher layers properly +# UTF-8 encode their fields before calling bencode +if UnicodeType: + encode_func[UnicodeType] = encode_unicode + +def bencode(x): + r = [] + try: + encode_func[type(x)](x, r) + except: + print >>sys.stderr,"bencode: *** error *** could not encode type %s (value: %s)" % (type(x), x) + print_stack() + + print_exc() + assert 0 + try: + return ''.join(r) + except: + if DEBUG: + print >>sys.stderr,"bencode: join error",x + for elem in r: + print >>sys.stderr,"elem",elem,"has type",type(elem) + print_exc() + return '' + +def test_bencode(): + assert bencode(4) == 'i4e' + assert bencode(0) == 'i0e' + assert bencode(-10) == 'i-10e' + assert bencode(12345678901234567890L) == 'i12345678901234567890e' + assert bencode('') == '0:' + assert bencode('abc') == '3:abc' + assert bencode('1234567890') == '10:1234567890' + assert bencode([]) == 'le' + assert bencode([1, 2, 3]) == 'li1ei2ei3ee' + assert bencode([['Alice', 'Bob'], [2, 3]]) == 'll5:Alice3:Bobeli2ei3eee' + assert bencode({}) == 'de' + assert bencode({'age': 25, 'eyes': 'blue'}) == 'd3:agei25e4:eyes4:bluee' + assert bencode({'spam.mp3': {'author': 'Alice', 'length': 100000}}) == 'd8:spam.mp3d6:author5:Alice6:lengthi100000eee' + try: + bencode({1: 'foo'}) + assert 0 + except AssertionError: + pass + + +try: + import psyco + psyco.bind(bdecode) + psyco.bind(bencode) +except ImportError: + pass diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/bitfield.py b/instrumentation/next-share/BaseLib/Core/BitTornado/bitfield.py new file mode 100644 index 0000000..e54d5de --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/bitfield.py @@ -0,0 +1,226 @@ +# Written by Bram Cohen, Uoti Urpala, and John Hoffman +# see LICENSE.txt for license information + +import sys + +try: + True +except: + True = 1 + False = 0 + bool = lambda x: not not x + +try: + sum([1]) + negsum = lambda a: len(a) - sum(a) +except: + negsum = lambda a: reduce(lambda x, y: x + (not y), a, 0) + +def _int_to_booleans(x): + r = [] + for i in range(8): + r.append(bool(x & 0x80)) + x <<= 1 + return tuple(r) + +lookup_table = [] +reverse_lookup_table = {} +for i in xrange(256): + x = _int_to_booleans(i) + lookup_table.append(x) + reverse_lookup_table[x] = chr(i) + + +class Bitfield: + def __init__(self, length = None, bitstring = None, copyfrom = None, fromarray = None, calcactiveranges=False): + """ + STBSPEED + @param calcactivetanges Calculate which parts of the piece-space + are non-zero, used an optimization for hooking in whilst live streaming. + Only works in combination with bitstring parameter. + """ + + self.activeranges = [] + + if copyfrom is not None: + self.length = copyfrom.length + self.array = copyfrom.array[:] + self.numfalse = copyfrom.numfalse + return + if length is None: + raise ValueError, "length must be provided unless copying from another array" + self.length = length + if bitstring is not None: + extra = len(bitstring) * 8 - length + if extra < 0 or extra >= 8: + raise ValueError + t = lookup_table + r = [] + + chr0 = chr(0) + inrange = False + startpiece = 0 + countpiece = 0 + for c in bitstring: + r.extend(t[ord(c)]) + + # STBSPEED + if calcactiveranges: + if c != chr0: + # Non-zero value, either start or continuation of range + if inrange: + # Stay in activerange + pass + else: + # Start activerange + startpiece = countpiece + inrange = True + else: + # Zero, either end or continuation of zeroness + if inrange: + # End of activerange + self.activeranges.append((startpiece,countpiece)) + inrange = False + else: + # Stay in zero + pass + countpiece += 8 + + if calcactiveranges: + if inrange: + # activerange ended at end of piece space + self.activeranges.append((startpiece,min(countpiece,self.length-1))) + + if extra > 0: + if r[-extra:] != [0] * extra: + raise ValueError + del r[-extra:] + self.array = r + self.numfalse = negsum(r) + + elif fromarray is not None: + self.array = fromarray + self.numfalse = negsum(self.array) + else: + self.array = [False] * length + self.numfalse = length + + def __setitem__(self, index, val): + val = bool(val) + self.numfalse += self.array[index]-val + self.array[index] = val + + def __getitem__(self, index): + return self.array[index] + + def __len__(self): + return self.length + + def tostring(self): + booleans = self.array + t = reverse_lookup_table + s = len(booleans) % 8 + r = [ t[tuple(booleans[x:x+8])] for x in xrange(0, len(booleans)-s, 8) ] + if s: + r += t[tuple(booleans[-s:] + ([0] * (8-s)))] + return ''.join(r) + + def complete(self): + return not self.numfalse + + def copy(self): + return self.array[:self.length] + + def toboollist(self): + bools = [False] * self.length + for piece in range(0,self.length): + bools[piece] = self.array[piece] + return bools + + def get_active_ranges(self): + # STBSPEED + return self.activeranges + + def get_numtrue(self): + return self.length - self.numfalse + + +def test_bitfield(): + try: + x = Bitfield(7, 'ab') + assert False + except ValueError: + pass + try: + x = Bitfield(7, 'ab') + assert False + except ValueError: + pass + try: + x = Bitfield(9, 'abc') + assert False + except ValueError: + pass + try: + x = Bitfield(0, 'a') + assert False + except ValueError: + pass + try: + x = Bitfield(1, '') + assert False + except ValueError: + pass + try: + x = Bitfield(7, '') + assert False + except ValueError: + pass + try: + x = Bitfield(8, '') + assert False + except ValueError: + pass + try: + x = Bitfield(9, 'a') + assert False + except ValueError: + pass + try: + x = Bitfield(7, chr(1)) + assert False + except ValueError: + pass + try: + x = Bitfield(9, chr(0) + chr(0x40)) + assert False + except ValueError: + pass + assert Bitfield(0, '').tostring() == '' + assert Bitfield(1, chr(0x80)).tostring() == chr(0x80) + assert Bitfield(7, chr(0x02)).tostring() == chr(0x02) + assert Bitfield(8, chr(0xFF)).tostring() == chr(0xFF) + assert Bitfield(9, chr(0) + chr(0x80)).tostring() == chr(0) + chr(0x80) + x = Bitfield(1) + assert x.numfalse == 1 + x[0] = 1 + assert x.numfalse == 0 + x[0] = 1 + assert x.numfalse == 0 + assert x.tostring() == chr(0x80) + x = Bitfield(7) + assert len(x) == 7 + x[6] = 1 + assert x.numfalse == 6 + assert x.tostring() == chr(0x02) + x = Bitfield(8) + x[7] = 1 + assert x.tostring() == chr(1) + x = Bitfield(9) + x[8] = 1 + assert x.numfalse == 8 + assert x.tostring() == chr(0) + chr(0x80) + x = Bitfield(8, chr(0xC4)) + assert len(x) == 8 + assert x.numfalse == 5 + assert x.tostring() == chr(0xC4) diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/clock.py b/instrumentation/next-share/BaseLib/Core/BitTornado/clock.py new file mode 100644 index 0000000..459e1ea --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/clock.py @@ -0,0 +1,30 @@ +# Written by John Hoffman +# see LICENSE.txt for license information + +import sys + +from time import time + +_MAXFORWARD = 100 +_FUDGE = 1 + +class RelativeTime: + def __init__(self): + self.time = time() + self.offset = 0 + + def get_time(self): + t = time() + self.offset + if t < self.time or t > self.time + _MAXFORWARD: + self.time += _FUDGE + self.offset += self.time - t + return self.time + self.time = t + return t + +if sys.platform != 'win32': + _RTIME = RelativeTime() + def clock(): + return _RTIME.get_time() +else: + from time import clock \ No newline at end of file diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/download_bt1.py b/instrumentation/next-share/BaseLib/Core/BitTornado/download_bt1.py new file mode 100644 index 0000000..82a71c0 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/download_bt1.py @@ -0,0 +1,837 @@ +# Written by Bram Cohen and Pawel Garbacki, George Milescu +# see LICENSE.txt for license information + +import sys +import os +import time +from zurllib import urlopen +from urlparse import urlparse +from BT1.btformats import check_message +from BT1.Choker import Choker +from BT1.Storage import Storage +from BT1.StorageWrapper import StorageWrapper +from BT1.FileSelector import FileSelector +from BT1.Uploader import Upload +from BT1.Downloader import Downloader +from BT1.GetRightHTTPDownloader import GetRightHTTPDownloader +from BT1.HoffmanHTTPDownloader import HoffmanHTTPDownloader +from BT1.Connecter import Connecter +from RateLimiter import RateLimiter +from BT1.Encrypter import Encoder +from RawServer import RawServer, autodetect_socket_style +from BT1.Rerequester import Rerequester +from BT1.DownloaderFeedback import DownloaderFeedback +from RateMeasure import RateMeasure +from CurrentRateMeasure import Measure +from BT1.PiecePicker import PiecePicker +from BT1.Statistics import Statistics +from bencode import bencode, bdecode +from BaseLib.Core.Utilities.Crypto import sha +from os import path, makedirs, listdir +from parseargs import parseargs, formatDefinitions, defaultargs +from socket import error as socketerror +from random import seed +from threading import Event +from clock import clock +import re +from traceback import print_exc,print_stack + +from BaseLib.Core.simpledefs import * +from BaseLib.Core.Merkle.merkle import create_fake_hashes +from BaseLib.Core.Utilities.unicode import bin2unicode, dunno2unicode +from BaseLib.Core.Video.PiecePickerStreaming import PiecePickerVOD +# Ric: added svc +from BaseLib.Core.Video.PiecePickerSVC import PiecePickerSVC +from BaseLib.Core.Video.SVCTransporter import SVCTransporter +from BaseLib.Core.Video.VideoOnDemand import MovieOnDemandTransporter +from BaseLib.Core.APIImplementation.maketorrent import torrentfilerec2savefilename,savefilenames2finaldest + +#ProxyService_ +# +from BaseLib.Core.ProxyService.Coordinator import Coordinator +from BaseLib.Core.ProxyService.Helper import Helper +from BaseLib.Core.ProxyService.RatePredictor import ExpSmoothRatePredictor +import sys +from traceback import print_exc,print_stack +# +#_ProxyService + +try: + True +except: + True = 1 + False = 0 + +DEBUG = False + +class BT1Download: + def __init__(self, statusfunc, finfunc, errorfunc, excfunc, logerrorfunc, doneflag, + config, response, infohash, id, rawserver, get_extip_func, port, + videoanalyserpath): + self.statusfunc = statusfunc + self.finfunc = finfunc + self.errorfunc = errorfunc + self.excfunc = excfunc + self.logerrorfunc = logerrorfunc + self.doneflag = doneflag + self.config = config + self.response = response + self.infohash = infohash + self.myid = id + self.rawserver = rawserver + self.get_extip_func = get_extip_func + self.port = port + self.info = self.response['info'] + + # Merkle: Create list of fake hashes. This will be filled if we're an + # initial seeder + if self.info.has_key('root hash') or self.info.has_key('live'): + self.pieces = create_fake_hashes(self.info) + else: + self.pieces = [self.info['pieces'][x:x+20] + for x in xrange(0, len(self.info['pieces']), 20)] + self.len_pieces = len(self.pieces) + self.piecesize = self.info['piece length'] + self.unpauseflag = Event() + self.unpauseflag.set() + self.downloader = None + self.storagewrapper = None + self.fileselector = None + self.super_seeding_active = False + self.filedatflag = Event() + self.spewflag = Event() + self.superseedflag = Event() + self.whenpaused = None + self.finflag = Event() + self.rerequest = None + self.tcp_ack_fudge = config['tcp_ack_fudge'] + # Ric added SVC case + self.svc_video = (config['mode'] == DLMODE_SVC) + self.play_video = (config['mode'] == DLMODE_VOD) + self.am_video_source = bool(config['video_source']) + # i.e. if VOD then G2G, if live then BT + self.use_g2g = self.play_video and not ('live' in response['info']) + self.videoinfo = None + self.videoanalyserpath = videoanalyserpath + self.voddownload = None + + + self.selector_enabled = config['selector_enabled'] + + self.excflag = self.rawserver.get_exception_flag() + self.failed = False + self.checking = False + self.started = False + + # ProxyService_ + # + self.helper = None + self.coordinator = None + self.rate_predictor = None + # + # _ProxyService + + # 2fastbt_ + try: + + if self.config['download_help']: + if DEBUG: + print >>sys.stderr,"BT1Download: coopdl_role is",self.config['coopdl_role'],`self.config['coopdl_coordinator_permid']` + + if self.config['coopdl_role'] == COOPDL_ROLE_COORDINATOR: + from BaseLib.Core.ProxyService.Coordinator import Coordinator + + self.coordinator = Coordinator(self.infohash, self.len_pieces) + #if self.config['coopdl_role'] == COOPDL_ROLE_COORDINATOR or self.config['coopdl_role'] == COOPDL_ROLE_HELPER: + # Arno, 2008-05-20: removed Helper when coordinator, shouldn't need it. + # Reason to remove it is because it messes up PiecePicking: when a + # helper, it calls _next() again after it returned None, probably + # to provoke a RESERVE_PIECE request to the coordinator. + # This change passes test_dlhelp.py + # + if self.config['coopdl_role'] == COOPDL_ROLE_HELPER: + from BaseLib.Core.ProxyService.Helper import Helper + + self.helper = Helper(self.infohash, self.len_pieces, self.config['coopdl_coordinator_permid'], coordinator = self.coordinator) + self.config['coopdl_role'] = '' + self.config['coopdl_coordinator_permid'] = '' + + + if self.am_video_source: + from BaseLib.Core.Video.VideoSource import PiecePickerSource + + self.picker = PiecePickerSource(self.len_pieces, config['rarest_first_cutoff'], + config['rarest_first_priority_cutoff'], helper = self.helper, coordinator = self.coordinator) + elif self.play_video: + # Jan-David: Start video-on-demand service + self.picker = PiecePickerVOD(self.len_pieces, config['rarest_first_cutoff'], + config['rarest_first_priority_cutoff'], helper = self.helper, coordinator = self.coordinator, piecesize=self.piecesize) + elif self.svc_video: + # Ric: Start SVC VoD service TODO + self.picker = PiecePickerSVC(self.len_pieces, config['rarest_first_cutoff'], + config['rarest_first_priority_cutoff'], helper = self.helper, coordinator = self.coordinator, piecesize=self.piecesize) + else: + self.picker = PiecePicker(self.len_pieces, config['rarest_first_cutoff'], + config['rarest_first_priority_cutoff'], helper = self.helper, coordinator = self.coordinator) + except: + print_exc() + print >> sys.stderr,"BT1Download: EXCEPTION in __init__ :'" + str(sys.exc_info()) + "' '" +# _2fastbt + + self.choker = Choker(config, rawserver.add_task, + self.picker, self.finflag.isSet) + + #print >>sys.stderr,"download_bt1.BT1Download: play_video is",self.play_video + + def set_videoinfo(self,videoinfo,videostatus): + self.videoinfo = videoinfo + self.videostatus = videostatus + + # Ric: added svc case + if self.play_video or self.svc_video: + self.picker.set_videostatus( self.videostatus ) + + def checkSaveLocation(self, loc): + if self.info.has_key('length'): + return path.exists(loc) + for x in self.info['files']: + if path.exists(path.join(loc, x['path'][0])): + return True + return False + + + def saveAs(self, filefunc, pathfunc = None): + """ Now throws Exceptions """ + def make(f, forcedir = False): + if not forcedir: + f = path.split(f)[0] + if f != '' and not path.exists(f): + makedirs(f) + + if self.info.has_key('length'): + file_length = self.info['length'] + file = filefunc(self.info['name'], file_length, + self.config['saveas'], False) + # filefunc throws exc if filename gives IOError + + make(file) + files = [(file, file_length)] + else: + file_length = 0L + for x in self.info['files']: + file_length += x['length'] + file = filefunc(self.info['name'], file_length, + self.config['saveas'], True) + # filefunc throws exc if filename gives IOError + + # if this path exists, and no files from the info dict exist, we assume it's a new download and + # the user wants to create a new directory with the default name + existing = 0 + if path.exists(file): + if not path.isdir(file): + raise IOError(file + 'is not a dir') + if listdir(file): # if it's not empty + for x in self.info['files']: + savepath1 = torrentfilerec2savefilename(x,1) + if path.exists(path.join(file, savepath1)): + existing = 1 + if not existing: + try: + file = path.join(file, self.info['name']) + except UnicodeDecodeError: + file = path.join(file, dunno2unicode(self.info['name'])) + if path.exists(file) and not path.isdir(file): + if file.endswith('.torrent') or file.endswith(TRIBLER_TORRENT_EXT): + (prefix,ext) = os.path.splitext(file) + file = prefix + if path.exists(file) and not path.isdir(file): + raise IOError("Can't create dir - " + self.info['name']) + make(file, True) + + # alert the UI to any possible change in path + if pathfunc != None: + pathfunc(file) + + files = [] + for x in self.info['files']: + savepath = torrentfilerec2savefilename(x) + full = savefilenames2finaldest(file,savepath) + # Arno: TODO: this sometimes gives too long filenames for + # Windows. When fixing this take into account that + # Download.get_dest_files() should still produce the same + # filenames as your modifications here. + files.append((full, x['length'])) + make(full) + + self.filename = file + self.files = files + self.datalength = file_length + + if DEBUG: + print >>sys.stderr,"BT1Download: saveas returning ",`file`,"self.files is",`self.files` + + return file + + def getFilename(self): + return self.filename + + def get_dest(self,index): + return self.files[index][0] + + def get_datalength(self): + return self.datalength + + def _finished(self): + self.finflag.set() + try: + self.storage.set_readonly() + except (IOError, OSError), e: + self.errorfunc('trouble setting readonly at end - ' + str(e)) + if self.superseedflag.isSet(): + self._set_super_seed() + self.choker.set_round_robin_period( + max( self.config['round_robin_period'], + self.config['round_robin_period'] * + self.info['piece length'] / 200000 ) ) + self.rerequest_complete() + self.finfunc() + + def _data_flunked(self, amount, index): + self.ratemeasure_datarejected(amount) + if not self.doneflag.isSet(): + self.logerrorfunc('piece %d failed hash check, re-downloading it' % index) + + def _piece_from_live_source(self,index,data): + if self.videostatus.live_streaming and self.voddownload is not None: + return self.voddownload.piece_from_live_source(index,data) + else: + return True + + def _failed(self, reason): + self.failed = True + self.doneflag.set() + if reason is not None: + self.errorfunc(reason) + + + def initFiles(self, old_style = False, statusfunc = None, resumedata = None): + """ Now throws exceptions """ + if self.doneflag.isSet(): + return None + if not statusfunc: + statusfunc = self.statusfunc + + disabled_files = None + if self.selector_enabled: + self.priority = self.config['priority'] + if self.priority: + try: + self.priority = self.priority.split(',') + assert len(self.priority) == len(self.files) + self.priority = [int(p) for p in self.priority] + for p in self.priority: + assert p >= -1 + assert p <= 2 + except: + raise ValueError('bad priority list given, ignored') + self.priority = None + try: + disabled_files = [x == -1 for x in self.priority] + except: + pass + + self.storage = Storage(self.files, self.info['piece length'], + self.doneflag, self.config, disabled_files) + + # Merkle: Are we dealing with a Merkle torrent y/n? + if self.info.has_key('root hash'): + root_hash = self.info['root hash'] + else: + root_hash = None + self.storagewrapper = StorageWrapper(self.videoinfo, self.storage, self.config['download_slice_size'], + self.pieces, self.info['piece length'], root_hash, + self._finished, self._failed, + statusfunc, self.doneflag, self.config['check_hashes'], + self._data_flunked, self._piece_from_live_source, self.rawserver.add_task, + self.config, self.unpauseflag) + + if self.selector_enabled: + self.fileselector = FileSelector(self.files, self.info['piece length'], + None, + self.storage, self.storagewrapper, + self.rawserver.add_task, + self._failed) + + if resumedata: + self.fileselector.unpickle(resumedata) + + self.checking = True + if old_style: + return self.storagewrapper.old_style_init() + return self.storagewrapper.initialize + + + def _make_upload(self, connection, ratelimiter, totalup): + return Upload(connection, ratelimiter, totalup, + self.choker, self.storagewrapper, self.picker, + self.config) + + def _kick_peer(self, connection): + def k(connection = connection): + connection.close() + self.rawserver.add_task(k, 0) + + def _ban_peer(self, ip): + self.encoder_ban(ip) + + def _received_raw_data(self, x): + if self.tcp_ack_fudge: + x = int(x*self.tcp_ack_fudge) + self.ratelimiter.adjust_sent(x) +# self.upmeasure.update_rate(x) + + def _received_data(self, x): + self.downmeasure.update_rate(x) + self.ratemeasure.data_came_in(x) + + def _received_http_data(self, x): + self.downmeasure.update_rate(x) + self.ratemeasure.data_came_in(x) + self.downloader.external_data_received(x) + + def _cancelfunc(self, pieces): + self.downloader.cancel_piece_download(pieces) + self.ghttpdownloader.cancel_piece_download(pieces) + self.hhttpdownloader.cancel_piece_download(pieces) + def _reqmorefunc(self, pieces): + self.downloader.requeue_piece_download(pieces) + + def startEngine(self, ratelimiter = None, vodeventfunc = None): + + if DEBUG: + print >>sys.stderr,"BT1Download: startEngine",`self.info['name']` + + if self.doneflag.isSet(): + return + + self.checking = False + + # Arno, 2010-08-11: STBSPEED: if at all, loop only over pieces I have, + # not piece range. + completeondisk = (self.storagewrapper.get_amount_left() == 0) + if DEBUG: + print >>sys.stderr,"BT1Download: startEngine: complete on disk?",completeondisk,"found",len(self.storagewrapper.get_pieces_on_disk_at_startup()) + self.picker.fast_initialize(completeondisk) + if not completeondisk: + for i in self.storagewrapper.get_pieces_on_disk_at_startup(): # empty when completeondisk + self.picker.complete(i) + + self.upmeasure = Measure(self.config['max_rate_period'], + self.config['upload_rate_fudge']) + self.downmeasure = Measure(self.config['max_rate_period']) + + if ratelimiter: + self.ratelimiter = ratelimiter + else: + self.ratelimiter = RateLimiter(self.rawserver.add_task, + self.config['upload_unit_size'], + self.setConns) + self.ratelimiter.set_upload_rate(self.config['max_upload_rate']) + + self.ratemeasure = RateMeasure() + self.ratemeasure_datarejected = self.ratemeasure.data_rejected + + self.downloader = Downloader(self.infohash, self.storagewrapper, self.picker, + self.config['request_backlog'], self.config['max_rate_period'], + self.len_pieces, self.config['download_slice_size'], + self._received_data, self.config['snub_time'], self.config['auto_kick'], + self._kick_peer, self._ban_peer, scheduler = self.rawserver.add_task) + self.downloader.set_download_rate(self.config['max_download_rate']) + + self.picker.set_downloader(self.downloader) +# 2fastbt_ + if self.coordinator is not None: + self.coordinator.set_downloader(self.downloader) + + self.connecter = Connecter(self.response, self._make_upload, self.downloader, self.choker, + self.len_pieces, self.piecesize, self.upmeasure, self.config, + self.ratelimiter, self.info.has_key('root hash'), + self.rawserver.add_task, self.coordinator, self.helper, self.get_extip_func, self.port, self.use_g2g,self.infohash,self.response.get('announce',None),self.info.has_key('live')) +# _2fastbt + self.encoder = Encoder(self.connecter, self.rawserver, + self.myid, self.config['max_message_length'], self.rawserver.add_task, + self.config['keepalive_interval'], self.infohash, + self._received_raw_data, self.config) + self.encoder_ban = self.encoder.ban + if "initial peers" in self.response: + if DEBUG: + print >> sys.stderr, "BT1Download: startEngine: Using initial peers", self.response["initial peers"] + self.encoder.start_connections([(address, 0) for address in self.response["initial peers"]]) +#--- 2fastbt_ + if DEBUG: + print str(self.config['exclude_ips']) + for ip in self.config['exclude_ips']: + if DEBUG: + print >>sys.stderr,"BT1Download: startEngine: Banning ip: " + str(ip) + self.encoder_ban(ip) + + if self.helper is not None: + from BaseLib.Core.ProxyService.RatePredictor import ExpSmoothRatePredictor + + self.helper.set_encoder(self.encoder) + self.rate_predictor = ExpSmoothRatePredictor(self.rawserver, + self.downmeasure, self.config['max_download_rate']) + self.picker.set_rate_predictor(self.rate_predictor) + self.rate_predictor.update() +# _2fastbt + + self.ghttpdownloader = GetRightHTTPDownloader(self.storagewrapper, self.picker, + self.rawserver, self.finflag, self.logerrorfunc, self.downloader, + self.config['max_rate_period'], self.infohash, self._received_http_data, + self.connecter.got_piece) + if self.response.has_key('url-list') and not self.finflag.isSet(): + for u in self.response['url-list']: + self.ghttpdownloader.make_download(u) + + self.hhttpdownloader = HoffmanHTTPDownloader(self.storagewrapper, self.picker, + self.rawserver, self.finflag, self.logerrorfunc, self.downloader, + self.config['max_rate_period'], self.infohash, self._received_http_data, + self.connecter.got_piece) + if self.response.has_key('httpseeds') and not self.finflag.isSet(): + for u in self.response['httpseeds']: + self.hhttpdownloader.make_download(u) + + if self.selector_enabled: + self.fileselector.tie_in(self.picker, self._cancelfunc, self._reqmorefunc) + if self.priority: + self.fileselector.set_priorities_now(self.priority) + # erase old data once you've started modifying it + + # Ric: added svc case TODO check with play_video + if self.svc_video: + if self.picker.am_I_complete(): + # TODO do something + pass + self.voddownload = SVCTransporter(self,self.videostatus,self.videoinfo,self.videoanalyserpath,vodeventfunc) + + elif self.play_video: + if self.picker.am_I_complete(): + if DEBUG: + print >>sys.stderr,"BT1Download: startEngine: VOD requested, but file complete on disk",self.videoinfo + # Added bitrate parameter for html5 playback + vodeventfunc( self.videoinfo, VODEVENT_START, { + "complete": True, + "filename": self.videoinfo["outpath"], + "mimetype": self.videoinfo["mimetype"], + "stream": None, + "length": self.videostatus.selected_movie["size"], + "bitrate": self.videoinfo["bitrate"] + } ) + else: + if DEBUG: + print >>sys.stderr,"BT1Download: startEngine: Going into VOD mode",self.videoinfo + + self.voddownload = MovieOnDemandTransporter(self,self.videostatus,self.videoinfo,self.videoanalyserpath,vodeventfunc,self.ghttpdownloader) + elif DEBUG: + print >>sys.stderr,"BT1Download: startEngine: Going into standard mode" + + if self.am_video_source: + from BaseLib.Core.Video.VideoSource import VideoSourceTransporter,RateLimitedVideoSourceTransporter + + if DEBUG: + print >>sys.stderr,"BT1Download: startEngine: Acting as VideoSource" + if self.config['video_ratelimit']: + self.videosourcetransporter = RateLimitedVideoSourceTransporter(self.config['video_ratelimit'],self.config['video_source'],self,self.config['video_source_authconfig'],self.config['video_source_restartstatefilename']) + else: + self.videosourcetransporter = VideoSourceTransporter(self.config['video_source'],self,self.config['video_source_authconfig'],self.config['video_source_restartstatefilename']) + self.videosourcetransporter.start() + elif DEBUG: + print >>sys.stderr,"BT1Download: startEngine: Not a VideoSource" + + if not self.doneflag.isSet(): + self.started = True + + def rerequest_complete(self): + if self.rerequest: + self.rerequest.announce(1) + + def rerequest_stopped(self): + if self.rerequest: + self.rerequest.announce(2) + + def rerequest_lastfailed(self): + if self.rerequest: + return self.rerequest.last_failed + return False + + def startRerequester(self, paused=False): + # RePEX: + # Moved the creation of the Rerequester to a separate method, + # allowing us to only create the Rerequester without starting + # it from SingleDownload. + if self.rerequest is None: + self.rerequest = self.createRerequester() + self.encoder.set_rerequester(self.rerequest) + + if not paused: + self.rerequest.start() + + + def createRerequester(self, callback=None): + if self.response.has_key ('announce-list'): + trackerlist = self.response['announce-list'] + for tier in range(len(trackerlist)): + for t in range(len(trackerlist[tier])): + trackerlist[tier][t] = bin2unicode(trackerlist[tier][t]) + else: + tracker = bin2unicode(self.response.get('announce', '')) + if tracker: + trackerlist = [[tracker]] + else: + trackerlist = [[]] + + if callback is None: + callback = self.encoder.start_connections + + rerequest = Rerequester(trackerlist, self.config['rerequest_interval'], + self.rawserver.add_task,self.connecter.how_many_connections, + self.config['min_peers'], callback, + self.rawserver.add_task, self.storagewrapper.get_amount_left, + self.upmeasure.get_total, self.downmeasure.get_total, self.port, self.config['ip'], + self.myid, self.infohash, self.config['http_timeout'], + self.logerrorfunc, self.excfunc, self.config['max_initiate'], + self.doneflag, self.upmeasure.get_rate, self.downmeasure.get_rate, + self.unpauseflag,self.config) + + if self.play_video and self.voddownload is not None: + rerequest.add_notifier( lambda x: self.voddownload.peers_from_tracker_report( len( x ) ) ) + + return rerequest + + + def _init_stats(self): + self.statistics = Statistics(self.upmeasure, self.downmeasure, + self.connecter, self.ghttpdownloader, self.hhttpdownloader, self.ratelimiter, + self.rerequest_lastfailed, self.filedatflag) + if self.info.has_key('files'): + self.statistics.set_dirstats(self.files, self.info['piece length']) + + def autoStats(self, displayfunc = None): + if not displayfunc: + displayfunc = self.statusfunc + + self._init_stats() + DownloaderFeedback(self.choker, self.ghttpdownloader, self.hhttpdownloader, self.rawserver.add_task, + self.upmeasure.get_rate, self.downmeasure.get_rate, + self.ratemeasure, self.storagewrapper.get_stats, + self.datalength, self.finflag, self.spewflag, self.statistics, + displayfunc, self.config['display_interval'], + infohash = self.infohash,voddownload=self.voddownload) + + def startStats(self): + self._init_stats() + self.spewflag.set() # start collecting peer cache + d = DownloaderFeedback(self.choker, self.ghttpdownloader, self.hhttpdownloader, self.rawserver.add_task, + self.upmeasure.get_rate, self.downmeasure.get_rate, + self.ratemeasure, self.storagewrapper.get_stats, + self.datalength, self.finflag, self.spewflag, self.statistics, + infohash = self.infohash,voddownload=self.voddownload) + return d.gather + + + def getPortHandler(self): + return self.encoder + + + def checkpoint(self): # Added by Arno + """ Called by network thread """ + if self.fileselector and self.started: + # self.fileselector.finish() does nothing at the moment, so as + # long as the network thread calls this, it should be OK. + return self.fileselector.pickle() + else: + return None + + def shutdown(self): + if self.checking or self.started: + self.storagewrapper.sync() + self.storage.close() + self.rerequest_stopped() + resumedata = None + if self.fileselector and self.started: + if not self.failed: + self.fileselector.finish() + resumedata = self.fileselector.pickle() + if self.voddownload is not None: + self.voddownload.stop() + return resumedata + + + def setUploadRate(self, rate, networkcalling=False): + try: + def s(self = self, rate = rate): + if DEBUG: + print >>sys.stderr,"BT1Download: set max upload to",rate + self.config['max_upload_rate'] = rate + self.ratelimiter.set_upload_rate(rate) + if networkcalling: + s() + else: + self.rawserver.add_task(s) + except AttributeError: + pass + + def setConns(self, conns, conns2 = None,networkcalling=False): + if not conns2: + conns2 = conns + try: + def s(self = self, conns = conns, conns2 = conns2): + self.config['min_uploads'] = conns + self.config['max_uploads'] = conns2 + if (conns > 30): + self.config['max_initiate'] = conns + 10 + if networkcalling: + s() + else: + self.rawserver.add_task(s) + except AttributeError: + pass + + def setDownloadRate(self, rate,networkcalling=False): + try: + def s(self = self, rate = rate): + self.config['max_download_rate'] = rate + self.downloader.set_download_rate(rate) + if networkcalling: + s() + else: + self.rawserver.add_task(s) + except AttributeError: + pass + + def startConnection(self, ip, port, id): + self.encoder._start_connection((ip, port), id) + + def _startConnection(self, ipandport, id): + self.encoder._start_connection(ipandport, id) + + def setInitiate(self, initiate,networkcalling=False): + try: + def s(self = self, initiate = initiate): + self.config['max_initiate'] = initiate + if networkcalling: + s() + else: + self.rawserver.add_task(s) + except AttributeError: + pass + + def setMaxConns(self,nconns,networkcalling=False): + try: + def s(self = self, nconns = nconns): + self.config['max_connections'] = nconns + if networkcalling: + s() + else: + self.rawserver.add_task(s) + except AttributeError: + pass + + + def getConfig(self): + return self.config + + def reannounce(self, special = None): + try: + def r(self = self, special = special): + if special is None: + self.rerequest.announce() + else: + self.rerequest.announce(specialurl = special) + self.rawserver.add_task(r) + except AttributeError: + pass + + def getResponse(self): + try: + return self.response + except: + return None + +# def Pause(self): +# try: +# if self.storagewrapper: +# self.rawserver.add_task(self._pausemaker, 0) +# except: +# return False +# self.unpauseflag.clear() +# return True +# +# def _pausemaker(self): +# self.whenpaused = clock() +# self.unpauseflag.wait() # sticks a monkey wrench in the main thread +# +# def Unpause(self): +# self.unpauseflag.set() +# if self.whenpaused and clock()-self.whenpaused > 60: +# def r(self = self): +# self.rerequest.announce(3) # rerequest automatically if paused for >60 seconds +# self.rawserver.add_task(r) + + def Pause(self): + if not self.storagewrapper: + return False + self.unpauseflag.clear() + self.rawserver.add_task(self.onPause) + return True + + def onPause(self): + self.whenpaused = clock() + if not self.downloader: + return + self.downloader.pause(True) + self.encoder.pause(True) + self.choker.pause(True) + + def Unpause(self): + self.unpauseflag.set() + self.rawserver.add_task(self.onUnpause) + + def onUnpause(self): + if not self.downloader: + return + self.downloader.pause(False) + self.encoder.pause(False) + self.choker.pause(False) + if self.rerequest and self.whenpaused and clock()-self.whenpaused > 60: + self.rerequest.announce(3) # rerequest automatically if paused for >60 seconds + + def set_super_seed(self,networkcalling=False): + self.superseedflag.set() + if networkcalling: + self._set_super_seed() + else: + self.rawserver.add_task(self._set_super_seed) + + def _set_super_seed(self): + if not self.super_seeding_active and self.finflag.isSet(): + self.super_seeding_active = True + self.logerrorfunc(' ** SUPER-SEED OPERATION ACTIVE **\n' + + ' please set Max uploads so each peer gets 6-8 kB/s') + def s(self = self): + self.downloader.set_super_seed() + self.choker.set_super_seed() + self.rawserver.add_task(s) + if self.finflag.isSet(): # mode started when already finished + def r(self = self): + self.rerequest.announce(3) # so after kicking everyone off, reannounce + self.rawserver.add_task(r) + + def am_I_finished(self): + return self.finflag.isSet() + + def get_transfer_stats(self): + return self.upmeasure.get_total(), self.downmeasure.get_total() + + def get_moviestreamtransport(self): + return self.voddownload diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/inifile.py b/instrumentation/next-share/BaseLib/Core/BitTornado/inifile.py new file mode 100644 index 0000000..4802f0e --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/inifile.py @@ -0,0 +1,169 @@ +# Written by John Hoffman +# see LICENSE.txt for license information + +__fool_epydoc = 481 +''' +reads/writes a Windows-style INI file +format: + + aa = "bb" + cc = 11 + + [eee] + ff = "gg" + +decodes to: +d = { '': {'aa':'bb','cc':'11'}, 'eee': {'ff':'gg'} } + +the encoder can also take this as input: + +d = { 'aa': 'bb, 'cc': 11, 'eee': {'ff':'gg'} } + +though it will only decode in the above format. Keywords must be strings. +Values that are strings are written surrounded by quotes, and the decoding +routine automatically strips any. +Booleans are written as integers. Anything else aside from string/int/float +may have unpredictable results. +''' + +from traceback import print_exc +from types import DictType, StringType +try: + from types import BooleanType +except ImportError: + BooleanType = None + +try: + True +except: + True = 1 + False = 0 + +DEBUG = False + +def ini_write(f, d, comment=''): + try: + a = {'':{}} + for k, v in d.items(): + assert type(k) == StringType + k = k.lower() + if type(v) == DictType: + if DEBUG: + print 'new section:' +k + if k: + assert not a.has_key(k) + a[k] = {} + aa = a[k] + for kk, vv in v: + assert type(kk) == StringType + kk = kk.lower() + assert not aa.has_key(kk) + if type(vv) == BooleanType: + vv = int(vv) + if type(vv) == StringType: + vv = '"'+vv+'"' + aa[kk] = str(vv) + if DEBUG: + print 'a['+k+']['+kk+'] = '+str(vv) + else: + aa = a[''] + assert not aa.has_key(k) + if type(v) == BooleanType: + v = int(v) + if type(v) == StringType: + v = '"'+v+'"' + aa[k] = str(v) + if DEBUG: + print 'a[\'\']['+k+'] = '+str(v) + r = open(f, 'w') + if comment: + for c in comment.split('\n'): + r.write('# '+c+'\n') + r.write('\n') + l = a.keys() + l.sort() + for k in l: + if k: + r.write('\n['+k+']\n') + aa = a[k] + ll = aa.keys() + ll.sort() + for kk in ll: + r.write(kk+' = '+aa[kk]+'\n') + success = True + except: + if DEBUG: + print_exc() + success = False + try: + r.close() + except: + pass + return success + + +if DEBUG: + def errfunc(lineno, line, err): + print '('+str(lineno)+') '+err+': '+line +else: + errfunc = lambda lineno, line, err: None + +def ini_read(f, errfunc = errfunc): + try: + r = open(f, 'r') + ll = r.readlines() + d = {} + dd = {'':d} + for i in xrange(len(ll)): + l = ll[i] + l = l.strip() + if not l: + continue + if l[0] == '#': + continue + if l[0] == '[': + if l[-1] != ']': + errfunc(i, l, 'syntax error') + continue + l1 = l[1:-1].strip().lower() + if not l1: + errfunc(i, l, 'syntax error') + continue + if dd.has_key(l1): + errfunc(i, l, 'duplicate section') + d = dd[l1] + continue + d = {} + dd[l1] = d + continue + try: + k, v = l.split('=', 1) + except: + try: + k, v = l.split(':', 1) + except: + errfunc(i, l, 'syntax error') + continue + k = k.strip().lower() + v = v.strip() + if len(v) > 1 and ( (v[0] == '"' and v[-1] == '"') or + (v[0] == "'" and v[-1] == "'") ): + v = v[1:-1] + if not k: + errfunc(i, l, 'syntax error') + continue + if d.has_key(k): + errfunc(i, l, 'duplicate entry') + continue + d[k] = v + if DEBUG: + print dd + except: + if DEBUG: + print_exc() + dd = None + try: + r.close() + except: + pass + return dd diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/iprangeparse.py b/instrumentation/next-share/BaseLib/Core/BitTornado/iprangeparse.py new file mode 100644 index 0000000..9304c39 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/iprangeparse.py @@ -0,0 +1,194 @@ +# Written by John Hoffman +# see LICENSE.txt for license information + +from bisect import bisect, insort + +try: + True +except: + True = 1 + False = 0 + bool = lambda x: not not x + + +def to_long_ipv4(ip): + ip = ip.split('.') + if len(ip) != 4: + raise ValueError, "bad address" + b = 0L + for n in ip: + b *= 256 + b += int(n) + return b + + +def to_long_ipv6(ip): + if not ip: + raise ValueError, "bad address" + if ip == '::': # boundary handling + ip = '' + elif ip[:2] == '::': + ip = ip[1:] + elif ip[0] == ':': + raise ValueError, "bad address" + elif ip[-2:] == '::': + ip = ip[:-1] + elif ip[-1] == ':': + raise ValueError, "bad address" + + b = [] + doublecolon = False + for n in ip.split(':'): + if n == '': # double-colon + if doublecolon: + raise ValueError, "bad address" + doublecolon = True + b.append(None) + continue + if n.find('.') >= 0: # IPv4 + n = n.split('.') + if len(n) != 4: + raise ValueError, "bad address" + for i in n: + b.append(int(i)) + continue + n = ('0'*(4-len(n))) + n + b.append(int(n[:2], 16)) + b.append(int(n[2:], 16)) + bb = 0L + for n in b: + if n is None: + for i in xrange(17-len(b)): + bb *= 256 + continue + bb *= 256 + bb += n + return bb + +ipv4addrmask = 65535L*256*256*256*256 + +class IP_List: + def __init__(self): + self.ipv4list = [] # starts of ranges + self.ipv4dict = {} # start: end of ranges + self.ipv6list = [] # " + self.ipv6dict = {} # " + + def __nonzero__(self): + return bool(self.ipv4list or self.ipv6list) + + + def append(self, ip_beg, ip_end = None): + if ip_end is None: + ip_end = ip_beg + else: + assert ip_beg <= ip_end + if ip_beg.find(':') < 0: # IPv4 + ip_beg = to_long_ipv4(ip_beg) + ip_end = to_long_ipv4(ip_end) + l = self.ipv4list + d = self.ipv4dict + else: + ip_beg = to_long_ipv6(ip_beg) + ip_end = to_long_ipv6(ip_end) + bb = ip_beg % (256*256*256*256) + if bb == ipv4addrmask: + ip_beg -= bb + ip_end -= bb + l = self.ipv4list + d = self.ipv4dict + else: + l = self.ipv6list + d = self.ipv6dict + + pos = bisect(l, ip_beg)-1 + done = pos < 0 + while not done: + p = pos + while p < len(l): + range_beg = l[p] + if range_beg > ip_end+1: + done = True + break + range_end = d[range_beg] + if range_end < ip_beg-1: + p += 1 + if p == len(l): + done = True + break + continue + # if neither of the above conditions is true, the ranges overlap + ip_beg = min(ip_beg, range_beg) + ip_end = max(ip_end, range_end) + del l[p] + del d[range_beg] + break + + insort(l, ip_beg) + d[ip_beg] = ip_end + + + def includes(self, ip): + if not (self.ipv4list or self.ipv6list): + return False + if ip.find(':') < 0: # IPv4 + ip = to_long_ipv4(ip) + l = self.ipv4list + d = self.ipv4dict + else: + ip = to_long_ipv6(ip) + bb = ip % (256*256*256*256) + if bb == ipv4addrmask: + ip -= bb + l = self.ipv4list + d = self.ipv4dict + else: + l = self.ipv6list + d = self.ipv6dict + for ip_beg in l[bisect(l, ip)-1:]: + if ip == ip_beg: + return True + ip_end = d[ip_beg] + if ip > ip_beg and ip <= ip_end: + return True + return False + + + # reads a list from a file in the format 'whatever:whatever:ip-ip' + # (not IPv6 compatible at all) + def read_rangelist(self, file): + f = open(file, 'r') + while 1: + line = f.readline() + if not line: + break + line = line.strip() + if not line or line[0] == '#': + continue + line = line.split(':')[-1] + try: + ip1, ip2 = line.split('-') + except: + ip1 = line + ip2 = line + try: + self.append(ip1.strip(), ip2.strip()) + except: + print '*** WARNING *** could not parse IP range: '+line + f.close() + +def is_ipv4(ip): + return ip.find(':') < 0 + +def is_valid_ip(ip): + try: + if is_ipv4(ip): + a = ip.split('.') + assert len(a) == 4 + for i in a: + chr(int(i)) + return True + to_long_ipv6(ip) + return True + except: + return False diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/natpunch.py b/instrumentation/next-share/BaseLib/Core/BitTornado/natpunch.py new file mode 100644 index 0000000..4cebfb6 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/natpunch.py @@ -0,0 +1,381 @@ +# Written by John Hoffman, Arno Bakker +# derived from NATPortMapping.py by Yejun Yang +# and from example code by Myers Carpenter +# see LICENSE.txt for license information + +import sys +import socket +from traceback import print_exc +from subnetparse import IP_List +from clock import clock +from __init__ import createPeerID + +from BaseLib.Core.NATFirewall.upnp import UPnPPlatformIndependent,UPnPError +from BaseLib.Core.NATFirewall.guessip import get_my_wan_ip +try: + True +except: + True = 1 + False = 0 + +DEBUG = False + +EXPIRE_CACHE = 30 # seconds +ID = "BT-"+createPeerID()[-4:] + +try: + import pythoncom, win32com.client + win32_imported = 1 +except ImportError: + if DEBUG and (sys.platform == 'win32'): + print >>sys.stderr,"natpunch: ERROR: pywin32 package not installed, UPnP mode 2 won't work now" + win32_imported = 0 + +UPnPError = UPnPError + +class _UPnP1: # derived from Myers Carpenter's code + # seems to use the machine's local UPnP + # system for its operation. Runs fairly fast + + def __init__(self): + self.map = None + self.last_got_map = -10e10 + + def _get_map(self): + if self.last_got_map + EXPIRE_CACHE < clock(): + try: + dispatcher = win32com.client.Dispatch("HNetCfg.NATUPnP") + self.map = dispatcher.StaticPortMappingCollection + self.last_got_map = clock() + except: + if DEBUG: + print_exc() + self.map = None + return self.map + + def test(self): + try: + assert self._get_map() # make sure a map was found + success = True + except: + if DEBUG: + print_exc() + success = False + return success + + + def open(self, ip, p, iproto='TCP'): + map = self._get_map() + try: + map.Add(p, iproto, p, ip, True, ID) + if DEBUG: + print >>sys.stderr,'upnp1: succesfully opened port: '+ip+':'+str(p) + success = True + except: + if DEBUG: + print >>sys.stderr,"upnp1: COULDN'T OPEN "+str(p) + print_exc() + success = False + return success + + + def close(self, p, iproto='TCP'): + map = self._get_map() + try: + map.Remove(p, iproto) + success = True + if DEBUG: + print >>sys.stderr,'upnp1: succesfully closed port: '+str(p) + except: + if DEBUG: + print >>sys.stderr,"upnp1: COULDN'T CLOSE "+str(p) + print_exc() + success = False + return success + + + def clean(self, retry = False, iproto='TCP'): + if not win32_imported: + return + try: + map = self._get_map() + ports_in_use = [] + for i in xrange(len(map)): + try: + mapping = map[i] + port = mapping.ExternalPort + prot = str(mapping.Protocol).lower() + desc = str(mapping.Description).lower() + except: + port = None + if port and prot == iproto.lower() and desc[:3] == 'bt-': + ports_in_use.append(port) + success = True + for port in ports_in_use: + try: + map.Remove(port, iproto) + except: + success = False + if not success and not retry: + self.clean(retry = True) + except: + pass + + def get_ext_ip(self): + return None + + +class _UPnP2: # derived from Yejun Yang's code + # apparently does a direct search for UPnP hardware + # may work in some cases where _UPnP1 won't, but is slow + # still need to implement "clean" method + + def __init__(self): + self.services = None + self.last_got_services = -10e10 + + def _get_services(self): + if not self.services or self.last_got_services + EXPIRE_CACHE < clock(): + self.services = [] + try: + f=win32com.client.Dispatch("UPnP.UPnPDeviceFinder") + for t in ( "urn:schemas-upnp-org:service:WANIPConnection:1", + "urn:schemas-upnp-org:service:WANPPPConnection:1" ): + try: + conns = f.FindByType(t, 0) + for c in xrange(len(conns)): + try: + svcs = conns[c].Services + for s in xrange(len(svcs)): + try: + self.services.append(svcs[s]) + except: + if DEBUG: + print_exc() + except: + if DEBUG: + print_exc() + except: + if DEBUG: + print_exc() + except: + if DEBUG: + print_exc() + self.last_got_services = clock() + return self.services + + def test(self): + try: + assert self._get_services() # make sure some services can be found + success = True + except: + success = False + return success + + + def open(self, ip, p, iproto='TCP'): + svcs = self._get_services() + success = False + for s in svcs: + try: + s.InvokeAction('AddPortMapping', ['', p, iproto, p, ip, True, ID, 0], '') + success = True + except: + if DEBUG: + print_exc() + if DEBUG and not success: + print >>sys.stderr,"upnp2: COULDN'T OPEN "+str(p) + print_exc() + return success + + + def close(self, p, iproto='TCP'): + svcs = self._get_services() + success = False + for s in svcs: + try: + s.InvokeAction('DeletePortMapping', ['', p, iproto], '') + success = True + except: + if DEBUG: + print_exc() + if DEBUG and not success: + print >>sys.stderr,"upnp2: COULDN'T CLOSE "+str(p) + print_exc() + return success + + + def get_ext_ip(self): + svcs = self._get_services() + success = None + for s in svcs: + try: + ret = s.InvokeAction('GetExternalIPAddress',[],'') + # With MS Internet Connection Sharing: + # - Good reply is: (None, (u'130.37.168.199',)) + # - When router disconnected from Internet: (None, (u'',)) + if DEBUG: + print >>sys.stderr,"upnp2: GetExternapIPAddress returned",ret + dns = ret[1] + if str(dns[0]) != '': + success = str(dns[0]) + elif DEBUG: + print >>sys.stderr,"upnp2: RETURNED IP ADDRESS EMPTY" + except: + if DEBUG: + print_exc() + if DEBUG and not success: + print >>sys.stderr,"upnp2: COULDN'T GET EXT IP ADDR" + return success + +class _UPnP3: + def __init__(self): + self.u = UPnPPlatformIndependent() + + def test(self): + try: + self.u.discover() + return self.u.found_wanted_services() + except: + if DEBUG: + print_exc() + return False + + def open(self,ip,p,iproto='TCP'): + """ Return False in case of network failure, + Raises UPnPError in case of a properly reported error from the server + """ + try: + self.u.add_port_map(ip,p,iproto=iproto) + return True + except UPnPError,e: + if DEBUG: + print_exc() + raise e + except: + if DEBUG: + print_exc() + return False + + def close(self,p,iproto='TCP'): + """ Return False in case of network failure, + Raises UPnPError in case of a properly reported error from the server + """ + try: + self.u.del_port_map(p,iproto=iproto) + return True + except UPnPError,e: + if DEBUG: + print_exc() + raise e + except: + if DEBUG: + print_exc() + return False + + def get_ext_ip(self): + """ Return False in case of network failure, + Raises UPnPError in case of a properly reported error from the server + """ + try: + return self.u.get_ext_ip() + except UPnPError,e: + if DEBUG: + print_exc() + raise e + except: + if DEBUG: + print_exc() + return None + +class UPnPWrapper: # master holding class + + __single = None + + def __init__(self): + if UPnPWrapper.__single: + raise RuntimeError, "UPnPWrapper is singleton" + UPnPWrapper.__single = self + + self.upnp1 = _UPnP1() + self.upnp2 = _UPnP2() + self.upnp3 = _UPnP3() + self.upnplist = (None, self.upnp1, self.upnp2, self.upnp3) + self.upnp = None + self.local_ip = None + self.last_got_ip = -10e10 + + def getInstance(*args, **kw): + if UPnPWrapper.__single is None: + UPnPWrapper(*args, **kw) + return UPnPWrapper.__single + getInstance = staticmethod(getInstance) + + def register(self,guessed_localip): + self.local_ip = guessed_localip + + def get_ip(self): + if self.last_got_ip + EXPIRE_CACHE < clock(): + if self.local_ip is None: + local_ips = IP_List() + local_ips.set_intranet_addresses() + try: + for info in socket.getaddrinfo(socket.gethostname(), 0, socket.AF_INET): + # exception if socket library isn't recent + self.local_ip = info[4][0] + if local_ips.includes(self.local_ip): + self.last_got_ip = clock() + if DEBUG: + print >>sys.stderr,'upnpX: Local IP found: '+self.local_ip + break + else: + raise ValueError('upnpX: couldn\'t find intranet IP') + except: + self.local_ip = None + if DEBUG: + print >>sys.stderr,'upnpX: Error finding local IP' + print_exc() + return self.local_ip + + def test(self, upnp_type): + if DEBUG: + print >>sys.stderr,'upnpX: testing UPnP type '+str(upnp_type) + if not upnp_type or self.get_ip() is None or (upnp_type <= 2 and not win32_imported): + if DEBUG: + print >>sys.stderr,'upnpX: UPnP not supported' + return 0 + if upnp_type != 3: + pythoncom.CoInitialize() # leave initialized + self.upnp = self.upnplist[upnp_type] # cache this + if self.upnp.test(): + if DEBUG: + print >>sys.stderr,'upnpX: ok' + return upnp_type + if DEBUG: + print >>sys.stderr,'upnpX: tested bad' + return 0 + + def open(self, p, iproto='TCP'): + assert self.upnp, "upnpX: must run UPnP_test() with the desired UPnP access type first" + return self.upnp.open(self.get_ip(), p, iproto=iproto) + + def close(self, p, iproto='TCP'): + assert self.upnp, "upnpX: must run UPnP_test() with the desired UPnP access type first" + return self.upnp.close(p,iproto=iproto) + + def clean(self,iproto='TCP'): + return self.upnp1.clean(iproto=iproto) + + def get_ext_ip(self): + assert self.upnp, "upnpX: must run UPnP_test() with the desired UPnP access type first" + return self.upnp.get_ext_ip() + +if __name__ == '__main__': + ip = get_my_wan_ip() + print >>sys.stderr,"guessed ip",ip + u = UPnPWrapper() + u.register(ip) + print >>sys.stderr,"TEST RETURNED",u.test(3) + print >>sys.stderr,"IGD says my external IP is",u.get_ext_ip() + print >>sys.stderr,"IGD open returned",u.open(6881) + print >>sys.stderr,"IGD close returned",u.close(6881) diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/parseargs.py b/instrumentation/next-share/BaseLib/Core/BitTornado/parseargs.py new file mode 100644 index 0000000..7d05819 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/parseargs.py @@ -0,0 +1,142 @@ +# Written by Bill Bumgarner and Bram Cohen +# see LICENSE.txt for license information + +from types import * +from cStringIO import StringIO + + +def splitLine(line, COLS=80, indent=10): + indent = " " * indent + width = COLS - (len(indent) + 1) + if indent and width < 15: + width = COLS - 2 + indent = " " + s = StringIO() + i = 0 + for word in line.split(): + if i == 0: + s.write(indent+word) + i = len(word) + continue + if i + len(word) >= width: + s.write('\n'+indent+word) + i = len(word) + continue + s.write(' '+word) + i += len(word) + 1 + return s.getvalue() + +def formatDefinitions(options, COLS, presets = {}): + s = StringIO() + for (longname, default, doc) in options: + s.write('--' + longname + ' \n') + default = presets.get(longname, default) + if type(default) in (IntType, LongType): + try: + default = int(default) + except: + pass + if default is not None: + doc += ' (defaults to ' + repr(default) + ')' + s.write(splitLine(doc, COLS, 10)) + s.write('\n\n') + return s.getvalue() + + +def usage(string): + raise ValueError(string) + + +def defaultargs(options): + l = {} + for (longname, default, doc) in options: + if default is not None: + l[longname] = default + return l + + +def parseargs(argv, options, minargs = None, maxargs = None, presets = {}): + config = {} + longkeyed = {} + for option in options: + longname, default, doc = option + longkeyed[longname] = option + config[longname] = default + for longname in presets.keys(): # presets after defaults but before arguments + config[longname] = presets[longname] + options = [] + args = [] + pos = 0 + while pos < len(argv): + if argv[pos][:2] != '--': + args.append(argv[pos]) + pos += 1 + else: + if pos == len(argv) - 1: + usage('parameter passed in at end with no value') + key, value = argv[pos][2:], argv[pos+1] + pos += 2 + if not longkeyed.has_key(key): + usage('unknown key --' + key) + longname, default, doc = longkeyed[key] + try: + t = type(config[longname]) + if t is NoneType or t is StringType: + config[longname] = value + elif t is IntType: + config[longname] = int(value) + elif t is LongType: + config[longname] = long(value) + elif t is FloatType: + config[longname] = float(value) + elif t is BooleanType: + config[longname] = bool(value) + else: + print 'parseargs: unknown type is',t + assert 0 + except ValueError, e: + usage('wrong format of --%s - %s' % (key, str(e))) + for key, value in config.items(): + if value is None: + usage("Option --%s is required." % key) + if minargs is not None and len(args) < minargs: + usage("Must supply at least %d args." % minargs) + if maxargs is not None and len(args) > maxargs: + usage("Too many args - %d max." % maxargs) + return (config, args) + +def test_parseargs(): + assert parseargs(('d', '--a', 'pq', 'e', '--b', '3', '--c', '4.5', 'f'), (('a', 'x', ''), ('b', 1, ''), ('c', 2.3, ''))) == ({'a': 'pq', 'b': 3, 'c': 4.5}, ['d', 'e', 'f']) + assert parseargs([], [('a', 'x', '')]) == ({'a': 'x'}, []) + assert parseargs(['--a', 'x', '--a', 'y'], [('a', '', '')]) == ({'a': 'y'}, []) + try: + parseargs([], [('a', 'x', '')]) + except ValueError: + pass + try: + parseargs(['--a', 'x'], []) + except ValueError: + pass + try: + parseargs(['--a'], [('a', 'x', '')]) + except ValueError: + pass + try: + parseargs([], [], 1, 2) + except ValueError: + pass + assert parseargs(['x'], [], 1, 2) == ({}, ['x']) + assert parseargs(['x', 'y'], [], 1, 2) == ({}, ['x', 'y']) + try: + parseargs(['x', 'y', 'z'], [], 1, 2) + except ValueError: + pass + try: + parseargs(['--a', '2.0'], [('a', 3, '')]) + except ValueError: + pass + try: + parseargs(['--a', 'z'], [('a', 2.1, '')]) + except ValueError: + pass + diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/parsedir.py b/instrumentation/next-share/BaseLib/Core/BitTornado/parsedir.py new file mode 100644 index 0000000..051123c --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/parsedir.py @@ -0,0 +1,162 @@ +# Written by John Hoffman and Uoti Urpala +# see LICENSE.txt for license information +import os +import sys +from traceback import print_exc + +from bencode import bencode, bdecode +from BT1.btformats import check_info +from BaseLib.Core.simpledefs import TRIBLER_TORRENT_EXT +from BaseLib.Core.TorrentDef import TorrentDef +from BaseLib.Core.Utilities.Crypto import sha + +try: + True +except: + True = 1 + False = 0 + +DEBUG = False + +def _errfunc(x): + print >>sys.stderr,"tracker: parsedir: "+x + +def parsedir(directory, parsed, files, blocked, + exts = ['.torrent', TRIBLER_TORRENT_EXT], return_metainfo = False, errfunc = _errfunc): + if DEBUG: + errfunc('checking dir') + dirs_to_check = [directory] + new_files = {} + new_blocked = {} + torrent_type = {} + while dirs_to_check: # first, recurse directories and gather torrents + directory = dirs_to_check.pop() + newtorrents = False + for f in os.listdir(directory): + newtorrent = None + for ext in exts: + if f.endswith(ext): + newtorrent = ext[1:] + break + if newtorrent: + newtorrents = True + p = os.path.join(directory, f) + new_files[p] = [(int(os.path.getmtime(p)), os.path.getsize(p)), 0] + torrent_type[p] = newtorrent + if not newtorrents: + for f in os.listdir(directory): + p = os.path.join(directory, f) + if os.path.isdir(p): + dirs_to_check.append(p) + + new_parsed = {} + to_add = [] + added = {} + removed = {} + # files[path] = [(modification_time, size), hash], hash is 0 if the file + # has not been successfully parsed + for p, v in new_files.items(): # re-add old items and check for changes + oldval = files.get(p) + if not oldval: # new file + to_add.append(p) + continue + h = oldval[1] + if oldval[0] == v[0]: # file is unchanged from last parse + if h: + if blocked.has_key(p): # parseable + blocked means duplicate + to_add.append(p) # other duplicate may have gone away + else: + new_parsed[h] = parsed[h] + new_files[p] = oldval + else: + new_blocked[p] = 1 # same broken unparseable file + continue + if parsed.has_key(h) and not blocked.has_key(p): + if DEBUG: + errfunc('removing '+p+' (will re-add)') + removed[h] = parsed[h] + to_add.append(p) + + to_add.sort() + for p in to_add: # then, parse new and changed torrents + new_file = new_files[p] + v, h = new_file + if new_parsed.has_key(h): # duplicate + if not blocked.has_key(p) or files[p][0] != v: + errfunc('**warning** '+ + p +' is a duplicate torrent for '+new_parsed[h]['path']) + new_blocked[p] = 1 + continue + + if DEBUG: + errfunc('adding '+p) + try: + # Arno: P2PURL + tdef = TorrentDef.load(p) + h = tdef.get_infohash() + d = tdef.get_metainfo() + + new_file[1] = h + if new_parsed.has_key(h): + errfunc('**warning** '+ + p +' is a duplicate torrent for '+new_parsed[h]['path']) + new_blocked[p] = 1 + continue + + a = {} + a['path'] = p + f = os.path.basename(p) + a['file'] = f + a['type'] = torrent_type[p] + if tdef.get_url_compat(): + a['url'] = tdef.get_url() + i = d['info'] + l = 0 + nf = 0 + if i.has_key('length'): + l = i.get('length', 0) + nf = 1 + elif i.has_key('files'): + for li in i['files']: + nf += 1 + if li.has_key('length'): + l += li['length'] + a['numfiles'] = nf + a['length'] = l + a['name'] = i.get('name', f) + def setkey(k, d = d, a = a): + if d.has_key(k): + a[k] = d[k] + setkey('failure reason') + setkey('warning message') + setkey('announce-list') + # Arno, LOOKUP SERVICE + if tdef.get_urllist() is not None: + httpseedhashes = [] + for url in tdef.get_urllist(): + # TODO: normalize? + urlhash = sha(url).digest() + httpseedhashes.append(urlhash) + a['url-hash-list'] = httpseedhashes + if return_metainfo: + a['metainfo'] = d + except: + print_exc() + errfunc('**warning** '+p+' has errors') + new_blocked[p] = 1 + continue + if DEBUG: + errfunc('... successful') + new_parsed[h] = a + added[h] = a + + for p, v in files.items(): # and finally, mark removed torrents + if not new_files.has_key(p) and not blocked.has_key(p): + if DEBUG: + errfunc('removing '+p) + removed[v[1]] = parsed[v[1]] + + if DEBUG: + errfunc('done checking') + return (new_parsed, new_files, new_blocked, added, removed) + diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/piecebuffer.py b/instrumentation/next-share/BaseLib/Core/BitTornado/piecebuffer.py new file mode 100644 index 0000000..75e3e07 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/piecebuffer.py @@ -0,0 +1,86 @@ +# Written by John Hoffman +# see LICENSE.txt for license information + +from array import array +from threading import Lock +# import inspect +try: + True +except: + True = 1 + False = 0 + +DEBUG = False + +class SingleBuffer: + def __init__(self, pool): + self.pool = pool + self.buf = array('c') + + def init(self): + if DEBUG: + print self.pool.count + ''' + for x in xrange(6,1,-1): + try: + f = inspect.currentframe(x).f_code + print (f.co_filename,f.co_firstlineno,f.co_name) + del f + except: + pass + print '' + ''' + self.length = 0 + + def append(self, s): + l = self.length+len(s) + self.buf[self.length:l] = array('c', s) + self.length = l + + def __len__(self): + return self.length + + def __getslice__(self, a, b): + if b > self.length: + b = self.length + if b < 0: + b += self.length + if a == 0 and b == self.length and len(self.buf) == b: + return self.buf # optimization + return self.buf[a:b] + + def getarray(self): + return self.buf[:self.length] + + def release(self): + if DEBUG: + print -self.pool.count + self.pool.release(self) + + +class BufferPool: + def __init__(self): + self.pool = [] + self.lock = Lock() + if DEBUG: + self.count = 0 + + def new(self): + self.lock.acquire() + if self.pool: + x = self.pool.pop() + else: + x = SingleBuffer(self) + if DEBUG: + self.count += 1 + x.count = self.count + x.init() + self.lock.release() + return x + + def release(self, x): + self.pool.append(x) + + +_pool = BufferPool() +PieceBuffer = _pool.new diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/selectpoll.py b/instrumentation/next-share/BaseLib/Core/BitTornado/selectpoll.py new file mode 100644 index 0000000..7be1a55 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/selectpoll.py @@ -0,0 +1,130 @@ +# Written by Bram Cohen +# see LICENSE.txt for license information +# Arno,2007-02-23: this poll class is used on win32 + +import sys +from select import select +from time import sleep +from types import IntType +from bisect import bisect +from sets import Set +POLLIN = 1 +POLLOUT = 2 +POLLERR = 8 +POLLHUP = 16 + +DEBUG = False + +class poll: + def __init__(self): + self.rlist = [] + self.wlist = [] + + def register(self, f, t): + if type(f) != IntType: + f = f.fileno() + if (t & POLLIN): + insert(self.rlist, f) + else: + remove(self.rlist, f) + if (t & POLLOUT): + insert(self.wlist, f) + else: + remove(self.wlist, f) + + def unregister(self, f): + if type(f) != IntType: + f = f.fileno() + remove(self.rlist, f) + remove(self.wlist, f) + + def poll(self, timeout = None): + if self.rlist or self.wlist: + try: + # Arno, 2007-02-23: The original code never checked for errors + # on any file descriptors. + elist = Set(self.rlist) + elist = elist.union(self.wlist) + elist = list(elist) # in Python2.3, elist must be a list type + if DEBUG: + print >>sys.stderr,"selectpoll: elist = ",elist + + #print >>sys.stderr,"selectpoll: rlist",self.rlist,"wlist",self.wlist,"elist",elist + + r, w, e = select(self.rlist, self.wlist, elist, timeout) + if DEBUG: + print >>sys.stderr,"selectpoll: e = ",e + except ValueError: + if DEBUG: + print >>sys.stderr,"selectpoll: select: bad param" + return None + else: + sleep(timeout) + return [] + result = [] + for s in r: + result.append((s, POLLIN)) + for s in w: + result.append((s, POLLOUT)) + for s in e: + result.append((s, POLLERR)) + return result + +def remove(list, item): + i = bisect(list, item) + if i > 0 and list[i-1] == item: + del list[i-1] + +def insert(list, item): + i = bisect(list, item) + if i == 0 or list[i-1] != item: + list.insert(i, item) + +def test_remove(): + x = [2, 4, 6] + remove(x, 2) + assert x == [4, 6] + x = [2, 4, 6] + remove(x, 4) + assert x == [2, 6] + x = [2, 4, 6] + remove(x, 6) + assert x == [2, 4] + x = [2, 4, 6] + remove(x, 5) + assert x == [2, 4, 6] + x = [2, 4, 6] + remove(x, 1) + assert x == [2, 4, 6] + x = [2, 4, 6] + remove(x, 7) + assert x == [2, 4, 6] + x = [2, 4, 6] + remove(x, 5) + assert x == [2, 4, 6] + x = [] + remove(x, 3) + assert x == [] + +def test_insert(): + x = [2, 4] + insert(x, 1) + assert x == [1, 2, 4] + x = [2, 4] + insert(x, 3) + assert x == [2, 3, 4] + x = [2, 4] + insert(x, 5) + assert x == [2, 4, 5] + x = [2, 4] + insert(x, 2) + assert x == [2, 4] + x = [2, 4] + insert(x, 4) + assert x == [2, 4] + x = [2, 3, 4] + insert(x, 3) + assert x == [2, 3, 4] + x = [] + insert(x, 3) + assert x == [3] diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/subnetparse.py b/instrumentation/next-share/BaseLib/Core/BitTornado/subnetparse.py new file mode 100644 index 0000000..18e4187 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/subnetparse.py @@ -0,0 +1,218 @@ +# Written by John Hoffman +# see LICENSE.txt for license information + +from bisect import bisect, insort + +try: + True +except: + True = 1 + False = 0 + bool = lambda x: not not x + +hexbinmap = { + '0': '0000', + '1': '0001', + '2': '0010', + '3': '0011', + '4': '0100', + '5': '0101', + '6': '0110', + '7': '0111', + '8': '1000', + '9': '1001', + 'a': '1010', + 'b': '1011', + 'c': '1100', + 'd': '1101', + 'e': '1110', + 'f': '1111', + 'x': '0000', +} + +chrbinmap = {} +for n in xrange(256): + b = [] + nn = n + for i in xrange(8): + if nn & 0x80: + b.append('1') + else: + b.append('0') + nn <<= 1 + chrbinmap[n] = ''.join(b) + + +def to_bitfield_ipv4(ip): + ip = ip.split('.') + if len(ip) != 4: + raise ValueError, "bad address" + b = [] + for i in ip: + b.append(chrbinmap[int(i)]) + return ''.join(b) + +def to_bitfield_ipv6(ip): + b = '' + doublecolon = False + + if not ip: + raise ValueError, "bad address" + if ip == '::': # boundary handling + ip = '' + elif ip[:2] == '::': + ip = ip[1:] + elif ip[0] == ':': + raise ValueError, "bad address" + elif ip[-2:] == '::': + ip = ip[:-1] + elif ip[-1] == ':': + raise ValueError, "bad address" + for n in ip.split(':'): + if n == '': # double-colon + if doublecolon: + raise ValueError, "bad address" + doublecolon = True + b += ':' + continue + if n.find('.') >= 0: # IPv4 + n = to_bitfield_ipv4(n) + b += n + '0'*(32-len(n)) + continue + n = ('x'*(4-len(n))) + n + for i in n: + b += hexbinmap[i] + if doublecolon: + pos = b.find(':') + b = b[:pos]+('0'*(129-len(b)))+b[pos+1:] + if len(b) != 128: # always check size + raise ValueError, "bad address" + return b + +ipv4addrmask = to_bitfield_ipv6('::ffff:0:0')[:96] + +class IP_List: + def __init__(self): + self.ipv4list = [] + self.ipv6list = [] + + def __nonzero__(self): + return bool(self.ipv4list or self.ipv6list) + + + def append(self, ip, depth = 256): + if ip.find(':') < 0: # IPv4 + insort(self.ipv4list, to_bitfield_ipv4(ip)[:depth]) + else: + b = to_bitfield_ipv6(ip) + if b.startswith(ipv4addrmask): + insort(self.ipv4list, b[96:][:depth-96]) + else: + insort(self.ipv6list, b[:depth]) + + + def includes(self, ip): + if not (self.ipv4list or self.ipv6list): + return False + if ip.find(':') < 0: # IPv4 + b = to_bitfield_ipv4(ip) + else: + b = to_bitfield_ipv6(ip) + if b.startswith(ipv4addrmask): + b = b[96:] + if len(b) > 32: + l = self.ipv6list + else: + l = self.ipv4list + for map in l[bisect(l, b)-1:]: + if b.startswith(map): + return True + if map > b: + return False + return False + + + def read_fieldlist(self, file): # reads a list from a file in the format 'ip/len ' + f = open(file, 'r') + while 1: + line = f.readline() + if not line: + break + line = line.strip().expandtabs() + if not line or line[0] == '#': + continue + try: + line, garbage = line.split(' ', 1) + except: + pass + try: + line, garbage = line.split('#', 1) + except: + pass + try: + ip, depth = line.split('/') + except: + ip = line + depth = None + try: + if depth is not None: + depth = int(depth) + self.append(ip, depth) + except: + print '*** WARNING *** could not parse IP range: '+line + f.close() + + + def set_intranet_addresses(self): + self.append('127.0.0.1', 8) + self.append('10.0.0.0', 8) + self.append('172.16.0.0', 12) + self.append('192.168.0.0', 16) + self.append('169.254.0.0', 16) + self.append('::1') + self.append('fe80::', 16) + self.append('fec0::', 16) + + def set_ipv4_addresses(self): + self.append('::ffff:0:0', 96) + +def ipv6_to_ipv4(ip): + ip = to_bitfield_ipv6(ip) + if not ip.startswith(ipv4addrmask): + raise ValueError, "not convertible to IPv4" + ip = ip[-32:] + x = '' + for i in range(4): + x += str(int(ip[:8], 2)) + if i < 3: + x += '.' + ip = ip[8:] + return x + +def to_ipv4(ip): + if is_ipv4(ip): + _valid_ipv4(ip) + return ip + return ipv6_to_ipv4(ip) + +def is_ipv4(ip): + return ip.find(':') < 0 + +def _valid_ipv4(ip): + ip = ip.split('.') + if len(ip) != 4: + raise ValueError + for i in ip: + chr(int(i)) + +def is_valid_ip(ip): + try: + if not ip: + return False + if is_ipv4(ip): + _valid_ipv4(ip) + return True + to_bitfield_ipv6(ip) + return True + except: + return False diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/torrentlistparse.py b/instrumentation/next-share/BaseLib/Core/BitTornado/torrentlistparse.py new file mode 100644 index 0000000..668c245 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/torrentlistparse.py @@ -0,0 +1,38 @@ +# Written by John Hoffman +# see LICENSE.txt for license information + +from binascii import unhexlify + +try: + True +except: + True = 1 + False = 0 + + +# parses a list of torrent hashes, in the format of one hash per line in hex format + +def parsetorrentlist(filename, parsed): + new_parsed = {} + added = {} + removed = parsed + f = open(filename, 'r') + while 1: + l = f.readline() + if not l: + break + l = l.strip() + try: + if len(l) != 40: + raise ValueError, 'bad line' + h = unhexlify(l) + except: + print '*** WARNING *** could not parse line in torrent list: '+l + if parsed.has_key(h): + del removed[h] + else: + added[h] = True + new_parsed[h] = True + f.close() + return (new_parsed, added, removed) + diff --git a/instrumentation/next-share/BaseLib/Core/BitTornado/zurllib.py b/instrumentation/next-share/BaseLib/Core/BitTornado/zurllib.py new file mode 100644 index 0000000..7efdd90 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BitTornado/zurllib.py @@ -0,0 +1,143 @@ +# Written by John Hoffman +# see LICENSE.txt for license information + +import sys +from httplib import HTTPConnection, HTTPSConnection, HTTPException +from urlparse import urlparse +from bencode import bdecode +from gzip import GzipFile +from StringIO import StringIO +from __init__ import product_name, version_short +from traceback import print_exc,print_stack + +from BaseLib.Core.Utilities.timeouturlopen import find_proxy + +VERSION = product_name+'/'+version_short +MAX_REDIRECTS = 10 + + +class btHTTPcon(HTTPConnection): # attempt to add automatic connection timeout + def connect(self): + HTTPConnection.connect(self) + try: + self.sock.settimeout(30) + except: + pass + +class btHTTPScon(HTTPSConnection): # attempt to add automatic connection timeout + def connect(self): + HTTPSConnection.connect(self) + try: + self.sock.settimeout(30) + except: + pass + +class urlopen: + def __init__(self, url): + self.tries = 0 + self._open(url.strip()) + self.error_return = None + + def _open(self, url): + try: + self.tries += 1 + if self.tries > MAX_REDIRECTS: + raise IOError, ('http error', 500, + "Internal Server Error: Redirect Recursion") + (scheme, netloc, path, pars, query, fragment) = urlparse(url) + if scheme != 'http' and scheme != 'https': + raise IOError, ('url error', 'unknown url type', scheme, url) + wanturl = path + if pars: + wanturl += ';'+pars + if query: + wanturl += '?'+query + # if fragment: + + proxyhost = find_proxy(url) + if proxyhost is None: + desthost = netloc + desturl = wanturl + else: + desthost = proxyhost + desturl = scheme+'://'+netloc+wanturl + try: + self.response = None + if scheme == 'http': + self.connection = btHTTPcon(desthost) + else: + self.connection = btHTTPScon(desthost) + self.connection.request('GET', desturl, None, + { 'Host': netloc, 'User-Agent': VERSION, + 'Accept-Encoding': 'gzip' } ) + self.response = self.connection.getresponse() + except HTTPException, e: + print_exc() + raise IOError, ('http error', str(e)) + status = self.response.status + if status in (301, 302): + try: + self.connection.close() + except: + pass + self._open(self.response.getheader('Location')) + return + if status != 200: + try: + data = self._read() + d = bdecode(data) + if d.has_key('failure reason'): + self.error_return = data + return + except: + pass + raise IOError, ('http error', status, self.response.reason) + except: + print_exc() + + + def read(self): + if self.error_return: + return self.error_return + return self._read() + + def _read(self): + data = self.response.read() + if self.response.getheader('Content-Encoding', '').find('gzip') >= 0: + try: + compressed = StringIO(data) + f = GzipFile(fileobj = compressed) + data = f.read() + except: + raise IOError, ('http error', 'got corrupt response') + return data + + def close(self): + self.connection.close() + +try: + import pycurl + + class curlopen: + def __init__(self,url): + + print >>sys.stderr,"CURL",url + + self.contents = '' + self.c = pycurl.Curl() + self.c.setopt(c.URL, url) + self.c.setopt(c.WRITEFUNCTION, t.body_callback) + self.c.perform() + self.c.close() + + def body_callback(self, buf): + self.contents = self.contents + buf + + def read(self): + return self.contents + + def close(self): + pass +except: + pass + diff --git a/instrumentation/next-share/BaseLib/Core/BuddyCast/TorrentCollecting.py b/instrumentation/next-share/BaseLib/Core/BuddyCast/TorrentCollecting.py new file mode 100644 index 0000000..a9288bd --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BuddyCast/TorrentCollecting.py @@ -0,0 +1,26 @@ +# Written by Jie Yang +# see LICENSE.txt for license information + +DEBUG = False + +class SimpleTorrentCollecting: + """ + Simplest torrent collecting policy: randomly collect a torrent when received + a buddycast message + """ + + def __init__(self, metadata_handler, data_handler): + self.metadata_handler = metadata_handler + self.data_handler = data_handler + self.torrent_db = data_handler.torrent_db + self.pref_db = data_handler.pref_db + self.cache_pool = {} + + + def trigger(self, permid, selversion, collect_candidate=None): + infohash = self.torrent_db.selectTorrentToCollect(permid, collect_candidate) + #print >> sys.stderr, '*****-----------***** trigger torrent collecting', `infohash` + if infohash and self.metadata_handler: + self.metadata_handler.send_metadata_request(permid, infohash, selversion) + + diff --git a/instrumentation/next-share/BaseLib/Core/BuddyCast/__init__.py b/instrumentation/next-share/BaseLib/Core/BuddyCast/__init__.py new file mode 100644 index 0000000..395f8fb --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BuddyCast/__init__.py @@ -0,0 +1,2 @@ +# Written by Arno Bakker +# see LICENSE.txt for license information diff --git a/instrumentation/next-share/BaseLib/Core/BuddyCast/bartercast.py b/instrumentation/next-share/BaseLib/Core/BuddyCast/bartercast.py new file mode 100644 index 0000000..dea4dab --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BuddyCast/bartercast.py @@ -0,0 +1,343 @@ +# Written by Michel Meulpolder +# see LICENSE.txt for license information +import sys, os + +from BaseLib.Core.BitTornado.bencode import bencode, bdecode +from BaseLib.Core.Statistics.Logger import OverlayLogger +from BaseLib.Core.BitTornado.BT1.MessageID import BARTERCAST #, KEEP_ALIVE +from BaseLib.Core.CacheDB.CacheDBHandler import BarterCastDBHandler +from BaseLib.Core.Utilities.utilities import * +from traceback import print_exc +from types import StringType, ListType, DictType +from time import time, gmtime, strftime, ctime + +from BaseLib.Core.Overlay.permid import permid_for_user +from BaseLib.Core.Overlay.SecureOverlay import OLPROTO_VER_FIFTH + + +MAX_BARTERCAST_LENGTH = 10 * 1024 * 1024 # TODO: give this length a reasonable value +NO_PEERS_IN_MSG = 10 +REFRESH_TOPN_INTERVAL = 30 * 60 + +DEBUG = False +LOG = False + +def now(): + return int(time()) + +class BarterCastCore: + + ################################ + def __init__(self, data_handler, overlay_bridge, log = '', dnsindb = None): + + if DEBUG: + print >> sys.stderr, "=================Initializing bartercast core" + + self.data_handler = data_handler + self.dnsindb = dnsindb + self.log = log + self.overlay_bridge = overlay_bridge + self.bartercastdb = BarterCastDBHandler.getInstance() + + self.network_delay = 30 + self.send_block_list = {} + self.recv_block_list = {} + self.block_interval = 1*60*60 # block interval for a peer to barter cast + + self.topn = self.bartercastdb.getTopNPeers(NO_PEERS_IN_MSG, local_only = True)['top'] + self.overlay_bridge.add_task(self.refreshTopN, REFRESH_TOPN_INTERVAL) + + if self.log: + self.overlay_log = OverlayLogger.getInstance(self.log) + + if LOG: + self.logfile = '/Users/michel/packages/bartercast_dataset/bartercast42.log' + if not os.path.exists(self.logfile): + log = open(self.logfile, 'w') + log.close() + + + ################################ + def refreshTopN(self): + + self.topn = self.bartercastdb.getTopNPeers(NO_PEERS_IN_MSG, local_only = True)['top'] + self.overlay_bridge.add_task(self.refreshTopN, REFRESH_TOPN_INTERVAL) + + + + ################################ + def createAndSendBarterCastMessage(self, target_permid, selversion, active = False): + + + # for older versions of Tribler (non-BarterCast): do nothing + if selversion <= OLPROTO_VER_FIFTH: + return + + if DEBUG: + print >> sys.stderr, "===========bartercast: Sending BarterCast msg to ", self.bartercastdb.getName(target_permid) + + # create a new bartercast message + bartercast_data = self.createBarterCastMessage(target_permid) + + if LOG: + self.logMsg(bartercast_data, target_permid, 'out', logfile = self.logfile) + + try: + bartercast_msg = bencode(bartercast_data) + except: + print_exc() + print >> sys.stderr, "error bartercast_data:", bartercast_data + return + + # send the message + self.overlay_bridge.send(target_permid, BARTERCAST+bartercast_msg, self.bartercastSendCallback) + + self.blockPeer(target_permid, self.send_block_list, self.block_interval) + + + + ################################ + def createBarterCastMessage(self, target_permid): + """ Create a bartercast message """ + + my_permid = self.bartercastdb.my_permid + local_top = self.topn + top_peers = map(lambda (permid, up, down): permid, local_top) + data = {} + totals = self.bartercastdb.getTotals() # (total_up, total_down) + + for permid in top_peers: + + item = self.bartercastdb.getItem((my_permid, permid)) + + if item is not None: + # retrieve what i have uploaded to permid + data_to = item['uploaded'] + # retrieve what i have downloaded from permid + data_from = item['downloaded'] + + data[permid] = {'u': data_to, 'd': data_from} + + bartercast_data = {'data': data, 'totals': totals} + + return bartercast_data + + + ################################ + def bartercastSendCallback(self, exc, target_permid, other=0): + if exc is None: + if DEBUG: + print "bartercast: %s *** msg was sent successfully to peer %s" % (ctime(now()), self.bartercastdb.getName(target_permid)) + else: + if DEBUG: + print "bartercast: %s *** warning - error in sending msg to %s" % (ctime(now()), self.bartercastdb.getName(target_permid)) + + + ################################ + def gotBarterCastMessage(self, recv_msg, sender_permid, selversion): + """ Received a bartercast message and handle it. Reply if needed """ + + if DEBUG: + print >>sys.stderr,'bartercast: %s Received a BarterCast msg from %s'% (ctime(now()), self.bartercastdb.getName(sender_permid)) + + if not sender_permid or sender_permid == self.bartercastdb.my_permid: + print >> sys.stderr, "bartercast: error - got BarterCastMsg from a None peer", \ + sender_permid, recv_msg + return False + + if MAX_BARTERCAST_LENGTH > 0 and len(recv_msg) > MAX_BARTERCAST_LENGTH: + print >> sys.stderr, "bartercast: warning - got large BarterCastMsg", len(recv_msg) + return False + + bartercast_data = {} + + try: + bartercast_data = bdecode(recv_msg) + except: + print >> sys.stderr, "bartercast: warning, invalid bencoded data" + return False + + try: # check bartercast message + self.validBarterCastMsg(bartercast_data) + except RuntimeError, msg: + print >> sys.stderr, msg + return False + + if LOG: + self.logMsg(bartercast_data, sender_permid, 'in', logfile = self.logfile) + + data = bartercast_data['data'] + + if 'totals' in bartercast_data: + totals = bartercast_data['totals'] + else: + totals = None + + if DEBUG: + st = time() + self.handleBarterCastMsg(sender_permid, data) + et = time() + diff = et - st + print >>sys.stderr,"bartercast: HANDLE took %.4f" % diff + else: + self.handleBarterCastMsg(sender_permid, data, totals) + + if not self.isBlocked(sender_permid, self.send_block_list): + self.replyBarterCast(sender_permid, selversion) + + return True + + + + ################################ + def validBarterCastMsg(self, bartercast_data): + + if not type(bartercast_data) == DictType: + raise RuntimeError, "bartercast: received data is not a dictionary" + return False + + if not bartercast_data.has_key('data'): + raise RuntimeError, "bartercast: 'data' key doesn't exist" + return False + + if not type(bartercast_data['data']) == DictType: + raise RuntimeError, "bartercast: 'data' value is not dictionary" + return False + + for permid in bartercast_data['data'].keys(): + + if not bartercast_data['data'][permid].has_key('u') or \ + not bartercast_data['data'][permid].has_key('d'): + raise RuntimeError, "bartercast: datafield doesn't contain 'u' or 'd' keys" + return False + + return True + + ################################ + def handleBarterCastMsg(self, sender_permid, data, totals = None): + """ process bartercast data in database """ + if DEBUG: + print >> sys.stderr, "bartercast: Processing bartercast msg from: ", self.bartercastdb.getName(sender_permid) + print >> sys.stderr, "totals: ", totals + + + permids = data.keys() + changed = False + + # 1. Add any unknown peers to the database in a single transaction + self.bartercastdb.addPeersBatch(permids) + + + # 2. Add totals to database (without committing) + if totals != None and len(totals) == 2: + up = int(totals[0]) + down = int(totals[1]) + self.bartercastdb.updateULDL((sender_permid, sender_permid), up, down, commit = False) + changed = True + + # 3. Add all the received records to the database in a single transaction + datalen = len(permids) + for i in range(0,datalen): + permid = permids[i] + + data_to = data[permid]['u'] + data_from = data[permid]['d'] + + if DEBUG: + print >> sys.stderr, "bartercast: data: (%s, %s) up = %d down = %d" % (self.bartercastdb.getName(sender_permid), self.bartercastdb.getName(permid),\ + data_to, data_from) + + # update database sender->permid and permid->sender + #commit = (i == datalen-1) + self.bartercastdb.updateULDL((sender_permid, permid), data_to, data_from, commit = False) + changed = True + + if changed: + self.bartercastdb.commit() + + + # ARNODB: + # get rid of index on DB? See where used + + + ################################ + def replyBarterCast(self, target_permid, selversion): + """ Reply a bartercast message """ + + if DEBUG: + st = time() + self.createAndSendBarterCastMessage(target_permid, selversion) + et = time() + diff = et - st + print >>sys.stderr,"bartercast: CREATE took %.4f" % diff + else: + self.createAndSendBarterCastMessage(target_permid, selversion) + + + # Blocking functions (similar to BuddyCast): + + ################################ + def isBlocked(self, peer_permid, block_list): + if peer_permid not in block_list: + return False + unblock_time = block_list[peer_permid] + if now() >= unblock_time - self.network_delay: # 30 seconds for network delay + block_list.pop(peer_permid) + return False + return True + + + + ################################ + def blockPeer(self, peer_permid, block_list, block_interval=None): + """ Add a peer to a block list """ + + if block_interval is None: + block_interval = self.block_interval + unblock_time = now() + block_interval + block_list[peer_permid] = unblock_time + + if DEBUG: + print >>sys.stderr,'bartercast: %s Blocked peer %s'% (ctime(now()), self.bartercastdb.getName(peer_permid)) + + + ################################ + def logMsg(self, msg_data, msg_permid, in_or_out, logfile): + + if in_or_out == 'in': + permid_from = permid_for_user(msg_permid) + + elif in_or_out == 'out': + permid_from = 'LOCAL' + + else: + return + + timestamp = now() + + log = open(logfile, 'a') + string = '%.1f %s %s' % (timestamp, in_or_out, permid_for_user(msg_permid)) + log.write(string + '\n') + print >> sys.stderr, string + + data = msg_data.get('data', []) + + for permid in data: + u = data[permid]['u'] + d = data[permid]['d'] + + string = '%.1f %s %s %d %d' % (timestamp, permid_from, permid_for_user(permid), u, d) + log.write(string + '\n') + print >> sys.stderr, string + + totals = msg_data.get('totals', None) + + if totals != None: + (u, d) = totals + + string = '%.1f TOT %s %d %d' % (timestamp, permid_from, u, d) + log.write(string + '\n') + print >> sys.stderr, string + + + log.close() diff --git a/instrumentation/next-share/BaseLib/Core/BuddyCast/buddycast.py b/instrumentation/next-share/BaseLib/Core/BuddyCast/buddycast.py new file mode 100644 index 0000000..f0f4082 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BuddyCast/buddycast.py @@ -0,0 +1,2706 @@ +# Written by Jie Yang +# see LICENSE.txt for license information +# + +__fool_epydoc = 481 +""" + BuddyCast2 epidemic protocol for p2p recommendation and semantic clustering + +Algorithm in LaTeX format: + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%% algorithm of the active peer %%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +\begin{figure*}[ht] +\begin{center} +\begin{algorithmic}[1] + +\LOOP +\STATE wait($\Delta T$ time units) \COMMENT{15 seconds in current implementation} +\STATE remove any peer from $B_S$ and $B_R$ if its block time was expired. +\STATE keep connection with all peers in $C_T$, $C_R$ and $C_U$ +\IF{$idle\_loops > 0$} + \STATE $idle\_loops \leftarrow idle\_loops - 1$ \COMMENT{skip this loop for rate control} +\ELSE + \IF{$C_C$ is empty} + \STATE $C_C \leftarrow$ select 5 peers recently seen from Mega Cache + \ENDIF + \STATE $Q \leftarrow$ select a most similar taste buddy or a random online peer from $C_C$ + \STATE connectPeer($Q$) + \STATE block($Q$, $B_S$, 4hours) + \STATE remove $Q$ from $C_C$ + \IF{$Q$ is connected successfully} + \STATE buddycast\_msg\_send $\leftarrow$ \textbf{createBuddycastMsg}() + \STATE send buddycast\_msg\_send to $Q$ + \STATE receive buddycast\_msg\_recv from $Q$ + \STATE $C_C \leftarrow$ fillPeers(buddycast\_msg\_recv) + \STATE \textbf{addConnectedPeer}($Q$) \COMMENT{add $Q$ into $C_T$, $C_R$ or $C_U$ according to its similarity} + \STATE blockPeer($Q$, $B_R$, 4hours) + \ENDIF + +\ENDIF +\ENDLOOP + +\end{algorithmic} +\caption{The protocol of an active peer.} +\label{Fig:buddycast_algorithm} +\end{center} +\end{figure*} + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%% algorithm of the passive peer %%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +\begin{figure*}[ht] +\begin{center} +\begin{algorithmic}[1] + +\LOOP + \STATE receive buddycast\_msg\_recv from $Q$ + \STATE $C_C \leftarrow$ fillPeers(buddycast\_msg\_recv) + \STATE \textbf{addConnectedPeer}($Q$) + \STATE blockPeer($Q$, $B_R$, 4hours) + \STATE buddycast\_msg\_send $\leftarrow$ \textbf{createBuddycastMsg}() + \STATE send buddycast\_msg\_send to $Q$ + \STATE blockPeer($Q$, $B_S$, 4hours) + \STATE remove $Q$ from $C_C$ + \STATE $idle\_loops \leftarrow idle\_loops + 1$ \COMMENT{idle for a loop for + rate control} +\ENDLOOP + +\end{algorithmic} +\caption{The protocol of an passive peer.} +\label{Fig:buddycast_algorithm} +\end{center} +\end{figure*} + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% algorithm of creating a buddycast message %% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +\begin{figure*}[ht] +\begin{center} +function \textbf{createBuddycastMsg}() +\begin{algorithmic} + \STATE $My\_Preferences \leftarrow$ the most recently 50 preferences of the active peer + \STATE $Taste\_Buddies \leftarrow$ all peers from $C_T$ + \STATE $Random\_Peers \leftarrow$ all peers from $C_R$ + \STATE $buddycast\_msg\_send \leftarrow$ create an empty message + \STATE $buddycast\_msg\_send$ attaches the active peer's address and $My\_Preferences$ + \STATE $buddycast\_msg\_send$ attaches addresses of $Taste\_Buddies$ + \STATE $buddycast\_msg\_send$ attaches at most 10 preferences of each peer in $Taste\_Buddies$ + \STATE $buddycast\_msg\_send$ attaches addresses of $Random\_Peers$ +\end{algorithmic} +\caption{The function of creating a buddycast message} +\label{Fig:buddycast_createBuddycastMsg} +\end{center} +\end{figure*} + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% algorithm of adding a peer into C_T or C_R or C_U %% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +\begin{figure*}[ht] +\begin{center} +function \textbf{addConnectedPeer}($Q$) +\begin{algorithmic} + \IF{$Q$ is connectable} + \STATE $Sim_Q \leftarrow$ getSimilarity($Q$) \COMMENT{similarity between $Q$ and the active peer} + \STATE $Min_{Sim} \leftarrow$ similarity of the least similar peer in $C_T$ + \IF{$Sim_Q \geq Min_{Sim}$ \textbf{or} ($C_T$ is not full \textbf{and} $Sim_Q>0$)} + \STATE $C_T \leftarrow C_T + Q$ + \STATE move the least similar peer to $C_R$ if $C_T$ overloads + \ELSE + \STATE $C_R \leftarrow C_R + Q$ + \STATE remove the oldest peer to $C_R$ if $C_R$ overloads + \ENDIF + \ELSE + \STATE $C_U \leftarrow C_U + Q$ + \ENDIF + +\end{algorithmic} +\caption{The function of adding a peer into $C_T$ or $C_R$} +\label{Fig:buddycast_addConnectedPeer} +\end{center} +\end{figure*} + +""" +""" + +BuddyCast 3: + No preferences for taste buddies; + don't accept preferences of taste buddies from incoming message either + 50 recent my prefs + 50 recent collected torrents + 50 ratings + +Torrent info + preferences: Recently downloaded torrents by the user {'seeders','leechers','check time'} + collected torrents: Recently collected torrents (include Subscribed torrents) + #ratings: Recently rated torrents and their ratings (negative rating means this torrent was deleted) +Taste Buddies + permid + ip + port + similarity +Random Peers + permid + ip + port + similarity + +""" + +import sys +from random import sample, randint, shuffle +from time import time, gmtime, strftime +from traceback import print_exc,print_stack +from sets import Set +from array import array +from bisect import insort +from copy import deepcopy +import gc +import socket + +from BaseLib.Core.simpledefs import BCCOLPOLICY_SIMPLE +from BaseLib.Core.BitTornado.bencode import bencode, bdecode +from BaseLib.Core.BitTornado.BT1.MessageID import BUDDYCAST, BARTERCAST, KEEP_ALIVE, VOTECAST, CHANNELCAST +from BaseLib.Core.Utilities.utilities import show_permid_short, show_permid,validPermid,validIP,validPort,validInfohash,readableBuddyCastMsg, hostname_or_ip2ip +from BaseLib.Core.Utilities.unicode import dunno2unicode +from BaseLib.Core.simpledefs import NTFY_ACT_MEET, NTFY_ACT_RECOMMEND, NTFY_MYPREFERENCES, NTFY_INSERT, NTFY_DELETE +from BaseLib.Core.NATFirewall.DialbackMsgHandler import DialbackMsgHandler +from BaseLib.Core.Overlay.SecureOverlay import OLPROTO_VER_FIRST, OLPROTO_VER_SECOND, OLPROTO_VER_THIRD, OLPROTO_VER_FOURTH, OLPROTO_VER_FIFTH, OLPROTO_VER_SIXTH, OLPROTO_VER_SEVENTH, OLPROTO_VER_EIGHTH, OLPROTO_VER_ELEVENTH , OLPROTO_VER_CURRENT, OLPROTO_VER_LOWEST +from BaseLib.Core.CacheDB.sqlitecachedb import bin2str, str2bin +from similarity import P2PSim_Single, P2PSim_Full, P2PSimColdStart +from TorrentCollecting import SimpleTorrentCollecting #, TiT4TaTTorrentCollecting +from BaseLib.Core.Statistics.Logger import OverlayLogger +from BaseLib.Core.Statistics.Crawler import Crawler + +from threading import currentThread + +from bartercast import BarterCastCore +from votecast import VoteCastCore +from channelcast import ChannelCastCore + +DEBUG = False # for errors +debug = False # for status +debugnic = False # for my temporary outputs +unblock = 0 + +# Nicolas: 10 KByte -- I set this to 1024 KByte. +# The term_id->term dictionary can become almost arbitrarily long +# would be strange if buddycast stopped working once a user has done a lot of searches... +# +# Arno, 2009-03-06: Too big: we don't want every peer to send out 1 MB messages +# every 15 secs. Set to 100K +# +# Nicolas, 2009-03-06: Ok this was really old. 10k in fact is enough with the new constraints on clicklog data +MAX_BUDDYCAST_LENGTH = 10*1024 + +REMOTE_SEARCH_PEER_NTORRENTS_THRESHOLD = 100 # speedup finding >=4.1 peers in this version + +# used for datahandler.peers +PEER_SIM_POS = 0 +PEER_LASTSEEN_POS = 1 +#PEER_PREF_POS = 2 #not needed since new similarity function + +def now(): + return int(time()) + +def ctime(t): + return strftime("%Y-%m-%d.%H:%M:%S", gmtime(t)) + +def validBuddyCastData(prefxchg, nmyprefs=50, nbuddies=10, npeers=10, nbuddyprefs=10, selversion=0): + + # + # + # Arno: TODO: make check version dependent + # + # + + def validPeer(peer): + validPermid(peer['permid']) + validIP(peer['ip']) + validPort(peer['port']) + + def validHisPeer(peer): + validIP(peer['ip']) + validPort(peer['port']) + + + def validPref(pref, num): + if not (isinstance(prefxchg, list) or isinstance(prefxchg, dict)): + raise RuntimeError, "bc: invalid pref type " + str(type(prefxchg)) + if num > 0 and len(pref) > num: + raise RuntimeError, "bc: length of pref exceeds " + str((len(pref), num)) + for p in pref: + validInfohash(p) + + validHisPeer(prefxchg) + if not (isinstance(prefxchg['name'], str)): + raise RuntimeError, "bc: invalid name type " + str(type(prefxchg['name'])) + + # Nicolas: create a validity check that doesn't have to know about the version + # just found out this function is not called anymore. well if it gets called one day, it should handle both + prefs = prefxchg['preferences'] + if prefs: + # >= OLPROTO_VER_EIGHT + if type(prefs[0])==list: + # list of lists: this is the new wire protocol. entry 0 of each list contains infohash + validPref([pref[0] for pref in prefs], nmyprefs) + else: + # old style + validPref(prefs, nmyprefs) + + if len(prefxchg['taste buddies']) > nbuddies: + raise RuntimeError, "bc: length of prefxchg['taste buddies'] exceeds " + \ + str(len(prefxchg['taste buddies'])) + for b in prefxchg['taste buddies']: + validPeer(b) + #validPref(b['preferences'], nbuddyprefs) # not used from version 4 + + if len(prefxchg['random peers']) > npeers: + raise RuntimeError, "bc: length of random peers " + \ + str(len(prefxchg['random peers'])) + for b in prefxchg['random peers']: + validPeer(b) + + if 'collected torrents' in prefxchg: + # 'collected torrents' must contain a list with 20 byte infohashes + if not isinstance(prefxchg['collected torrents'], list): + raise RuntimeError, "bc: invalid 'collected torrents' type " + str(type(prefxchg['collected torrents'])) + for value in prefxchg['collected torrents']: + if selversion >= OLPROTO_VER_ELEVENTH: + if not isinstance(value, list): + raise RuntimeError, "bc: invalid 'collected torrents' type of list elem should be list, not " + str(type(value)) + # infohash + # number of seeders + # number of leechers + # age of checking + # number of sources seen + if len(value) != 5: + raise RuntimeError, "bc: invalid 'collected torrents' length of list elem should be 5" + infohash = value[0] + seeders = value[1] + leechers = value[2] + age = value[3] + sources = value[4] + if not len(infohash) == 20: + raise RuntimeError, "bc: invalid infohash length " + str(len(infohash)) + else: + infohash = value + if not isinstance(infohash, str): + raise RuntimeError, "bc: invalid infohash type " + str(type(infohash)) + if not len(infohash) == 20: + raise RuntimeError, "bc: invalid infohash length " + str(len(infohash)) + + return True + + +class BuddyCastFactory: + __single = None + + def __init__(self, superpeer=False, log=''): + if BuddyCastFactory.__single: + raise RuntimeError, "BuddyCastFactory is singleton" + BuddyCastFactory.__single = self + self.registered = False + self.buddycast_core = None + self.buddycast_interval = 15 # MOST IMPORTANT PARAMETER + self.superpeer = superpeer + self.log = log + self.running = False + self.data_handler = None + self.started = False # did call do_buddycast() at least once + self.max_peers = 2500 # was 2500 + self.ranonce = False # Nicolas: had the impression that BuddyCast can be tested more reliably if I wait until it has gone through buddycast_core.work() successfully once + if self.superpeer: + print >>sys.stderr,"bc: Starting in SuperPeer mode" + + def getInstance(*args, **kw): + if BuddyCastFactory.__single is None: + BuddyCastFactory(*args, **kw) + return BuddyCastFactory.__single + getInstance = staticmethod(getInstance) + + def register(self, overlay_bridge, launchmany, errorfunc, + metadata_handler, torrent_collecting_solution, running, + max_peers=2500,amcrawler=False): + if self.registered: + return + self.overlay_bridge = overlay_bridge + self.launchmany = launchmany + self.metadata_handler = metadata_handler + self.torrent_collecting_solution = torrent_collecting_solution + self.errorfunc = errorfunc + + # BuddyCast is always started, but only active when this var is set. + self.running = bool(running) + self.max_peers = max_peers + self.amcrawler = amcrawler + + self.registered = True + + def register2(self): + # Arno: only start using overlay thread when normal init is finished to + # prevent concurrencty on singletons + if self.registered: + if debug: + print >> sys.stderr, "bc: Register BuddyCast", currentThread().getName() + self.overlay_bridge.add_task(self.olthread_register, 0) + + def olthread_register(self, start=True): + if debug: + print >> sys.stderr, "bc: OlThread Register", currentThread().getName() + + self.data_handler = DataHandler(self.launchmany, self.overlay_bridge, max_num_peers=self.max_peers) + + # ARNOCOMMENT: get rid of this dnsindb / get_dns_from_peerdb abuse off SecureOverlay + self.bartercast_core = BarterCastCore(self.data_handler, self.overlay_bridge, self.log, self.launchmany.secure_overlay.get_dns_from_peerdb) + + self.votecast_core = VoteCastCore(self.data_handler, self.overlay_bridge, self.launchmany.session, self.getCurrrentInterval, self.log, self.launchmany.secure_overlay.get_dns_from_peerdb) + self.channelcast_core = ChannelCastCore(self.data_handler, self.overlay_bridge, self.launchmany.session, self.getCurrrentInterval, self.log, self.launchmany.secure_overlay.get_dns_from_peerdb) + + self.buddycast_core = BuddyCastCore(self.overlay_bridge, self.launchmany, + self.data_handler, self.buddycast_interval, self.superpeer, + self.metadata_handler, self.torrent_collecting_solution, self.bartercast_core, self.votecast_core, self.channelcast_core, self.log, self.amcrawler) + + self.data_handler.register_buddycast_core(self.buddycast_core) + + if start: + self.start_time = now() + # Arno, 2007-02-28: BC is now started self.buddycast_interval after client + # startup. This is assumed to give enough time for UPnP to open the firewall + # if any. So when you change this time, make sure it allows for UPnP to + # do its thing, or add explicit coordination between UPnP and BC. + # See BitTornado/launchmany.py + self.overlay_bridge.add_task(self.data_handler.postInit, 0) + self.overlay_bridge.add_task(self.doBuddyCast, 0.1) + # Arno: HYPOTHESIS: if set to small, we'll only ask superpeers at clean start. + if self.data_handler.torrent_db.size() > 0: + waitt = 1.0 + else: + waitt = 3.0 + self.overlay_bridge.add_task(self.data_handler.initRemoteSearchPeers,waitt) + + #Nitin: While booting up, we try to update the channels that we are subscribed to + # after 6 seconds initially and later, at every 2 hour interval + self.overlay_bridge.add_task(self.channelcast_core.updateMySubscribedChannels, 6) + + print >> sys.stderr, "BuddyCast starts up",waitt + + def doBuddyCast(self): + if not self.running: + return + + if debug: + print >>sys.stderr,"bc: doBuddyCast!", currentThread().getName() + + # Reschedule ourselves for next round + buddycast_interval = self.getCurrrentInterval() + self.overlay_bridge.add_task(self.doBuddyCast, buddycast_interval) + if not self.started: + self.started = True + # Do our thang. + self.buddycast_core.work() + self.ranonce = True # Nicolas: now we can start testing and stuff works better + + def pauseBuddyCast(self): + self.running = False + + def restartBuddyCast(self): + if self.registered and not self.running: + self.running = True + self.doBuddyCast() + + def getCurrrentInterval(self): + """ + install [#(peers - superpeers)==0] & start < 2min: interval = 1 + start < 30min: interval = 5 + start > 24hour: interval = 60 + other: interval = 15 + """ + + #return 3 ### DEBUG, remove it before release!! + + past = now() - self.start_time + if past < 2*60: + if len(self.buddycast_core.connected_connectable_peers)<10: + interval = 0.2 + elif self.data_handler.get_npeers() < 20: + interval = 2 + else: + interval = 5 + elif past < 30*60: + if len(self.buddycast_core.connected_connectable_peers)<10: + interval = 2 + else: + interval = 5 + elif past > 24*60*60: + interval = 60 + else: + interval = 15 + return interval + + + def handleMessage(self, permid, selversion, message): + + if not self.registered or not self.running: + if DEBUG: + print >> sys.stderr, "bc: handleMessage got message, but we're not enabled or running" + return False + + t = message[0] + + if t == BUDDYCAST: + return self.gotBuddyCastMessage(message[1:], permid, selversion) + elif t == KEEP_ALIVE: + if message[1:] == '': + return self.gotKeepAliveMessage(permid) + else: + return False + + elif t == VOTECAST: + if DEBUG: + print >> sys.stderr, "bc: Received votecast message" + if self.votecast_core != None: + return self.votecast_core.gotVoteCastMessage(message[1:], permid, selversion) + + elif t == CHANNELCAST: + if DEBUG: + print >> sys.stderr, "bc: Received channelcast message" + if self.channelcast_core != None: + return self.channelcast_core.gotChannelCastMessage(message[1:], permid, selversion) + + elif t == BARTERCAST: + if DEBUG: + print >> sys.stderr, "bc: Received bartercast message" + if self.bartercast_core != None: + return self.bartercast_core.gotBarterCastMessage(message[1:], permid, selversion) + + else: + if DEBUG: + print >> sys.stderr, "bc: wrong message to buddycast", ord(t), "Round", self.buddycast_core.round + return False + + def gotBuddyCastMessage(self, msg, permid, selversion): + if self.registered and self.running: + return self.buddycast_core.gotBuddyCastMessage(msg, permid, selversion) + else: + return False + + def gotKeepAliveMessage(self, permid): + if self.registered and self.running: + return self.buddycast_core.gotKeepAliveMessage(permid) + else: + return False + + def handleConnection(self,exc,permid,selversion,locally_initiated): + + if DEBUG: + print >> sys.stderr, "bc: handleConnection",exc,show_permid_short(permid),selversion,locally_initiated,currentThread().getName() + + if not self.registered: + return + + if DEBUG: + nconn = 0 + conns = self.buddycast_core.connections + print >> sys.stderr, "\nbc: conn in buddycast", len(conns) + for peer_permid in conns: + _permid = show_permid_short(peer_permid) + nconn += 1 + print >> sys.stderr, "bc: ", nconn, _permid, conns[peer_permid] + + if self.running or exc is not None: # if not running, only close connection + self.buddycast_core.handleConnection(exc,permid,selversion,locally_initiated) + + def addMyPref(self, torrent): + """ Called by OverlayThread (as should be everything) """ + if self.registered: + self.data_handler.addMyPref(torrent) + + def delMyPref(self, torrent): + if self.registered: + self.data_handler.delMyPref(torrent) + + + +class BuddyCastCore: + + TESTASSERVER = False # for unit testing + + def __init__(self, overlay_bridge, launchmany, data_handler, + buddycast_interval, superpeer, + metadata_handler, torrent_collecting_solution, bartercast_core, votecast_core, channelcast_core, log=None, amcrawler=False): + self.overlay_bridge = overlay_bridge + self.launchmany = launchmany + self.data_handler = data_handler + self.buddycast_interval = buddycast_interval + self.superpeer = superpeer + #print_stack() + #print >> sys.stderr, 'debug buddycast' + #superpeer # change it for superpeers + #self.superpeer_set = Set(self.data_handler.getSuperPeers()) + self.log = log + self.dialback = DialbackMsgHandler.getInstance() + + self.ip = self.data_handler.getMyIp() + self.port = self.data_handler.getMyPort() + self.permid = self.data_handler.getMyPermid() + self.nameutf8 = self.data_handler.getMyName().encode("UTF-8") + + # --- parameters --- + #self.timeout = 5*60 + self.block_interval = 4*60*60 # block interval for a peer to buddycast + self.short_block_interval = 4*60*60 # block interval if failed to connect the peer + self.num_myprefs = 50 # num of my preferences in buddycast msg + self.max_collected_torrents = 50 # num of recently collected torrents (from BuddyCast 3) + self.num_tbs = 10 # num of taste buddies in buddycast msg + self.num_tb_prefs = 10 # num of taset buddy's preferences in buddycast msg + self.num_rps = 10 # num of random peers in buddycast msg + # time to check connection and send keep alive message + #self.check_connection_round = max(1, 120/self.buddycast_interval) + self.max_conn_cand = 100 # max number of connection candidates + self.max_conn_tb = 10 # max number of connectable taste buddies + self.max_conn_rp = 10 # max number of connectable random peers + self.max_conn_up = 10 # max number of unconnectable peers + self.bootstrap_num = 10 # max number of peers to fill when bootstrapping + self.bootstrap_interval = 5*60 # 5 min + self.network_delay = self.buddycast_interval*2 # 30 seconds + self.check_period = 120 # how many seconds to send keep alive message and check updates + self.num_search_cand = 10 # max number of remote search peer candidates + self.num_remote_peers_in_msg = 2 # number of remote search peers in msg + + # --- memory --- + self.send_block_list = {} # permid:unlock_time + self.recv_block_list = {} + self.connections = {} # permid: overlay_version + self.connected_taste_buddies = [] # [permid] + self.connected_random_peers = [] # [permid] + self.connected_connectable_peers = {} # permid: {'connect_time', 'ip', 'port', 'similarity', 'oversion', 'num_torrents'} + self.connected_unconnectable_peers = {} # permid: connect_time + self.connection_candidates = {} # permid: last_seen + self.remote_search_peer_candidates = [] # [last_seen,permid,selversion], sorted, the first one in the list is the oldest one + + # --- stats --- + self.target_type = 0 + self.next_initiate = 0 + self.round = 0 # every call to work() is a round + self.bootstrapped = False # bootstrap once every 1 hours + self.bootstrap_time = 0 # number of times to bootstrap + self.total_bootstrapped_time = 0 + self.last_bootstrapped = now() # bootstrap time of the last time + self.start_time = now() + self.last_check_time = 0 + + # --- dependent modules --- + self.metadata_handler = metadata_handler + self.torrent_collecting = None + if torrent_collecting_solution == BCCOLPOLICY_SIMPLE: + self.torrent_collecting = SimpleTorrentCollecting(metadata_handler, data_handler) + + # -- misc --- + self.dnsindb = launchmany.secure_overlay.get_dns_from_peerdb + if self.log: + self.overlay_log = OverlayLogger.getInstance(self.log) + + # Bartercast + self.bartercast_core = bartercast_core + #self.bartercast_core.buddycast_core = self + + self.votecast_core = votecast_core + self.channelcast_core = channelcast_core + + # Crawler + self.amcrawler = amcrawler + + + def get_peer_info(self, target_permid, include_permid=True): + + if not target_permid: + return ' None ' + dns = self.dnsindb(target_permid) + if not dns: + return ' None ' + try: + ip = dns[0] + port = dns[1] + sim = self.data_handler.getPeerSim(target_permid) + if include_permid: + s_pid = show_permid_short(target_permid) + return ' %s %s:%s %.3f ' % (s_pid, ip, port, sim) + else: + return ' %s:%s %.3f' % (ip, port, sim) + except: + return ' ' + repr(dns) + ' ' + + def work(self): + """ + The worker of buddycast epidemic protocol. + In every round, it selects a target and initates a buddycast exchange, + or idles due to replying messages in the last rounds. + """ + + try: + self.round += 1 + if DEBUG: + print >> sys.stderr, 'bc: Initiate exchange' + self.print_debug_info('Active', 2) + if self.log: + nPeer, nPref, nCc, nBs, nBr, nSO, nCo, nCt, nCr, nCu = self.get_stats() + self.overlay_log('BUCA_STA', self.round, (nPeer,nPref,nCc), (nBs,nBr), (nSO,nCo), (nCt,nCr,nCu)) + + self.print_debug_info('Active', 3) + #print >> sys.stderr, 'bc: ************ working buddycast 2' + self.updateSendBlockList() + + _now = now() + if _now - self.last_check_time >= self.check_period: + self.print_debug_info('Active', 4) + self.keepConnections() + #self.data_handler.checkUpdate() + gc.collect() + self.last_check_time = _now + + if self.next_initiate > 0: + # It replied some meesages in the last rounds, so it doesn't initiate Buddycast + self.print_debug_info('Active', 6) + self.next_initiate -= 1 + else: + if len(self.connection_candidates) == 0: + self.booted = self._bootstrap(self.bootstrap_num) + self.print_debug_info('Active', 9) + + # It didn't reply any message in the last rounds, so it can initiate BuddyCast + if len(self.connection_candidates) > 0: + r, target_permid = self.selectTarget() + self.print_debug_info('Active', 11, target_permid, r=r) + self.startBuddyCast(target_permid) + + if debug: + print + except: + print_exc() + + # -------------- bootstrap -------------- # + def _bootstrap(self, number): + """ Select a number of peers from recent online peers which are not + in send_block_list to fill connection_candidates. + When to call this function is an issue to study. + """ + + _now = now() + # bootstrapped recently, so wait for a while + if self.bootstrapped and _now - self.last_bootstrapped < self.bootstrap_interval: + self.bootstrap_time = 0 # let it read the most recent peers next time + return -1 + + #ARNODB: self.data_handler.peers is a map from peer_id to something, i.e., not + # permid. send_block_list is a list of permids + send_block_list_ids = [] + for permid in self.send_block_list: + peer_id = self.data_handler.getPeerID(permid) + send_block_list_ids.append(peer_id) + + target_cands_ids = Set(self.data_handler.peers) - Set(send_block_list_ids) + recent_peers_ids = self.selectRecentPeers(target_cands_ids, number, + startfrom=self.bootstrap_time*number) + + for peer_id in recent_peers_ids: + last_seen = self.data_handler.getPeerIDLastSeen(peer_id) + self.addConnCandidate(self.data_handler.getPeerPermid(peer_id), last_seen) + self.limitConnCandidate() + + self.bootstrap_time += 1 + self.total_bootstrapped_time += 1 + self.last_bootstrapped = _now + if len(self.connection_candidates) < self.bootstrap_num: + self.bootstrapped = True # don't reboot until self.bootstrap_interval later + else: + self.bootstrapped = False # reset it to allow read more peers if needed + return 1 + + def selectRecentPeers(self, cand_ids, number, startfrom=0): + """ select a number of most recently online peers + @return a list of peer_ids + """ + + if not cand_ids: + return [] + peerids = [] + last_seens = [] + for peer_id in cand_ids: + peerids.append(peer_id) + last_seens.append(self.data_handler.getPeerIDLastSeen(peer_id)) + npeers = len(peerids) + if npeers == 0: + return [] + aux = zip(last_seens, peerids) + aux.sort() + aux.reverse() + peers = [] + i = 0 + + # roll back when startfrom is bigger than npeers + startfrom = startfrom % npeers + endat = startfrom + number + for _, peerid in aux[startfrom:endat]: + peers.append(peerid) + return peers + + def addConnCandidate(self, peer_permid, last_seen): + """ add a peer to connection_candidates, and only keep a number of + the most fresh peers inside. + """ + + if self.isBlocked(peer_permid, self.send_block_list) or peer_permid == self.permid: + return + self.connection_candidates[peer_permid] = last_seen + + def limitConnCandidate(self): + if len(self.connection_candidates) > self.max_conn_cand: + tmp_list = zip(self.connection_candidates.values(),self.connection_candidates.keys()) + tmp_list.sort() + while len(self.connection_candidates) > self.max_conn_cand: + ls,peer_permid = tmp_list.pop(0) + self.removeConnCandidate(peer_permid) + + def removeConnCandidate(self, peer_permid): + if peer_permid in self.connection_candidates: + self.connection_candidates.pop(peer_permid) + + # -------------- routines in each round -------------- # + def updateSendBlockList(self): + """ Remove expired peers in send block list """ + + _now = now() + for p in self.send_block_list.keys(): # don't call isBlocked() for performance reason + if _now >= self.send_block_list[p] - self.network_delay: + if debug: + print >>sys.stderr,"bc: *** unblock peer in send block list" + self.get_peer_info(p) + \ + "expiration:", ctime(self.send_block_list[p]) + self.send_block_list.pop(p) + + def keepConnections(self): + """ Close expired connections, and extend the expiration of + peers in connection lists + """ + + timeout_list = [] + for peer_permid in self.connections: + # we don't close connection here, because if no incoming msg, + # sockethandler will close connection in 5-6 min. + + if (peer_permid in self.connected_connectable_peers or \ + peer_permid in self.connected_unconnectable_peers): + timeout_list.append(peer_permid) + + # 04/08/10 boudewijn: a crawler can no longer disconnect. + # Staying connected means that the crawler is returned in + # buddycast messages, otherwise not. + for peer_permid in timeout_list: + self.sendKeepAliveMsg(peer_permid) + + def sendKeepAliveMsg(self, peer_permid): + """ Send keep alive message to a peer, and extend its expiration """ + + if self.isConnected(peer_permid): + overlay_protocol_version = self.connections[peer_permid] + if overlay_protocol_version >= OLPROTO_VER_THIRD: + # From this version, support KEEP_ALIVE message in secure overlay + keepalive_msg = '' + self.overlay_bridge.send(peer_permid, KEEP_ALIVE+keepalive_msg, + self.keepaliveSendCallback) + if debug: + print >>sys.stderr,"bc: *** Send keep alive to peer", self.get_peer_info(peer_permid), \ + "overlay version", overlay_protocol_version + + def isConnected(self, peer_permid): + return peer_permid in self.connections + + def keepaliveSendCallback(self, exc, peer_permid, other=0): + if exc is None: + pass + else: + if debug: + print >> sys.stderr, "bc: error - send keep alive msg", exc, \ + self.get_peer_info(peer_permid), "Round", self.round + self.closeConnection(peer_permid, 'keepalive:'+str(exc)) + + def gotKeepAliveMessage(self, peer_permid): + if self.isConnected(peer_permid): + if debug: + print >> sys.stderr, "bc: Got keep alive from", self.get_peer_info(peer_permid) + # 04/08/10 boudewijn: a crawler can no longer disconnect. + # Staying connected means that the crawler is returned in + # buddycast messages, otherwise not. + return True + else: + if DEBUG: + print >> sys.stderr, "bc: error - got keep alive from a not connected peer. Round", \ + self.round + return False + + # -------------- initiate buddycast, active thread -------------- # + # ------ select buddycast target ------ # + def selectTarget(self): + """ select a most similar taste buddy or a most likely online random peer + from connection candidates list by 50/50 chance to initate buddycast exchange. + """ + + def selectTBTarget(): + # Select the most similar taste buddy + max_sim = (-1, None) + for permid in self.connection_candidates: + peer_id = self.data_handler.getPeerID(permid) + if peer_id: + sim = self.data_handler.getPeerSim(permid) + max_sim = max(max_sim, (sim, permid)) + selected_permid = max_sim[1] + if selected_permid is None: + return None + else: + return selected_permid + + def selectRPTarget(): + # Randomly select a random peer + selected_permid = None + while len(self.connection_candidates) > 0: + selected_permid = sample(self.connection_candidates, 1)[0] + selected_peer_id = self.data_handler.getPeerID(selected_permid) + if selected_peer_id is None: + self.removeConnCandidate(selected_permid) + selected_permid = None + elif selected_peer_id: + break + + return selected_permid + + self.target_type = 1 - self.target_type + if self.target_type == 0: # select a taste buddy + target_permid = selectTBTarget() + else: # select a random peer + target_permid = selectRPTarget() + + return self.target_type, target_permid + + # ------ start buddycast exchange ------ # + def startBuddyCast(self, target_permid): + """ Connect to a peer, create a buddycast message and send it """ + + if not target_permid or target_permid == self.permid: + return + + if not self.isBlocked(target_permid, self.send_block_list): + if debug: + print >> sys.stderr, 'bc: connect a peer', show_permid_short(target_permid), currentThread().getName() + self.overlay_bridge.connect(target_permid, self.buddycastConnectCallback) + + self.print_debug_info('Active', 12, target_permid) + if self.log: + dns = self.dnsindb(target_permid) + if dns: + ip,port = dns + self.overlay_log('CONN_TRY', ip, port, show_permid(target_permid)) + + # always block the target for a while not matter succeeded or not + #self.blockPeer(target_permid, self.send_block_list, self.short_block_interval) + self.print_debug_info('Active', 13, target_permid) + + # remove it from candidates no matter if it has been connected + self.removeConnCandidate(target_permid) + self.print_debug_info('Active', 14, target_permid) + + else: + if DEBUG: + print >> sys.stderr, 'buddycast: peer', self.get_peer_info(target_permid), \ + 'is blocked while starting buddycast to it.', "Round", self.round + + def buddycastConnectCallback(self, exc, dns, target_permid, selversion): + if exc is None: + self.addConnection(target_permid, selversion, True) + + ## Create message depending on selected protocol version + try: + # 04/08/10 boudewijn: the self.isConnected check fails + # in certain threading conditions, namely when the + # callback to self.buddycastConnectCallback is made + # before the callback to self.handleConnection where + # the peer is put in the connection list. However, + # since self.buddycastConnectCallback already + # indicates a successfull connection, this check is + # not needed. + # if not self.isConnected(target_permid): + # if debug: + # raise RuntimeError, 'buddycast: not connected while calling connect_callback' + # return + + self.print_debug_info('Active', 15, target_permid, selversion) + + self.createAndSendBuddyCastMessage(target_permid, selversion, active=True) + + except: + print_exc() + print >> sys.stderr, "bc: error in reply buddycast msg",\ + exc, dns, show_permid_short(target_permid), selversion, "Round", self.round, + + else: + if debug: + print >> sys.stderr, "bc: warning - connecting to",\ + show_permid_short(target_permid),exc,dns, ctime(now()) + + def createAndSendBuddyCastMessage(self, target_permid, selversion, active): + + #print >>sys.stderr,"bc: SENDING BC to",show_permid_short(target_permid) + #target_permid ="""MFIwEAYHKoZIzj0CAQYFK4EEABoDPgAEAGbSaE3xVUvdMYGkj+x/mE24f/f4ZId7kNPVkALbAa2bQNjCKRDSPt+oE1nzr7It/CfxvCTK+sjOYAjr""" + + #selversion = 12 # for test + + buddycast_data = self.createBuddyCastMessage(target_permid, selversion) + if debug: + print >> sys.stderr, "bc: createAndSendBuddyCastMessage", len(buddycast_data), currentThread().getName() + try: + buddycast_data['permid'] = self.permid + #validBuddyCastData(buddycast_data, self.num_myprefs, + # self.num_tbs, self.num_rps, self.num_tb_prefs) + buddycast_data.pop('permid') + buddycast_msg = bencode(buddycast_data) + except: + print_exc() + print >> sys.stderr, "error buddycast_data:", buddycast_data + return + + if active: + self.print_debug_info('Active', 16, target_permid) + else: + self.print_debug_info('Passive', 6, target_permid) + + self.overlay_bridge.send(target_permid, BUDDYCAST+buddycast_msg, self.buddycastSendCallback) + self.blockPeer(target_permid, self.send_block_list, self.short_block_interval) + self.removeConnCandidate(target_permid) + + if debug: + print >> sys.stderr, '****************--------------'*2 + print >> sys.stderr, 'sent buddycast message to', show_permid_short(target_permid), len(buddycast_msg) + + if active: + self.print_debug_info('Active', 17, target_permid) + else: + self.print_debug_info('Passive', 7, target_permid) + + # Bartercast + if self.bartercast_core != None and active: + try: + self.bartercast_core.createAndSendBarterCastMessage(target_permid, selversion, active) + except: + print_exc() + + # As of March 5, 2009, VoteCast Messages are sent in lock-step with BuddyCast. + # (only if there are any votes to send.) + # Update (July 24, 2009): ChannelCast is used in place of ModerationCast + + if self.votecast_core != None: + try: + self.votecast_core.createAndSendVoteCastMessage(target_permid, selversion) + except: + print_exc() + + + if self.channelcast_core != None: + try: + self.channelcast_core.createAndSendChannelCastMessage(target_permid, selversion) + except: + print_exc() + + if self.log: + dns = self.dnsindb(target_permid) + if dns: + ip,port = dns + if active: + MSG_ID = 'ACTIVE_BC' + else: + MSG_ID = 'PASSIVE_BC' + msg = repr(readableBuddyCastMsg(buddycast_data,selversion)) # from utilities + self.overlay_log('SEND_MSG', ip, port, show_permid(target_permid), selversion, MSG_ID, msg) + + #print >>sys.stderr,"bc: Created BC",`buddycast_data` + + return buddycast_data # Nicolas: for testing + + def createBuddyCastMessage(self, target_permid, selversion, target_ip=None, target_port=None): + """ Create a buddycast message for a target peer on selected protocol version """ + # Nicolas: added manual target_ip, target_port parameters for testing + ## Test + try: + target_ip,target_port = self.dnsindb(target_permid) + except: + if not self.TESTASSERVER: + raise # allow manual ips during unit-testing if dnsindb fails + if not target_ip or not target_port: + return {} + + my_pref = self.data_handler.getMyLivePreferences(selversion, self.num_myprefs) #[pref] + + if debug: + print >> sys.stderr, " bc:Amended preference list is:", str(my_pref) + + taste_buddies = self.getTasteBuddies(self.num_tbs, self.num_tb_prefs, target_permid, target_ip, target_port, selversion) + random_peers = self.getRandomPeers(self.num_rps, target_permid, target_ip, target_port, selversion) #{peer:last_seen} + buddycast_data = {'ip':self.ip, + 'port':self.port, + 'name':self.nameutf8, + 'preferences':my_pref, + 'taste buddies':taste_buddies, + 'random peers':random_peers} + + if selversion >= OLPROTO_VER_THIRD: + # From this version, add 'connectable' entry in buddycast message + connectable = self.isConnectable() + buddycast_data['connectable'] = connectable + + if selversion >= OLPROTO_VER_FOURTH: + recent_collect = self.metadata_handler.getRecentlyCollectedTorrents(self.max_collected_torrents, selversion) + + buddycast_data['collected torrents'] = recent_collect + + if selversion >= OLPROTO_VER_SIXTH: + npeers = self.data_handler.get_npeers() + ntorrents = self.data_handler.get_ntorrents() + nmyprefs = self.data_handler.get_nmyprefs() + buddycast_data['npeers'] = npeers + buddycast_data['nfiles'] = ntorrents + buddycast_data['ndls'] = nmyprefs + + + return buddycast_data + + def getTasteBuddies(self, ntbs, ntbprefs, target_permid, target_ip, target_port, selversion): + """ Randomly select a number of peers from connected_taste_buddies. """ + + if not self.connected_taste_buddies: + return [] + tb_list = self.connected_taste_buddies[:] + if target_permid in tb_list: + tb_list.remove(target_permid) + + peers = [] + for permid in tb_list: + # keys = ('ip', 'port', 'oversion', 'num_torrents') + peer = deepcopy(self.connected_connectable_peers[permid]) + if peer['ip'] == target_ip and peer['port'] == target_port: + continue + peer['similarity'] = self.data_handler.getPeerSim(permid) + peer['permid'] = permid + # Arno, 2010-01-28: St*pid Unicode handling causes IP addresses to be Unicode, fix. + peer['ip'] = str(peer['ip']) + peers.append(peer) + +# peers = self.data_handler.getPeers(tb_list, ['permid', 'ip', 'port', 'similarity', 'oversion', 'num_torrents']) +# # filter peers with the same ip and port +# peers = filter(lambda p:p['ip']!=target_ip or int(p['port'])!=target_port, peers) +# +# for i in range(len(peers)): +# peers[i]['port'] = int(peers[i]['port']) + + # In overlay version 2, buddycast has 'age' field + if selversion <= OLPROTO_VER_SECOND: + for i in range(len(peers)): + peers[i]['age'] = 0 + + # In overlay version 2 and 3, buddycast doesn't have similarity field, and taste buddy has preferences + if selversion <= OLPROTO_VER_THIRD: + for i in range(len(peers)): + peers[i].pop('similarity') + peers[i]['preferences'] = [] # don't support from now on + + # From overlay version 4, buddycast includes similarity for peers + if selversion >= OLPROTO_VER_FOURTH: + for i in range(len(peers)): + peers[i]['similarity'] = int(peers[i]['similarity']+0.5) # bencode doesn't accept float type + + + + # Every peer >= 6 in message attachs nfiles and oversion for remote search from version 6 + for i in range(len(peers)): + oversion = peers[i].pop('oversion') + nfiles = peers[i].pop('num_torrents') + if selversion >= OLPROTO_VER_SIXTH and oversion >= OLPROTO_VER_SIXTH and nfiles >= REMOTE_SEARCH_PEER_NTORRENTS_THRESHOLD: + peers[i]['oversion'] = oversion + # ascribe it to the inconsistent name of the same concept in msg and db + peers[i]['nfiles'] = nfiles + + return peers + + def getRandomPeers(self, nrps, target_permid, target_ip, target_port, selversion): + """ Randomly select a number of peers from connected_random_peers. """ + + if not self.connected_random_peers: + return [] + rp_list = self.connected_random_peers[:] + + # From version 6, two (might be offline) remote-search-peers must be included in msg + if selversion >= OLPROTO_VER_SIXTH: + remote_search_peers = self.getRemoteSearchPeers(self.num_remote_peers_in_msg) + rp_list += remote_search_peers + if len(rp_list) > nrps: + rp_list = sample(rp_list, nrps) + + if target_permid in rp_list: + rp_list.remove(target_permid) + + peers = [] + if DEBUG: + print >> sys.stderr, 'bc: ******** rplist nconn', len(rp_list), len(self.connected_connectable_peers) + #print >> sys.stderr, rp_list, self.connected_connectable_peers + for permid in rp_list: + # keys = ('ip', 'port', 'oversion', 'num_torrents') + #print >> sys.stderr, '**************', `self.connected_connectable_peers`, `rp_list` + # TODO: Fix this bug: not consisitent + if permid not in self.connected_connectable_peers: + continue + peer = deepcopy(self.connected_connectable_peers[permid]) + if peer['ip'] == target_ip and peer['port'] == target_port: + continue + peer['similarity'] = self.data_handler.getPeerSim(permid) + peer['permid'] = permid + # Arno, 2010-01-28: St*pid Unicode handling causes IP addresses to be Unicode, fix. + peer['ip'] = str(peer['ip']) + peers.append(peer) + +# peers = self.data_handler.getPeers(rp_list, ['permid', 'ip', 'port', 'similarity', 'oversion', 'num_torrents']) +# peers = filter(lambda p:p['ip']!=target_ip or int(p['port'])!=target_port, peers) +# +# for i in range(len(peers)): +# peers[i]['port'] = int(peers[i]['port']) + + if selversion <= OLPROTO_VER_SECOND: + for i in range(len(peers)): + peers[i]['age'] = 0 + + # random peer also attachs similarity from 4 + if selversion <= OLPROTO_VER_THIRD: + for i in range(len(peers)): + peers[i].pop('similarity') + + if selversion >= OLPROTO_VER_FOURTH: + for i in range(len(peers)): + old_sim = peers[i]['similarity'] + if old_sim is None: + old_sim = 0.0 + peers[i]['similarity'] = int(old_sim+0.5) + + # Every peer >= 6 in message attachs nfiles and oversion for remote search from version 6 + for i in range(len(peers)): + oversion = peers[i].pop('oversion') + nfiles = peers[i].pop('num_torrents') + # only include remote-search-peers + if selversion >= OLPROTO_VER_SIXTH and oversion >= OLPROTO_VER_SIXTH and nfiles >= REMOTE_SEARCH_PEER_NTORRENTS_THRESHOLD: + peers[i]['oversion'] = oversion + # ascribe it to the inconsistent name of the same concept in msg and db + peers[i]['nfiles'] = nfiles + + return peers + + def isConnectable(self): + return bool(self.dialback.isConnectable()) + + def buddycastSendCallback(self, exc, target_permid, other=0): + if exc is None: + if debug: + print >>sys.stderr,"bc: *** msg was sent successfully to peer", \ + self.get_peer_info(target_permid) + else: + if debug: + print >>sys.stderr,"bc: *** warning - error in sending msg to",\ + self.get_peer_info(target_permid), exc + self.closeConnection(target_permid, 'buddycast:'+str(exc)) + + def blockPeer(self, peer_permid, block_list, block_interval=None): + """ Add a peer to a block list """ + + peer_id = peer_permid # ARNODB: confusing! + if block_interval is None: + block_interval = self.block_interval + unblock_time = now() + block_interval + block_list[peer_id] = unblock_time + + + + def isBlocked(self, peer_permid, block_list): + if self.TESTASSERVER: + return False # we do not want to be blocked when sending various messages + + peer_id = peer_permid + if peer_id not in block_list: + return False + + unblock_time = block_list[peer_id] + if now() >= unblock_time - self.network_delay: # 30 seconds for network delay + block_list.pop(peer_id) + return False + return True + + + + # ------ receive a buddycast message, for both active and passive thread ------ # + def gotBuddyCastMessage(self, recv_msg, sender_permid, selversion): + """ Received a buddycast message and handle it. Reply if needed """ + + if debug: + print >> sys.stderr, "bc: got and handle buddycast msg", currentThread().getName() + + if not sender_permid or sender_permid == self.permid: + print >> sys.stderr, "bc: error - got BuddyCastMsg from a None peer", \ + sender_permid, recv_msg, "Round", self.round + return False + + blocked = self.isBlocked(sender_permid, self.recv_block_list) + + if blocked: + if DEBUG: + print >> sys.stderr, "bc: warning - got BuddyCastMsg from a recv blocked peer", \ + show_permid(sender_permid), "Round", self.round + return True # allow the connection to be kept. That peer may have restarted in 4 hours + + # Jie: Because buddycast message is implemented as a dictionary, anybody can + # insert any content in the message. It isn't secure if someone puts + # some fake contents inside and make the message very large. The same + # secure issue could happen in other protocols over the secure overlay layer. + # Therefore, I'd like to set a limitation of the length of buddycast message. + # The receiver should close the connection if the length of the message + # exceeds the limitation. According to my experience, the biggest + # buddycast message should be around 6~7KBytes. So the reasonable + # length limitation might be 10KB for buddycast message. + if MAX_BUDDYCAST_LENGTH > 0 and len(recv_msg) > MAX_BUDDYCAST_LENGTH: + print >> sys.stderr, "bc: warning - got large BuddyCastMsg", len(recv_msg), "Round", self.round + return False + + active = self.isBlocked(sender_permid, self.send_block_list) + + if active: + self.print_debug_info('Active', 18, sender_permid) + else: + self.print_debug_info('Passive', 2, sender_permid) + + buddycast_data = {} + try: + try: + buddycast_data = bdecode(recv_msg) + except ValueError, msg: + try: + errmsg = str(msg) + except: + errmsg = repr(msg) + if DEBUG: + print >> sys.stderr, "bc: warning, got invalid BuddyCastMsg:", errmsg, \ + "Round", self.round # ipv6 + return False + buddycast_data.update({'permid':sender_permid}) + + try: # check buddycast message + validBuddyCastData(buddycast_data, 0, + self.num_tbs, self.num_rps, self.num_tb_prefs, selversion) # RCP 2 + except RuntimeError, msg: + try: + errmsg = str(msg) + except: + errmsg = repr(msg) + if DEBUG: + dns = self.dnsindb(sender_permid) + print >> sys.stderr, "bc: warning, got invalid BuddyCastMsg:", errmsg, "From", dns, "Round", self.round # ipv6 + + return False + + # update sender's ip and port in buddycast + dns = self.dnsindb(sender_permid) + if dns != None: + sender_ip = dns[0] + sender_port = dns[1] + buddycast_data.update({'ip':sender_ip}) + buddycast_data.update({'port':sender_port}) + + if self.log: + if active: + MSG_ID = 'ACTIVE_BC' + else: + MSG_ID = 'PASSIVE_BC' + msg = repr(readableBuddyCastMsg(buddycast_data,selversion)) # from utilities + self.overlay_log('RECV_MSG', sender_ip, sender_port, show_permid(sender_permid), selversion, MSG_ID, msg) + + # store discovered peers/preferences/torrents to cache and db + conn = buddycast_data.get('connectable', 0) # 0 - unknown + + self.handleBuddyCastMessage(sender_permid, buddycast_data, selversion) + if active: + conn = 1 + + if active: + self.print_debug_info('Active', 19, sender_permid) + else: + self.print_debug_info('Passive', 3, sender_permid) + + # update sender and other peers in connection list + addto = self.addPeerToConnList(sender_permid, conn) + + if active: + self.print_debug_info('Active', 20, sender_permid) + else: + self.print_debug_info('Passive', 4, sender_permid) + + except Exception, msg: + print_exc() + raise Exception, msg + return True # don't close connection, maybe my problem in handleBuddyCastMessage + + self.blockPeer(sender_permid, self.recv_block_list) + + # update torrent collecting module + #self.data_handler.checkUpdate() + collectedtorrents = buddycast_data.get('collected torrents', []) + if selversion >= OLPROTO_VER_ELEVENTH: + collected_infohashes = [] + for value in collectedtorrents: + infohash = value['infohash'] + collected_infohashes.append(infohash) + else: + collected_infohashes = collectedtorrents + + if self.torrent_collecting and not self.superpeer: + collected_infohashes += self.getPreferenceHashes(buddycast_data) + self.torrent_collecting.trigger(sender_permid, selversion, collected_infohashes) + + if active: + self.print_debug_info('Active', 21, sender_permid) + else: + self.print_debug_info('Passive', 5, sender_permid) + + if not active: + self.replyBuddyCast(sender_permid, selversion) + + # show activity + buf = dunno2unicode('"'+buddycast_data['name']+'"') + self.launchmany.set_activity(NTFY_ACT_RECOMMEND, buf) + + if DEBUG: + print >> sys.stderr, "bc: Got BUDDYCAST message from",self.get_peer_info(sender_permid),active + + return True + + + def createPreferenceDictionaryList(self, buddycast_data): + """as of OL 8, preferences are no longer lists of infohashes, but lists of lists containing + infohashes and associated metadata. this method checks which overlay version has been used + and replaces either format by a list of dictionaries, such that the rest of the code can remain + version-agnostic and additional information like torrent ids can be stored along the way""" + + prefs = buddycast_data.get('preferences',[]) + # assume at least one entry below here + if len(prefs) == 0: + return [] + d = [] + + try: + + if not type(prefs[0])==list: + # pre-OLPROTO_VER_EIGHTH + # create dictionary from list of info hashes, extended fields simply aren't set + + d = [dict({'infohash': pref}) for pref in prefs] + + # we shouldn't receive these lists if the peer says he's OL 8. + # let's accept it but complain + if buddycast_data['oversion'] >= OLPROTO_VER_EIGHTH: + if DEBUG: + print >> sys.stderr, 'buddycast: received OLPROTO_VER_EIGHTH buddycast data containing old style preferences. only ok if talking to an earlier non-release version' + return d + + # if the single prefs entries are lists, we have a more modern wire format + # currently, there is only one possibility + if buddycast_data['oversion'] >= OLPROTO_VER_ELEVENTH: + # Rahim: This part extracts swarm size info from the BC message + # and then returns it in the result list. + # create dictionary from list of lists + d = [dict({'infohash': pref[0], + 'search_terms': pref[1], + 'position': pref[2], + 'reranking_strategy': pref[3], + 'num_seeders':pref[4], + 'num_leechers':pref[5], + 'calc_age':pref[6], + 'num_sources_seen':pref[7]}) + for pref in prefs] + + elif buddycast_data['oversion'] >= OLPROTO_VER_EIGHTH: + # create dictionary from list of lists + d = [dict({'infohash': pref[0], + 'search_terms': pref[1], + 'position': pref[2], + 'reranking_strategy': pref[3]}) + for pref in prefs] + else: + raise RuntimeError, 'buddycast: unknown preference protocol, pref entries are lists but oversion= %s:\n%s' % (buddycast_data['oversion'], prefs) + + return d + + except Exception, msg: + print_exc() + raise Exception, msg + return d + + + def getPreferenceHashes(self, buddycast_data): + """convenience function returning the infohashes from the preferences. + returns a list of infohashes, i.e. replaces old calls to buddycast_data.get('preferences')""" + return [preference.get('infohash',"") for preference in buddycast_data.get('preferences', [])] + + def handleBuddyCastMessage(self, sender_permid, buddycast_data, selversion): + """ Handle received buddycast message + Add peers, torrents and preferences into database and update last seen + Add fresh peers to candidate list + All database updates caused by buddycast msg should be handled here + """ + + _now = now() + + cache_db_data = {'peer':{},'infohash':Set(),'pref':[], 'coll':[]} # peer, updates / pref, pairs, Rahim: coll for colleected torrents + cache_peer_data = {} + + tbs = buddycast_data.pop('taste buddies') + rps = buddycast_data.pop('random peers') + buddycast_data['oversion'] = selversion + + # print >> sys.stderr, "bc: \n" * 10 + # print >> sys.stderr, "bc: received", len(tbs), "and", len(rps), "tastebudies and randompeers, respectively" + # for peer in tbs: + # print >> sys.stderr, "bc: tastebuddy", peer + # for peer in rps: + # print >> sys.stderr, "bc: randompeer", peer + + max_tb_sim = 1 + + # include sender itself + bc_data = [buddycast_data] + tbs + rps + for peer in bc_data: + + #print >>sys.stderr,"bc: Learned about peer",peer['ip'] + + peer_permid = peer['permid'] + if peer_permid == self.permid: + continue + age = max(peer.get('age', 0), 0) # From secure overlay version 3, it doesn't include 'age' + last_seen = _now - age + old_last_seen = self.data_handler.getPeerLastSeen(peer_permid) + last_seen = min(max(old_last_seen, last_seen), _now) + oversion = peer.get('oversion', 0) + nfiles = peer.get('nfiles', 0) + self.addRemoteSearchPeer(peer_permid, oversion, nfiles, last_seen) + + cache_peer_data[peer_permid] = {} + cache_peer_data[peer_permid]['last_seen'] = last_seen + #self.data_handler._addPeerToCache(peer_permid, last_seen) + #if selversion >= OLPROTO_VER_FOURTH: + sim = peer.get('similarity', 0) + max_tb_sim = max(max_tb_sim, sim) + if sim > 0: + cache_peer_data[peer_permid]['sim'] = sim + #self.data_handler.addRelativeSim(sender_permid, peer_permid, sim, max_tb_sim) + + if peer_permid != sender_permid: + self.addConnCandidate(peer_permid, last_seen) + + new_peer_data = {} + #new_peer_data['permid'] = peer['permid'] + new_peer_data['ip'] = hostname_or_ip2ip(peer['ip']) + new_peer_data['port'] = peer['port'] + new_peer_data['last_seen'] = last_seen + if peer.has_key('name'): + new_peer_data['name'] = dunno2unicode(peer['name']) # store in db as unicode + cache_db_data['peer'][peer_permid] = new_peer_data + #self.data_handler.addPeer(peer_permid, last_seen, new_peer_data, commit=True) # new peer + + self.limitConnCandidate() + if len(self.connection_candidates) > self.bootstrap_num: + self.bootstrapped = True + + # database stuff + if selversion >= OLPROTO_VER_SIXTH: + stats = {'num_peers':buddycast_data['npeers'],'num_torrents':buddycast_data['nfiles'],'num_prefs':buddycast_data['ndls']} + cache_db_data['peer'][sender_permid].update(stats) + + cache_db_data['peer'][sender_permid]['last_buddycast'] = _now + + prefs = self.createPreferenceDictionaryList(buddycast_data) + + #Rahim: Since overlay version 11 , the collected torrents contain + # swarm size info. The code below handles it and changes list of list + # to a list of dictionary, same as preference. + # + if selversion >= OLPROTO_VER_ELEVENTH: + collecteds = self.createCollectedDictionaryList(buddycast_data, selversion) + buddycast_data['collected torrents'] = collecteds + infohashes = Set(self.getCollectedHashes(buddycast_data, selversion)) + else: + infohashes = Set(buddycast_data.get('collected torrents', [])) + + # Nicolas: store this back into buddycast_data because it's used later on gotBuddyCastMessage again + buddycast_data['preferences'] = prefs + prefhashes = Set(self.getPreferenceHashes(buddycast_data)) # only accept sender's preference, to avoid pollution + infohashes = infohashes.union(prefhashes) + + cache_db_data['infohash'] = infohashes + if prefs: + cache_db_data['pref'] = prefs + + + if selversion >= OLPROTO_VER_ELEVENTH: + if collecteds: + cache_db_data['coll'] = collecteds + + + self.data_handler.handleBCData(cache_db_data, cache_peer_data, sender_permid, max_tb_sim, selversion, _now) + + def getCollectedHashes(self, buddycast_data, selversion): + """ + @author: Rahim + @param buddycast_data: A dictionary structure that contains received buddycast message. + @param selversion: The selected overlay version between peers. + @return: The infohash of the collected torrents is returned as a list. + """ + return [collected.get('infohash',"") for collected in buddycast_data.get('collected torrents', [])] + + + def createCollectedDictionaryList(self, buddycast_data, selversion): + """ + Processes the list of the collected torrents and then returns back a list of dictionaries. + @author: Rahim + @param buddycast_data: Received BC message. + @param selversion: Version of the agreed OL protocol. + @return: List of dictionaries. Each item in the dictionary is like : + """ + collecteds = buddycast_data.get('collected torrents',[]) + + if len(collecteds) == 0: + return [] + d = [] + + try: + d = [dict({'infohash': coll[0], + 'num_seeders': coll[1], + 'num_leechers': coll[2], + 'calc_age': coll[3], + 'num_sources_seen':coll[4]}) + for coll in collecteds] + + return d + except Exception, msg: + print_exc() + raise Exception, msg + return d + + def removeFromConnList(self, peer_permid): + removed = 0 + if peer_permid in self.connected_connectable_peers: # Ct + self.connected_connectable_peers.pop(peer_permid) + try: + self.connected_taste_buddies.remove(peer_permid) + except ValueError: + pass + try: + self.connected_random_peers.remove(peer_permid) + except ValueError: + pass + removed = 1 + if peer_permid in self.connected_unconnectable_peers: # Cu + self.connected_unconnectable_peers.pop(peer_permid) + removed = 2 + return removed + + def addPeerToConnList(self, peer_permid, connectable=0): + """ Add the peer to Ct, Cr or Cu """ + + # remove the existing peer from lists so that its status can be updated later + self.removeFromConnList(peer_permid) + + if not self.isConnected(peer_permid): + #print >> sys.stderr, "bc: cannot add a unconnected peer to conn list", "Round", self.round + return + + _now = now() + + if connectable == 1: + self.addPeerToConnCP(peer_permid, _now) + addto = '(reachable peer)' + else: + self.addPeerToConnUP(peer_permid, _now) + addto = '(peer deemed unreachable)' + + return addto + + def updateTBandRPList(self): + """ Select the top 10 most similar (sim>0) peer to TB and others to RP """ + + """ In early September 2009, it has been decided that, out of 10 taste buddies, 3 peers are selected which has an overlay + same or better of the current version; another 3 peers are selected each of which has an overlay better than 8. Rest + of the slots are filled with highest similarity (just as before). The process of the selection of random peers is not changed!""" + + nconnpeers = len(self.connected_connectable_peers) + if nconnpeers == 0: + self.connected_taste_buddies = [] + self.connected_random_peers = [] + return + + # we need at least 3 peers of the same or better versions, among taste buddies + better_version_peers = 0 + + # we also need at least 4 peers of the recent versions (here, OL>=8), among taste buddies + recent_version_peers = 0 + + tmplist = [] + tmpverlist = [] + tmplist2 = [] + tbs = [] + rps = [] + for permid in self.connected_connectable_peers: + sim = self.data_handler.getPeerSim(permid) + version = self.connected_connectable_peers[permid]['oversion'] + if sim > 0: + tmplist.append([version,sim,permid]) + else: + rps.append(permid) + + #ntb = self.max_conn_tb # 10 tb & 10 rp + ntb = min((nconnpeers+1)/2, self.max_conn_tb) # half tb and half rp + + """ tmplist now contains all peers which have sim > 0, + because of new similarity function we add X peers until ntb is reached + """ + if len(tmplist) < ntb: + cold_start_peers = P2PSimColdStart(self.connected_connectable_peers, tmplist, ntb - len(tmplist)) + tmplist.extend(cold_start_peers) + + #remove cold_start_peers from rps + for version, sim, permid in cold_start_peers: + if permid in rps: + rps.remove(permid) + + """ sort tmplist, emphasis is on overlay version, then on similarity. + thus we try to select top-(self.max_conn_tb) with the highest overlay/similarity + """ + tmplist.sort() + tmplist.reverse() + + if len(tmplist) > 0: + for version,sim,permid in tmplist: + if version >= OLPROTO_VER_CURRENT and better_version_peers<=3: #OLPROTO_VER_ELEVENTH + better_version_peers += 1 + tmpverlist.append(permid) + elif version >= OLPROTO_VER_EIGHTH and recent_version_peers<=3: + recent_version_peers += 1 + tmpverlist.append(permid) + else: + tmplist2.append([sim,permid]) + tmplist2.sort() + tmplist2.reverse() + tbs = tmpverlist + for sim, permid in tmplist2[:ntb-better_version_peers-recent_version_peers]: + tbs.append(permid) + + ntb = len(tbs) + if len(tmplist) > ntb: + rps = [permid for sim,permid in tmplist2[ntb-better_version_peers-recent_version_peers:]] + rps + + tmplist = [] + # remove the oldest peer from both random peer list and connected_connectable_peers + if len(rps) > self.max_conn_rp: + # then select recently seen peers + tmplist = [] + for permid in rps: + connect_time = self.connected_connectable_peers[permid]['connect_time'] + tmplist.append([connect_time, permid]) + tmplist.sort() + tmplist.reverse() + rps = [] + for last_seen,permid in tmplist[:self.max_conn_rp]: + rps.append(permid) + for last_seen,permid in tmplist[self.max_conn_rp:]: + self.connected_connectable_peers.pop(permid) + + self.connected_taste_buddies = tbs + self.connected_random_peers = rps + #print >> sys.stderr, "#tbs:",len(tbs), ";#rps:", len(rps) + #for p in self.connected_taste_buddies: + # assert p in self.connected_connectable_peers + #for p in self.connected_random_peers: + # assert p in self.connected_connectable_peers + #assert len(self.connected_taste_buddies) + len(self.connected_random_peers) <= len(self.connected_connectable_peers) + + + def addPeerToConnCP(self, peer_permid, conn_time): + keys = ('ip', 'port', 'oversion', 'num_torrents') + res = self.data_handler.getPeer(peer_permid, keys) + peer = dict(zip(keys,res)) + peer['connect_time'] = conn_time + self.connected_connectable_peers[peer_permid] = peer + self.updateTBandRPList() + + def addNewPeerToConnList(self, conn_list, max_num, peer_permid, conn_time): + """ Add a peer to a connection list, and pop the oldest peer out """ + + if max_num <= 0 or len(conn_list) < max_num: + conn_list[peer_permid] = conn_time + return None + + else: + oldest_peer = (conn_time+1, None) + initial = 'abcdefghijklmnopqrstuvwxyz' + separator = ':-)' + for p in conn_list: + _conn_time = conn_list[p] + r = randint(0, self.max_conn_tb) + name = initial[r] + separator + p + to_cmp = (_conn_time, name) + oldest_peer = min(oldest_peer, to_cmp) + + if conn_time >= oldest_peer[0]: # add it + out_peer = oldest_peer[1].split(separator)[1] + conn_list.pop(out_peer) + conn_list[peer_permid] = conn_time + return out_peer + return peer_permid + + def addPeerToConnUP(self, peer_permid, conn_time): + ups = self.connected_unconnectable_peers + if peer_permid not in ups: + out_peer = self.addNewPeerToConnList(ups, + self.max_conn_up, peer_permid, conn_time) + if out_peer != peer_permid: + return True + return False + + # -------------- reply buddycast, passive thread -------------- # + def replyBuddyCast(self, target_permid, selversion): + """ Reply a buddycast message """ + + #print >> sys.stderr, '*************** replay buddycast message', show_permid_short(target_permid), self.isConnected(target_permid) + + if not self.isConnected(target_permid): + #print >> sys.stderr, 'buddycast: lost connection while replying buddycast', \ + # "Round", self.round + return + + self.createAndSendBuddyCastMessage(target_permid, selversion, active=False) + + self.print_debug_info('Passive', 8, target_permid) + self.print_debug_info('Passive', 9, target_permid) + + self.next_initiate += 1 # Be idel in next round + self.print_debug_info('Passive', 10) + + + # -------------- handle overlay connections from SecureOverlay ---------- # + def handleConnection(self,exc,permid,selversion,locally_initiated): + if exc is None and permid != self.permid: # add a connection + self.addConnection(permid, selversion, locally_initiated) + else: + self.closeConnection(permid, 'overlayswarm:'+str(exc)) + + if debug: + print >> sys.stderr, "bc: handle conn from overlay", exc, \ + self.get_peer_info(permid), "selversion:", selversion, \ + "local_init:", locally_initiated, ctime(now()), "; #connections:", len(self.connected_connectable_peers), \ + "; #TB:", len(self.connected_taste_buddies), "; #RP:", len(self.connected_random_peers) + + def addConnection(self, peer_permid, selversion, locally_initiated): + # add connection to connection list + _now = now() + if DEBUG: + print >> sys.stderr, "bc: addConnection", self.isConnected(peer_permid) + if not self.isConnected(peer_permid): + # SecureOverlay has already added the peer to db + self.connections[peer_permid] = selversion # add a new connection + addto = self.addPeerToConnList(peer_permid, locally_initiated) + + dns = self.get_peer_info(peer_permid, include_permid=False) + buf = '%s %s'%(dns, addto) + self.launchmany.set_activity(NTFY_ACT_MEET, buf) # notify user interface + + if self.torrent_collecting and not self.superpeer: + try: + # Arno, 2009-10-09: Torrent Collecting errors should not kill conn. + self.torrent_collecting.trigger(peer_permid, selversion) + except: + print_exc() + + if debug: + print >> sys.stderr, "bc: add connection", \ + self.get_peer_info(peer_permid), "to", addto + if self.log: + dns = self.dnsindb(peer_permid) + if dns: + ip,port = dns + self.overlay_log('CONN_ADD', ip, port, show_permid(peer_permid), selversion) + + def closeConnection(self, peer_permid, reason): + """ Close connection with a peer, and remove it from connection lists """ + + if debug: + print >> sys.stderr, "bc: close connection:", self.get_peer_info(peer_permid) + + if self.isConnected(peer_permid): + self.connections.pop(peer_permid) + removed = self.removeFromConnList(peer_permid) + if removed == 1: + self.updateTBandRPList() + + if self.log: + dns = self.dnsindb(peer_permid) + if dns: + ip,port = dns + self.overlay_log('CONN_DEL', ip, port, show_permid(peer_permid), reason) + + # -------------- print debug info ---------- # + def get_stats(self): + nPeer = len(self.data_handler.peers) + nPref = nPeer #len(self.data_handler.preferences) + nCc = len(self.connection_candidates) + nBs = len(self.send_block_list) + nBr = len(self.recv_block_list) + nSO = -1 # TEMP ARNO len(self.overlay_bridge.debug_get_live_connections()) + nCo = len(self.connections) + nCt = len(self.connected_taste_buddies) + nCr = len(self.connected_random_peers) + nCu = len(self.connected_unconnectable_peers) + return nPeer, nPref, nCc, nBs, nBr, nSO, nCo, nCt, nCr, nCu + + def print_debug_info(self, thread, step, target_permid=None, selversion=0, r=0, addto=''): + if not debug: + return + if DEBUG: + print >>sys.stderr,"bc: *****", thread, str(step), "-", + if thread == 'Active': + if step == 2: + print >> sys.stderr, "Working:", now() - self.start_time, \ + "seconds since start. Round", self.round, "Time:", ctime(now()) + nPeer, nPref, nCc, nBs, nBr, nSO, nCo, nCt, nCr, nCu = self.get_stats() + print >> sys.stderr, "bc: *** Status: nPeer nPref nCc: %d %d %d nBs nBr: %d %d nSO nCo nCt nCr nCu: %d %d %d %d %d" % \ + (nPeer,nPref,nCc, nBs,nBr, nSO,nCo, nCt,nCr,nCu) + if nSO != nCo: + print >> sys.stderr, "bc: warning - nSo and nCo is inconsistent" + if nCc > self.max_conn_cand or nCt > self.max_conn_tb or nCr > self.max_conn_rp or nCu > self.max_conn_up: + print >> sys.stderr, "bc: warning - nCC or nCt or nCr or nCu overloads" + _now = now() + buf = "" + i = 1 + for p in self.connected_taste_buddies: + buf += "bc: %d taste buddies: "%i + self.get_peer_info(p) + str(_now-self.connected_connectable_peers[p]['connect_time']) + " version: " + str(self.connections[p]) + "\n" + i += 1 + print >> sys.stderr, buf + + buf = "" + i = 1 + for p in self.connected_random_peers: + buf += "bc: %d random peers: "%i + self.get_peer_info(p) + str(_now-self.connected_connectable_peers[p]['connect_time']) + " version: " + str(self.connections[p]) + "\n" + i += 1 + print >> sys.stderr, buf + + buf = "" + i = 1 + for p in self.connected_unconnectable_peers: + buf += "bc: %d unconnectable peers: "%i + self.get_peer_info(p) + str(_now-self.connected_unconnectable_peers[p]) + " version: " + str(self.connections[p]) + "\n" + i += 1 + print >> sys.stderr, buf + buf = "" + totalsim = 0 + nsimpeers = 0 + minsim = 1e10 + maxsim = 0 + sims = [] + for p in self.data_handler.peers: + sim = self.data_handler.peers[p][PEER_SIM_POS] + if sim > 0: + sims.append(sim) + if sims: + minsim = min(sims) + maxsim = max(sims) + nsimpeers = len(sims) + totalsim = sum(sims) + if nsimpeers > 0: + meansim = totalsim/nsimpeers + else: + meansim = 0 + print >> sys.stderr, "bc: * sim peer: %d %.3f %.3f %.3f %.3f\n" % (nsimpeers, totalsim, meansim, minsim, maxsim) + + elif step == 3: + print >> sys.stderr, "check blocked peers: Round", self.round + + elif step == 4: + print >> sys.stderr, "keep connections with peers: Round", self.round + + elif step == 6: + print >> sys.stderr, "idle loop:", self.next_initiate + + elif step == 9: + print >> sys.stderr, "bootstrapping: select", self.bootstrap_num, \ + "peers recently seen from Mega Cache" + if self.booted < 0: + print >> sys.stderr, "bc: *** bootstrapped recently, so wait for a while" + elif self.booted == 0: + print >> sys.stderr, "bc: *** no peers to bootstrap. Try next time" + else: + print >> sys.stderr, "bc: *** bootstrapped, got", len(self.connection_candidates), \ + "peers in Cc. Times of bootstrapped", self.total_bootstrapped_time + buf = "" + for p in self.connection_candidates: + buf += "bc: * cand:" + `p` + "\n" + buf += "\nbc: Remote Search Peer Candidates:\n" + for p in self.remote_search_peer_candidates: + buf += "bc: * remote: %d "%p[0] + self.get_peer_info(p[1]) + "\n" + print >> sys.stderr, buf + + elif step == 11: + buf = "select " + if r == 0: + buf += "a most similar taste buddy" + else: + buf += "a most likely online random peer" + buf += " from Cc for buddycast out\n" + + if target_permid: + buf += "bc: *** got target %s sim: %s last_seen: %s" % \ + (self.get_peer_info(target_permid), + self.data_handler.getPeerSim(target_permid), + ctime(self.data_handler.getPeerLastSeen(target_permid))) + else: + buf += "bc: *** no target to select. Skip this round" + print >> sys.stderr, buf + + elif step == 12: + print >> sys.stderr, "connect a peer to start buddycast", self.get_peer_info(target_permid) + + elif step == 13: + print >> sys.stderr, "block connected peer in send block list", \ + self.get_peer_info(target_permid)#, self.send_block_list[target_permid] + + elif step == 14: + print >> sys.stderr, "remove connected peer from Cc", \ + self.get_peer_info(target_permid)#, "removed?", target_permid not in self.connection_candidates + + elif step == 15: + print >> sys.stderr, "peer is connected", \ + self.get_peer_info(target_permid), "overlay version", selversion, currentThread().getName() + + elif step == 16: + print >> sys.stderr, "create buddycast to send to", self.get_peer_info(target_permid) + + elif step == 17: + print >> sys.stderr, "send buddycast msg to", self.get_peer_info(target_permid) + + elif step == 18: + print >> sys.stderr, "receive buddycast message from peer %s" % self.get_peer_info(target_permid) + + elif step == 19: + print >> sys.stderr, "store peers from incoming msg to cache and db" + + elif step == 20: + print >> sys.stderr, "add connected peer %s to connection list %s" % (self.get_peer_info(target_permid), addto) + + elif step == 21: + print >> sys.stderr, "block connected peer in recv block list", \ + self.get_peer_info(target_permid), self.recv_block_list[target_permid] + + if thread == 'Passive': + if step == 2: + print >> sys.stderr, "receive buddycast message from peer %s" % self.get_peer_info(target_permid) + + elif step == 3: + print >> sys.stderr, "store peers from incoming msg to cache and db" + + elif step == 4: + print >> sys.stderr, "add connected peer %s to connection list %s" % (self.get_peer_info(target_permid), addto) + + elif step == 5: + print >> sys.stderr, "block connected peer in recv block list", \ + self.get_peer_info(target_permid), self.recv_block_list[target_permid] + + elif step == 6: + print >> sys.stderr, "create buddycast to reply to", self.get_peer_info(target_permid) + + elif step == 7: + print >> sys.stderr, "reply buddycast msg to", self.get_peer_info(target_permid) + + elif step == 8: + print >> sys.stderr, "block connected peer in send block list", \ + self.get_peer_info(target_permid), self.send_block_list[target_permid] + + elif step == 9: + print >> sys.stderr, "remove connected peer from Cc", \ + self.get_peer_info(target_permid)#, "removed?", target_permid not in self.connection_candidates + + elif step == 10: + print >> sys.stderr, "add idle loops", self.next_initiate + sys.stdout.flush() + sys.stderr.flush() + if DEBUG: + print >> sys.stderr, "bc: *****", thread, str(step), "-", + + def getAllTasteBuddies(self): + return self.connected_taste_buddies + + def addRemoteSearchPeer(self, permid, oversion, ntorrents, last_seen): + if oversion >= OLPROTO_VER_SIXTH and ntorrents >= REMOTE_SEARCH_PEER_NTORRENTS_THRESHOLD: + insort(self.remote_search_peer_candidates, [last_seen,permid,oversion]) + if len(self.remote_search_peer_candidates) > self.num_search_cand: + self.remote_search_peer_candidates.pop(0) + + def getRemoteSearchPeers(self, npeers,minoversion=None): + """ Return some peers that are remote-search capable """ + if len(self.remote_search_peer_candidates) > npeers: + _peers = sample(self.remote_search_peer_candidates, npeers) # randomly select + else: + _peers = self.remote_search_peer_candidates + peers = [] + for p in _peers: + (last_seen,permid,selversion) = p + if minoversion is None or selversion >= minoversion: + peers.append(permid) + + # Also add local peers (they should be cheap) + # TODO: How many peers? Should these be part of the npeers? + local_peers = self.data_handler.getLocalPeerList(max_peers=5,minoversion=minoversion) + if DEBUG: + print >> sys.stderr, "bc: getRemoteSearchPeers: Selected %d local peers" % len(local_peers) + + return local_peers + peers + + +class DataHandler: + def __init__(self, launchmany, overlay_bridge, max_num_peers=2500): + self.launchmany = launchmany + self.overlay_bridge = overlay_bridge + self.config = self.launchmany.session.sessconfig # should be safe at startup + # --- database handlers --- + self.peer_db = launchmany.peer_db + self.superpeer_db = launchmany.superpeer_db + self.torrent_db = launchmany.torrent_db + self.mypref_db = launchmany.mypref_db + self.pref_db = launchmany.pref_db + self.simi_db = launchmany.simi_db + # self.term_db = launchmany.term_db + self.friend_db = launchmany.friend_db + self.pops_db = launchmany.pops_db + self.myfriends = Set() # FIXME: implement friends + self.myprefs = [] # torrent ids + self.peers = {} # peer_id: [similarity, last_seen, prefs(array('l',[torrent_id])] + self.default_peer = [0, 0, None] + self.permid = self.getMyPermid() + self.ntorrents = 0 + self.last_check_ntorrents = 0 + #self.total_pref_changed = 0 + # how many peers to load into cache from db + #self.max_peer_in_db = max_num_peers + self.max_num_peers = min(max(max_num_peers, 100), 2500) # at least 100, at most 2500 + #self.time_sim_weight = 4*60*60 # every 4 hours equals to a point of similarity + # after added some many (user, item) pairs, update sim of item to item + #self.update_i2i_threshold = 100 + #self.npeers = self.peer_db.size() - self.superpeer_db.size() + self.old_peer_num = 0 + self.buddycast_core = None + self.all_peer_list = None + self.num_peers_ui = None + self.num_torrents_ui = None + self.cached_updates = {'peer':{},'torrent':{}} + + # Subscribe BC to updates to MyPreferences, such that we can add/remove + # them from our download history that we send to other peers. + self.launchmany.session.add_observer(self.sesscb_ntfy_myprefs,NTFY_MYPREFERENCES,[NTFY_INSERT,NTFY_DELETE]) + + def commit(self): + self.peer_db.commit() + + def register_buddycast_core(self, buddycast_core): + self.buddycast_core = buddycast_core + + def getMyName(self, name=''): + return self.config.get('nickname', name) + + def getMyIp(self, ip=''): + return self.launchmany.get_ext_ip() + + def getMyPort(self, port=0): + return self.launchmany.listen_port + + def getMyPermid(self, permid=''): + return self.launchmany.session.get_permid() + + def getPeerID(self, permid): + if isinstance(permid, int) and permid > 0: + return permid + else: + return self.peer_db.getPeerID(permid) + + def getTorrentID(self, infohash): + if isinstance(infohash, int) and infohash > 0: + return infohash + else: + return self.torrent_db.getTorrentID(infohash) + + def getPeerPermid(self, peer_id): + return self.peer_db.getPermid(peer_id) + + def getLocalPeerList(self, max_peers,minoversion=None): + return self.peer_db.getLocalPeerList(max_peers,minoversion=minoversion) + + def postInit(self, delay=4, batch=50, update_interval=10, npeers=None, updatesim=True): + # build up a cache layer between app and db + if npeers is None: + npeers = self.max_num_peers + self.updateMyPreferences() + self.loadAllPeers(npeers) + if updatesim: + self.updateAllSim(delay, batch, update_interval) + + def updateMyPreferences(self, num_pref=None): + # get most recent preferences, and sort by torrent id + res = self.mypref_db.getAll('torrent_id', order_by='creation_time desc', limit=num_pref) + self.myprefs = [p[0] for p in res] + + def loadAllPeers(self, num_peers=None): + """ Read peers from db and put them in self.peers. + At most num_peers (=self.max_num_peers) recently seen peers can be cached. + + """ + peer_values = self.peer_db.getAll(['peer_id','similarity','last_seen'], order_by='last_connected desc', limit=num_peers) + self.peers = dict(zip([p[0] for p in peer_values], [[p[1],p[2],array('l', [])] for p in peer_values])) + + """ Not needed due to new similarity function + user_item_pairs = self.pref_db.getRecentPeersPrefs('last_connected',num_peers) + for pid,tid in user_item_pairs: + self.peers[pid][PEER_PREF_POS].append(tid) + """ + #print >> sys.stderr, '**************** loadAllPeers', len(self.peers) + +# for pid in self.peers: +# self.peers[pid][PEER_PREF_POS].sort() # keep in order + + def updateAllSim(self, delay=4, batch=50, update_interval=10): + self._updateAllPeerSim(delay, batch, update_interval) # 0.156 second + + #Disabled Torrent Relevancy since 5.0 + #self._updateAllItemRel(delay, batch, update_interval) # 0.875 second + # Tuning batch (without index relevance) + + # batch = 25: 0.00 0.22 0.58 + # batch = 50: min/avg/max execution time: 0.09 0.29 0.63 second + # batch = 100: 0.16 0.47 0.95 + # update_interval=10 + # 50000 updates take: 50000 / 50 * (10+0.3) / 3600 = 3 hours + # cpu load: 0.3/10 = 3% + + # With index relevance: + # batch = 50: min/avg/max execution time: 0.08 0.62 1.39 second + # batch = 25: 0.00 0.41 1.67 + # update_interval=5, batch=25 + # 50000 updates take: 50000 / 25 * (5+0.4) / 3600 = 3 hours + # cpu load: 0.4/5 = 8% + + def cacheSimUpdates(self, update_table, updates, delay, batch, update_interval): + self.cached_updates[update_table].update(updates) + self.overlay_bridge.add_task(lambda:self.checkSimUpdates(batch, update_interval), delay, 'checkSimUpdates') + + def checkSimUpdates(self, batch, update_interval): + last_update = 0 + if self.cached_updates['peer']: + updates = [] + update_peers = self.cached_updates['peer'] + keys = update_peers.keys() + shuffle(keys) # to avoid always update the same items when cacheSimUpdates is called frequently + for key in keys[:batch]: + updates.append((update_peers.pop(key), key)) + self.overlay_bridge.add_task(lambda:self.peer_db.updatePeerSims(updates), last_update + update_interval, 'updatePeerSims') + last_update += update_interval + + if self.cached_updates['torrent']: + updates = [] + update_peers = self.cached_updates['torrent'] + keys = update_peers.keys() + shuffle(keys) + for key in keys[:batch]: + updates.append((update_peers.pop(key), key)) + self.overlay_bridge.add_task(lambda:self.torrent_db.updateTorrentRelevances(updates), last_update + update_interval, 'updateTorrentRelevances') + last_update += update_interval + + if self.cached_updates['peer'] or self.cached_updates['torrent']: + self.overlay_bridge.add_task(lambda:self.checkSimUpdates(batch, update_interval), last_update+0.001, 'checkSimUpdates') + + def _updateAllPeerSim(self, delay, batch, update_interval): + # update similarity to all peers to keep consistent + #if self.old_peer_num == len(self.peers): # if no new peers, don't update + # return + + #call full_update + updates = {} + if len(self.myprefs) > 0: + not_peer_id = self.getPeerID(self.permid) + similarities = P2PSim_Full(self.simi_db.getPeersWithOverlap(not_peer_id, self.myprefs), len(self.myprefs)) + + for peer_id in self.peers: + if peer_id in similarities: + oldsim = self.peers[peer_id][PEER_SIM_POS] + sim = similarities[peer_id] + updates[peer_id] = sim + + #print >> sys.stderr, '****************** update peer sim', len(updates), len(self.peers) + if updates: + self.cacheSimUpdates('peer', updates, delay, batch, update_interval) + + def _updateAllItemRel(self, delay, batch, update_interval): + # update all item's relevance + # Relevance of I = Sum(Sim(Users who have I)) + Poplarity(I) + # warning: this function may take 5 seconds to commit to the database + + """ + Disabled, not in use since v5.0 + + if len(self.peers) == 0: + return + tids = {} + nsimpeers = 0 + for peer_id in self.peers: + if self.peers[peer_id][PEER_PREF_POS]: + sim = self.peers[peer_id][PEER_SIM_POS] + if sim > 0: + nsimpeers += 1 + prefs = self.peers[peer_id][PEER_PREF_POS] + for tid in prefs: + if tid not in tids: + tids[tid] = [0,0] + tids[tid][0] += sim + tids[tid][1] += 1 + + if len(tids) == 1: + return + + res = self.torrent_db.getTorrentRelevances(tids) + if res: + old_rels = dict(res) + else: + old_rels = {} + #print >> sys.stderr, '********* update all item rel', len(old_rels), len(tids) #, old_rels[:10] + + for tid in tids.keys(): + tids[tid] = tids[tid][0]/tids[tid][1] + tids[tid][1] + old_rel = old_rels.get(tid, None) + if old_rel != None and abs(old_rel - tids[tid]) <= old_rel*0.05: + tids.pop(tid) # don't update db + + #print >> sys.stderr, '**************--- update all item rel', len(tids), len(old_rels) #, len(self.peers), nsimpeers, tids.items()[:10] # 37307 2500 + if tids: + self.cacheSimUpdates('torrent', tids, delay, batch, update_interval) + """ + + def sesscb_ntfy_myprefs(self,subject,changeType,objectID,*args): + """ Called by SessionCallback thread """ + if DEBUG: + print >>sys.stderr,"bc: sesscb_ntfy_myprefs:",subject,changeType,`objectID` + if subject == NTFY_MYPREFERENCES: + infohash = objectID + if changeType == NTFY_INSERT: + op_my_pref_lambda = lambda:self.addMyPref(infohash) + elif changeType == NTFY_DELETE: + op_my_pref_lambda = lambda:self.delMyPref(infohash) + # Execute on OverlayThread + self.overlay_bridge.add_task(op_my_pref_lambda, 0) + + + def addMyPref(self, infohash): + infohash_str=bin2str(infohash) + torrentdata = self.torrent_db.getOne(('secret', 'torrent_id'), infohash=infohash_str) + if not torrentdata: + return + + secret = torrentdata[0] + torrent_id = torrentdata[1] + if secret: + if DEBUG: + print >> sys.stderr, 'bc: Omitting secret download: %s' % torrentdata.get('info', {}).get('name', 'unknown') + return # do not buddycast secret downloads + + if torrent_id not in self.myprefs: + insort(self.myprefs, torrent_id) + self.old_peer_num = 0 + self.updateAllSim() # time-consuming + #self.total_pref_changed += self.update_i2i_threshold + + def delMyPref(self, infohash): + torrent_id = self.torrent_db.getTorrentID(infohash) + if torrent_id in self.myprefs: + self.myprefs.remove(torrent_id) + self.old_peer_num = 0 + self.updateAllSim() + #self.total_pref_changed += self.update_i2i_threshold + + def initRemoteSearchPeers(self, num_peers=10): + peer_values = self.peer_db.getAll(['permid','oversion','num_torrents','last_seen'], order_by='last_seen desc', limit=num_peers) + for p in peer_values: + p = list(p) + p[0] = str2bin(p[0]) + self.buddycast_core.addRemoteSearchPeer(*tuple(p)) + pass + + def getMyLivePreferences(self, selversion, num=0): + """ Get a number of my preferences. Get all if num==0 """ + #Rahim + if selversion >= OLPROTO_VER_ELEVENTH: + return self.mypref_db.getRecentLivePrefListOL11(num) # return a list of preferences with clicklog and swarm size info. + + elif selversion>=OLPROTO_VER_EIGHTH: + return self.mypref_db.getRecentLivePrefListWithClicklog(num) + + else: + return self.mypref_db.getRecentLivePrefList(num) + + def getPeerSim(self, peer_permid, read_db=False, raw=False): + if read_db: + sim = self.peer_db.getPeerSim(peer_permid) + else: + peer_id = self.getPeerID(peer_permid) + if peer_id is None or peer_id not in self.peers: + sim = 0 + else: + sim = self.peers[peer_id][PEER_SIM_POS] + if sim is None: + sim = 0 + if not raw: + # negative value means it is calculated from other peers, + # not itself. See addRelativeSim() + return abs(sim) + else: + return sim + + def getPeerLastSeen(self, peer_permid): + peer_id = self.getPeerID(peer_permid) + return self.getPeerIDLastSeen(peer_id) + + def getPeerIDLastSeen(self, peer_id): + if not peer_id or peer_id not in self.peers: + return 0 + #print >> sys.stderr, '***** getPeerLastSeen', self.peers[pefer_permid], `peer_permid` + return self.peers[peer_id][PEER_LASTSEEN_POS] + + def getPeerPrefList(self, peer_permid): + """ Get a number of peer's preference list. Get all if num==0. + If live==True, dead torrents won't include + """ + return self.pref_db.getPrefList(peer_permid) + +# def addPeer(self, peer_permid, last_seen, peer_data=None, commit=True): +# """ add a peer from buddycast message to both cache and db """ +# +# if peer_permid != self.permid: +# if peer_data is not None: +# self._addPeerToDB(peer_permid, last_seen, peer_data, commit=commit) +# self._addPeerToCache(peer_permid, last_seen) + + def _addPeerToCache(self, peer_permid, last_seen): + """ add a peer to cache """ + # Secure Overlay should have added this peer to database. + if peer_permid == self.permid: + return + peer_id = self.getPeerID(peer_permid) + assert peer_id != None, `peer_permid` + if peer_id not in self.peers: + sim = self.peer_db.getPeerSim(peer_permid) + peerprefs = self.pref_db.getPrefList(peer_permid) # [torrent_id] + self.peers[peer_id] = [last_seen, sim, array('l', peerprefs)] # last_seen, similarity, pref + else: + self.peers[peer_id][PEER_LASTSEEN_POS] = last_seen + + def _addPeerToDB(self, peer_permid, peer_data, commit=True): + + if peer_permid == self.permid: + return + new_peer_data = {} + try: + new_peer_data['permid'] = peer_data['permid'] + new_peer_data['ip'] = hostname_or_ip2ip(peer_data['ip']) + new_peer_data['port'] = peer_data['port'] + new_peer_data['last_seen'] = peer_data['last_seen'] + if peer_data.has_key('name'): + new_peer_data['name'] = dunno2unicode(peer_data['name']) # store in db as unicode + + self.peer_db.addPeer(peer_permid, new_peer_data, update_dns=True, commit=commit) + + except KeyError: + print_exc() + print >> sys.stderr, "bc: _addPeerToDB has KeyError" + except socket.gaierror: + print >> sys.stderr, "bc: _addPeerToDB cannot find host by name", peer_data['ip'] + except: + print_exc() + + def addInfohashes(self, infohash_list, commit=True): + for infohash in infohash_list: + self.torrent_db.addInfohash(infohash, commit=False) # it the infohash already exists, it will skip it + if commit: + self.torrent_db.commit() + + def addPeerPreferences(self, peer_permid, prefs, selversion, recvTime, commit=True): + """ add a peer's preferences to both cache and db """ + + if peer_permid == self.permid: + return 0 + + cur_prefs = self.getPeerPrefList(peer_permid) + if not cur_prefs: + cur_prefs = [] + prefs2add = [] + #Rahim: It is possible that, a peer receive info about same torrent in + # different rounds. New torrents are handled by adding them to prefs2add + # list and adding them. If the peer receive same torrent more than + # once, the current version ignores it. But the swarm size is + # dynamic so the next torrents may have different swarm size info. So + # we should handle them as well. + # + pops2update = [] # a new list that contains already available torrents. + for pref in prefs: + infohash = pref['infohash'] # Nicolas: new dictionary format of OL 8 preferences + torrent_id = self.torrent_db.getTorrentID(infohash) + if not torrent_id: + print >> sys.stderr, "buddycast: DB Warning: infohash", bin2str(infohash), "should have been inserted into db, but was not found" + continue + pref['torrent_id'] = torrent_id + if torrent_id not in cur_prefs: + prefs2add.append(pref) + cur_prefs.append(torrent_id) + elif selversion >= OLPROTO_VER_ELEVENTH: + pops2update.append(pref) # already available preference is appended to this list. + + + if len(prefs2add) > 0: + self.pref_db.addPreferences(peer_permid, prefs2add, recvTime, is_torrent_id=True, commit=commit) + peer_id = self.getPeerID(peer_permid) + self.updateSimilarity(peer_id, commit=commit) + + if len(pops2update)>0: + self.pops_db.addPopularityRecord(peer_permid, pops2update, selversion, recvTime, is_torrent_id=True, commit=commit) + + def addCollectedTorrentsPopularity(self, peer_permid, colls, selversion, recvTime, commit=True): + """ + This method adds/updats the popularity of the collected torrents that is received + through BuddyCast message. + @param peer_permid: perm_id of the sender of BC message. + @param param: colls: A dictionary that contains a subset of collected torrents by the sender of BC. + @param selversion: The overlay protocol version that both sides agreed on. + @param recvTime: receive time of the message. + @param commit: whether or not to do database commit. + @author: Rahim 11-02-2010 + """ + if peer_permid == self.permid: + return 0 + + if selversion < OLPROTO_VER_ELEVENTH: + return 0 + + pops2update = [] + + for coll in colls: + infohash = coll['infohash'] + torrent_id = self.torrent_db.getTorrentID(infohash) + if not torrent_id: + print >> sys.stderr, "buddycast: DB Warning: infohash", bin2str(infohash), "should have been inserted into db, but was not found" + continue + coll['torrent_id'] = torrent_id + pops2update.append(coll) + + if len(pops2update)>0: + self.pops_db.addPopularityRecord(peer_permid, pops2update, selversion, recvTime, is_torrent_id=True, commit=commit) + + + def updateSimilarity(self, peer_id, update_db=True, commit=True): + """ update a peer's similarity """ + + if len(self.myprefs) == 0: + return + + sim = P2PSim_Single(self.simi_db.getOverlapWithPeer(peer_id, self.myprefs), len(self.myprefs)); + self.peers[peer_id][PEER_SIM_POS] = sim + if update_db and sim>0: + self.peer_db.updatePeerSims([(sim,peer_id)], commit=commit) + +# def increaseBuddyCastTimes(self, peer_permid, commit=True): +# self.peer_db.updateTimes(peer_permid, 'buddycast_times', 1, commit=False) +# self.peer_db.updatePeer(peer_permid, commit=commit, last_buddycast=now()) + + def getPeer(self, permid, keys=None): + return self.peer_db.getPeer(permid, keys) + + def addRelativeSim(self, sender_permid, peer_permid, sim, max_sim): + # Given Sim(I, A) and Sim(A, B), predict Sim(I, B) + # Sim(I, B) = Sim(I, A)*Sim(A, B)/Max(Sim(A,B)) for all B + old_sim = self.getPeerSim(peer_permid, raw=True) + if old_sim > 0: # its similarity has been calculated based on its preferences + return + old_sim = abs(old_sim) + sender_sim = self.getPeerSim(sender_permid) + new_sim = sender_sim*sim/max_sim + if old_sim == 0: + peer_sim = new_sim + else: + peer_sim = (new_sim + old_sim)/2 + peer_sim = -1*peer_sim + # using negative value to indicate this sim comes from others + peer_id = self.getPeerID(peer_permid) + self.peers[peer_id][PEER_SIM_POS] = peer_sim + + def get_npeers(self): + if self.num_peers_ui is None: + return len(self.peers) # changed to this according to Maarten's suggestion + else: + return self.num_peers_ui + + def get_ntorrents(self): + if self.num_torrents_ui is None: + _now = now() + if _now - self.last_check_ntorrents > 5*60: + self.ntorrents = self.torrent_db.getNumberCollectedTorrents() + self.last_check_ntorrents = _now + return self.ntorrents + else: + return self.num_torrents_ui + + def get_nmyprefs(self): + return len(self.myprefs) + +# def updatePeerLevelStats(self,permid,npeers,ntorrents,nprefs,commit=True): +# d = {'num_peers':npeers,'num_torrents':ntorrents,'num_prefs':nprefs} +# self.peer_db.updatePeer(permid, commit=commit, **d) + +# def getAllPeerList(self): +# return self.all_peer_list +# +# def removeAllPeerList(self): +# self.all_peer_list = None +# +# def setNumPeersFromUI(self, num): +# self.num_peers_ui = num +# +# def setNumTorrentsFromUI(self, num): # not thread safe +# self.num_torrents_ui = num + + def handleBCData(self, cache_db_data, cache_peer_data, sender_permid, max_tb_sim, selversion, recvTime): + #self.data_handler.addPeer(peer_permid, last_seen, new_peer_data, commit=True) # new peer + #self.data_handler.increaseBuddyCastTimes(sender_permid, commit=True) + #self.data_handler.addInfohashes(infohashes, commit=True) + + #self.data_handler._addPeerToCache(peer_permid, last_seen) + #self.data_handler.addRelativeSim(sender_permid, peer_permid, sim, max_tb_sim) + + #self.data_handler.addPeerPreferences(sender_permid, prefs) + + #print >>sys.stderr,"bc: handleBCData:",`cache_db_data` + + + ADD_PEER = 1 + UPDATE_PEER = 2 + ADD_INFOHASH = 3 + + peer_data = cache_db_data['peer'] + db_writes = [] + for permid in peer_data: + new_peer = peer_data[permid] + old_peer = self.peer_db.getPeer(permid) + if not old_peer: + if permid == sender_permid: + new_peer['buddycast_times'] = 1 + db_writes.append((ADD_PEER, permid, new_peer)) + else: + #print old_peer + old_last_seen = old_peer['last_seen'] + new_last_seen = new_peer['last_seen'] + if permid == sender_permid: + if not old_peer['buddycast_times']: + new_peer['buddycast_times'] = 1 + else: + new_peer['buddycast_times'] = + 1 + if not old_last_seen or new_last_seen > old_last_seen + 4*60*60: + # don't update if it was updated in 4 hours + for k in new_peer.keys(): + if old_peer[k] == new_peer[k]: + new_peer.pop(k) + if new_peer: + db_writes.append((UPDATE_PEER, permid, new_peer)) + + for infohash in cache_db_data['infohash']: + tid = self.torrent_db.getTorrentID(infohash) + if tid is None: + db_writes.append((ADD_INFOHASH, infohash)) + + for item in db_writes: + if item[0] == ADD_PEER: + permid = item[1] + new_peer = item[2] + # Arno, 2008-09-17: Don't use IP data from BC message, network info gets precedence + updateDNS = (permid != sender_permid) + self.peer_db.addPeer(permid, new_peer, update_dns=updateDNS, commit=False) + elif item[0] == UPDATE_PEER: + permid = item[1] + new_peer = item[2] + # Arno, 2008-09-17: Don't use IP data from BC message, network info gets precedence + updateDNS = (permid != sender_permid) + if not updateDNS: + if 'ip' in new_peer: + del new_peer['ip'] + if 'port' in new_peer: + del new_peer['port'] + self.peer_db.updatePeer(permid, commit=False, **new_peer) + elif item[0] == ADD_INFOHASH: + infohash = item[1] + self.torrent_db.addInfohash(infohash, commit=False) + + #self.torrent_db._db.show_sql(1) + self.torrent_db.commit() + #self.torrent_db._db.show_sql(0) + + for item in db_writes: + if item[0] == ADD_PEER or item[0] == UPDATE_PEER: + permid = item[1] + new_peer = item[2] + last_seen = new_peer['last_seen'] + self._addPeerToCache(permid, last_seen) + + for permid in peer_data: + if 'sim' in peer_data[permid]: + sim = peer_data[permid]['sim'] + self.addRelativeSim(sender_permid, permid, sim, max_tb_sim) + + #self.torrent_db._db.show_sql(1) + self.torrent_db.commit() + #self.torrent_db._db.show_sql(0) + + # Nicolas: moved this block *before* the call to addPeerPreferences because with the clicklog, + # this in fact writes to several different databases, so it's easier to tell it to commit + # right away. hope this is ok + + # Nicolas 2009-03-30: thing is that we need to create terms and their generated ids, forcing at least one commit in-between + # have to see later how this might be optimized. right now, there's three commits: + # before addPeerPreferences, after bulk_insert, and after storing clicklog data + + if cache_db_data['pref']: + self.addPeerPreferences(sender_permid, + cache_db_data['pref'], selversion, recvTime, + commit=True) + + # Arno, 2010-02-04: Since when are collected torrents also a peer pref? + + if cache_db_data['coll']: + self.addCollectedTorrentsPopularity(sender_permid, + cache_db_data['coll'], selversion, recvTime, + commit=True) + + + #print hash(k), peer_data[k] + #cache_db_data['infohash'] + #cache_db_data['pref'] diff --git a/instrumentation/next-share/BaseLib/Core/BuddyCast/channelcast.py b/instrumentation/next-share/BaseLib/Core/BuddyCast/channelcast.py new file mode 100644 index 0000000..871ca03 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BuddyCast/channelcast.py @@ -0,0 +1,382 @@ +# Written by Nitin Chiluka +# see LICENSE.txt for license information + +import sys +from time import time, ctime, sleep +from zlib import compress, decompress +from binascii import hexlify +from traceback import print_exc, print_stack +from types import StringType, ListType, DictType +from random import randint, sample, seed, random +from sha import sha +from sets import Set + +from BaseLib.Core.BitTornado.bencode import bencode, bdecode +from BaseLib.Core.Statistics.Logger import OverlayLogger +from BaseLib.Core.BitTornado.BT1.MessageID import CHANNELCAST, BUDDYCAST +from BaseLib.Core.CacheDB.CacheDBHandler import ChannelCastDBHandler, VoteCastDBHandler +from BaseLib.Core.Utilities.unicode import str2unicode +from BaseLib.Core.Utilities.utilities import * +from BaseLib.Core.Overlay.permid import permid_for_user,sign_data,verify_data +from BaseLib.Core.CacheDB.sqlitecachedb import SQLiteCacheDB, bin2str, str2bin, NULL +from BaseLib.Core.CacheDB.Notifier import Notifier +from BaseLib.Core.SocialNetwork.RemoteTorrentHandler import RemoteTorrentHandler +from BaseLib.Core.BuddyCast.moderationcast_util import * +from BaseLib.Core.Overlay.SecureOverlay import OLPROTO_VER_THIRTEENTH,\ + OLPROTO_VER_FOURTEENTH +from BaseLib.Core.simpledefs import NTFY_CHANNELCAST, NTFY_UPDATE +from BaseLib.Core.Subtitles.RichMetadataInterceptor import RichMetadataInterceptor +from BaseLib.Core.CacheDB.MetadataDBHandler import MetadataDBHandler +from BaseLib.Core.Subtitles.PeerHaveManager import PeersHaveManager +from BaseLib.Core.Subtitles.SubtitlesSupport import SubtitlesSupport + +DEBUG = False + +NUM_OWN_RECENT_TORRENTS = 15 +NUM_OWN_RANDOM_TORRENTS = 10 +NUM_OTHERS_RECENT_TORRENTS = 15 +NUM_OTHERS_RECENT_TORRENTS = 10 + +RELOAD_FREQUENCY = 2*60*60 + +class ChannelCastCore: + __single = None + TESTASSERVER = False # for unit testing + + def __init__(self, data_handler, overlay_bridge, session, buddycast_interval_function, log = '', dnsindb = None): + """ Returns an instance of this class """ + #Keep reference to interval-function of BuddycastFactory + self.interval = buddycast_interval_function + self.data_handler = data_handler + self.dnsindb = dnsindb + self.log = log + self.overlay_bridge = overlay_bridge + self.channelcastdb = ChannelCastDBHandler.getInstance() + self.votecastdb = VoteCastDBHandler.getInstance() + self.rtorrent_handler = RemoteTorrentHandler.getInstance() + self.my_permid = self.channelcastdb.my_permid + self.session = session + + self.network_delay = 30 + #Reference to buddycast-core, set by the buddycast-core (as it is created by the + #buddycast-factory after calling this constructor). + self.buddycast_core = None + + #Extend logging with ChannelCast-messages and status + if self.log: + self.overlay_log = OverlayLogger.getInstance(self.log) + self.dnsindb = self.data_handler.get_dns_from_peerdb + + self.hits = [] + + self.notifier = Notifier.getInstance() + + self.metadataDbHandler = MetadataDBHandler.getInstance() + + #subtitlesHandler = SubtitlesHandler.getInstance() + subtitleSupport = SubtitlesSupport.getInstance() + # better if an instance of RMDInterceptor was provided from the + # outside + self.peersHaveManger = PeersHaveManager.getInstance() + if not self.peersHaveManger.isRegistered(): + self.peersHaveManger.register(self.metadataDbHandler, self.overlay_bridge) + self.richMetadataInterceptor = RichMetadataInterceptor(self.metadataDbHandler,self.votecastdb, + self.my_permid, subtitleSupport, self.peersHaveManger, + self.notifier) + + + + + def initialized(self): + return self.buddycast_core is not None + + + + def getInstance(*args, **kw): + if ChannelCastCore.__single is None: + ChannelCastCore(*args, **kw) + return ChannelCastCore.__single + getInstance = staticmethod(getInstance) + + + def createAndSendChannelCastMessage(self, target_permid, selversion): + """ Create and send a ChannelCast Message """ + # ChannelCast feature starts from eleventh version; hence, do not send to lower version peers + # Arno, 2010-02-05: v12 uses a different on-the-wire format, ignore those. + + # Andrea, 2010-04-08: sending the "old-style" channelcast message to older + # peers, and enriched channelcast messages to new versions, for full backward + # compatibility + if selversion < OLPROTO_VER_THIRTEENTH: + if DEBUG: + print >> sys.stderr, "channelcast: Do not send to lower version peer:", selversion + return + + # 3/5/2010 Andrea: adding the destination parameters to createChannelCastMessage for + # logging reasons only. When logging will be disabled, that parameter will + # become useless + channelcast_data = self.createChannelCastMessage(selversion, target_permid) + if channelcast_data is None or len(channelcast_data)==0: + if DEBUG: + print >>sys.stderr, "channelcast: No channels there.. hence we do not send" + return + channelcast_msg = bencode(channelcast_data) + + if self.log: + dns = self.dnsindb(target_permid) + if dns: + ip,port = dns + MSG_ID = "CHANNELCAST" + msg = repr(channelcast_data) + self.overlay_log('SEND_MSG', ip, port, show_permid(target_permid), selversion, MSG_ID, msg) + + data = CHANNELCAST + channelcast_msg + self.overlay_bridge.send(target_permid, data, self.channelCastSendCallback) + #if DEBUG: print >> sys.stderr, "channelcast: Sent channelcastmsg",repr(channelcast_data) + + def createChannelCastMessage(self, selversion, dest_permid=None): + """ + Create a ChannelCast Message + + @param selversion: the protocol version of the destination + @param dest_permid: the destination of the message. Actually this parameter is not really needed. If + not none, it is used for logging purposes only + + @return a channelcast message, possibly enrich with rich metadata content in the + case selversion is sufficiently high + """ + # 09-04-2010 Andrea: I addedd the selversion param, to intercept and modify + # the ChannelCast message contents if the protocol version allows rich metadata + # enrichment + + if DEBUG: + print >> sys.stderr, "channelcast: Creating channelcastmsg..." + + hits = self.channelcastdb.getRecentAndRandomTorrents(NUM_OWN_RECENT_TORRENTS,NUM_OWN_RANDOM_TORRENTS,NUM_OTHERS_RECENT_TORRENTS,NUM_OTHERS_RECENT_TORRENTS) + # 3/5/2010 Andrea: + # hits is of the form: [(mod_id, mod_name, infohash, torrenthash, torrent_name, time_stamp, signature)] + # adding the destination parameter to buildChannelcastMessageFrom Hits for + # logging reasons only. When logging will be disabled, that parameter will + # become useless + d = self.buildChannelcastMessageFromHits(hits, selversion, dest_permid) +# #assert validChannelCastMsg(d) + return d + + def channelCastSendCallback(self, exc, target_permid, other=0): + if DEBUG: + if exc is None: + print >> sys.stderr,"channelcast: *** msg was sent successfully to peer", show_permid_short(target_permid) + else: + print >> sys.stderr, "channelcast: *** warning - error in sending msg to", show_permid_short(target_permid), exc + + def gotChannelCastMessage(self, recv_msg, sender_permid, selversion): + """ Receive and handle a ChannelCast message """ + # ChannelCast feature starts from eleventh version; hence, do not receive from lower version peers + # Arno, 2010-02-05: v12 uses a different on-the-wire format, ignore those. + + # Andrea: 2010-04-08: v14 can still receive v13 channelcast messages + if selversion < OLPROTO_VER_THIRTEENTH: + if DEBUG: + print >> sys.stderr, "channelcast: Do not receive from lower version peer:", selversion + return True + + if DEBUG: + print >> sys.stderr,'channelcast: Received a msg from ', show_permid_short(sender_permid) + print >> sys.stderr,"channelcast: my_permid=", show_permid_short(self.my_permid) + + if not sender_permid or sender_permid == self.my_permid: + if DEBUG: + print >> sys.stderr, "channelcast: warning - got channelcastMsg from a None/Self peer", \ + show_permid_short(sender_permid), recv_msg + return False + + #if len(recv_msg) > self.max_length: + # if DEBUG: + # print >> sys.stderr, "channelcast: warning - got large channelCastHaveMsg", len(recv_msg) + # return False + + channelcast_data = {} + + try: + channelcast_data = bdecode(recv_msg) + except: + print >> sys.stderr, "channelcast: warning, invalid bencoded data" + return False + + # check message-structure + if not validChannelCastMsg(channelcast_data): + print >> sys.stderr, "channelcast: invalid channelcast_message" + return False + + # 19/02/10 Boudewijn: validChannelCastMsg passes when + # PUBLISHER_NAME and TORRENTNAME are either string or + # unicode-string. However, all further code requires that + # these are unicode! + for ch in channelcast_data.values(): + if isinstance(ch["publisher_name"], str): + ch["publisher_name"] = str2unicode(ch["publisher_name"]) + if isinstance(ch["torrentname"], str): + ch["torrentname"] = str2unicode(ch["torrentname"]) + + self.handleChannelCastMsg(sender_permid, channelcast_data) + + #Log RECV_MSG of uncompressed message + if self.log: + dns = self.dnsindb(sender_permid) + if dns: + ip,port = dns + MSG_ID = "CHANNELCAST" + # 08/04/10 Andrea: representing the whole channelcast + metadata message + msg = repr(channelcast_data) + self.overlay_log('RECV_MSG', ip, port, show_permid(sender_permid), selversion, MSG_ID, msg) + + if self.TESTASSERVER: + self.createAndSendChannelCastMessage(sender_permid, selversion) + + return True + + def handleChannelCastMsg(self, sender_permid, data): + self._updateChannelInternal(sender_permid, None, data) + + def updateChannel(self,query_permid, query, hits): + """ + This function is called when there is a reply from remote peer regarding updating of a channel + @param query_permid: the peer who returned the results + @param query: the query string (None if this is not the results of a query) + @param hits: details of all matching results related to the query + """ + + return self._updateChannelInternal(query_permid, query, hits) + + + + def _updateChannelInternal(self, query_permid, query, hits): + listOfAdditions = list() + + # a single read from the db is more efficient + all_spam_channels = self.votecastdb.getPublishersWithNegVote(bin2str(self.session.get_permid())) + for k,v in hits.items(): + #check if the record belongs to a channel who we have "reported spam" (negative vote) + if bin2str(v['publisher_id']) in all_spam_channels: + # if so, ignore the incoming record + continue + + # make everything into "string" format, if "binary" + hit = (bin2str(v['publisher_id']),v['publisher_name'],bin2str(v['infohash']),bin2str(v['torrenthash']),v['torrentname'],v['time_stamp'],bin2str(k)) + + listOfAdditions.append(hit) + + # Arno, 2010-06-11: We're on the OverlayThread + self._updateChannelcastDB(query_permid, query, hits, listOfAdditions) + + ##return listOfAdditions + + + def _updateChannelcastDB(self, query_permid, query, hits, listOfAdditions): + + publisher_ids = Set() + + #08/04/10: Andrea: processing rich metadata part. + self.richMetadataInterceptor.handleRMetadata(query_permid, hits, fromQuery = query is not None) + + + tmp_hits = {} #"binary" key + + def usercallback(infohash,metadata,filename): + if tmp_hits.has_key(infohash): + hit = tmp_hits[infohash] + if self.channelcastdb.addTorrent(hit): + self.hits.append(hit) + else: + print >> sys.stderr, "channelcast: updatechannel: could not find infohash", bin2str(infohash) + + + for hit in listOfAdditions: + publisher_ids.add(hit[0]) + infohash = str2bin(hit[2]) + tmp_hits[infohash] = hit + # effectively v['infohash'] == str2bin(hit[2]) + + + if self.channelcastdb.existsTorrent(infohash): + if self.channelcastdb.addTorrent(hit): + self.hits.append(hit) + else: + self.rtorrent_handler.download_torrent(query_permid,infohash,usercallback) + + # Arno, 2010-02-24: Generate event + for publisher_id in publisher_ids: + try: + self.notifier.notify(NTFY_CHANNELCAST, NTFY_UPDATE, publisher_id) + except: + print_exc() + + + + + + def updateMySubscribedChannels(self): + subscribed_channels = self.channelcastdb.getMySubscribedChannels() + for permid, channel_name, num_subscriptions, notused in subscribed_channels: + # query the remote peers, based on permid, to update the channel content + q = "CHANNEL p "+permid + self.session.query_connected_peers(q,usercallback=self.updateChannel) + + self.overlay_bridge.add_task(self.updateMySubscribedChannels, RELOAD_FREQUENCY) + + + def buildChannelcastMessageFromHits(self, hits, selversion, dest_permid=None, fromQuery=False): + ''' + Creates a channelcast message from database hits. + + This method is used to create channel results both when a channelcast message + is created in the "normal" buddycast epidemic protocol, and when a remote + query for channels arrives and is processed. It substitutes a lot of duplicated + code in the old versions. + + @param hits: a tuple (publisher_id, publisher_name, infohash, + torrenthash, torrentname, time_stamp, signature) representing + a channelcast entry in the db + @param selversion: the protocol version of the destination + @param dest_permid: the permid of the destination of the message. Actually this parameter + is used for logging purposes only, when not None. If None, nothing + bad happens. + ''' + # 09-04-2010 Andrea : I introduced this separate method because this code was + # duplicated in RemoteQueryMessageHandler + enrichWithMetadata = False + + if selversion >= OLPROTO_VER_FOURTEENTH: + enrichWithMetadata = True + if DEBUG: + print >> sys.stderr, "channelcast: creating enriched messages"\ + "since peer has version: ", selversion + d = {} + for hit in hits: + # ARNOUNICODE: temp fixes until data is sent not Base64-encoded + + # 08/04/10 Andrea: I substituted the keys with constnats, otherwise a change here + # would break my code in the RichMetadataInterceptor + r = {} + r['publisher_id'] = str(hit[0]) # ARNOUNICODE: must be str + r['publisher_name'] = hit[1].encode("UTF-8") # ARNOUNICODE: must be explicitly UTF-8 encoded + r['infohash'] = str(hit[2]) # ARNOUNICODE: must be str + r['torrenthash'] = str(hit[3]) # ARNOUNICODE: must be str + r['torrentname'] = hit[4].encode("UTF-8") # ARNOUNICODE: must be explicitly UTF-8 encoded + r['time_stamp'] = int(hit[5]) + # hit[6]: signature, which is unique for any torrent published by a user + signature = hit[6] + d[signature] = r + + + # 08/04/10 Andrea: intercepting a channelcast message and enriching it with + # subtitles information + # 3/5/2010 Andrea: adding the destination parameter to addRichMetadataContent for + # logging reasons only. When logging will be disabled, that parameter will + # become useless + if enrichWithMetadata: + d = self.richMetadataInterceptor.addRichMetadataContent(d, dest_permid, fromQuery) + + return d + + diff --git a/instrumentation/next-share/BaseLib/Core/BuddyCast/moderationcast_util.py b/instrumentation/next-share/BaseLib/Core/BuddyCast/moderationcast_util.py new file mode 100644 index 0000000..39c77df --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BuddyCast/moderationcast_util.py @@ -0,0 +1,147 @@ +# Written by Vincent Heinink and Rameez Rahman +# see LICENSE.txt for license information +# +#Utilities for moderationcast (including databases) +# +import sys + +from BaseLib.Core.CacheDB.sqlitecachedb import bin2str, str2bin +#For validity-checks +from types import StringType, ListType, DictType +from time import time +from BaseLib.Core.BitTornado.bencode import bencode +from BaseLib.Core.Overlay.permid import verify_data +from os.path import exists, isfile +from BaseLib.Core.Subtitles.RichMetadataInterceptor import validMetadataEntry + + +DEBUG = False + +TIMESTAMP_IN_FUTURE = 5 * 60 # 5 minutes is okay + +#*****************Validity-checks***************** +def validInfohash(infohash): + """ Returns True iff infohash is a valid infohash """ + r = isinstance(infohash, str) and len(infohash) == 20 + if not r: + if DEBUG: + print >>sys.stderr, "Invalid infohash: type(infohash) ==", str(type(infohash))+\ + ", infohash ==", `infohash` + return r + +def validPermid(permid): + """ Returns True iff permid is a valid Tribler Perm-ID """ + r = type(permid) == str and len(permid) <= 125 + if not r: + if DEBUG: + print >>sys.stderr, "Invalid permid: type(permid) ==", str(type(permid))+\ + ", permid ==", `permid` + return r + +def now(): + """ Returns current-system-time in UTC, seconds since the epoch (type==int) """ + return int(time()) + +def validTimestamp(timestamp): + """ Returns True iff timestamp is a valid timestamp """ + r = timestamp is not None and type(timestamp) == int and timestamp > 0 and timestamp <= now() + TIMESTAMP_IN_FUTURE + if not r: + if DEBUG: + print >>sys.stderr, "Invalid timestamp" + return r + +def validVoteCastMsg(data): + """ Returns True if VoteCastMsg is valid, ie, be of type [(mod_id,vote)] """ + if data is None: + print >> sys.stderr, "data is None" + return False + + if not type(data) == DictType: + print >> sys.stderr, "data is not Dictionary" + return False + + for key,value in data.items(): + #if DEBUG: + # print >>sys.stderr, "validvotecastmsg: ", repr(record) + if not validPermid(key): + if DEBUG: + print >> sys.stderr, "not valid permid: ", repr(key) + return False + if not ('vote' in value and 'time_stamp' in value): + if DEBUG: + print >> sys.stderr, "validVoteCastMsg: key missing, got", value.keys() + return False + if not type(value['vote']) == int: + if DEBUG: + print >> sys.stderr, "Vote is not int: ", repr(value['vote']) + return False + if not(value['vote']==2 or value['vote']==-1): + if DEBUG: + print >> sys.stderr, "Vote is not -1 or 2: ", repr(value['vote']) + return False + if not type(value['time_stamp']) == int: + if DEBUG: + print >> sys.stderr, "time_stamp is not int: ", repr(value['time_stamp']) + return False + return True + + +def validChannelCastMsg(channelcast_data): + """ Returns true if ChannelCastMsg is valid, + format: {'signature':{'publisher_id':, 'publisher_name':, 'infohash':, 'torrenthash':, 'torrent_name':, 'timestamp':, 'signature':}} + """ + + + if not isinstance(channelcast_data,dict): + return False + for signature, ch in channelcast_data.items(): + if not isinstance(ch,dict): + if DEBUG: + print >>sys.stderr,"validChannelCastMsg: value not dict" + return False + + # 08-04-2010 We accept both 6 and 7 fields to allow + # compatibility with messages from older versions + # the rich metadata field + length = len(ch) + if not 6 <= length <= 7: + if DEBUG: + print >>sys.stderr,"validChannelCastMsg: #keys!=7" + return False + if not ('publisher_id' in ch and 'publisher_name' in ch and 'infohash' in ch and 'torrenthash' in ch \ + and 'torrentname' in ch and 'time_stamp' in ch): + if DEBUG: + print >>sys.stderr,"validChannelCastMsg: key missing" + return False + + if length == 7: + if 'rich_metadata' not in ch: #enriched Channelcast + if DEBUG: + print >>sys.stderr,"validChannelCastMsg: key missing" + return False + else: + if not validMetadataEntry(ch['rich_metadata']): + print >> sys.stderr, "validChannelCastMsg: invalid rich metadata" + return False + + + + if not (validPermid(ch['publisher_id']) and isinstance(ch['publisher_name'],str) \ + and validInfohash(ch['infohash']) and validInfohash(ch['torrenthash']) + and isinstance(ch['torrentname'],str) and validTimestamp(ch['time_stamp'])): + if DEBUG: + print >>sys.stderr,"validChannelCastMsg: something not valid" + return False + # now, verify signature + # Nitin on Feb 5, 2010: Signature is validated using binary forms of permid, infohash, torrenthash fields + l = (ch['publisher_id'],ch['infohash'], ch['torrenthash'], ch['time_stamp']) + if not verify_data(bencode(l),ch['publisher_id'],signature): + if DEBUG: + print >>sys.stderr, "validChannelCastMsg: verification failed!" + return False + return True + +#************************************************* + +def voteCastMsgToString(data): + return repr(data) diff --git a/instrumentation/next-share/BaseLib/Core/BuddyCast/similarity.py b/instrumentation/next-share/BaseLib/Core/BuddyCast/similarity.py new file mode 100644 index 0000000..e1d97a4 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BuddyCast/similarity.py @@ -0,0 +1,139 @@ +# Written by Jun Wang, Jie Yang +# see LICENSE.txt for license information + +__fool_epydoc = 481 +""" +Formulas: + P(I|U) = sum{U'<-I} P(U'|U) # U' has I in his profile + P(U'|U) = Sum{I}Pbs(U'|I)Pml(I|U) # P2PSim + Pbs(U|I) = (c(U,I) + mu*Pml(U))/(Sum{U}c(U,I) + mu) # mu=1 by tuning on tribler dataset + Pml(I|U) = c(U,I)/Sum{I}c(U,I) + Pml(U) = Sum{I}c(U,I) / Sum{U,I}c(U,I) + +Data Structur: + preferences - U:{I|c(U,I)>0}, # c(U,I) # Sum{I}c(U,I) = len(preferences[U]) + owners - I:{U|c(U,I)>0} # I:I:Sum{U}c(U,I) = len(owners[I]) + userSim - U':P(U'|U) + itemSim - I:P(I|U) + total - Sum{U,I}c(U,I) # Pml(U) = len(preferences[U])/total + +Test: + Using hash(permid) as user id, hash(infohash) as torrent id + Incremental change == overall change +""" + +from sets import Set + +def P2PSim(pref1, pref2): + """ Calculate simple similarity between peers """ + + cooccurrence = len(Set(pref1) & Set(pref2)) + if cooccurrence == 0: + return 0 + normValue = (len(pref1)*len(pref2))**0.5 + _sim = cooccurrence/normValue + sim = int(_sim*1000) # use integer for bencode + return sim + +def getCooccurrence(pref1, pref2): # pref1 and pref2 are sorted + i = 0 + j = 0 + co = 0 + size1 = len(pref1) + size2 = len(pref2) + if size1 == 0 or size2 == 0: + return 0 + while 1: + if (i>= size1) or (j>=size2): break + Curr_ID1 = pref1[i] + Curr_ID2 = pref2[j] + if Curr_ID1 < Curr_ID2 : + i=i+1 + elif Curr_ID1 > Curr_ID2 : + j=j+1 + else: + co +=1 + i+=1 + j+=1 + return co + +def P2PSimSorted(pref1, pref2): + """ Calculate similarity between peers """ + + cooccurrence = getCooccurrence(pref1, pref2) + if cooccurrence == 0: + return 0 + normValue = (len(pref1)*len(pref2))**0.5 + _sim = cooccurrence/normValue + sim = int(_sim*1000) # use integer for bencode + return sim + +def P2PSimLM(peer_permid, my_pref, peer_pref, owners, total_prefs, mu=1.0): + """ + Calculate similarity between two peers using Bayesian Smooth. + P(U|U') = Sum{I}Pbs(U|I)Pml(I|U') + Pbs(U|I) = (c(U,I) + mu*Pml(U))/(Sum{U}c(U,I) + mu) + Pml(U) = Sum{I}c(U,I) / Sum{U,I}c(U,I) + Pml(I|U') = c(U',I)/Sum{I}c(U',I) + """ + + npeerprefs = len(peer_pref) + if npeerprefs == 0 or total_prefs == 0: + return 0 + + nmyprefs = len(my_pref) + if nmyprefs == 0: + return 0 + + PmlU = float(npeerprefs) / total_prefs + PmlIU = 1.0 / nmyprefs + peer_sim = 0.0 + for item in owners: + nowners = len(owners[item]) + 1 # add myself + cUI = item in peer_pref + PbsUI = float(cUI + mu*PmlU)/(nowners + mu) + peer_sim += PbsUI*PmlIU + return peer_sim * 100000 + + +def P2PSim_Single(db_row, nmyprefs): + sim = 0 + if db_row: + peer_id, nr_items, overlap = db_row + + # Arno, 2010-01-14: Safety catch for weird by reported by Johan + if (nr_items is None) or (nmyprefs is None): + return sim + if (nr_items is 0) or (nmyprefs is 0): + return sim + + #Cosine Similarity With Emphasis on users with profilelength >= 40 + sim = overlap * ((1.0/(nmyprefs ** .5)) * (1.0/(nr_items ** .5))) + if nr_items < 40: + sim = (nr_items/40.0) * sim + return sim + +def P2PSim_Full(db_rows, nmyprefs): + similarity = {} + for db_row in db_rows: + similarity[db_row[0]] = P2PSim_Single(db_row, nmyprefs) + return similarity + +def P2PSimColdStart(choose_from, not_in, nr): + """ + choose_from has keys: ip port oversion num_torrents + not_in is [version, permid] + return a list containing [version, permid] + """ + allready_choosen = [permid for version,sim,permid in not_in] + options = [] + for permid in choose_from: + if permid not in allready_choosen: + options.append([choose_from[permid]['num_torrents'],[choose_from[permid]['oversion'],0.0,permid]]) + options.sort() + options.reverse() + + options = [row[1] for row in options[:nr]] + return options + + \ No newline at end of file diff --git a/instrumentation/next-share/BaseLib/Core/BuddyCast/votecast.py b/instrumentation/next-share/BaseLib/Core/BuddyCast/votecast.py new file mode 100644 index 0000000..3a7887b --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/BuddyCast/votecast.py @@ -0,0 +1,216 @@ +# Written by Rameez Rahman +# see LICENSE.txt for license information +# + +import sys +from time import time +from sets import Set + +from BaseLib.Core.BitTornado.bencode import bencode, bdecode +from BaseLib.Core.Statistics.Logger import OverlayLogger +from BaseLib.Core.BitTornado.BT1.MessageID import VOTECAST +from BaseLib.Core.CacheDB.CacheDBHandler import VoteCastDBHandler +from BaseLib.Core.Utilities.utilities import * +from BaseLib.Core.Overlay.permid import permid_for_user +from BaseLib.Core.CacheDB.sqlitecachedb import bin2str, str2bin +from BaseLib.Core.BuddyCast.moderationcast_util import * +from BaseLib.Core.Overlay.SecureOverlay import OLPROTO_VER_THIRTEENTH +from BaseLib.Core.CacheDB.Notifier import Notifier +from BaseLib.Core.simpledefs import NTFY_VOTECAST, NTFY_UPDATE + +DEBUG_UI = False +DEBUG = False #Default debug +debug = False #For send-errors and other low-level stuff + + +SINGLE_VOTECAST_LENGTH = 130 + +class VoteCastCore: + """ VoteCastCore is responsible for sending and receiving VOTECAST-messages """ + + TESTASSERVER = False # for unit testing + + ################################ + def __init__(self, data_handler, secure_overlay, session, buddycast_interval_function, log = '', dnsindb = None): + """ Returns an instance of this class + """ + #Keep reference to interval-function of BuddycastFactory + self.interval = buddycast_interval_function + self.data_handler = data_handler + self.dnsindb = dnsindb + self.log = log + self.secure_overlay = secure_overlay + self.votecastdb = VoteCastDBHandler.getInstance() + self.my_permid = self.votecastdb.my_permid + self.session = session + self.max_length = SINGLE_VOTECAST_LENGTH * (session.get_votecast_random_votes() + session.get_votecast_recent_votes()) + + self.network_delay = 30 + #Reference to buddycast-core, set by the buddycast-core (as it is created by the + #buddycast-factory after calling this constructor). + self.buddycast_core = None + + + self.notifier = Notifier.getInstance() + + #Extend logging with VoteCast-messages and status + if self.log: + self.overlay_log = OverlayLogger.getInstance(self.log) + + def initialized(self): + return self.buddycast_core is not None + + ################################ + def createAndSendVoteCastMessage(self, target_permid, selversion): + """ Creates and sends a VOTECAST message """ + # Arno, 2010-02-05: v12 uses a different on-the-wire format, ignore those. + if selversion < OLPROTO_VER_THIRTEENTH: + if DEBUG: + print >> sys.stderr, "votecast: Do not send to lower version peer:", selversion + return + + votecast_data = self.createVoteCastMessage() + if len(votecast_data) == 0: + if DEBUG: + print >>sys.stderr, "votecast: No votes there.. hence we do not send" + return + + votecast_msg = bencode(votecast_data) + + if self.log: + dns = self.dnsindb(target_permid) + if dns: + ip,port = dns + MSG_ID = "VOTECAST" + # msg = voteCastReplyMsgToString(votecast_data) + self.overlay_log('SEND_MSG', ip, port, show_permid(target_permid), selversion, MSG_ID) + + if DEBUG: print >> sys.stderr, "votecast: Sending votecastmsg",voteCastMsgToString(votecast_data) +# data = VOTECAST + votecast_msg +# self.secure_overlay.send(target_permid, data, self.voteCastSendCallback) + self.secure_overlay.send(target_permid, VOTECAST + votecast_msg, self.voteCastSendCallback) + + + ################################ + def createVoteCastMessage(self): + """ Create a VOTECAST message """ + + if DEBUG: print >> sys.stderr, "votecast: Creating votecastmsg..." + + NO_RANDOM_VOTES = self.session.get_votecast_random_votes() + NO_RECENT_VOTES = self.session.get_votecast_recent_votes() + records = self.votecastdb.getRecentAndRandomVotes() + + data = {} + for record in records: + # record is of the format: (publisher_id, vote, time_stamp) + if DEBUG: + print >>sys.stderr,"votecast: publisher id",`record[0]`,type(record[0]) + publisher_id = record[0] + data[publisher_id] = {'vote':record[1], 'time_stamp':record[2]} + if DEBUG: print >>sys.stderr, "votecast to be sent:", repr(data) + return data + + + ################################ + def voteCastSendCallback(self, exc, target_permid, other=0): + if DEBUG: + if exc is None: + print >> sys.stderr,"votecast: *** msg was sent successfully to peer", show_permid_short(target_permid) + else: + print >> sys.stderr, "votecast: *** warning - error in sending msg to", show_permid_short(target_permid), exc + + ################################ + def gotVoteCastMessage(self, recv_msg, sender_permid, selversion): + """ Receives VoteCast message and handles it. """ + # VoteCast feature is renewed in eleventh version; hence, do not receive from lower version peers + # Arno, 2010-02-05: v12 uses a different on-the-wire format, ignore those. + if selversion < OLPROTO_VER_THIRTEENTH: + if DEBUG: + print >> sys.stderr, "votecast: Do not receive from lower version peer:", selversion + return True + + if DEBUG: + print >> sys.stderr,'votecast: Received a msg from ', show_permid_short(sender_permid) + + if not sender_permid or sender_permid == self.my_permid: + if DEBUG: + + print >> sys.stderr, "votecast: error - got votecastMsg from a None peer", \ + show_permid_short(sender_permid), recv_msg + return False + + if self.max_length > 0 and len(recv_msg) > self.max_length: + if DEBUG: + print >> sys.stderr, "votecast: warning - got large voteCastHaveMsg; msg_size:", len(recv_msg) + return False + + votecast_data = {} + + try: + votecast_data = bdecode(recv_msg) + except: + print >> sys.stderr, "votecast: warning, invalid bencoded data" + return False + + # check message-structure + if not validVoteCastMsg(votecast_data): + print >> sys.stderr, "votecast: warning, invalid votecast_message" + return False + + self.handleVoteCastMsg(sender_permid, votecast_data) + + #Log RECV_MSG of uncompressed message + if self.log: + dns = self.dnsindb(sender_permid) + if dns: + ip,port = dns + MSG_ID = "VOTECAST" + msg = voteCastMsgToString(votecast_data) + self.overlay_log('RECV_MSG', ip, port, show_permid(sender_permid), selversion, MSG_ID, msg) + + if self.TESTASSERVER: + self.createAndSendVoteCastMessage(sender_permid, selversion) + return True + + ################################ + ################################ + def handleVoteCastMsg(self, sender_permid, data): + """ Handles VoteCast message """ + if DEBUG: + print >> sys.stderr, "votecast: Processing VOTECAST msg from: ", show_permid_short(sender_permid), "; data: ", repr(data) + + mod_ids = Set() + for key, value in data.items(): + vote = {} + vote['mod_id'] = bin2str(key) + vote['voter_id'] = permid_for_user(sender_permid) + vote['vote'] = value['vote'] + vote['time_stamp'] = value['time_stamp'] + self.votecastdb.addVote(vote) + + mod_ids.add(vote['mod_id']) + + # Arno, 2010-02-24: Generate event + for mod_id in mod_ids: + try: + self.notifier.notify(NTFY_VOTECAST, NTFY_UPDATE, mod_id) + except: + print_exc() + + if DEBUG: + print >> sys.stderr,"votecast: Processing VOTECAST msg from: ", show_permid_short(sender_permid), "DONE; data:" + + def showAllVotes(self): + """ Currently this function is only for testing, to show all votes """ + if DEBUG: + records = self.votecastdb.getAll() + print >>sys.stderr, "Existing votes..." + for record in records: + print >>sys.stderr, " mod_id:",record[0],"; voter_id:", record[1], "; votes:",record[2],"; timestamp:", record[3] + print >>sys.stderr, "End of votes..." + + + + + ################################ diff --git a/instrumentation/next-share/BaseLib/Core/CacheDB/CacheDBHandler.py b/instrumentation/next-share/BaseLib/Core/CacheDB/CacheDBHandler.py new file mode 100644 index 0000000..038ed32 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/CacheDB/CacheDBHandler.py @@ -0,0 +1,4 @@ +# Written by Jie Yang +# see LICENSE.txt for license information + +from SqliteCacheDBHandler import * diff --git a/instrumentation/next-share/BaseLib/Core/CacheDB/EditDist.py b/instrumentation/next-share/BaseLib/Core/CacheDB/EditDist.py new file mode 100644 index 0000000..1244ece --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/CacheDB/EditDist.py @@ -0,0 +1,54 @@ +# Written by Maarten Clemens, Jelle Roozenburg +# see LICENSE.txt for license information + +#http://en.wikipedia.org/wiki/Damerau-Levenshtein_distance + +def editDist(str1,str2, maxlength=14): + # If fast is set: only calculate titles with same #fast initial chars + if not str1 or not str2: # protect against empty strings + return 1.0 + + str1 = str1[:maxlength].lower() + str2 = str2[:maxlength].lower() + + lenStr1 = len(str1) + lenStr2 = len(str2) + + d = [range(lenStr2+1)] + row = [] + + for i in range(lenStr1): + row.append(i+1) + for j in range(lenStr2): + penalty = 1./max(i+1,j+1) + ##penalty = 1 + if str1[i] == str2[j]: + cost = 0 + else: + cost = penalty + deletion = d[i][j+1] + penalty + insertion = row[j] + penalty + substitution = d[i][j] + cost + row.append(min(deletion,insertion,substitution)) + (deletion,insertion,substitution) + if i>0 and j>0 and str1[i] == str2[j-1] and str1[i-1] == str2[j]: + row[j+1] = min(row[j+1], d[i-1][j-1]+cost) # transposition + d.append(row) + row = [] + + ##maxi = max(lenStr1,lenStr2) # for penalty = 1 + maxi = sum([1./j for j in range(max(lenStr1,lenStr2)+1)[1:]]) + return 1.*d[lenStr1][lenStr2]/ maxi + + +if __name__ == '__main__': + import sys + str1 = sys.argv[1] + str2 = sys.argv[2] + print editDist(str1, str2) + + +## d,e = EditDist('mamamstein','levenstein') +## print e +## for i in d: +## print i diff --git a/instrumentation/next-share/BaseLib/Core/CacheDB/MetadataDBHandler.py b/instrumentation/next-share/BaseLib/Core/CacheDB/MetadataDBHandler.py new file mode 100644 index 0000000..4545e44 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/CacheDB/MetadataDBHandler.py @@ -0,0 +1,1125 @@ +# Written by Andrea Reale +# see LICENSE.txt for license information + +from BaseLib.Core.Subtitles.MetadataDomainObjects.SubtitleInfo import SubtitleInfo +from BaseLib.Core.Subtitles.MetadataDomainObjects.MetadataDTO import MetadataDTO +from BaseLib.Core.CacheDB.SqliteCacheDBHandler import BasicDBHandler +import threading +from BaseLib.Core.CacheDB.sqlitecachedb import SQLiteCacheDB + +import sys +from BaseLib.Core.Subtitles.MetadataDomainObjects.MetadataExceptions import SignatureException, \ + MetadataDBException +from BaseLib.Core.Utilities.utilities import bin2str, str2bin +import sqlite3 +import time + + +SUBTITLE_LANGUAGE_CODE = "lang" +SUBTITLE_PATH = "path" + +METADATA_TABLE = "Metadata" + +MD_ID_KEY = "metadata_id" +MD_PUBLISHER_KEY = "publisher_id" +MD_INFOHASH_KEY = "infohash" +MD_DESCRIPTION_KEY = "description" +MD_TIMESTAMP_KEY = "timestamp" +MD_SIGNATURE_KEY = "signature" + + +SUBTITLES_TABLE = "Subtitles" + +SUB_MD_FK_KEY = "metadata_id_fk" +SUB_LANG_KEY = "subtitle_lang" +SUB_LOCATION_KEY = "subtitle_location" +SUB_CHECKSUM_KEY = "checksum" + +SUBTITLES_HAVE_TABLE = "SubtitlesHave" + +SH_MD_FK_KEY = "metadata_id_fk" +SH_PEER_ID_KEY = "peer_id" +SH_HAVE_MASK_KEY = "have_mask" +SH_TIMESTAMP = "received_ts" + + +# maximum number of have entries returned +# by the database (-1 for unlimited) +SH_RESULTS_LIMIT = 200 + +DEBUG = False + +#it's good to have all of the queries in one place: +#the code is more easy to read, and if some query is wrong +#it is easier to correct them all +SELECT_SUBS_JOIN_BASE = "SELECT sub." + SUB_MD_FK_KEY + ", sub." + SUB_LANG_KEY \ + + ", sub." + SUB_LOCATION_KEY \ + + ", sub." + SUB_CHECKSUM_KEY \ + + " FROM " + METADATA_TABLE + " AS md " \ + + "INNER JOIN " \ + + SUBTITLES_TABLE + " AS sub " \ + + "ON md." + MD_ID_KEY + " = sub." + SUB_MD_FK_KEY + +MD_SH_JOIN_CLAUSE = \ + METADATA_TABLE + " AS md " \ + + "INNER JOIN " \ + + SUBTITLES_HAVE_TABLE + " AS sh " \ + + "ON md." + MD_ID_KEY + " = sh." + SH_MD_FK_KEY + + +QUERIES = { + "SELECT SUBS JOIN HASH ALL" : + SELECT_SUBS_JOIN_BASE + + " WHERE md." + MD_INFOHASH_KEY + " = ?"\ + + " AND md." + MD_PUBLISHER_KEY + " = ?;", + + "SELECT SUBS JOIN HASH ONE" : + SELECT_SUBS_JOIN_BASE + + " WHERE md." + MD_INFOHASH_KEY + " = ?"\ + + " AND md." + MD_PUBLISHER_KEY + " = ?"\ + + " AND sub." + SUB_LANG_KEY + " = ?;", + + "SELECT SUBS FK ALL" : + "SELECT * FROM " + SUBTITLES_TABLE + + " WHERE " + SUB_MD_FK_KEY + " = ?;", + + "SELECT SUBS FK ONE" : + "SELECT * FROM " + SUBTITLES_TABLE + + " WHERE " + SUB_MD_FK_KEY + " = ?"\ + + " AND " + SUB_LANG_KEY + " = ?;", + + "SELECT METADATA" : + "SELECT * FROM " \ + + METADATA_TABLE + " WHERE " + MD_INFOHASH_KEY + " = ?" \ + + " AND " + MD_PUBLISHER_KEY + " = ?;", + + "SELECT PUBLISHERS FROM INFOHASH": + "SELECT " + MD_PUBLISHER_KEY + " FROM " + METADATA_TABLE \ + + " WHERE " + MD_INFOHASH_KEY + " = ?;", + + "UPDATE METADATA" : + "UPDATE " + METADATA_TABLE \ + + " SET " \ + + MD_DESCRIPTION_KEY + " = ?, " \ + + MD_TIMESTAMP_KEY + " = ?, " \ + + MD_SIGNATURE_KEY + " = ?" \ + + " WHERE " + MD_INFOHASH_KEY + " = ?" \ + + " AND " + MD_PUBLISHER_KEY + " = ?;", + + "UPDATE SUBTITLES" : + "UPDATE " + SUBTITLES_TABLE \ + + " SET " + SUB_LOCATION_KEY + "= ?, " \ + + SUB_CHECKSUM_KEY + "= ?" \ + + " WHERE " + SUB_MD_FK_KEY + "= ?" \ + + " AND " + SUB_LANG_KEY + "= ?;", + + "DELETE ONE SUBTITLES" : + "DELETE FROM " + SUBTITLES_TABLE \ + + " WHERE " + SUB_MD_FK_KEY + "= ? " \ + + " AND " + SUB_LANG_KEY + "= ?;", + + "DELETE ONE SUBTITLE JOIN" : + "DELETE FROM " + SUBTITLES_TABLE \ + + " WHERE " + SUB_MD_FK_KEY \ + + " IN ( SELECT " + MD_ID_KEY + " FROM " + METADATA_TABLE \ + + " WHERE " + MD_PUBLISHER_KEY + " = ?" \ + + " AND " + MD_INFOHASH_KEY + " = ? )" \ + + " AND " + SUB_LANG_KEY + "= ?;", + + "DELETE ALL SUBTITLES" : + "DELETE FROM " + SUBTITLES_TABLE \ + + " WHERE " + SUB_MD_FK_KEY + "= ?;", + + "DELETE METADATA PK" : + "DELETE FROM " + METADATA_TABLE \ + + " WHERE " + MD_ID_KEY + " = ?;", + + "INSERT METADATA" : + "INSERT INTO " + METADATA_TABLE + " VALUES " \ + + "(NULL,?,?,?,?,?)", + + "INSERT SUBTITLES" : + "INSERT INTO " + SUBTITLES_TABLE + " VALUES (?, ?, ?, ?);", + + "SELECT SUBTITLES WITH PATH": + "SELECT sub." + SUB_MD_FK_KEY + ", sub." + SUB_LOCATION_KEY + ", sub." \ + + SUB_LANG_KEY + ", sub." + SUB_CHECKSUM_KEY \ + + ", m." + MD_PUBLISHER_KEY + ", m." + MD_INFOHASH_KEY \ + + " FROM " + METADATA_TABLE + " AS m " \ + +"INNER JOIN " + SUBTITLES_TABLE + " AS sub "\ + + "ON m." + MD_ID_KEY + " = " + " sub." + SUB_MD_FK_KEY \ + + " WHERE " \ + + SUB_LOCATION_KEY + " IS NOT NULL;", + + "SELECT SUBTITLES WITH PATH BY CHN INFO": + "SELECT sub." + SUB_LOCATION_KEY + ", sub." \ + + SUB_LANG_KEY + ", sub." + SUB_CHECKSUM_KEY \ + + " FROM " + METADATA_TABLE + " AS m " \ + +"INNER JOIN " + SUBTITLES_TABLE + " AS sub "\ + + "ON m." + MD_ID_KEY + " = " + " sub." + SUB_MD_FK_KEY \ + + " WHERE sub." \ + + SUB_LOCATION_KEY + " IS NOT NULL" \ + + " AND m." + MD_PUBLISHER_KEY + " = ?"\ + + " AND m." + MD_INFOHASH_KEY + " = ?;" , + + "INSERT HAVE MASK": + "INSERT INTO " + SUBTITLES_HAVE_TABLE + " VALUES " \ + + "(?, ?, ?, ?);", + + "GET ALL HAVE MASK": + "SELECT sh." + SH_PEER_ID_KEY + ", sh." + SH_HAVE_MASK_KEY \ + + ", sh." + SH_TIMESTAMP \ + + " FROM " + MD_SH_JOIN_CLAUSE + " WHERE md." + MD_PUBLISHER_KEY \ + + " = ? AND md." + MD_INFOHASH_KEY + " = ? "\ + + "ORDER BY sh." + SH_TIMESTAMP + " DESC" \ + + " LIMIT " + str(SH_RESULTS_LIMIT) + ";", + + "GET ONE HAVE MASK": + "SELECT sh." + SH_HAVE_MASK_KEY \ + + ", sh." + SH_TIMESTAMP \ + + " FROM " + MD_SH_JOIN_CLAUSE + " WHERE md." + MD_PUBLISHER_KEY \ + + " = ? AND md." + MD_INFOHASH_KEY + " = ? AND sh." + SH_PEER_ID_KEY \ + + " = ?;", + + "UPDATE HAVE MASK": + "UPDATE " + SUBTITLES_HAVE_TABLE \ + + " SET " + SH_HAVE_MASK_KEY + " = ?, " \ + + SH_TIMESTAMP + " = ?" \ + + " WHERE " + SH_PEER_ID_KEY + " = ?" \ + + " AND " + SH_MD_FK_KEY + " IN " \ + + "( SELECT + " + MD_ID_KEY+ " FROM " \ + + METADATA_TABLE + " WHERE + "\ + + MD_PUBLISHER_KEY + " = ?"\ + + " AND " + MD_INFOHASH_KEY + " = ? );", + + "DELETE HAVE": + "DELETE FROM " + SUBTITLES_HAVE_TABLE \ + + " WHERE " + SH_PEER_ID_KEY + " = ?" \ + + " AND " + SH_MD_FK_KEY + " IN " \ + + "( SELECT + " + MD_ID_KEY+ " FROM " \ + + METADATA_TABLE + " WHERE + "\ + + MD_PUBLISHER_KEY + " = ?"\ + + " AND " + MD_INFOHASH_KEY + " = ? );", + + "CLEANUP OLD HAVE": + "DELETE FROM " + SUBTITLES_HAVE_TABLE \ + + " WHERE " + SH_TIMESTAMP + " < ? " \ + + " AND " + SH_PEER_ID_KEY + " NOT IN " \ + + "( SELECT md." + MD_PUBLISHER_KEY + " FROM " \ + + METADATA_TABLE + " AS md WHERE md." + MD_ID_KEY \ + + " = " + SH_MD_FK_KEY + " );" + } + +class MetadataDBHandler (object, BasicDBHandler): + + """ + Data Access Layer for the subtitles database. + """ + + __single = None # used for multithreaded singletons pattern + _lock = threading.RLock() + + @staticmethod + def getInstance(*args, **kw): + if MetadataDBHandler.__single is None: + MetadataDBHandler._lock.acquire() + try: + if MetadataDBHandler.__single is None: + MetadataDBHandler(*args, **kw) + finally: + MetadataDBHandler._lock.release() + return MetadataDBHandler.__single + + + def __init__(self, db=SQLiteCacheDB.getInstance()): + # notice that singleton pattern is not enforced. + # This way the code is more easy + # to test. + + try: + MetadataDBHandler._lock.acquire() + MetadataDBHandler.__single = self + finally: + MetadataDBHandler._lock.release() + + try: + self._db = db + # Don't know what those life should know. Assuming I don't need + # them 'till a countrary proof! (Ask Nitin) + # BasicDBHandler.__init__(self,db,METADATA_TABLE) + # BasicDBHandler.__init__(self,db,SUBTITLES_TABLE) + print >> sys.stderr, "Metadata: DB made" + except: + print >> sys.stderr, "Metadata: couldn't make the tables" + + + print >> sys.stderr, "Metadata DB Handler initialized" + + def commit(self): + self._db.commit() + +# Commented for the sake of API simplicity +# But then uncommented for coding simplicity :P + def getAllSubtitles(self, channel, infohash): + """ + Get all the available subtitles for a channel and infohash. + + Returns a list representing subtitles that are available for + a givenchannel and infohash. + + @param channel: the perm_id of the channel owner (binary) + @param infohash: the infhash of a channel elements as it + is announced in ChannelCast (binary) + @return: a dictionary of { lang : SubtitleInfo instance} + """ + + query = QUERIES["SELECT SUBS JOIN HASH ALL"] + infohash = bin2str(infohash) + channel = bin2str(channel) + + results = self._db.fetchall(query, (infohash, channel)) + + subsDict = {} + for entry in results: + subsDict[entry[1]] = SubtitleInfo(entry[1], entry[2], entry[3]) + + return subsDict + + def _deleteSubtitleByChannel(self, channel, infohash, lang): + ''' + Remove a subtitle for a channel infohash + + @param channel: the channel where the subtitle is (binary) + @param infohash: the infohash of the torrent referred by the subtitle + (binary) + @param lang: ISO-639-2 language code of the subtitle to remove + + ''' + + query = QUERIES["DELETE ONE SUBTITLE JOIN"] + + infohash = bin2str(infohash) + channel = bin2str(channel) + + self._db.execute_write(query,(channel, infohash, lang)) + + + def _getAllSubtitlesByKey(self, metadataKey): + ''' + Retrieves every subtitles given a Metadata table key + + Given an instance of the Metadata table artificial key, retrieves + every subtitle instance associated to that key + + @param metadataKey: a value of an artificial key in the Metadata table + @return : a dictionary of type {lang : SubtitleInfo}, empty if no results + ''' + query = QUERIES["SELECT SUBS FK ALL"] + + + results = self._db.fetchall(query, (metadataKey,)) + subsDict = {} + for entry in results: + subsDict[entry[1]] = SubtitleInfo(entry[1], entry[2], str2bin(entry[3])) + + return subsDict + + +# commented for the sake of API simplicity +# def hasSubtitleInLang(self,channel,infohash, lang): +# """ +# Checks whether an item in a channel as available subitltles. +# +# @param channel: a perm_id identifying the owner of the channel. +# @param infohash: the infohash of an item, as announced in channelcast +# messages. +# @param lang: a 3 characters ISO 639-2 language code, identifying +# the desired subtitle langugage +# @return: bool +# """ +# sub = self.getSubtitle(channel, infohash, lang) +# return sub is not None +# + +# commented for the sake of api simplicity +# But then uncommented for coding simplicity :P + def getSubtitle(self, channel, infohash, lang): + """ + Get a subtitle for a language for a given item in a given channel. + + Returns the details reguarding a subtitles in a given language for a + given item in a given channel, if it exists. Otherwise it returns + None. + + @param channel: a perm_id identifying the owner of the channel. + @param infohash: the infohash of an item, as announced in channelcast + messages. + @param lang: a 3 characters ISO 639-2 language code, identifying + the desired subtitle langugage + @return: a SubtitleInfo instance + """ + query = QUERIES["SELECT SUBS JOIN HASH ONE"] + + infohash = bin2str(infohash) + channel = bin2str(channel) + + + res = self._db.fetchall(query, (infohash, channel, lang)) + if len(res) == 0 : + return None + elif len(res) == 1 : + checksum = str2bin(res[0][3]) + return SubtitleInfo(res[0][1], res[0][2], checksum) + else : + # This should be not possible to database constraints + raise MetadataDBException("Metadata DB Constraint violeted!") + + + + def _getSubtitleByKey(self, metadata_fk, lang): + """ + Return a subtitle in a given language for a key of the Metadata table. + + Given an instance of the artificial key in the metadata table, + retrieves a SubtitleInfo instance for that key and the language passed in. + + @param metadata_fk: a key in the metadata table + @param lang: a language code for the subtitle to be retrieved + + @return: a SubtitleInfo instance, or None + """ + query = QUERIES["SELECT SUBS FK ONE"] + + + res = self._db.fetchall(query, (metadata_fk, lang)) + if len(res) == 0 : + return None + elif len(res) == 1 : + checksum = str2bin(res[0][3]) + return SubtitleInfo(res[0][1], res[0][2], checksum) + else : + # This should be not possible to database constraints + raise MetadataDBException("Metadata DB Constraint violeted!") + + + def getMetadata(self, channel, infohash): + """ + Returns a MetadataDTO instance for channel/infohash if available in DB + + Given a channel/infhash couple returns a MetadataDTO instance, built + with the values retrieved from the Metadata and Subtitles DB. If + no result returns None + + @param channel: the permid of the channel's owner (binary) + @param infohash: the infohash of the item the metadata refers to + (binary) + @return: a MetadataDTO instance comprehensive of subtitles if any + metadata is found in the DB. None otherwise. + """ + + query = QUERIES["SELECT METADATA"] + + infohash = bin2str(infohash) + channel = bin2str(channel) + + res = self._db.fetchall(query, (infohash, channel)) + + if len(res) == 0: + return None + if len(res) > 1: + raise MetadataDBException("Metadata DB Constraint violated") + + metaTuple = res[0] + + subsDictionary = self._getAllSubtitlesByKey(metaTuple[0]) + + publisher = str2bin(metaTuple[1]) + infohash = str2bin(metaTuple[2]) + timestamp = int(metaTuple[4]) + description = unicode(metaTuple[3]) + signature = str2bin(metaTuple[5]) + + toReturn = MetadataDTO(publisher, infohash, + timestamp, description, None, + signature) + + for sub in subsDictionary.itervalues(): + toReturn.addSubtitle(sub) + + return toReturn + + + def getAllMetadataForInfohash(self, infohash): + """ + Returns a list of MetadataDTO instances for a given infohash + + Given a torrent infohash returns a list of MetadataDTO instances for + that infohash. Each one of the MetadataDTO refers to a different + channel. + + @param infohash: the infohash for the torrent (binary) + @return: a list of MetadataDTO isntances (or empty list if nothing + is found) + """ + + assert infohash is not None + + strinfohash = bin2str(infohash) + + query = QUERIES["SELECT PUBLISHERS FROM INFOHASH"] + + channels = self._db.fetchall(query, (strinfohash,)) + + return [self.getMetadata(str2bin(entry[0]), infohash) for entry in channels] + + + + + def hasMetadata(self, channel, infohash): + """ + Checks whether there exists some metadata for an item in a channel. + + @param channel: a perm_id identifying the owner of the channel. + @param infohash: the infohash of an item, as announced in channelcast + messages. + @return boolean + """ + query = QUERIES["SELECT METADATA"] + + infohash = bin2str(infohash) + channel = bin2str(channel) + + res = self._db.fetchall(query, (infohash, channel)) + return len(res) != 0 + + + def insertMetadata(self, metadata_dto): + ''' + Insert the metadata contained in a Metadata DTO in the database. + + If an entry relative to the same channel and infohash of the provided + dto already exists in the db, the db is updated only if the timestamp + of the new dto is newer then the entry in the database. + If there is no such an entry, a new wan in the Metadata DB is created + along with the required entries in the SubtitleInfo DB + + @type metadata_dto: MetadataDTO + @param metada_dto: an instance of MetadataDTO describing metadata + + @return True if an existing entry was updated, false if a new entry + was interested. Otherwise None. + + ''' + assert metadata_dto is not None + assert isinstance(metadata_dto, MetadataDTO) + #try to retrieve a correspindng record for channel,infhoash + + #won't do nothing if the metadata_dto is not correctly signed + if not metadata_dto.verifySignature(): + raise SignatureException("Metadata to insert is not properly" \ + "signed") + + select_query = QUERIES["SELECT METADATA"] + + signature = bin2str(metadata_dto.signature) + infohash = bin2str(metadata_dto.infohash) + channel = bin2str(metadata_dto.channel) + + res = self._db.fetchall(select_query, + (infohash, channel)) + + isUpdate = False + + if len(res) != 0 : + #updated if the new message is newer + if metadata_dto.timestamp > res[0][4] : + query = QUERIES["UPDATE METADATA"] + + + self._db.execute_write(query, + (metadata_dto.description, + metadata_dto.timestamp, + signature, + infohash, + channel,), + False) #I don't want the transaction to commit now + + fk_key = res[0][0] + + isUpdate = True + + else: + return + + else: #if is this a whole new metadata item + query = QUERIES["INSERT METADATA"] + + self._db.execute_write(query, + (channel, + infohash, + metadata_dto.description, + metadata_dto.timestamp, + signature, + ), + True) + + if DEBUG: + print >> sys.stderr, "Performing query on db: " + query + + newRows = self._db.fetchall(select_query, + (infohash, channel)) + + + if len(newRows) == 0 : + raise IOError("No results, while there should be one") + + fk_key = newRows[0][0] + + + self._insertOrUpdateSubtitles(fk_key, metadata_dto.getAllSubtitles(), \ + False) + + self._db.commit() #time to commit everything + + return isUpdate + + + + def _insertOrUpdateSubtitles(self, fk_key, subtitles, commitNow=True): + """ + Given a dictionary of subtitles updates the corrisponding entries. + + This method takes as input a foreign key for the Metadata table, + and a dictionary of type {lang : SubtitleInfo}. Then it updates the + SubtitleInfo table, updating existing entries, deleting entries that are + in the db but not in the passed dictionary, and inserting entries + that are in the dictionary but not in the db. + + @param fk_key: a foreign key from the Metadata table. Notice that + sqlite does not enforce the fk constraint. Be careful! + @param subtitles: a dictionary {lang : SubtitleInfo} (subtitle must be + an instance of SubtitleInfo) + @param commitNow: if False the transaction is not committed + """ + + + allSubtitles = self._getAllSubtitlesByKey(fk_key) + oldSubsSet = frozenset(allSubtitles.keys()) + newSubsSet = frozenset(subtitles.keys()) + + commonLangs = oldSubsSet & newSubsSet + newLangs = newSubsSet - oldSubsSet + toDelete = oldSubsSet - newSubsSet + + #update existing subtitles + for lang in commonLangs: + self._updateSubtitle(fk_key, subtitles[lang], False) + + + #remove subtitles that are no more in the set + for lang in toDelete: + self._deleteSubtitle(fk_key, lang, False) + + #insert new subtitles + for lang in newLangs: + self._insertNewSubtitle(fk_key, subtitles[lang], False) + + if commitNow: + self._db.commit() + + + + + def _updateSubtitle(self, metadata_fk, subtitle, commitNow=True): + """ + Update an entry in the Subtitles database. + + If the entry identified by metadata_fk, subtitle.lang does not exist + in the subtitle database this method does nothing. + + @param metadata_fk: foreign key of the metadata table + @param subtitle: instance of Subitle containing the data to insert + @param commitNow: if False, this method does not commit the changes to + the database + """ + assert metadata_fk is not None + assert subtitle is not None + assert isinstance(subtitle, SubtitleInfo) + + toUpdate = self._getSubtitleByKey(metadata_fk, subtitle.lang) + + if toUpdate is None: + return + + + query = QUERIES["UPDATE SUBTITLES"] + + checksum = bin2str(subtitle.checksum) + + self._db.execute_write(query, (subtitle.path, + checksum, metadata_fk, subtitle.lang), + commitNow) + + def updateSubtitlePath(self, channel, infohash, lang, newPath, commitNow=True): + """ + Updates a subtitle entry in the database if it exists. + + Given the channel, the infohash, and a SubtitleInfo instance, + the entry relative to that subtitle is updated accordingly + to the details in the SubtitleInfo instance. + If an instance for the provided channel, infohash, and language + does not already exist in the db, nothing is done. + + @param channel: the channel id (permid) of the channel for the + subtitle (binary) + @param infohash: the infohash of the item the subtitle refrs to + (binary) + @param lang: the language of the subtitle to update + @param path: the new path of the subtitle. None to indicate that the + subtitle is not available + @return True if an entry was updated in the db. False if nothing + got written on the db + + @precondition: subtitle.lang is not None + """ + query = QUERIES["SELECT SUBS JOIN HASH ONE"] + + channel = bin2str(channel) + infohash = bin2str(infohash) + + res = self._db.fetchall(query, (infohash, channel, lang)) + + if len(res) > 1 : + raise MetadataDBException("Metadata DB constraint violated") + elif len(res) == 0 : + if DEBUG: + print >> sys.stderr, "Nothing to update for channel %s, infohash %s, lang"\ + " %s. Doing nothing." % (channel[-10:],\ + infohash, lang) + return False + else: + query = QUERIES["UPDATE SUBTITLES"] + self._db.execute_write(query, (newPath, + res[0][3], res[0][0], lang), + commitNow) + return True + + + + + + + def _deleteSubtitle(self, metadata_fk, lang, commitNow=True): + """ + Delete an entry from the subtitles table. + + Given a foreign key from the metadata table and a language delets + the corresponding entry in the subtitle table. If the entry + is not found, it does nothing. + + @param metadata_fk: a foreign key from the Metadata table + @param lang: a 3 characters language code + @param commitNow: if False does not commit the transaction + """ + assert metadata_fk is not None + assert lang is not None + + query = QUERIES["DELETE ONE SUBTITLES"] + self._db.execute_write(query, (metadata_fk, lang), commitNow) + + + def _insertNewSubtitle(self, metadata_fk, subtitle, commitNow=True) : + """ + Insert a new subtitle entry in the Subtitles table. + + Given a foreign key from the Metadata table, and a SubtitleInfo instance + describing the subtitle to insert, adds it to the metadata table. + This method assumes that that entry does not already exist in the + table. + NOTICE that sqlite does not enforce the foreign key constraint, + so be careful about integrity + """ + assert metadata_fk is not None + assert subtitle is not None + assert isinstance(subtitle, SubtitleInfo) + + query = QUERIES["INSERT SUBTITLES"] + + checksum = bin2str(subtitle.checksum) + self._db.execute_write(query, (metadata_fk, subtitle.lang, + subtitle.path, checksum), + commitNow) + + def deleteMetadata(self, channel, infohash): + """ + Removes all the metadata associated to a channel/infohash. + + Everything is dropped from both the Metadata and Subtitles db. + + @param channel: the permid of the channel's owner + @param infohash: the infhoash of the entry + """ + + assert channel is not None + assert infohash is not None + + channel = bin2str(channel) + infohash = bin2str(infohash) + + query = QUERIES["SELECT METADATA"] + + if DEBUG: + print >> sys.stderr, "Performing query on db: " + query + + res = self._db.fetchall(query, (infohash, channel)) + + if len(res) == 0 : + return + if len(res) > 1 : + raise IOError("Metadata DB constraint violated") + + metadata_fk = res[0][0] + + self._deleteAllSubtitles(metadata_fk, False) + + query = QUERIES["DELETE METADATA PK"] + + self._db.execute_write(query, (metadata_fk,), False) + + self._db.commit() + + + + + + def _deleteAllSubtitles(self, metadata_fk, commitNow): + query = QUERIES["DELETE ALL SUBTITLES"] + + self._db.execute_write(query, (metadata_fk,), commitNow) + + def getAllLocalSubtitles(self): + ''' + Returns a structure containing all the subtitleInfos that are pointing + to a local path + + @return a dictionary like this: + { ... + channel1 : { infohash1 : [ SubtitleInfo1, ...] } + ... + } + ''' + query = QUERIES["SELECT SUBTITLES WITH PATH"] + res = self._db.fetchall(query) + + result = {} + + for entry in res: + # fk = entry[0] + path = entry[1] + lang = entry[2] + checksum = str2bin(entry[3]) + channel = str2bin(entry[4]) + infohash = str2bin(entry[5]) + + s = SubtitleInfo(lang, path, checksum) + + if channel not in result: + result[channel] = {} + if infohash not in result[channel]: + result[channel][infohash] = [] + + result[channel][infohash].append(s) + + return result + + def getLocalSubtitles(self, channel, infohash): + ''' + Returns a dictionary containing all the subtitles pointing + to a local pathm for the given channel, infohash + @param channel: binary channel_id(permid) + @param infohash: binary infohash + + @rtype: dict + @return: a dictionary like this: + { + ... + langCode : SubtitleInfo, + ... + } + The dictionary will be empty if no local subtitle + is available. + ''' + query = QUERIES["SELECT SUBTITLES WITH PATH BY CHN INFO"] + + channel = bin2str(channel) + infohash = bin2str(infohash) + res = self._db.fetchall(query,(channel,infohash)) + + result = {} + + for entry in res: + location = entry[0] + language = entry[1] + checksum = str2bin(entry[2]) + subInfo = SubtitleInfo(language, location, checksum) + result[language] = subInfo + + return result + + + def insertHaveMask(self, channel, infohash, peer_id, havemask, timestamp=None): + ''' + Store a received have mask in the db + + Each inserted rows represent a delcaration of subtitle + availability from peer_id, for some subtitles for + a torrent identified by infohash in a channel identified + by channel. + + @type channel: str + @param channel: channel_id (binary) + + @type infohash: str + @param infohash: the infohash of a torrent (binary) + + @type peer_id: str + @param peer_id: peer from whom the infomask was received.(ie its binary permid) + + @type havemask: int + @param havemask: a non-negative integer. It must be smaller + then 2**32. + + @precondition: an entry for (channel, infohash) must already + exist in the database + ''' + query = QUERIES["SELECT METADATA"] + + if timestamp is None: + timestamp = int(time.time()) + + channel = bin2str(channel) + infohash = bin2str(infohash) + peer_id = bin2str(peer_id) + + res = self._db.fetchall(query, (infohash, channel)) + + if len(res) != 1: + raise MetadataDBException("No entry in the MetadataDB for %s, %s" %\ + (channel[-10:],infohash)) + + metadata_fk = res[0][0] + + insertQuery = QUERIES["INSERT HAVE MASK"] + + try: + self._db.execute_write(insertQuery, (metadata_fk, peer_id, havemask, timestamp)) + except sqlite3.IntegrityError,e: + raise MetadataDBException(str(e)) + + + def updateHaveMask(self,channel,infohash,peer_id, newMask, timestamp=None): + ''' + Store a received have mask in the db + + (See insertHaveMask for description) + + @type channel: str + @param channel: channel_id (binary) + + @type infohash: str + @param infohash: the infohash of a torrent (binary) + + @type peer_id: str + @param peer_id: peer from whom the infomask was received.(ie its binary permid) + + @type havemask: int + "param havemask: a non-negative integer. It must be smaller + then 2**32. + ''' + channel = bin2str(channel) + infohash = bin2str(infohash) + peer_id = bin2str(peer_id) + + updateQuery = QUERIES["UPDATE HAVE MASK"] + if timestamp is None: + timestamp = int(time.time()) + self._db.execute_write(updateQuery, + (newMask,timestamp,peer_id, channel, infohash)) + + def deleteHaveEntry(self, channel, infohash, peer_id): + ''' + Delete a row from the SubtitlesHave db. + + If the row is not in the db nothing happens. + + @type channel: str + @param channel: channel_id (binary) + + @type infohash: str + @param infohash: the infohash of a torrent (binary) + + @type peer_id: str + @param peer_id: peer from whom the infomask was received.(ie its binary permid) + + @postcondition: if a row identified by channel, infohash, peer_id + was in the database, it will no longer be there + at the end of this method call + + ''' + channel = bin2str(channel) + infohash = bin2str(infohash) + peer_id = bin2str(peer_id) + deleteQuery = QUERIES["DELETE HAVE"] + self._db.execute_write(deleteQuery, + (peer_id,channel,infohash)) + + def getHaveMask(self, channel, infohash, peer_id): + ''' + Returns the have mask for a single peer if available. + + @type channel: str + @param channel: channel_id (binary) + + @type infohash: str + @param infohash: the infohash of a torrent (binary) + + @type peer_id: str + @param peer_id: peer from whom the infomask was received.(ie its binary permid) + + @rtype: int + @return: the have mask relative to channel, infohash, and peer. + If not available returns None + + @postcondition: the return value is either None or a non-negative + integer smaller then 2**32 + ''' + + query = QUERIES["GET ONE HAVE MASK"] + + channel = bin2str(channel) + infohash = bin2str(infohash) + peer_id = bin2str(peer_id) + + res = self._db.fetchall(query,(channel,infohash,peer_id)) + + if len(res) <= 0: + return None + elif len(res) > 1: + raise AssertionError("channel,infohash,peer_id should be unique") + else: + return res[0][0] + + + def getHaveEntries(self, channel, infohash): + ''' + Return a list of have entries for subtitles for a torrent + in a channel. + + This method returns a list of tuple, like: + [ + ... + (peer_id, haveMask, timestamp), + ... + ] + + (peer_id) is the perm_id of a Tribler + Peer, haveMask is an integer value representing a + bitmask of subtitles owned by that peer. + Timestamp is the timestamp at the time the havemask + was received. + The results are ordered by descending timestamp. + If there are no + entris for the givenn channel,infohash pair, the returned + list will be empty + + @type channel: str + @param channel: channel_id (binary) + + @type infohash: str + @param infohash: the infohash of a torrent (binary) + + @rtype: list + @return: see description + + ''' + query = QUERIES["GET ALL HAVE MASK"] + + channel = bin2str(channel) + infohash = bin2str(infohash) + + res = self._db.fetchall(query,(channel,infohash)) + returnlist = list() + + for entry in res: + peer_id = str2bin(entry[0]) + haveMask = entry[1] + timestamp = entry[2] + returnlist.append((peer_id, haveMask, timestamp)) + + return returnlist + + def cleanupOldHave(self, limit_ts): + ''' + Remove from the SubtitlesHave database every entry + received at a timestamp that is (strictly) less then limit_ts + + This method does not remove have messages sent by + the publisher of the channel. + + @type limit_ts: int + @param limit_ts: a timestamp. All the entries in the + database having timestamp lessere then + limit_ts will be removed, excpet if + they were received by the publisher + of the channel + ''' + cleanupQuery = QUERIES["CLEANUP OLD HAVE"] + + self._db.execute_write(cleanupQuery,(limit_ts,)) + + + def insertOrUpdateHave(self, channel, infohash, peer_id, havemask, timestamp=None): + ''' + Store a received have mask in the db + + Each inserted rows represent a delcaration of subtitle + availability from peer_id, for some subtitles for + a torrent identified by infohash in a channel identified + by channel. + + If a row for the given (channel, infohash, peer_id) it + is updated accordingly to the parameters. Otherwise + a new row is added to the db + + @type channel: str + @param channel: channel_id (binary) + + @type infohash: str + @param infohash: the infohash of a torrent (binary) + + @type peer_id: str + @param peer_id: peer from whom the infomask was received.(ie its binary permid) + + @type havemask: int + @param havemask: a non-negative integer. It must be smaller + then 2**32. + + @precondition: an entry for (channel, infohash) must already + exist in the database + ''' + + if timestamp is None: + timestamp = int(time.time()) + + + if self.getHaveMask(channel, infohash, peer_id) is not None: + self.updateHaveMask(channel, infohash, peer_id, havemask, timestamp) + else: + self.insertHaveMask(channel, infohash, peer_id, havemask, timestamp) + + + + + + + diff --git a/instrumentation/next-share/BaseLib/Core/CacheDB/Notifier.py b/instrumentation/next-share/BaseLib/Core/CacheDB/Notifier.py new file mode 100644 index 0000000..329a082 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/CacheDB/Notifier.py @@ -0,0 +1,90 @@ +# Written by Jelle Roozenburg +# see LICENSE.txt for license information + +import sys +import threading +from traceback import print_exc, print_stack + +from BaseLib.Core.simpledefs import * + +class Notifier: + + SUBJECTS = [NTFY_PEERS, NTFY_TORRENTS, NTFY_PREFERENCES, NTFY_MYPREFERENCES, NTFY_ACTIVITIES, NTFY_REACHABLE, NTFY_CHANNELCAST, NTFY_VOTECAST, NTFY_RICH_METADATA, NTFY_SUBTITLE_CONTENTS] + + #. . . + # todo: add all datahandler types+other observables + __single = None + + def __init__(self, pool = None): + if Notifier.__single: + raise RuntimeError, "Notifier is singleton" + self.pool = pool + self.observers = [] + self.observerLock = threading.Lock() + Notifier.__single = self + + def getInstance(*args, **kw): + if Notifier.__single is None: + Notifier(*args, **kw) + return Notifier.__single + getInstance = staticmethod(getInstance) + + def add_observer(self, func, subject, changeTypes = [NTFY_UPDATE, NTFY_INSERT, NTFY_DELETE], id = None): + """ + Add observer function which will be called upon certain event + Example: + addObserver(NTFY_PEERS, [NTFY_INSERT,NTFY_DELETE]) -> get callbacks + when peers are added or deleted + addObserver(NTFY_PEERS, [NTFY_SEARCH_RESULT], 'a_search_id') -> get + callbacks when peer-searchresults of of search + with id=='a_search_id' come in + """ + assert type(changeTypes) == list + assert subject in self.SUBJECTS + + obs = (func, subject, changeTypes, id) + self.observerLock.acquire() + self.observers.append(obs) + self.observerLock.release() + + def remove_observer(self, func): + """ Remove all observers with function func + """ + + self.observerLock.acquire() + i=0 + while i < len(self.observers): + ofunc = self.observers[i][0] + if ofunc == func: + del self.observers[i] + else: + i+=1 + self.observerLock.release() + + def notify(self, subject, changeType, obj_id, *args): + """ + Notify all interested observers about an event with threads from the pool + """ + tasks = [] + assert subject in self.SUBJECTS + + self.observerLock.acquire() + for ofunc, osubject, ochangeTypes, oid in self.observers: + try: + if (subject == osubject and + changeType in ochangeTypes and + (oid is None or oid == obj_id)): + tasks.append(ofunc) + except: + print_stack() + print_exc() + print >>sys.stderr,"notify: OIDs were",`oid`,`obj_id` + + self.observerLock.release() + args = [subject, changeType, obj_id] + list(args) + for task in tasks: + if self.pool: + self.pool.queueTask(task, args) + else: + task(*args) # call observer function in this thread + diff --git a/instrumentation/next-share/BaseLib/Core/CacheDB/SqliteCacheDBHandler.py b/instrumentation/next-share/BaseLib/Core/CacheDB/SqliteCacheDBHandler.py new file mode 100644 index 0000000..c10e830 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/CacheDB/SqliteCacheDBHandler.py @@ -0,0 +1,4274 @@ +# Written by Jie Yang +# see LICENSE.txt for license information +# Note for Developers: Please write a unittest in Tribler/Test/test_sqlitecachedbhandler.py +# for any function you add to database. +# Please reuse the functions in sqlitecachedb as much as possible + +from BaseLib.Core.CacheDB.sqlitecachedb import SQLiteCacheDB, bin2str, str2bin, NULL +from copy import deepcopy,copy +from traceback import print_exc +from time import time +from BaseLib.Core.Utilities.Crypto import sha +from BaseLib.Core.TorrentDef import TorrentDef +import sys +import os +import socket +import threading +import base64 +from random import randint, sample +from sets import Set +import math +import re + +from maxflow import Network +from math import atan, pi + + +from BaseLib.Core.BitTornado.bencode import bencode, bdecode +from Notifier import Notifier +from BaseLib.Core.simpledefs import * +from BaseLib.Core.BuddyCast.moderationcast_util import * +from BaseLib.Core.Overlay.permid import sign_data, verify_data, permid_for_user +from BaseLib.Core.Search.SearchManager import split_into_keywords +from BaseLib.Core.Utilities.unicode import name2unicode, dunno2unicode +from BaseLib.Category.Category import Category + +# maxflow constants +MAXFLOW_DISTANCE = 2 +ALPHA = float(1)/30000 + +DEBUG = False +SHOW_ERROR = False + +MAX_KEYWORDS_STORED = 5 +MAX_KEYWORD_LENGTH = 50 + +#Rahim: +MAX_POPULARITY_REC_PER_TORRENT = 5 # maximum number of records in popularity table for each torrent +MAX_POPULARITY_REC_PER_TORRENT_PEER = 3 # maximum number of records per each combination of torrent and peer + +from BaseLib.Core.Search.SearchManager import split_into_keywords + +def show_permid_shorter(permid): + if not permid: + return 'None' + s = base64.encodestring(permid).replace("\n","") + return s[-5:] + +class BasicDBHandler: + def __init__(self,db, table_name): ## self, table_name + self._db = db ## SQLiteCacheDB.getInstance() + self.table_name = table_name + self.notifier = Notifier.getInstance() + + def __del__(self): + try: + self.sync() + except: + if SHOW_ERROR: + print_exc() + + def close(self): + try: + self._db.close() + except: + if SHOW_ERROR: + print_exc() + + def size(self): + return self._db.size(self.table_name) + + def sync(self): + self._db.commit() + + def commit(self): + self._db.commit() + + def getOne(self, value_name, where=None, conj='and', **kw): + return self._db.getOne(self.table_name, value_name, where=where, conj=conj, **kw) + + def getAll(self, value_name, where=None, group_by=None, having=None, order_by=None, limit=None, offset=None, conj='and', **kw): + return self._db.getAll(self.table_name, value_name, where=where, group_by=group_by, having=having, order_by=order_by, limit=limit, offset=offset, conj=conj, **kw) + + +class MyDBHandler(BasicDBHandler): + + __single = None # used for multithreaded singletons pattern + lock = threading.Lock() + + def getInstance(*args, **kw): + # Singleton pattern with double-checking + if MyDBHandler.__single is None: + MyDBHandler.lock.acquire() + try: + if MyDBHandler.__single is None: + MyDBHandler(*args, **kw) + finally: + MyDBHandler.lock.release() + return MyDBHandler.__single + + getInstance = staticmethod(getInstance) + + def __init__(self): + if MyDBHandler.__single is not None: + raise RuntimeError, "MyDBHandler is singleton" + MyDBHandler.__single = self + db = SQLiteCacheDB.getInstance() + BasicDBHandler.__init__(self,db,'MyInfo') ## self,db,'MyInfo' + # keys: version, torrent_dir + + def get(self, key, default_value=None): + value = self.getOne('value', entry=key) + if value is not NULL: + return value + else: + if default_value is not None: + return default_value + else: + raise KeyError, key + + def put(self, key, value, commit=True): + if self.getOne('value', entry=key) is NULL: + self._db.insert(self.table_name, commit=commit, entry=key, value=value) + else: + where = "entry=" + repr(key) + self._db.update(self.table_name, where, commit=commit, value=value) + +class FriendDBHandler(BasicDBHandler): + + __single = None # used for multithreaded singletons pattern + lock = threading.Lock() + + def getInstance(*args, **kw): + # Singleton pattern with double-checking + if FriendDBHandler.__single is None: + FriendDBHandler.lock.acquire() + try: + if FriendDBHandler.__single is None: + FriendDBHandler(*args, **kw) + finally: + FriendDBHandler.lock.release() + return FriendDBHandler.__single + + getInstance = staticmethod(getInstance) + + def __init__(self): + if FriendDBHandler.__single is not None: + raise RuntimeError, "FriendDBHandler is singleton" + FriendDBHandler.__single = self + db = SQLiteCacheDB.getInstance() + BasicDBHandler.__init__(self,db, 'Peer') ## self,db,'Peer' + + def setFriendState(self, permid, state=1, commit=True): + self._db.update(self.table_name, 'permid='+repr(bin2str(permid)), commit=commit, friend=state) + self.notifier.notify(NTFY_PEERS, NTFY_UPDATE, permid, 'friend', state) + + def getFriends(self,state=1): + where = 'friend=%d ' % state + res = self._db.getAll('Friend', 'permid',where=where) + return [str2bin(p[0]) for p in res] + #raise Exception('Use PeerDBHandler getGUIPeers(category = "friend")!') + + def getFriendState(self, permid): + res = self.getOne('friend', permid=bin2str(permid)) + return res + + def deleteFriend(self,permid): + self.setFriendState(permid,0) + + def searchNames(self,kws): + return doPeerSearchNames(self,'Friend',kws) + + def getRanks(self): + # TODO + return [] + + def size(self): + return self._db.size('Friend') + + def addExternalFriend(self, peer): + peerdb = PeerDBHandler.getInstance() + peerdb.addPeer(peer['permid'], peer) + self.setFriendState(peer['permid']) + +NETW_MIME_TYPE = 'image/jpeg' + +class PeerDBHandler(BasicDBHandler): + + __single = None # used for multithreaded singletons pattern + lock = threading.Lock() + + gui_value_name = ('permid', 'name', 'ip', 'port', 'similarity', 'friend', + 'num_peers', 'num_torrents', 'num_prefs', + 'connected_times', 'buddycast_times', 'last_connected', + 'is_local') + + def getInstance(*args, **kw): + # Singleton pattern with double-checking + if PeerDBHandler.__single is None: + PeerDBHandler.lock.acquire() + try: + if PeerDBHandler.__single is None: + PeerDBHandler(*args, **kw) + finally: + PeerDBHandler.lock.release() + return PeerDBHandler.__single + + getInstance = staticmethod(getInstance) + + def __init__(self): + if PeerDBHandler.__single is not None: + raise RuntimeError, "PeerDBHandler is singleton" + PeerDBHandler.__single = self + db = SQLiteCacheDB.getInstance() + BasicDBHandler.__init__(self, db,'Peer') ## self, db ,'Peer' + self.pref_db = PreferenceDBHandler.getInstance() + self.online_peers = set() + + + def __len__(self): + return self.size() + + def getPeerID(self, permid): + return self._db.getPeerID(permid) + + def getPeer(self, permid, keys=None): + if keys is not None: + res = self.getOne(keys, permid=bin2str(permid)) + return res + else: + # return a dictionary + # make it compatible for calls to old bsddb interface + value_name = ('permid', 'name', 'ip', 'port', 'similarity', 'friend', + 'num_peers', 'num_torrents', 'num_prefs', 'num_queries', + 'connected_times', 'buddycast_times', 'last_connected', 'last_seen', 'last_buddycast') + + item = self.getOne(value_name, permid=bin2str(permid)) + if not item: + return None + peer = dict(zip(value_name, item)) + peer['permid'] = str2bin(peer['permid']) + return peer + + def getPeerSim(self, permid): + permid_str = bin2str(permid) + sim = self.getOne('similarity', permid=permid_str) + if sim is None: + sim = 0 + return sim + + def getPeerList(self, peerids=None): # get the list of all peers' permid + if peerids is None: + permid_strs = self.getAll('permid') + return [str2bin(permid_str[0]) for permid_str in permid_strs] + else: + if not peerids: + return [] + s = str(peerids).replace('[','(').replace(']',')') +# if len(peerids) == 1: +# s = '(' + str(peerids[0]) + ')' # tuple([1]) = (1,), syntax error for sql +# else: +# s = str(tuple(peerids)) + sql = 'select permid from Peer where peer_id in ' + s + permid_strs = self._db.fetchall(sql) + return [str2bin(permid_str[0]) for permid_str in permid_strs] + + + def getPeers(self, peer_list, keys): # get a list of dictionaries given peer list + # BUG: keys must contain 2 entries, otherwise the records in all are single values?? + value_names = ",".join(keys) + sql = 'select %s from Peer where permid=?;'%value_names + all = [] + for permid in peer_list: + permid_str = bin2str(permid) + p = self._db.fetchone(sql, (permid_str,)) + all.append(p) + + peers = [] + for i in range(len(all)): + p = all[i] + peer = dict(zip(keys,p)) + peer['permid'] = peer_list[i] + peers.append(peer) + + return peers + + def getLocalPeerList(self, max_peers,minoversion=None): # return a list of peer_ids + """Return a list of peerids for local nodes, friends first, then random local nodes""" + + sql = 'select permid from Peer where is_local=1 ' + if minoversion is not None: + sql += 'and oversion >= '+str(minoversion)+' ' + sql += 'ORDER BY friend DESC, random() limit %d'%max_peers + list = [] + for row in self._db.fetchall(sql): + list.append(base64.b64decode(row[0])) + return list + + + def addPeer(self, permid, value, update_dns=True, update_connected=False, commit=True): + # add or update a peer + # ARNO: AAARGGH a method that silently changes the passed value param!!! + # Jie: deepcopy(value)? + + _permid = _last_seen = _ip = _port = None + if 'permid' in value: + _permid = value.pop('permid') + + if not update_dns: + if value.has_key('ip'): + _ip = value.pop('ip') + if value.has_key('port'): + _port = value.pop('port') + + if update_connected: + old_connected = self.getOne('connected_times', permid=bin2str(permid)) + if not old_connected: + value['connected_times'] = 1 + else: + value['connected_times'] = old_connected + 1 + + + peer_existed = self._db.insertPeer(permid, commit=commit, **value) + + if _permid is not None: + value['permid'] = permid + if _last_seen is not None: + value['last_seen'] = _last_seen + if _ip is not None: + value['ip'] = _ip + if _port is not None: + value['port'] = _port + + if peer_existed: + self.notifier.notify(NTFY_PEERS, NTFY_UPDATE, permid) + # Jie: only notify the GUI when a peer was connected + if 'connected_times' in value: + self.notifier.notify(NTFY_PEERS, NTFY_INSERT, permid) + + #print >>sys.stderr,"sqldbhand: addPeer",`permid`,self._db.getPeerID(permid),`value` + #print_stack() + + + def hasPeer(self, permid): + return self._db.hasPeer(permid) + + + def findPeers(self, key, value): + # only used by Connecter + if key == 'permid': + value = bin2str(value) + res = self.getAll('permid', **{key:value}) + if not res: + return [] + ret = [] + for p in res: + ret.append({'permid':str2bin(p[0])}) + return ret + + def setPeerLocalFlag(self, permid, is_local, commit=True): + # argv = {"is_local":int(is_local)} + # updated = self._db.update(self.table_name, 'permid='+repr(bin2str(permid)), **argv) + # if commit: + # self.commit() + # return updated + self._db.update(self.table_name, 'permid='+repr(bin2str(permid)), commit=commit, is_local=int(is_local)) + + def updatePeer(self, permid, commit=True, **argv): + self._db.update(self.table_name, 'permid='+repr(bin2str(permid)), commit=commit, **argv) + self.notifier.notify(NTFY_PEERS, NTFY_UPDATE, permid) + + #print >>sys.stderr,"sqldbhand: updatePeer",`permid`,argv + #print_stack() + + def deletePeer(self, permid=None, peer_id=None, force=False, commit=True): + # don't delete friend of superpeers, except that force is True + # to do: add transaction + #self._db._begin() # begin a transaction + if peer_id is None: + peer_id = self._db.getPeerID(permid) + if peer_id is None: + return + deleted = self._db.deletePeer(permid=permid, peer_id=peer_id, force=force, commit=commit) + if deleted: + self.pref_db._deletePeer(peer_id=peer_id, commit=commit) + self.notifier.notify(NTFY_PEERS, NTFY_DELETE, permid) + + def updateTimes(self, permid, key, change=1, commit=True): + permid_str = bin2str(permid) + sql = "SELECT peer_id,%s FROM Peer WHERE permid==?"%key + find = self._db.fetchone(sql, (permid_str,)) + if find: + peer_id,value = find + if value is None: + value = 1 + else: + value += change + sql_update_peer = "UPDATE Peer SET %s=? WHERE peer_id=?"%key + self._db.execute_write(sql_update_peer, (value, peer_id), commit=commit) + self.notifier.notify(NTFY_PEERS, NTFY_UPDATE, permid) + + def updatePeerSims(self, sim_list, commit=True): + sql_update_sims = 'UPDATE Peer SET similarity=? WHERE peer_id=?' + s = time() + self._db.executemany(sql_update_sims, sim_list, commit=commit) + + def getPermIDByIP(self,ip): + permid = self.getOne('permid', ip=ip) + if permid is not None: + return str2bin(permid) + else: + return None + + def getPermid(self, peer_id): + permid = self.getOne('permid', peer_id=peer_id) + if permid is not None: + return str2bin(permid) + else: + return None + + def getNumberPeers(self, category_name = 'all'): + # 28/07/08 boudewijn: counting the union from two seperate + # select statements is faster than using a single select + # statement with an OR in the WHERE clause. Note that UNION + # returns a distinct list of peer_id's. + if category_name == 'friend': + sql = 'SELECT COUNT(peer_id) FROM Peer WHERE last_connected > 0 AND friend = 1' + else: + sql = 'SELECT COUNT(peer_id) FROM (SELECT peer_id FROM Peer WHERE last_connected > 0 UNION SELECT peer_id FROM Peer WHERE friend = 1)' + res = self._db.fetchone(sql) + if not res: + res = 0 + return res + + def getGUIPeers(self, category_name = 'all', range = None, sort = None, reverse = False, get_online=False, get_ranks=True): + # + # ARNO: WHY DIFF WITH NORMAL getPeers?????? + # load peers for GUI + #print >> sys.stderr, 'getGUIPeers(%s, %s, %s, %s)' % (category_name, range, sort, reverse) + """ + db keys: peer_id, permid, name, ip, port, thumbnail, oversion, + similarity, friend, superpeer, last_seen, last_connected, + last_buddycast, connected_times, buddycast_times, num_peers, + num_torrents, num_prefs, num_queries, is_local, + + @in: get_online: boolean: if true, give peers a key 'online' if there is a connection now + """ + value_name = PeerDBHandler.gui_value_name + + where = '(last_connected>0 or friend=1 or friend=2 or friend=3) ' + if category_name in ('friend', 'friends'): + # Show mutual, I invited and he invited + where += 'and (friend=1 or friend=2 or friend=3) ' + if range: + offset= range[0] + limit = range[1] - range[0] + else: + limit = offset = None + if sort: + # Arno, 2008-10-6: buggy: not reverse??? + desc = (reverse) and 'desc' or '' + if sort in ('name'): + order_by = ' lower(%s) %s' % (sort, desc) + else: + order_by = ' %s %s' % (sort, desc) + else: + order_by = None + + # Must come before query + if get_ranks: + ranks = self.getRanks() + # Arno, 2008-10-23: Someone disabled ranking of people, why? + + res_list = self.getAll(value_name, where, offset= offset, limit=limit, order_by=order_by) + + #print >>sys.stderr,"getGUIPeers: where",where,"offset",offset,"limit",limit,"order",order_by + #print >>sys.stderr,"getGUIPeers: returned len",len(res_list) + + peer_list = [] + for item in res_list: + peer = dict(zip(value_name, item)) + peer['name'] = dunno2unicode(peer['name']) + peer['simRank'] = ranksfind(ranks,peer['permid']) + peer['permid'] = str2bin(peer['permid']) + peer_list.append(peer) + + if get_online: + # Arno, 2010-01-28: Disabled this. Maybe something wrong with setOnline + # observer. + #self.checkOnline(peer_list) + raise ValueError("getGUIPeers get_online parameter currently disabled") + + + # peer_list consumes about 1.5M for 1400 peers, and this function costs about 0.015 second + + return peer_list + + + def getRanks(self): + value_name = 'permid' + order_by = 'similarity desc' + rankList_size = 20 + where = '(last_connected>0 or friend=1) ' + res_list = self._db.getAll('Peer', value_name, where=where, limit=rankList_size, order_by=order_by) + return [a[0] for a in res_list] + + def checkOnline(self, peerlist): + # Add 'online' key in peers when their permid + # Called by any thread, accesses single online_peers-dict + # Peers will never be sorted by 'online' because it is not in the db. + # Do not sort here, because then it would be sorted with a partial select (1 page in the grid) + self.lock.acquire() + for peer in peerlist: + peer['online'] = (peer['permid'] in self.online_peers) + self.lock.release() + + + + def setOnline(self,subject,changeType,permid,*args): + """Called by callback threads + with NTFY_CONNECTION, args[0] is boolean: connection opened/closed + """ + self.lock.acquire() + if args[0]: # connection made + self.online_peers.add(permid) + else: # connection closed + self.online_peers.remove(permid) + self.lock.release() + #print >> sys.stderr, (('#'*50)+'\n')*5+'%d peers online' % len(self.online_peers) + + def registerConnectionUpdater(self, session): + # Arno, 2010-01-28: Disabled this. Maybe something wrong with setOnline + # observer. ThreadPool may somehow not be executing the calls to setOnline + # session.add_observer(self.setOnline, NTFY_PEERS, [NTFY_CONNECTION], None) + pass + + def updatePeerIcon(self, permid, icontype, icondata, commit = True): + # save thumb in db + self.updatePeer(permid, thumbnail=bin2str(icondata)) + #if self.mm is not None: + # self.mm.save_data(permid, icontype, icondata) + + + def getPeerIcon(self, permid): + item = self.getOne('thumbnail', permid=bin2str(permid)) + if item: + return NETW_MIME_TYPE, str2bin(item) + else: + return None, None + #if self.mm is not None: + # return self.mm.load_data(permid) + #3else: + # return None + + + def searchNames(self,kws): + return doPeerSearchNames(self,'Peer',kws) + + + +class SuperPeerDBHandler(BasicDBHandler): + + __single = None # used for multithreaded singletons pattern + lock = threading.Lock() + + def getInstance(*args, **kw): + # Singleton pattern with double-checking + if SuperPeerDBHandler.__single is None: + SuperPeerDBHandler.lock.acquire() + try: + if SuperPeerDBHandler.__single is None: + SuperPeerDBHandler(*args, **kw) + finally: + SuperPeerDBHandler.lock.release() + return SuperPeerDBHandler.__single + + getInstance = staticmethod(getInstance) + + def __init__(self): + if SuperPeerDBHandler.__single is not None: + raise RuntimeError, "SuperPeerDBHandler is singleton" + SuperPeerDBHandler.__single = self + db = SQLiteCacheDB.getInstance() + BasicDBHandler.__init__(self, db, 'SuperPeer') + self.peer_db_handler = PeerDBHandler.getInstance() + + def loadSuperPeers(self, config, refresh=False): + filename = os.path.join(config['install_dir'], config['superpeer_file']) + superpeer_list = self.readSuperPeerList(filename) + self.insertSuperPeers(superpeer_list, refresh) + + def readSuperPeerList(self, filename=u''): + """ read (superpeer_ip, superpeer_port, permid [, name]) lines from a text file """ + + try: + filepath = os.path.abspath(filename) + file = open(filepath, "r") + except IOError: + print >> sys.stderr, "superpeer: cannot open superpeer file", filepath + return [] + + superpeers = file.readlines() + file.close() + superpeers_info = [] + for superpeer in superpeers: + if superpeer.strip().startswith("#"): # skip commended lines + continue + superpeer_line = superpeer.split(',') + superpeer_info = [a.strip() for a in superpeer_line] + try: + superpeer_info[2] = base64.decodestring(superpeer_info[2]+'\n' ) + except: + print_exc() + continue + try: + ip = socket.gethostbyname(superpeer_info[0]) + superpeer = {'ip':ip, 'port':superpeer_info[1], + 'permid':superpeer_info[2], 'superpeer':1} + if len(superpeer_info) > 3: + superpeer['name'] = superpeer_info[3] + superpeers_info.append(superpeer) + except: + print_exc() + + return superpeers_info + + def insertSuperPeers(self, superpeer_list, refresh=False): + for superpeer in superpeer_list: + superpeer = deepcopy(superpeer) + if not isinstance(superpeer, dict) or 'permid' not in superpeer: + continue + permid = superpeer.pop('permid') + self.peer_db_handler.addPeer(permid, superpeer, commit=False) + self.peer_db_handler.commit() + + def getSuperPeers(self): + # return list with permids of superpeers + res_list = self._db.getAll(self.table_name, 'permid') + return [str2bin(a[0]) for a in res_list] + + def addExternalSuperPeer(self, peer): + _peer = deepcopy(peer) + permid = _peer.pop('permid') + _peer['superpeer'] = 1 + self._db.insertPeer(permid, **_peer) + + +class CrawlerDBHandler: + """ + The CrawlerDBHandler is not an actual handle to a + database. Instead it uses a local file (usually crawler.txt) to + identify crawler processes. + """ + + __single = None # used for multithreaded singletons pattern + lock = threading.Lock() + + def getInstance(*args, **kw): + # Singleton pattern with double-checking + if CrawlerDBHandler.__single is None: + CrawlerDBHandler.lock.acquire() + try: + if CrawlerDBHandler.__single is None: + CrawlerDBHandler(*args, **kw) + finally: + CrawlerDBHandler.lock.release() + return CrawlerDBHandler.__single + + getInstance = staticmethod(getInstance) + + def __init__(self): + if CrawlerDBHandler.__single is not None: + raise RuntimeError, "CrawlerDBHandler is singleton" + CrawlerDBHandler.__single = self + self._crawler_list = [] + + def loadCrawlers(self, config, refresh=False): + filename = os.path.join(config['crawler_file']) + self._crawler_list = self.readCrawlerList(filename) + + def readCrawlerList(self, filename=''): + """ + read (permid [, name]) lines from a text file + returns a list containing permids + """ + + try: + filepath = os.path.abspath(filename) + file = open(filepath, "r") + except IOError: + print >> sys.stderr, "crawler: cannot open crawler file", filepath + return [] + + crawlers = file.readlines() + file.close() + crawlers_info = [] + for crawler in crawlers: + if crawler.strip().startswith("#"): # skip commended lines + continue + crawler_info = [a.strip() for a in crawler.split(",")] + try: + crawler_info[0] = base64.decodestring(crawler_info[0]+'\n') + except: + print_exc() + continue + crawlers_info.append(str2bin(crawler)) + + return crawlers_info + + def temporarilyAddCrawler(self, permid): + """ + Because of security reasons we will not allow crawlers to be + added to the crawler.txt list. This temporarilyAddCrawler + method can be used to add one for the running session. Usefull + for debugging and testing. + """ + if not permid in self._crawler_list: + self._crawler_list.append(permid) + + def getCrawlers(self): + """ + returns a list with permids of crawlers + """ + return self._crawler_list + + + +class PreferenceDBHandler(BasicDBHandler): + + __single = None # used for multithreaded singletons pattern + lock = threading.Lock() + + def getInstance(*args, **kw): + # Singleton pattern with double-checking + if PreferenceDBHandler.__single is None: + PreferenceDBHandler.lock.acquire() + try: + if PreferenceDBHandler.__single is None: + PreferenceDBHandler(*args, **kw) + finally: + PreferenceDBHandler.lock.release() + return PreferenceDBHandler.__single + + getInstance = staticmethod(getInstance) + + def __init__(self): + if PreferenceDBHandler.__single is not None: + raise RuntimeError, "PreferenceDBHandler is singleton" + PreferenceDBHandler.__single = self + db = SQLiteCacheDB.getInstance() + BasicDBHandler.__init__(self,db, 'Preference') ## self,db,'Preference' + + self.popularity_db = PopularityDBHandler.getInstance() + + + def _getTorrentOwnersID(self, torrent_id): + sql_get_torrent_owners_id = u"SELECT peer_id FROM Preference WHERE torrent_id==?" + res = self._db.fetchall(sql_get_torrent_owners_id, (torrent_id,)) + return [t[0] for t in res] + + def getPrefList(self, permid, return_infohash=False): + # get a peer's preference list of infohash or torrent_id according to return_infohash + peer_id = self._db.getPeerID(permid) + if peer_id is None: + return [] + + if not return_infohash: + sql_get_peer_prefs_id = u"SELECT torrent_id FROM Preference WHERE peer_id==?" + res = self._db.fetchall(sql_get_peer_prefs_id, (peer_id,)) + return [t[0] for t in res] + else: + sql_get_infohash = u"SELECT infohash FROM Torrent WHERE torrent_id IN (SELECT torrent_id FROM Preference WHERE peer_id==?)" + res = self._db.fetchall(sql_get_infohash, (peer_id,)) + return [str2bin(t[0]) for t in res] + + def _deletePeer(self, permid=None, peer_id=None, commit=True): # delete a peer from pref_db + # should only be called by PeerDBHandler + if peer_id is None: + peer_id = self._db.getPeerID(permid) + if peer_id is None: + return + + self._db.delete(self.table_name, commit=commit, peer_id=peer_id) + + def addPreference(self, permid, infohash, data={}, commit=True): + assert isinstance(infohash, str), "INFOHASH has invalid type: %s" % type(infohash) + assert len(infohash) == INFOHASH_LENGTH, "INFOHASH has invalid length: %d" % len(infohash) + # This function should be replaced by addPeerPreferences + # peer_permid and prefs are binaries, the peer must have been inserted in Peer table + # Nicolas: did not change this function as it seems addPreference*s* is getting called + peer_id = self._db.getPeerID(permid) + if peer_id is None: + print >> sys.stderr, 'PreferenceDBHandler: add preference of a peer which is not existed in Peer table', `permid` + return + + sql_insert_peer_torrent = u"INSERT INTO Preference (peer_id, torrent_id) VALUES (?,?)" + torrent_id = self._db.getTorrentID(infohash) + if not torrent_id: + self._db.insertInfohash(infohash) + torrent_id = self._db.getTorrentID(infohash) + try: + self._db.execute_write(sql_insert_peer_torrent, (peer_id, torrent_id), commit=commit) + except Exception, msg: # duplicated + print_exc() + + + + def addPreferences(self, peer_permid, prefs, recvTime=0.0, is_torrent_id=False, commit=True): + # peer_permid and prefs are binaries, the peer must have been inserted in Peer table + # boudewijn: for buddycast version >= OLPROTO_VER_EIGTH the + # prefs list may contain both strings (indicating an infohash) + # or dictionaries (indicating an infohash with metadata) + peer_id = self._db.getPeerID(peer_permid) + if peer_id is None: + print >> sys.stderr, 'PreferenceDBHandler: add preference of a peer which is not existed in Peer table', `peer_permid` + return + + prefs = [type(pref) is str and {"infohash":pref} or pref + for pref + in prefs] + + if __debug__: + for pref in prefs: + assert isinstance(pref["infohash"], str), "INFOHASH has invalid type: %s" % type(pref["infohash"]) + assert len(pref["infohash"]) == INFOHASH_LENGTH, "INFOHASH has invalid length: %d" % len(pref["infohash"]) + + torrent_id_swarm_size =[] + torrent_id_prefs =[] + if is_torrent_id: + for pref in prefs: + torrent_id_prefs.append((peer_id, + pref['torrent_id'], + pref.get('position', -1), + pref.get('reranking_strategy', -1)) + ) + #Rahim : Since overlay version 11 swarm size information is + # appended and should be added to the database . The code below + # does this. torrent_id, recv_time, calc_age, num_seeders, + # num_leechers, num_sources + # + #torrent_id_swarm_size =[] + if pref.get('calc_age') is not None: + tempAge= pref.get('calc_age') + tempSeeders = pref.get('num_seeders') + tempLeechers = pref.get('num_leechers') + if tempAge > 0 and tempSeeders >= 0 and tempLeechers >= 0: + torrent_id_swarm_size.append([pref['torrent_id'], + recvTime, + tempAge, + tempSeeders, + tempLeechers, + pref.get('num_sources_seen', -1)])# -1 means invalud value + else: + # Nicolas: do not know why this would be called, but let's handle + # it smoothly + torrent_id_prefs = [] + #Rahim: I also don't know when this part is run, I just follow the + # way that Nicolas has done. + #torrent_id_swarm_size = [] + for pref in prefs: + infohash = pref["infohash"] + torrent_id = self._db.getTorrentID(infohash) + if not torrent_id: + self._db.insertInfohash(infohash) + torrent_id = self._db.getTorrentID(infohash) + torrent_id_prefs.append((peer_id, torrent_id, -1, -1)) + #Rahim: Amended for handling and adding swarm size info. + #torrent_id_swarm_size.append((torrent_id, recvTime,0, -1, -1, -1)) + + + sql_insert_peer_torrent = u"INSERT INTO Preference (peer_id, torrent_id, click_position, reranking_strategy) VALUES (?,?,?,?)" + if len(prefs) > 0: + try: + self._db.executemany(sql_insert_peer_torrent, torrent_id_prefs, commit=commit) + popularity_db = PopularityDBHandler.getInstance() + if len(torrent_id_swarm_size) > 0: + popularity_db.storePeerPopularity(peer_id, torrent_id_swarm_size, commit=commit) + except Exception, msg: # duplicated + print_exc() + print >> sys.stderr, 'dbhandler: addPreferences:', Exception, msg + + # now, store search terms + + # Nicolas: if maximum number of search terms is exceeded, abort storing them. + # Although this may seem a bit strict, this means that something different than a genuine Tribler client + # is on the other side, so we might rather err on the side of caution here and simply let clicklog go. + nums_of_search_terms = [len(pref.get('search_terms',[])) for pref in prefs] + if max(nums_of_search_terms)>MAX_KEYWORDS_STORED: + if DEBUG: + print >>sys.stderr, "peer %d exceeds max number %d of keywords per torrent, aborting storing keywords" % \ + (peer_id, MAX_KEYWORDS_STORED) + return + + all_terms_unclean = Set([]) + for pref in prefs: + newterms = Set(pref.get('search_terms',[])) + all_terms_unclean = all_terms_unclean.union(newterms) + + all_terms = [] + for term in all_terms_unclean: + cleanterm = '' + for i in range(0,len(term)): + c = term[i] + if c.isalnum(): + cleanterm += c + if len(cleanterm)>0: + all_terms.append(cleanterm) + # maybe we haven't received a single key word, no need to loop again over prefs then + if len(all_terms)==0: + return + + termdb = TermDBHandler.getInstance() + searchdb = SearchDBHandler.getInstance() + + # insert all unknown terms NOW so we can rebuild the index at once + termdb.bulkInsertTerms(all_terms) + + # get local term ids for terms. + foreign2local = dict([(str(foreign_term), termdb.getTermID(foreign_term)) + for foreign_term + in all_terms]) + + # process torrent data + for pref in prefs: + torrent_id = pref.get('torrent_id', None) + search_terms = pref.get('search_terms', []) + + if search_terms==[]: + continue + if not torrent_id: + if DEBUG: + print >> sys.stderr, "torrent_id not set, retrieving manually!" + torrent_id = TorrentDBHandler.getInstance().getTorrentID(infohash) + + term_ids = [foreign2local[str(foreign)] for foreign in search_terms if str(foreign) in foreign2local] + searchdb.storeKeywordsByID(peer_id, torrent_id, term_ids, commit=False) + if commit: + searchdb.commit() + + def getAllEntries(self): + """use with caution,- for testing purposes""" + return self.getAll("rowid, peer_id, torrent_id, click_position,reranking_strategy", order_by="peer_id, torrent_id") + + + def getRecentPeersPrefs(self, key, num=None): + # get the recently seen peers' preference. used by buddycast + sql = "select peer_id,torrent_id from Preference where peer_id in (select peer_id from Peer order by %s desc)"%key + if num is not None: + sql = sql[:-1] + " limit %d)"%num + res = self._db.fetchall(sql) + return res + + def getPositionScore(self, torrent_id, keywords): + """returns a tuple (num, positionScore) stating how many times the torrent id was found in preferences, + and the average position score, where each click at position i receives 1-(1/i) points""" + + if not keywords: + return (0,0) + + term_db = TermDBHandler.getInstance() + term_ids = [term_db.getTermID(keyword) for keyword in keywords] + s_term_ids = str(term_ids).replace("[","(").replace("]",")").replace("L","") + + # we're not really interested in the peer_id here, + # just make sure we don't count twice if we hit more than one keyword in a search + # ... one might treat keywords a bit more strictly here anyway (AND instead of OR) + sql = """ +SELECT DISTINCT Preference.peer_id, Preference.click_position +FROM Preference +INNER JOIN ClicklogSearch +ON + Preference.torrent_id = ClicklogSearch.torrent_id + AND + Preference.peer_id = ClicklogSearch.peer_id +WHERE + ClicklogSearch.term_id IN %s + AND + ClicklogSearch.torrent_id = %s""" % (s_term_ids, torrent_id) + res = self._db.fetchall(sql) + scores = [1.0-1.0/float(click_position+1) + for (peer_id, click_position) + in res + if click_position>-1] + if len(scores)==0: + return (0,0) + score = float(sum(scores))/len(scores) + return (len(scores), score) + + +class TorrentDBHandler(BasicDBHandler): + + __single = None # used for multithreaded singletons pattern + lock = threading.Lock() + + def getInstance(*args, **kw): + # Singleton pattern with double-checking + if TorrentDBHandler.__single is None: + TorrentDBHandler.lock.acquire() + try: + if TorrentDBHandler.__single is None: + TorrentDBHandler(*args, **kw) + finally: + TorrentDBHandler.lock.release() + return TorrentDBHandler.__single + + getInstance = staticmethod(getInstance) + + def __init__(self): + if TorrentDBHandler.__single is not None: + raise RuntimeError, "TorrentDBHandler is singleton" + TorrentDBHandler.__single = self + db = SQLiteCacheDB.getInstance() + BasicDBHandler.__init__(self,db, 'Torrent') ## self,db,torrent + + self.mypref_db = MyPreferenceDBHandler.getInstance() + + self.status_table = {'good':1, 'unknown':0, 'dead':2} + self.status_table.update(self._db.getTorrentStatusTable()) + self.id2status = dict([(x,y) for (y,x) in self.status_table.items()]) + self.torrent_dir = None + # 0 - unknown + # 1 - good + # 2 - dead + + self.category_table = {'Video':1, + 'VideoClips':2, + 'Audio':3, + 'Compressed':4, + 'Document':5, + 'Picture':6, + 'xxx':7, + 'other':8,} + self.category_table.update(self._db.getTorrentCategoryTable()) + self.category_table['unknown'] = 0 + self.id2category = dict([(x,y) for (y,x) in self.category_table.items()]) + # 1 - Video + # 2 - VideoClips + # 3 - Audio + # 4 - Compressed + # 5 - Document + # 6 - Picture + # 7 - xxx + # 8 - other + + self.src_table = self._db.getTorrentSourceTable() + self.id2src = dict([(x,y) for (y,x) in self.src_table.items()]) + # 0 - '' # local added + # 1 - BC + # 2,3,4... - URL of RSS feed + self.keys = ['torrent_id', 'name', 'torrent_file_name', + 'length', 'creation_date', 'num_files', 'thumbnail', + 'insert_time', 'secret', 'relevance', + 'source_id', 'category_id', 'status_id', + 'num_seeders', 'num_leechers', 'comment'] + self.existed_torrents = Set() + + + self.value_name = ['C.torrent_id', 'category_id', 'status_id', 'name', 'creation_date', 'num_files', + 'num_leechers', 'num_seeders', 'length', + 'secret', 'insert_time', 'source_id', 'torrent_file_name', + 'relevance', 'infohash', 'tracker', 'last_check'] + + self.value_name_for_channel = ['C.torrent_id', 'infohash', 'name', 'torrent_file_name', 'length', 'creation_date', 'num_files', 'thumbnail', 'insert_time', 'secret', 'relevance', 'source_id', 'category_id', 'status_id', 'num_seeders', 'num_leechers', 'comment'] + + + def register(self, category, torrent_dir): + self.category = category + self.torrent_dir = torrent_dir + + def getTorrentID(self, infohash): + assert isinstance(infohash, str), "INFOHASH has invalid type: %s" % type(infohash) + assert len(infohash) == INFOHASH_LENGTH, "INFOHASH has invalid length: %d" % len(infohash) + return self._db.getTorrentID(infohash) + + def getInfohash(self, torrent_id): + return self._db.getInfohash(torrent_id) + + def hasTorrent(self, infohash): + assert isinstance(infohash, str), "INFOHASH has invalid type: %s" % type(infohash) + assert len(infohash) == INFOHASH_LENGTH, "INFOHASH has invalid length: %d" % len(infohash) + if infohash in self.existed_torrents: #to do: not thread safe + return True + infohash_str = bin2str(infohash) + existed = self._db.getOne('CollectedTorrent', 'torrent_id', infohash=infohash_str) + if existed is None: + return False + else: + self.existed_torrents.add(infohash) + return True + + def addExternalTorrent(self, torrentdef, source="BC", extra_info={}, commit=True): + assert isinstance(torrentdef, TorrentDef), "TORRENTDEF has invalid type: %s" % type(torrentdef) + assert torrentdef.is_finalized(), "TORRENTDEF is not finalized" + if torrentdef.is_finalized(): + infohash = torrentdef.get_infohash() + if not self.hasTorrent(infohash): + self._addTorrentToDB(torrentdef, source, extra_info, commit) + self.notifier.notify(NTFY_TORRENTS, NTFY_INSERT, infohash) + + def addInfohash(self, infohash, commit=True): + assert isinstance(infohash, str), "INFOHASH has invalid type: %s" % type(infohash) + assert len(infohash) == INFOHASH_LENGTH, "INFOHASH has invalid length: %d" % len(infohash) + if self._db.getTorrentID(infohash) is None: + self._db.insert('Torrent', commit=commit, infohash=bin2str(infohash)) + + def _getStatusID(self, status): + return self.status_table.get(status.lower(), 0) + + def _getCategoryID(self, category_list): + if len(category_list) > 0: + category = category_list[0].lower() + cat_int = self.category_table[category] + else: + cat_int = 0 + return cat_int + + def _getSourceID(self, src): + if src in self.src_table: + src_int = self.src_table[src] + else: + src_int = self._insertNewSrc(src) # add a new src, e.g., a RSS feed + self.src_table[src] = src_int + self.id2src[src_int] = src + return src_int + + def _get_database_dict(self, torrentdef, source="BC", extra_info={}): + assert isinstance(torrentdef, TorrentDef), "TORRENTDEF has invalid type: %s" % type(torrentdef) + assert torrentdef.is_finalized(), "TORRENTDEF is not finalized" + mime, thumb = torrentdef.get_thumbnail() + + return {"infohash":bin2str(torrentdef.get_infohash()), + "name":torrentdef.get_name_as_unicode(), + "torrent_file_name":extra_info.get("filename", None), + "length":torrentdef.get_length(), + "creation_date":torrentdef.get_creation_date(), + "num_files":len(torrentdef.get_files()), + "thumbnail":bool(thumb), + "insert_time":long(time()), + "secret":0, # todo: check if torrent is secret + "relevance":0.0, + "source_id":self._getSourceID(source), + # todo: the category_id is calculated directly from + # torrentdef.metainfo, the category checker should use + # the proper torrentdef api + "category_id":self._getCategoryID(self.category.calculateCategory(torrentdef.metainfo, torrentdef.get_name_as_unicode())), + "status_id":self._getStatusID(extra_info.get("status", "unknown")), + "num_seeders":extra_info.get("seeder", -1), + "num_leechers":extra_info.get("leecher", -1), + "comment":torrentdef.get_comment_as_unicode()} + + def _addTorrentToDB(self, torrentdef, source, extra_info, commit): + assert isinstance(torrentdef, TorrentDef), "TORRENTDEF has invalid type: %s" % type(torrentdef) + assert torrentdef.is_finalized(), "TORRENTDEF is not finalized" + + # ARNO: protect against injection attacks + # 27/01/10 Boudewijn: all inserts are done using '?' in the + # sql query. The sqlite database will ensure that the values + # are properly escaped. + + infohash = torrentdef.get_infohash() + torrent_name = torrentdef.get_name_as_unicode() + database_dict = self._get_database_dict(torrentdef, source, extra_info) + + # see if there is already a torrent in the database with this + # infohash + torrent_id = self._db.getTorrentID(infohash) + + if torrent_id is None: # not in database + self._db.insert("Torrent", commit=True, **database_dict) + torrent_id = self._db.getTorrentID(infohash) + + else: # infohash in db + where = 'torrent_id = %d' % torrent_id + self._db.update('Torrent', where=where, commit=False, **database_dict) + + # boudewijn: we are using a Set to ensure that all keywords + # are unique. no use having the database layer figuring this + # out when we can do it now, in memory + keywords = Set(split_into_keywords(torrent_name)) + + # search through the .torrent file for potential keywords in + # the filenames + for filename in torrentdef.get_files_as_unicode(): + keywords.update(split_into_keywords(filename)) + + # store the keywords in the InvertedIndex table in the database + if len(keywords) > 0: + values = [(keyword, torrent_id) for keyword in keywords] + self._db.executemany(u"INSERT OR REPLACE INTO InvertedIndex VALUES(?, ?)", values, commit=False) + if DEBUG: + print >> sys.stderr, "torrentdb: Extending the InvertedIndex table with", len(values), "new keywords for", torrent_name + + self._addTorrentTracker(torrent_id, torrentdef, extra_info, commit=False) + if commit: + self.commit() + return torrent_id + + def getInfohashFromTorrentName(self, name): ## + sql = "select infohash from Torrent where name='" + str2bin(name) + "'" + infohash = self._db.fetchone(sql) + return infohash + + def _insertNewSrc(self, src, commit=True): + desc = '' + if src.startswith('http') and src.endswith('xml'): + desc = 'RSS' + self._db.insert('TorrentSource', commit=commit, name=src, description=desc) + src_id = self._db.getOne('TorrentSource', 'source_id', name=src) + return src_id + + def _addTorrentTracker(self, torrent_id, torrentdef, extra_info={}, add_all=False, commit=True): + # Set add_all to True if you want to put all multi-trackers into db. + # In the current version (4.2) only the main tracker is used. + exist = self._db.getOne('TorrentTracker', 'tracker', torrent_id=torrent_id) + if exist: + return + + # announce = data['announce'] + # ignore_number = data['ignore_number'] + # retry_number = data['retry_number'] + # last_check_time = data['last_check_time'] + # announce_list = data['announce-list'] + + announce = torrentdef.get_tracker() + announce_list = torrentdef.get_tracker_hierarchy() + ignore_number = 0 + retry_number = 0 + last_check_time = 0 + if "last_check_time" in extra_info: + last_check_time = int(time() - extra_info["last_check_time"]) + + sql_insert_torrent_tracker = """ + INSERT INTO TorrentTracker + (torrent_id, tracker, announce_tier, + ignored_times, retried_times, last_check) + VALUES (?,?,?, ?,?,?) + """ + + values = [(torrent_id, announce, 1, ignore_number, retry_number, last_check_time)] + # each torrent only has one announce with tier number 1 + tier_num = 2 + trackers = {announce:None} + if add_all: + for tier in announce_list: + for tracker in tier: + if tracker in trackers: + continue + value = (torrent_id, tracker, tier_num, 0, 0, 0) + values.append(value) + trackers[tracker] = None + tier_num += 1 + + self._db.executemany(sql_insert_torrent_tracker, values, commit=commit) + + def updateTorrent(self, infohash, commit=True, **kw): # watch the schema of database + if 'category' in kw: + cat_id = self._getCategoryID(kw.pop('category')) + kw['category_id'] = cat_id + if 'status' in kw: + status_id = self._getStatusID(kw.pop('status')) + kw['status_id'] = status_id + if 'progress' in kw: + self.mypref_db.updateProgress(infohash, kw.pop('progress'), commit=False)# commit at end of function + if 'seeder' in kw: + kw['num_seeders'] = kw.pop('seeder') + if 'leecher' in kw: + kw['num_leechers'] = kw.pop('leecher') + if 'last_check_time' in kw or 'ignore_number' in kw or 'retry_number' in kw \ + or 'retried_times' in kw or 'ignored_times' in kw: + self.updateTracker(infohash, kw, commit=False) + + for key in kw.keys(): + if key not in self.keys: + kw.pop(key) + + if len(kw) > 0: + infohash_str = bin2str(infohash) + where = "infohash='%s'"%infohash_str + self._db.update(self.table_name, where, commit=False, **kw) + + if commit: + self.commit() + # to.do: update the torrent panel's number of seeders/leechers + self.notifier.notify(NTFY_TORRENTS, NTFY_UPDATE, infohash) + + def updateTracker(self, infohash, kw, tier=1, tracker=None, commit=True): + torrent_id = self._db.getTorrentID(infohash) + if torrent_id is None: + return + update = {} + assert type(kw) == dict and kw, 'updateTracker error: kw should be filled dict, but is: %s' % kw + if 'last_check_time' in kw: + update['last_check'] = kw.pop('last_check_time') + if 'ignore_number' in kw: + update['ignored_times'] = kw.pop('ignore_number') + if 'ignored_times' in kw: + update['ignored_times'] = kw.pop('ignored_times') + if 'retry_number' in kw: + update['retried_times'] = kw.pop('retry_number') + if 'retried_times' in kw: + update['retried_times'] = kw.pop('retried_times') + + if tracker is None: + where = 'torrent_id=%d AND announce_tier=%d'%(torrent_id, tier) + else: + where = 'torrent_id=%d AND tracker=%s'%(torrent_id, repr(tracker)) + self._db.update('TorrentTracker', where, commit=commit, **update) + + def deleteTorrent(self, infohash, delete_file=False, commit = True): + if not self.hasTorrent(infohash): + return False + + if self.mypref_db.hasMyPreference(infohash): # don't remove torrents in my pref + return False + + if delete_file: + deleted = self.eraseTorrentFile(infohash) + else: + deleted = True + + if deleted: + self._deleteTorrent(infohash, commit=commit) + + self.notifier.notify(NTFY_TORRENTS, NTFY_DELETE, infohash) + return deleted + + def _deleteTorrent(self, infohash, keep_infohash=True, commit=True): + torrent_id = self._db.getTorrentID(infohash) + if torrent_id is not None: + if keep_infohash: + self._db.update(self.table_name, where="torrent_id=%d"%torrent_id, commit=commit, torrent_file_name=None) + else: + self._db.delete(self.table_name, commit=commit, torrent_id=torrent_id) + if infohash in self.existed_torrents: + self.existed_torrents.remove(infohash) + self._db.delete('TorrentTracker', commit=commit, torrent_id=torrent_id) + #print '******* delete torrent', torrent_id, `infohash`, self.hasTorrent(infohash) + + def eraseTorrentFile(self, infohash): + torrent_id = self._db.getTorrentID(infohash) + if torrent_id is not None: + torrent_dir = self.getTorrentDir() + torrent_name = self.getOne('torrent_file_name', torrent_id=torrent_id) + src = os.path.join(torrent_dir, torrent_name) + if not os.path.exists(src): # already removed + return True + + try: + os.remove(src) + except Exception, msg: + print >> sys.stderr, "cachedbhandler: failed to erase torrent", src, Exception, msg + return False + + return True + + def getTracker(self, infohash, tier=0): + torrent_id = self._db.getTorrentID(infohash) + if torrent_id is not None: + sql = "SELECT tracker, announce_tier FROM TorrentTracker WHERE torrent_id==%d"%torrent_id + if tier > 0: + sql += " AND announce_tier<=%d"%tier + return self._db.fetchall(sql) + + def getSwarmInfo(self, torrent_id): + """ + returns info about swarm size from Torrent and TorrentTracker tables. + @author: Rahim + @param torrentId: The index of the torrent. + @return: A tuple of the form:(torrent_id, num_seeders, num_leechers, num_sources_seen, last_check) + """ + if torrent_id is not None: + sql = """SELECT tr.torrent_id, tr.num_seeders, tr.num_leechers, tt.last_check + FROM TorrentTracker tt, Torrent tr WHERE tr.torrent_id=tt.torrent_id AND tr.torrent_id==%d"""%torrent_id + sql +=" order by tt.last_check DESC limit 1" + sizeInfo = self._db.fetchall(sql) + + if len(sizeInfo) == 1: + num_seeders = sizeInfo[0][1] + num_leechers = sizeInfo[0][2] + last_check = sizeInfo[0][3] + + sql1= """SELECT COUNT(*) FROM Preference WHERE torrent_id=%d"""%torrent_id + mySeenSources = self._db.fetchone(sql1) + + return [(torrent_id, num_seeders, num_leechers, last_check, mySeenSources, sizeInfo)] + + return [()] + + + def getLargestSourcesSeen(self, torrent_id, timeNow, freshness=-1): + """ + Returns the largest number of the sources that have seen the torrent. + @author: Rahim + @param torrent_id: the id of the torrent. + @param freshness: A parameter that filters old records. The assumption is that those popularity reports that are + older than a rate are not reliable + @return: The largest number of the torrents that have seen the torrent. + """ + + if freshness == -1: + sql2 = """SELECT MAX(num_of_sources) FROM Popularity WHERE torrent_id=%d"""%torrent_id + else: + latestValidTime = timeNow - freshness + sql2 = """SELECT MAX(num_of_sources) FROM Popularity WHERE torrent_id=%d AND msg_receive_time > %d"""%(torrent_id, latestValidTime) + + othersSeenSources = self._db.fetchone(sql2) + if othersSeenSources is None: + othersSeenSources =0 + return othersSeenSources + + def getTorrentDir(self): + return self.torrent_dir + + def getTorrent(self, infohash, keys=None, include_mypref=True): + assert isinstance(infohash, str), "INFOHASH has invalid type: %s" % type(infohash) + assert len(infohash) == INFOHASH_LENGTH, "INFOHASH has invalid length: %d" % len(infohash) + # to do: replace keys like source -> source_id and status-> status_id ?? + + if keys is None: + keys = deepcopy(self.value_name) + #('torrent_id', 'category_id', 'status_id', 'name', 'creation_date', 'num_files', + # 'num_leechers', 'num_seeders', 'length', + # 'secret', 'insert_time', 'source_id', 'torrent_file_name', + # 'relevance', 'infohash', 'torrent_id') + else: + keys = list(keys) + where = 'C.torrent_id = T.torrent_id and announce_tier=1 ' + + res = self._db.getOne('CollectedTorrent C, TorrentTracker T', keys, where=where, infohash=bin2str(infohash)) + if not res: + return None + torrent = dict(zip(keys, res)) + if 'source_id' in torrent: + torrent['source'] = self.id2src[torrent['source_id']] + del torrent['source_id'] + if 'category_id' in torrent: + torrent['category'] = [self.id2category[torrent['category_id']]] + del torrent['category_id'] + if 'status_id' in torrent: + torrent['status'] = self.id2status[torrent['status_id']] + del torrent['status_id'] + torrent['infohash'] = infohash + if 'last_check' in torrent: + torrent['last_check_time'] = torrent['last_check'] + del torrent['last_check'] + + if include_mypref: + tid = torrent['C.torrent_id'] + stats = self.mypref_db.getMyPrefStats(tid) + del torrent['C.torrent_id'] + if stats: + torrent['myDownloadHistory'] = True + torrent['creation_time'] = stats[tid][0] + torrent['progress'] = stats[tid][1] + torrent['destination_path'] = stats[tid][2] + + + return torrent + + def getNumberTorrents(self, category_name = 'all', library = False): + table = 'CollectedTorrent' + value = 'count(torrent_id)' + where = '1 ' + + if category_name != 'all': + where += ' and category_id= %d' % self.category_table.get(category_name.lower(), -1) # unkown category_name returns no torrents + if library: + where += ' and torrent_id in (select torrent_id from MyPreference where destination_path != "")' + else: + where += ' and status_id=%d ' % self.status_table['good'] + # add familyfilter + where += self.category.get_family_filter_sql(self._getCategoryID) + + number = self._db.getOne(table, value, where) + if not number: + number = 0 + return number + + def getTorrents(self, category_name = 'all', range = None, library = False, sort = None, reverse = False): + """ + get Torrents of some category and with alive status (opt. not in family filter) + + @return Returns a list of dicts with keys: + torrent_id, infohash, name, category, status, creation_date, num_files, num_leechers, num_seeders, + length, secret, insert_time, source, torrent_filename, relevance, simRank, tracker, last_check + (if in library: myDownloadHistory, download_started, progress, dest_dir) + + """ + + #print >> sys.stderr, 'TorrentDBHandler: getTorrents(%s, %s, %s, %s, %s)' % (category_name, range, library, sort, reverse) + s = time() + + value_name = deepcopy(self.value_name) + + where = 'T.torrent_id = C.torrent_id and announce_tier=1 ' + + if category_name != 'all': + where += ' and category_id= %d' % self.category_table.get(category_name.lower(), -1) # unkown category_name returns no torrents + if library: + if sort in value_name: + where += ' and C.torrent_id in (select torrent_id from MyPreference where destination_path != "")' + else: + value_name[0] = 'C.torrent_id' + where += ' and C.torrent_id = M.torrent_id and announce_tier=1' + else: + where += ' and status_id=%d ' % self.status_table['good'] # if not library, show only good files + # add familyfilter + where += self.category.get_family_filter_sql(self._getCategoryID) + if range: + offset= range[0] + limit = range[1] - range[0] + else: + limit = offset = None + if sort: + # Arno, 2008-10-6: buggy: not reverse??? + desc = (reverse) and 'desc' or '' + if sort in ('name'): + order_by = ' lower(%s) %s' % (sort, desc) + else: + order_by = ' %s %s' % (sort, desc) + else: + order_by = None + + #print >>sys.stderr,"TorrentDBHandler: GET TORRENTS val",value_name,"where",where,"limit",limit,"offset",offset,"order",order_by + #print_stack + + # Must come before query + ranks = self.getRanks() + + #self._db.show_execute = True + if library and sort not in value_name: + res_list = self._db.getAll('CollectedTorrent C, MyPreference M, TorrentTracker T', value_name, where, limit=limit, offset=offset, order_by=order_by) + else: + res_list = self._db.getAll('CollectedTorrent C, TorrentTracker T', value_name, where, limit=limit, offset=offset, order_by=order_by) + #self._db.show_execute = False + + mypref_stats = self.mypref_db.getMyPrefStats() + + #print >>sys.stderr,"TorrentDBHandler: getTorrents: getAll returned ###################",len(res_list) + + torrent_list = self.valuelist2torrentlist(value_name,res_list,ranks,mypref_stats) + del res_list + del mypref_stats + return torrent_list + + def valuelist2torrentlist(self,value_name,res_list,ranks,mypref_stats): + + torrent_list = [] + for item in res_list: + value_name[0] = 'torrent_id' + torrent = dict(zip(value_name, item)) + + try: + torrent['source'] = self.id2src[torrent['source_id']] + except: + print_exc() + # Arno: RSS subscription and id2src issue + torrent['source'] = 'http://some/RSS/feed' + + torrent['category'] = [self.id2category[torrent['category_id']]] + torrent['status'] = self.id2status[torrent['status_id']] + torrent['simRank'] = ranksfind(ranks,torrent['infohash']) + torrent['infohash'] = str2bin(torrent['infohash']) + #torrent['num_swarm'] = torrent['num_seeders'] + torrent['num_leechers'] + torrent['last_check_time'] = torrent['last_check'] + del torrent['last_check'] + del torrent['source_id'] + del torrent['category_id'] + del torrent['status_id'] + torrent_id = torrent['torrent_id'] + if mypref_stats is not None and torrent_id in mypref_stats: + # add extra info for torrent in mypref + torrent['myDownloadHistory'] = True + data = mypref_stats[torrent_id] #(create_time,progress,destdir) + torrent['download_started'] = data[0] + torrent['progress'] = data[1] + torrent['destdir'] = data[2] + + #print >>sys.stderr,"TorrentDBHandler: GET TORRENTS",`torrent` + + torrent_list.append(torrent) + return torrent_list + + def getRanks(self): + value_name = 'infohash' + order_by = 'relevance desc' + rankList_size = 20 + where = 'status_id=%d ' % self.status_table['good'] + res_list = self._db.getAll('Torrent', value_name, where = where, limit=rankList_size, order_by=order_by) + return [a[0] for a in res_list] + + def getNumberCollectedTorrents(self): + #return self._db.size('CollectedTorrent') + return self._db.getOne('CollectedTorrent', 'count(torrent_id)') + + def freeSpace(self, torrents2del): +# if torrents2del > 100: # only delete so many torrents each time +# torrents2del = 100 + sql = """ + select torrent_file_name, torrent_id, infohash, relevance, + min(relevance,2500) + min(500,num_leechers) + 4*min(500,num_seeders) - (max(0,min(500,(%d-creation_date)/86400)) ) as weight + from CollectedTorrent + where torrent_id not in (select torrent_id from MyPreference) + order by weight + limit %d + """ % (int(time()), torrents2del) + res_list = self._db.fetchall(sql) + if len(res_list) == 0: + return False + + # delete torrents from db + sql_del_torrent = "delete from Torrent where torrent_id=?" + sql_del_tracker = "delete from TorrentTracker where torrent_id=?" + sql_del_pref = "delete from Preference where torrent_id=?" + tids = [(torrent_id,) for torrent_file_name, torrent_id, infohash, relevance, weight in res_list] + + self._db.executemany(sql_del_torrent, tids, commit=False) + self._db.executemany(sql_del_tracker, tids, commit=False) + self._db.executemany(sql_del_pref, tids, commit=False) + + self._db.commit() + + # but keep the infohash in db to maintain consistence with preference db + #torrent_id_infohashes = [(torrent_id,infohash_str,relevance) for torrent_file_name, torrent_id, infohash_str, relevance, weight in res_list] + #sql_insert = "insert into Torrent (torrent_id, infohash, relevance) values (?,?,?)" + #self._db.executemany(sql_insert, torrent_id_infohashes, commit=True) + + torrent_dir = self.getTorrentDir() + deleted = 0 # deleted any file? + for torrent_file_name, torrent_id, infohash, relevance, weight in res_list: + torrent_path = os.path.join(torrent_dir, torrent_file_name) + try: + os.remove(torrent_path) + print >> sys.stderr, "Erase torrent:", os.path.basename(torrent_path) + deleted += 1 + except Exception, msg: + #print >> sys.stderr, "Error in erase torrent", Exception, msg + pass + + self.notifier.notify(NTFY_TORRENTS, NTFY_DELETE, str2bin(infohash)) # refresh gui + + return deleted + + def hasMetaData(self, infohash): + return self.hasTorrent(infohash) + + def getTorrentRelevances(self, tids): + sql = 'SELECT torrent_id, relevance from Torrent WHERE torrent_id in ' + str(tuple(tids)) + return self._db.fetchall(sql) + + def updateTorrentRelevance(self, infohash, relevance): + self.updateTorrent(infohash, relevance=relevance) + + def updateTorrentRelevances(self, tid_rel_pairs, commit=True): + if len(tid_rel_pairs) > 0: + sql_update_sims = 'UPDATE Torrent SET relevance=? WHERE torrent_id=?' + self._db.executemany(sql_update_sims, tid_rel_pairs, commit=commit) + + def searchNames(self,kws,local=True): + t1 = time() + value_name = ['torrent_id', + 'infohash', + 'name', + 'torrent_file_name', + 'length', + 'creation_date', + 'num_files', + 'thumbnail', + 'insert_time', + 'secret', + 'relevance', + 'source_id', + 'category_id', + 'status_id', + 'num_seeders', + 'num_leechers', + 'comment', + 'channel_permid', + 'channel_name'] + + sql = "" + count = 0 + for word in kws: + word = word.lower() + count += 1 + sql += " select torrent_id from InvertedIndex where word='" + word + "' " + if count < len(kws): + sql += " intersect " + + mainsql = """select T.*, C.publisher_id as channel_permid, C.publisher_name as channel_name + from Torrent T LEFT OUTER JOIN ChannelCast C on T.infohash = C.infohash + where T.torrent_id in (%s) order by T.num_seeders desc """ % (sql) + if not local: + mainsql += " limit 20" + + results = self._db.fetchall(mainsql) + t2 = time() + sql = "select mod_id, sum(vote), count(*) from VoteCast group by mod_id order by 2 desc" + votecast_records = self._db.fetchall(sql) + + votes = {} + for vote in votecast_records: + votes[vote[0]] = (vote[1], vote[2]) + t3 = time() + + torrents_dict = {} + for result in results: + a = time() + torrent = dict(zip(value_name,result)) + + #bug fix: If channel_permid and/or channel_name is None, it cannot bencode + #bencode(None) is an Error + if torrent['channel_permid'] is None: + torrent['channel_permid'] = "" + if torrent['channel_name'] is None: + torrent['channel_name'] = "" + + # check if this torrent belongs to more than one channel + if torrent['infohash'] in torrents_dict: + old_record = torrents_dict[torrent['infohash']] + # check if this channel has votes and if so, is it better than previous channel + if torrent['channel_permid'] in votes: + sum, count = votes[torrent['channel_permid']] + numsubscriptions = (sum + count)/3 + negvotes = (2*count-sum)/3 + if numsubscriptions-negvotes > old_record['subscriptions'] - old_record['neg_votes']: + #print >> sys.stderr, "overridden", torrent['channel_name'], old_record['channel_name'] + old_record['channel_permid'] = torrent['channel_permid'] + old_record['channel_name'] = torrent['channel_name'] + old_record['subscriptions'] = numsubscriptions + old_record['neg_votes'] = negvotes + else: + if old_record['subscriptions'] - old_record['neg_votes'] < 0: # SPAM cutoff + old_record['channel_permid'] = torrent['channel_permid'] + old_record['channel_name'] = torrent['channel_name'] + old_record['subscriptions'] = 0 + old_record['neg_votes'] = 0 + continue + + torrents_dict[torrent['infohash']] = torrent + try: + torrent['source'] = self.id2src[torrent['source_id']] + except: + print_exc() + # Arno: RSS subscription and id2src issue + torrent['source'] = 'http://some/RSS/feed' + + torrent['category'] = [self.id2category[torrent['category_id']]] + torrent['status'] = self.id2status[torrent['status_id']] + torrent['simRank'] = ranksfind(None,torrent['infohash']) + torrent['infohash'] = str2bin(torrent['infohash']) + #torrent['num_swarm'] = torrent['num_seeders'] + torrent['num_leechers'] + torrent['last_check_time'] = 0 #torrent['last_check'] + #del torrent['last_check'] + del torrent['source_id'] + del torrent['category_id'] + del torrent['status_id'] + torrent_id = torrent['torrent_id'] + + torrent['neg_votes']=0 + torrent['subscriptions']=0 + if torrent['channel_permid'] in votes: + sum, count = votes[torrent['channel_permid']] + numsubscriptions = (sum + count)/3 + negvotes = (2*count-sum)/3 + torrent['neg_votes']=negvotes + torrent['subscriptions']=numsubscriptions + + #print >> sys.stderr, "hello.. %.3f,%.3f" %((time()-a), time()) + def compare(a,b): + return -1*cmp(a['num_seeders'], b['num_seeders']) + torrent_list = torrents_dict.values() + torrent_list.sort(compare) + #print >> sys.stderr, "# hits:%d; search time:%.3f,%.3f,%.3f" % (len(torrent_list),t2-t1, t3-t2, time()-t3 ) + return torrent_list + + + def selectTorrentToCollect(self, permid, candidate_list=None): + """ + select a torrent to collect from a given candidate list + If candidate_list is not present or None, all torrents of + this peer will be used for sampling. + Return: the infohashed of selected torrent + """ + + if candidate_list is None: + sql = """SELECT similarity, infohash FROM Peer, Preference, Torrent + WHERE Peer.peer_id = Preference.peer_id + AND Torrent.torrent_id = Preference.torrent_id + AND Peer.peer_id IN(Select peer_id from Peer WHERE similarity > 0 ORDER By similarity DESC,last_connected DESC Limit ?) + AND Preference.torrent_id IN(Select torrent_id from Peer, Preference WHERE Peer.peer_id = Preference.peer_id AND Peer.permid = ?) + AND torrent_file_name is NULL + """ + permid_str = bin2str(permid) + results = self._db.fetchall(sql, (50, permid_str)) + else: + #print >>sys.stderr,"torrentdb: selectTorrentToCollect: cands",`candidate_list` + + cand_str = [bin2str(infohash) for infohash in candidate_list] + s = repr(cand_str).replace('[','(').replace(']',')') + sql = """SELECT similarity, infohash FROM Peer, Preference, Torrent + WHERE Peer.peer_id = Preference.peer_id + AND Torrent.torrent_id = Preference.torrent_id + AND Peer.peer_id IN(Select peer_id from Peer WHERE similarity > 0 ORDER By similarity DESC Limit ?) + AND infohash in """+s+""" + AND torrent_file_name is NULL + """ + results = self._db.fetchall(sql, (50,)) + + res = None + #convert top-x similarities into item recommendations + infohashes = {} + for sim, infohash in results: + infohashes[infohash] = infohashes.get(infohash,0) + sim + + keys = infohashes.keys() + if len(keys) > 0: + keys.sort(lambda a,b: cmp(infohashes[b], infohashes[a])) + + #add all items with highest relevance to candidate_list + candidate_list = [] + for infohash in keys: + if infohashes[infohash] == infohashes[keys[0]]: + candidate_list.append(str2bin(infohash)) + + #if only 1 candidate use that as result + if len(candidate_list) == 1: + res = keys[0] + candidate_list = None + + #No torrent found with relevance, fallback to most downloaded torrent + if res is None: + if candidate_list is None or len(candidate_list) == 0: + sql = """SELECT infohash FROM Torrent, Peer, Preference + WHERE Peer.permid == ? + AND Peer.peer_id == Preference.peer_id + AND Torrent.torrent_id == Preference.torrent_id + AND torrent_file_name is NULL + GROUP BY Preference.torrent_id + ORDER BY Count(Preference.torrent_id) DESC + LIMIT 1""" + permid_str = bin2str(permid) + res = self._db.fetchone(sql, (permid_str,)) + else: + cand_str = [bin2str(infohash) for infohash in candidate_list] + s = repr(cand_str).replace('[','(').replace(']',')') + sql = """SELECT infohash FROM Torrent, Preference + WHERE Torrent.torrent_id == Preference.torrent_id + AND torrent_file_name is NULL + AND infohash IN """ + s + """ + GROUP BY Preference.torrent_id + ORDER BY Count(Preference.torrent_id) DESC + LIMIT 1""" + res = self._db.fetchone(sql) + + if res is None: + return None + return str2bin(res) + + def selectTorrentToCheck(self, policy='random', infohash=None, return_value=None): # for tracker checking + """ select a torrent to update tracker info (number of seeders and leechers) + based on the torrent checking policy. + RETURN: a dictionary containing all useful info. + + Policy 1: Random [policy='random'] + Randomly select a torrent to collect (last_check < 5 min ago) + + Policy 2: Oldest (unknown) first [policy='oldest'] + Select the non-dead torrent which was not been checked for the longest time (last_check < 5 min ago) + + Policy 3: Popular first [policy='popular'] + Select the non-dead most popular (3*num_seeders+num_leechers) one which has not been checked in last N seconds + (The default N = 4 hours, so at most 4h/torrentchecking_interval popular peers) + """ + + #import threading + #print >> sys.stderr, "****** selectTorrentToCheck", threading.currentThread().getName() + + if infohash is None: + # create a view? + sql = """select T.torrent_id, ignored_times, retried_times, torrent_file_name, infohash, status_id, num_seeders, num_leechers, last_check + from CollectedTorrent T, TorrentTracker TT + where TT.torrent_id=T.torrent_id and announce_tier=1 """ + if policy.lower() == 'random': + ntorrents = self.getNumberCollectedTorrents() + if ntorrents == 0: + rand_pos = 0 + else: + rand_pos = randint(0, ntorrents-1) + last_check_threshold = int(time()) - 300 + sql += """and last_check < %d + limit 1 offset %d """%(last_check_threshold, rand_pos) + elif policy.lower() == 'oldest': + last_check_threshold = int(time()) - 300 + sql += """ and last_check < %d and status_id <> 2 + order by last_check + limit 1 """%last_check_threshold + elif policy.lower() == 'popular': + last_check_threshold = int(time()) - 4*60*60 + sql += """ and last_check < %d and status_id <> 2 + order by 3*num_seeders+num_leechers desc + limit 1 """%last_check_threshold + res = self._db.fetchone(sql) + else: + sql = """select T.torrent_id, ignored_times, retried_times, torrent_file_name, infohash, status_id, num_seeders, num_leechers, last_check + from CollectedTorrent T, TorrentTracker TT + where TT.torrent_id=T.torrent_id and announce_tier=1 + and infohash=? + """ + infohash_str = bin2str(infohash) + res = self._db.fetchone(sql, (infohash_str,)) + + if res: + torrent_file_name = res[3] + torrent_dir = self.getTorrentDir() + torrent_path = os.path.join(torrent_dir, torrent_file_name) + if res is not None: + res = {'torrent_id':res[0], + 'ignored_times':res[1], + 'retried_times':res[2], + 'torrent_path':torrent_path, + 'infohash':str2bin(res[4]) + } + return_value['torrent'] = res + return_value['event'].set() + + + def getTorrentsFromSource(self,source): + """ Get all torrents from the specified Subscription source. + Return a list of dictionaries. Each dict is in the NEWDBSTANDARD format. + """ + id = self._getSourceID(source) + + where = 'C.source_id = %d and C.torrent_id = T.torrent_id and announce_tier=1' % (id) + # add familyfilter + where += self.category.get_family_filter_sql(self._getCategoryID) + + value_name = deepcopy(self.value_name) + + res_list = self._db.getAll('Torrent C, TorrentTracker T', value_name, where) + + torrent_list = self.valuelist2torrentlist(value_name,res_list,None,None) + del res_list + + return torrent_list + + + def setSecret(self,infohash,secret): + kw = {'secret': secret} + self.updateTorrent(infohash, commit=True, **kw) + + +class MyPreferenceDBHandler(BasicDBHandler): + + __single = None # used for multithreaded singletons pattern + lock = threading.Lock() + + def getInstance(*args, **kw): + # Singleton pattern with double-checking + if MyPreferenceDBHandler.__single is None: + MyPreferenceDBHandler.lock.acquire() + try: + if MyPreferenceDBHandler.__single is None: + MyPreferenceDBHandler(*args, **kw) + finally: + MyPreferenceDBHandler.lock.release() + return MyPreferenceDBHandler.__single + + getInstance = staticmethod(getInstance) + + def __init__(self): + if MyPreferenceDBHandler.__single is not None: + raise RuntimeError, "MyPreferenceDBHandler is singleton" + MyPreferenceDBHandler.__single = self + db = SQLiteCacheDB.getInstance() + BasicDBHandler.__init__(self,db, 'MyPreference') ## self,db,'MyPreference' + + self.status_table = {'good':1, 'unknown':0, 'dead':2} + self.status_table.update(self._db.getTorrentStatusTable()) + self.status_good = self.status_table['good'] + # Arno, 2010-02-04: ARNOCOMMENT ARNOTODO Get rid of this g*dd*mn caching + # or keep it consistent with the DB! + self.recent_preflist = None + self.recent_preflist_with_clicklog = None + self.recent_preflist_with_swarmsize = None + self.rlock = threading.RLock() + + self.popularity_db = PopularityDBHandler.getInstance() + + + def loadData(self): + """ Arno, 2010-02-04: Brute force update method for the self.recent_ + caches, because people don't seem to understand that caches need + to be kept consistent with the database. Caches are evil in the first place. + """ + self.rlock.acquire() + try: + self.recent_preflist = self._getRecentLivePrefList() + self.recent_preflist_with_clicklog = self._getRecentLivePrefListWithClicklog() + self.recent_preflist_with_swarmsize = self._getRecentLivePrefListOL11() + finally: + self.rlock.release() + + def getMyPrefList(self, order_by=None): + res = self.getAll('torrent_id', order_by=order_by) + return [p[0] for p in res] + + def getMyPrefListInfohash(self): + sql = 'select infohash from Torrent where torrent_id in (select torrent_id from MyPreference)' + res = self._db.fetchall(sql) + return [str2bin(p[0]) for p in res] + + def getMyPrefStats(self, torrent_id=None): + # get the full {torrent_id:(create_time,progress,destdir)} + value_name = ('torrent_id','creation_time','progress','destination_path') + if torrent_id is not None: + where = 'torrent_id=%s' % torrent_id + else: + where = None + res = self.getAll(value_name, where) + mypref_stats = {} + for pref in res: + torrent_id,creation_time,progress,destination_path = pref + mypref_stats[torrent_id] = (creation_time,progress,destination_path) + return mypref_stats + + def getCreationTime(self, infohash): + torrent_id = self._db.getTorrentID(infohash) + if torrent_id is not None: + ct = self.getOne('creation_time', torrent_id=torrent_id) + return ct + else: + return None + + def getRecentLivePrefListWithClicklog(self, num=0): + """returns OL 8 style preference list: a list of lists, with each of the inner lists + containing infohash, search terms, click position, and reranking strategy""" + + if self.recent_preflist_with_clicklog is None: + self.rlock.acquire() + try: + if self.recent_preflist_with_clicklog is None: + self.recent_preflist_with_clicklog = self._getRecentLivePrefListWithClicklog() + finally: + self.rlock.release() + if num > 0: + return self.recent_preflist_with_clicklog[:num] + else: + return self.recent_preflist_with_clicklog + + def getRecentLivePrefListOL11(self, num=0): + """ + Returns OL 11 style preference list. It contains all info from previous + versions like clickLog info and some additional info related to swarm size. + @author: Rahim + @param num: if num be equal to zero the lenghth of the return list is unlimited, otherwise it's maximum lenght will be num. + @return: a list of lists. Each inner list is like: + [previous info , num_seeders, num_leechers, swarm_size_calc_age, number_of_sources] + """ + if self.recent_preflist_with_swarmsize is None: + self.rlock.acquire() + try: + #if self.recent_preflist_with_swarmsize is None: + self.recent_preflist_with_swarmsize = self._getRecentLivePrefListOL11() + finally: + self.rlock.release() + if num > 0: + return self.recent_preflist_with_swarmsize[:num] + else: + return self.recent_preflist_with_swarmsize + + + def getRecentLivePrefList(self, num=0): + if self.recent_preflist is None: + self.rlock.acquire() + try: + if self.recent_preflist is None: + self.recent_preflist = self._getRecentLivePrefList() + finally: + self.rlock.release() + if num > 0: + return self.recent_preflist[:num] + else: + return self.recent_preflist + + + + def addClicklogToMyPreference(self, infohash, clicklog_data, commit=True): + torrent_id = self._db.getTorrentID(infohash) + clicklog_already_stored = False # equivalent to hasMyPreference TODO + if torrent_id is None or clicklog_already_stored: + return False + + d = {} + # copy those elements of the clicklog data which are used in the update command + for clicklog_key in ["click_position", "reranking_strategy"]: + if clicklog_key in clicklog_data: + d[clicklog_key] = clicklog_data[clicklog_key] + + if d=={}: + if DEBUG: + print >> sys.stderr, "no updatable information given to addClicklogToMyPreference" + else: + if DEBUG: + print >> sys.stderr, "addClicklogToMyPreference: updatable clicklog data: %s" % d + self._db.update(self.table_name, 'torrent_id=%d' % torrent_id, commit=commit, **d) + + # have keywords stored by SearchDBHandler + if 'keywords' in clicklog_data: + if not clicklog_data['keywords']==[]: + searchdb = SearchDBHandler.getInstance() + searchdb.storeKeywords(peer_id=0, + torrent_id=torrent_id, + terms=clicklog_data['keywords'], + commit=commit) + + + + def _getRecentLivePrefListWithClicklog(self, num=0): + """returns a list containing a list for each torrent: [infohash, [seach terms], click position, reranking strategy]""" + + sql = """ + select infohash, click_position, reranking_strategy, m.torrent_id from MyPreference m, Torrent t + where m.torrent_id == t.torrent_id + and status_id == %d + order by creation_time desc + """ % self.status_good + + recent_preflist_with_clicklog = self._db.fetchall(sql) + if recent_preflist_with_clicklog is None: + recent_preflist_with_clicklog = [] + else: + recent_preflist_with_clicklog = [[str2bin(t[0]), + t[3], # insert search terms in next step, only for those actually required, store torrent id for now + t[1], # click position + t[2]] # reranking strategy + for t in recent_preflist_with_clicklog] + + if num != 0: + recent_preflist_with_clicklog = recent_preflist_with_clicklog[:num] + + # now that we only have those torrents left in which we are actually interested, + # replace torrent id by user's search terms for torrent id + termdb = TermDBHandler.getInstance() + searchdb = SearchDBHandler.getInstance() + for pref in recent_preflist_with_clicklog: + torrent_id = pref[1] + search_terms = searchdb.getMyTorrentSearchTerms(torrent_id) + # Arno, 2010-02-02: Explicit encoding + pref[1] = self.searchterms2utf8pref(termdb,search_terms) + + return recent_preflist_with_clicklog + + def searchterms2utf8pref(self,termdb,search_terms): + terms = [termdb.getTerm(search_term) for search_term in search_terms] + eterms = [] + for term in terms: + eterms.append(term.encode("UTF-8")) + return eterms + + + def _getRecentLivePrefListOL11(self, num=0): + """ + first calls the previous method to get a list of torrents and related info from MyPreference db + (_getRecentLivePrefListWithClicklog) and then appendes it with swarm size info or ( num_seeders, num_leechers, calc_age, num_seeders). + @author: Rahim + @param num: if num=0 it returns all items otherwise it restricts the return result to num. + @return: a list that each item conatins below info: + [infohash, [seach terms], click position, reranking strategy, num_seeders, num_leechers, calc_age, num_of_sources] + """ + + sql = """ + select infohash, click_position, reranking_strategy, m.torrent_id from MyPreference m, Torrent t + where m.torrent_id == t.torrent_id + and status_id == %d + order by creation_time desc + """ % self.status_good + + recent_preflist_with_swarmsize = self._db.fetchall(sql) + if recent_preflist_with_swarmsize is None: + recent_preflist_with_swarmsize = [] + else: + recent_preflist_with_swarmsize = [[str2bin(t[0]), + t[3], # insert search terms in next step, only for those actually required, store torrent id for now + t[1], # click position + t[2]] # reranking strategy + for t in recent_preflist_with_swarmsize] + + if num != 0: + recent_preflist_with_swarmsize = recent_preflist_with_swarmsize[:num] + + # now that we only have those torrents left in which we are actually interested, + # replace torrent id by user's search terms for torrent id + termdb = TermDBHandler.getInstance() + searchdb = SearchDBHandler.getInstance() + tempTorrentList = [] + for pref in recent_preflist_with_swarmsize: + torrent_id = pref[1] + tempTorrentList.append(torrent_id) + search_terms = searchdb.getMyTorrentSearchTerms(torrent_id) + # Arno, 2010-02-02: Explicit encoding + pref[1] = self.searchterms2utf8pref(termdb,search_terms) + + #Step 3: appending swarm size info to the end of the inner lists + swarmSizeInfoList= self.popularity_db.calculateSwarmSize(tempTorrentList, 'TorrentIds', toBC=True) # returns a list of items [torrent_id, num_seeders, num_leechers, num_sources_seen] + + index = 0 + for index in range(0,len(swarmSizeInfoList)): + recent_preflist_with_swarmsize[index].append(swarmSizeInfoList[index][1]) # number of seeders + recent_preflist_with_swarmsize[index].append(swarmSizeInfoList[index][2])# number of leechers + recent_preflist_with_swarmsize[index].append(swarmSizeInfoList[index][3]) # age of the report + recent_preflist_with_swarmsize[index].append(swarmSizeInfoList[index][4]) # number of sources seen this torrent + return recent_preflist_with_swarmsize + + def _getRecentLivePrefList(self, num=0): # num = 0: all files + # get recent and live torrents + sql = """ + select infohash from MyPreference m, Torrent t + where m.torrent_id == t.torrent_id + and status_id == %d + order by creation_time desc + """ % self.status_good + + recent_preflist = self._db.fetchall(sql) + if recent_preflist is None: + recent_preflist = [] + else: + recent_preflist = [str2bin(t[0]) for t in recent_preflist] + + if num != 0: + return recent_preflist[:num] + else: + return recent_preflist + + def hasMyPreference(self, infohash): + torrent_id = self._db.getTorrentID(infohash) + if torrent_id is None: + return False + res = self.getOne('torrent_id', torrent_id=torrent_id) + if res is not None: + return True + else: + return False + + def addMyPreference(self, infohash, data, commit=True): + # keys in data: destination_path, progress, creation_time, torrent_id + torrent_id = self._db.getTorrentID(infohash) + if torrent_id is None or self.hasMyPreference(infohash): + # Arno, 2009-03-09: Torrent already exists in myrefs. + # Hack for hiding from lib while keeping in myprefs. + # see standardOverview.removeTorrentFromLibrary() + # + self.updateDestDir(infohash,data.get('destination_path'),commit=commit) + return False + d = {} + d['destination_path'] = data.get('destination_path') + d['progress'] = data.get('progress', 0) + d['creation_time'] = data.get('creation_time', int(time())) + d['torrent_id'] = torrent_id + + self._db.insert(self.table_name, commit=commit, **d) + self.notifier.notify(NTFY_MYPREFERENCES, NTFY_INSERT, infohash) + + # Arno, 2010-02-04: Update self.recent_ caches :-( + self.loadData() + + return True + + def deletePreference(self, infohash, commit=True): + # Arno: when deleting a preference, you may also need to do + # some stuff in BuddyCast: see delMyPref() + torrent_id = self._db.getTorrentID(infohash) + if torrent_id is None: + return + self._db.delete(self.table_name, commit=commit, **{'torrent_id':torrent_id}) + self.notifier.notify(NTFY_MYPREFERENCES, NTFY_DELETE, infohash) + + # Arno, 2010-02-04: Update self.recent_ caches :-( + self.loadData() + + + def updateProgress(self, infohash, progress, commit=True): + torrent_id = self._db.getTorrentID(infohash) + if torrent_id is None: + return + self._db.update(self.table_name, 'torrent_id=%d'%torrent_id, commit=commit, progress=progress) + #print >> sys.stderr, '********* update progress', `infohash`, progress, commit + + def getAllEntries(self): + """use with caution,- for testing purposes""" + return self.getAll("torrent_id, click_position, reranking_strategy", order_by="torrent_id") + + def updateDestDir(self, infohash, destdir, commit=True): + torrent_id = self._db.getTorrentID(infohash) + if torrent_id is None: + return + self._db.update(self.table_name, 'torrent_id=%d'%torrent_id, commit=commit, destination_path=destdir) + + +# def getAllTorrentCoccurrence(self): +# # should be placed in PreferenceDBHandler, but put here to be convenient for TorrentCollecting +# sql = """select torrent_id, count(torrent_id) as coocurrency from Preference where peer_id in +# (select peer_id from Preference where torrent_id in +# (select torrent_id from MyPreference)) and torrent_id not in +# (select torrent_id from MyPreference) +# group by torrent_id +# """ +# coccurrence = dict(self._db.fetchall(sql)) +# return coccurrence + + +class BarterCastDBHandler(BasicDBHandler): + + __single = None # used for multithreaded singletons pattern + lock = threading.Lock() + + def getInstance(*args, **kw): + + if BarterCastDBHandler.__single is None: + BarterCastDBHandler.lock.acquire() + try: + if BarterCastDBHandler.__single is None: + BarterCastDBHandler(*args, **kw) + finally: + BarterCastDBHandler.lock.release() + return BarterCastDBHandler.__single + + getInstance = staticmethod(getInstance) + + def __init__(self): + BarterCastDBHandler.__single = self + db = SQLiteCacheDB.getInstance() + BasicDBHandler.__init__(self, db,'BarterCast') ## self,db,'BarterCast' + self.peer_db = PeerDBHandler.getInstance() + + # create the maxflow network + self.network = Network({}) + self.update_network() + + if DEBUG: + print >> sys.stderr, "bartercastdb:" + + + ##def registerSession(self, session): + ## self.session = session + + # Retrieve MyPermid + ## self.my_permid = session.get_permid() + + + def registerSession(self, session): + self.session = session + + # Retrieve MyPermid + self.my_permid = session.get_permid() + + if DEBUG: + print >> sys.stderr, "bartercastdb: MyPermid is ", `self.my_permid` + + if self.my_permid is None: + raise ValueError('Cannot get permid from Session') + + # Keep administration of total upload and download + # (to include in BarterCast message) + self.my_peerid = self.getPeerID(self.my_permid) + + if self.my_peerid != None: + where = "peer_id_from=%s" % (self.my_peerid) + item = self.getOne(('sum(uploaded)', 'sum(downloaded)'), where=where) + else: + item = None + + if item != None and len(item) == 2 and item[0] != None and item[1] != None: + self.total_up = int(item[0]) + self.total_down = int(item[1]) + else: + self.total_up = 0 + self.total_down = 0 + +# if DEBUG: +# print >> sys.stderr, "My reputation: ", self.getMyReputation() + + + def getTotals(self): + return (self.total_up, self.total_down) + + def getName(self, permid): + + if permid == 'non-tribler': + return "non-tribler" + elif permid == self.my_permid: + return "local_tribler" + + name = self.peer_db.getPeer(permid, 'name') + + if name == None or name == '': + return 'peer %s' % show_permid_shorter(permid) + else: + return name + + def getNameByID(self, peer_id): + permid = self.getPermid(peer_id) + return self.getName(permid) + + + def getPermid(self, peer_id): + + # by convention '-1' is the id of non-tribler peers + if peer_id == -1: + return 'non-tribler' + else: + return self.peer_db.getPermid(peer_id) + + + def getPeerID(self, permid): + + # by convention '-1' is the id of non-tribler peers + if permid == "non-tribler": + return -1 + else: + return self.peer_db.getPeerID(permid) + + def getItem(self, (permid_from, permid_to), default=False): + + # ARNODB: now converting back to dbid! just did reverse in getItemList + peer_id1 = self.getPeerID(permid_from) + peer_id2 = self.getPeerID(permid_to) + + if peer_id1 is None: + self._db.insertPeer(permid_from) # ARNODB: database write + peer_id1 = self.getPeerID(permid_from) # ARNODB: database write + + if peer_id2 is None: + self._db.insertPeer(permid_to) + peer_id2 = self.getPeerID(permid_to) + + return self.getItemByIDs((peer_id1,peer_id2),default=default) + + + def getItemByIDs(self, (peer_id_from, peer_id_to), default=False): + if peer_id_from is not None and peer_id_to is not None: + + where = "peer_id_from=%s and peer_id_to=%s" % (peer_id_from, peer_id_to) + item = self.getOne(('downloaded', 'uploaded', 'last_seen'), where=where) + + if item is None: + return None + + if len(item) != 3: + return None + + itemdict = {} + itemdict['downloaded'] = item[0] + itemdict['uploaded'] = item[1] + itemdict['last_seen'] = item[2] + itemdict['peer_id_from'] = peer_id_from + itemdict['peer_id_to'] = peer_id_to + + return itemdict + + else: + return None + + + def getItemList(self): # get the list of all peers' permid + + keys = self.getAll(('peer_id_from','peer_id_to')) + # ARNODB: this dbid -> permid translation is more efficiently done + # on the final top-N list. + keys = map(lambda (id_from, id_to): (self.getPermid(id_from), self.getPermid(id_to)), keys) + return keys + + + def addItem(self, (permid_from, permid_to), item, commit=True): + +# if value.has_key('last_seen'): # get the latest last_seen +# old_last_seen = 0 +# old_data = self.getPeer(permid) +# if old_data: +# old_last_seen = old_data.get('last_seen', 0) +# last_seen = value['last_seen'] +# value['last_seen'] = max(last_seen, old_last_seen) + + # get peer ids + peer_id1 = self.getPeerID(permid_from) + peer_id2 = self.getPeerID(permid_to) + + # check if they already exist in database; if not: add + if peer_id1 is None: + self._db.insertPeer(permid_from) + peer_id1 = self.getPeerID(permid_from) + if peer_id2 is None: + self._db.insertPeer(permid_to) + peer_id2 = self.getPeerID(permid_to) + + item['peer_id_from'] = peer_id1 + item['peer_id_to'] = peer_id2 + + self._db.insert(self.table_name, commit=commit, **item) + + def updateItem(self, (permid_from, permid_to), key, value, commit=True): + + if DEBUG: + print >> sys.stderr, "bartercastdb: update (%s, %s) [%s] += %s" % (self.getName(permid_from), self.getName(permid_to), key, str(value)) + + itemdict = self.getItem((permid_from, permid_to)) + + # if item doesn't exist: add it + if itemdict == None: + self.addItem((permid_from, permid_to), {'uploaded':0, 'downloaded': 0, 'last_seen': int(time())}, commit=True) + itemdict = self.getItem((permid_from, permid_to)) + + # get peer ids + peer_id1 = itemdict['peer_id_from'] + peer_id2 = itemdict['peer_id_to'] + + if key in itemdict.keys(): + + where = "peer_id_from=%s and peer_id_to=%s" % (peer_id1, peer_id2) + item = {key: value} + self._db.update(self.table_name, where = where, commit=commit, **item) + + def incrementItem(self, (permid_from, permid_to), key, value, commit=True): + if DEBUG: + print >> sys.stderr, "bartercastdb: increment (%s, %s) [%s] += %s" % (self.getName(permid_from), self.getName(permid_to), key, str(value)) + + # adjust total_up and total_down + if permid_from == self.my_permid: + if key == 'uploaded': + self.total_up += int(value) + if key == 'downloaded': + self.total_down += int(value) + + itemdict = self.getItem((permid_from, permid_to)) + + # if item doesn't exist: add it + if itemdict == None: + self.addItem((permid_from, permid_to), {'uploaded':0, 'downloaded': 0, 'last_seen': int(time())}, commit=True) + itemdict = self.getItem((permid_from, permid_to)) + + # get peer ids + peer_id1 = itemdict['peer_id_from'] + peer_id2 = itemdict['peer_id_to'] + + if key in itemdict.keys(): + old_value = itemdict[key] + new_value = old_value + value + + where = "peer_id_from=%s and peer_id_to=%s" % (peer_id1, peer_id2) + + item = {key: new_value} + self._db.update(self.table_name, where = where, commit=commit, **item) + return new_value + + return None + + def addPeersBatch(self,permids): + """ Add unknown permids as batch -> single transaction """ + if DEBUG: + print >> sys.stderr, "bartercastdb: addPeersBatch: n=",len(permids) + + for permid in permids: + peer_id = self.getPeerID(permid) + # check if they already exist in database; if not: add + if peer_id is None: + self._db.insertPeer(permid,commit=False) + self._db.commit() + + def updateULDL(self, (permid_from, permid_to), ul, dl, commit=True): + """ Add ul/dl record to database as a single write """ + + if DEBUG: + print >> sys.stderr, "bartercastdb: updateULDL (%s, %s) ['ul'] += %s ['dl'] += %s" % (self.getName(permid_from), self.getName(permid_to), str(ul), str(dl)) + + itemdict = self.getItem((permid_from, permid_to)) + + # if item doesn't exist: add it + if itemdict == None: + itemdict = {'uploaded':ul, 'downloaded': dl, 'last_seen': int(time())} + self.addItem((permid_from, permid_to), itemdict, commit=commit) + return + + # get peer ids + peer_id1 = itemdict['peer_id_from'] + peer_id2 = itemdict['peer_id_to'] + + if 'uploaded' in itemdict.keys() and 'downloaded' in itemdict.keys(): + where = "peer_id_from=%s and peer_id_to=%s" % (peer_id1, peer_id2) + item = {'uploaded': ul, 'downloaded':dl} + self._db.update(self.table_name, where = where, commit=commit, **item) + + def getPeerIDPairs(self): + keys = self.getAll(('peer_id_from','peer_id_to')) + return keys + + def getTopNPeers(self, n, local_only = False): + """ + Return (sorted) list of the top N peers with the highest (combined) + values for the given keys. This version uses batched reads and peer_ids + in calculation + @return a dict containing a 'top' key with a list of (permid,up,down) + tuples, a 'total_up', 'total_down', 'tribler_up', 'tribler_down' field. + Sizes are in kilobytes. + """ + + # TODO: this won't scale to many interactions, as the size of the DB + # is NxN + + if DEBUG: + print >> sys.stderr, "bartercastdb: getTopNPeers: local = ", local_only + #print_stack() + + n = max(1, n) + my_peer_id = self.getPeerID(self.my_permid) + total_up = {} + total_down = {} + # Arno, 2008-10-30: I speculate this is to count transfers only once, + # i.e. the DB stored (a,b) and (b,a) and we want to count just one. + + processed = Set() + + + value_name = '*' + increment = 500 + + nrecs = self.size() + #print >>sys.stderr,"NEXTtopN: size is",nrecs + + for offset in range(0,nrecs,increment): + if offset+increment > nrecs: + limit = nrecs-offset + else: + limit = increment + #print >>sys.stderr,"NEXTtopN: get",offset,limit + + reslist = self.getAll(value_name, offset=offset, limit=limit) + #print >>sys.stderr,"NEXTtopN: res len is",len(reslist),`reslist` + for res in reslist: + (peer_id_from,peer_id_to,downloaded,uploaded,last_seen,value) = res + + if local_only: + if not (peer_id_to == my_peer_id or peer_id_from == my_peer_id): + # get only items of my local dealings + continue + + if (not (peer_id_to, peer_id_from) in processed) and (not peer_id_to == peer_id_from): + #if (not peer_id_to == peer_id_from): + + up = uploaded *1024 # make into bytes + down = downloaded *1024 + + if DEBUG: + print >> sys.stderr, "bartercastdb: getTopNPeers: DB entry: (%s, %s) up = %d down = %d" % (self.getNameByID(peer_id_from), self.getNameByID(peer_id_to), up, down) + + processed.add((peer_id_from, peer_id_to)) + + # fix for multiple my_permids + if peer_id_from == -1: # 'non-tribler': + peer_id_to = my_peer_id + if peer_id_to == -1: # 'non-tribler': + peer_id_from = my_peer_id + + # process peer_id_from + total_up[peer_id_from] = total_up.get(peer_id_from, 0) + up + total_down[peer_id_from] = total_down.get(peer_id_from, 0) + down + + # process peer_id_to + total_up[peer_id_to] = total_up.get(peer_id_to, 0) + down + total_down[peer_id_to] = total_down.get(peer_id_to, 0) + up + + + # create top N peers + top = [] + min = 0 + + for peer_id in total_up.keys(): + + up = total_up[peer_id] + down = total_down[peer_id] + + if DEBUG: + print >> sys.stderr, "bartercastdb: getTopNPeers: total of %s: up = %d down = %d" % (self.getName(peer_id), up, down) + + # we know rank on total upload? + value = up + + # check if peer belongs to current top N + if peer_id != -1 and peer_id != my_peer_id and (len(top) < n or value > min): + + top.append((peer_id, up, down)) + + # sort based on value + top.sort(cmp = lambda (p1, u1, d1), (p2, u2, d2): cmp(u2, u1)) + + # if list contains more than N elements: remove the last (=lowest value) + if len(top) > n: + del top[-1] + + # determine new minimum of values + min = top[-1][1] + + # Now convert to permid + permidtop = [] + for peer_id,up,down in top: + permid = self.getPermid(peer_id) + permidtop.append((permid,up,down)) + + result = {} + + result['top'] = permidtop + + # My total up and download, including interaction with non-tribler peers + result['total_up'] = total_up.get(my_peer_id, 0) + result['total_down'] = total_down.get(my_peer_id, 0) + + # My up and download with tribler peers only + result['tribler_up'] = result['total_up'] - total_down.get(-1, 0) # -1 = 'non-tribler' + result['tribler_down'] = result['total_down'] - total_up.get(-1, 0) # -1 = 'non-tribler' + + if DEBUG: + print >> sys.stderr, result + + return result + + + ################################ + def update_network(self): + + + keys = self.getPeerIDPairs() #getItemList() + + + ################################ + def getMyReputation(self, alpha = ALPHA): + + rep = atan((self.total_up - self.total_down) * alpha)/(0.5 * pi) + return rep + + +class VoteCastDBHandler(BasicDBHandler): + + __single = None # used for multithreaded singletons pattern + lock = threading.Lock() + + def getInstance(*args, **kw): + + if VoteCastDBHandler.__single is None: + VoteCastDBHandler.lock.acquire() + try: + if VoteCastDBHandler.__single is None: + VoteCastDBHandler(*args, **kw) + finally: + VoteCastDBHandler.lock.release() + return VoteCastDBHandler.__single + + getInstance = staticmethod(getInstance) + + def __init__(self): + VoteCastDBHandler.__single = self + try: + db = SQLiteCacheDB.getInstance() + BasicDBHandler.__init__(self,db,'VoteCast') + if DEBUG: print >> sys.stderr, "votecast: DB made" + except: + print >> sys.stderr, "votecast: couldn't make the table" + + self.peer_db = PeerDBHandler.getInstance() + if DEBUG: + print >> sys.stderr, "votecast: " + + def registerSession(self, session): + self.session = session + self.my_permid = session.get_permid() + + if DEBUG: + print >> sys.stderr, "votecast: My permid is",`self.my_permid` + + def getAllVotes(self, permid): + sql = 'select * from VoteCast where mod_id==?' + + records = self._db.fetchall(sql, (permid,)) + return records + + def getAll(self): + sql = 'select * from VoteCast' + + records = self._db.fetchall(sql) + return records + + def getPosNegVotes(self, permid): + sql = 'select * from VoteCast where mod_id==?' + + records = self._db.fetchall(sql, (permid[0],)) + pos_votes = 0 + neg_votes = 0 + + if records is None: + return(pos_votes,neg_votes) + + for vote in records: + + if vote[2] == "1": + pos_votes +=1 + else: + neg_votes +=1 + return (pos_votes, neg_votes) + + + def hasVote(self, permid, voter_peerid): + sql = 'select mod_id, voter_id from VoteCast where mod_id==? and voter_id==?' + item = self._db.fetchone(sql,(permid,voter_peerid,)) + #print >> sys.stderr,"well well well",infohash," sdd",item + if item is None: + return False + else: + return True + + def getBallotBox(self): + sql = 'select * from VoteCast' + items = self._db.fetchall(sql) + return items + + def addVotes(self, votes): + sql = 'insert into VoteCast Values (?,?,?,?)' + self._db.executemany(sql,votes,commit=True) + + def addVote(self, vote, clone=True): + if self.hasVote(vote['mod_id'],vote['voter_id']): + self.deleteVote(vote['mod_id'],vote['voter_id']) + self._db.insert(self.table_name, **vote) + + def deleteVotes(self, permid): + sql = 'Delete From VoteCast where mod_id==?' + self._db.execute_write(sql,(permid,)) + + def deleteVote(self, permid, voter_id): + sql = 'Delete From VoteCast where mod_id==? and voter_id==?' + self._db.execute_write(sql,(permid,voter_id,)) + + def getPermid(self, peer_id): + + # by convention '-1' is the id of non-tribler peers + if peer_id == -1: + return 'non-tribler' + else: + return self.peer_db.getPermid(peer_id) + + + def getPeerID(self, permid): + # by convention '-1' is the id of non-tribler peers + if permid == "non-tribler": + return -1 + else: + return self.peer_db.getPeerID(permid) + + + def hasPeer(self, permid): + return self.peer_db.hasPeer(permid) + + def getRecentAndRandomVotes(self, recent=25, random=25): + allrecords = [] + + sql = "SELECT mod_id, vote, time_stamp from VoteCast where voter_id==? order by time_stamp desc limit ?" + myrecentvotes = self._db.fetchall(sql,(permid_for_user(self.my_permid),recent,)) + allrecords.extend(myrecentvotes) + + if myrecentvotes is not None and len(myrecentvotes)>=recent: + t = myrecentvotes[len(myrecentvotes)-1][2] + sql = "select mod_id, vote, time_stamp from VoteCast where voter_id==? and time_stamp=0 and vote<=2: + sql = "update VoteCast set vote=-1 where mod_id==? and voter_id==?" + self._db.execute_write(sql,(permid,bin2str(self.my_permid),)) + + def getVote(self,publisher_id,subscriber_id): + """ return the vote status if such record exists, otherwise None """ + sql = "select vote from VoteCast where mod_id==? and voter_id==?" + return self._db.fetchone(sql, (publisher_id,subscriber_id,)) + + def getPublishersWithNegVote(self, subscriber_id): + ''' return the publisher_ids having a negative vote from subscriber_id ''' + sql = "select mod_id from VoteCast where voter_id==? and vote=-1" + res = self._db.fetchall(sql,(subscriber_id,)) + result_list = Set() + for entry in res: + result_list.add(entry[0]) + return result_list + + def getNegVotes(self,publisher_id): + """returns the number of negative votes in integer format""" + sql = "select count(*) from VoteCast where mod_id==? and vote=-1" + return self._db.fetchone(sql, (publisher_id,)) + + def getNumSubscriptions(self,publisher_id): ### + """returns the number of subscribers in integer format""" + sql = "select count(*) from VoteCast where mod_id==? and vote=2" # before select vote + return self._db.fetchone(sql, (publisher_id,)) + + def getVotes(self, publisher_id): + """ returns (sum, count) from VoteCast """ + sql = "select sum(vote), count(*) from VoteCast where mod_id==?" + return self._db.fetchone(sql, (publisher_id,)) + + def getEffectiveVote(self, publisher_id): + """ returns positive - negative votes """ + sql = "select count(*) from VoteCast where mod_id==? and vote=2" + subscriptions = self._db.fetchone(sql, (publisher_id,)) + sql = "select count(*) from VoteCast where mod_id==? and vote=-1" + negative_votes = self._db.fetchone(sql, (publisher_id,)) + return (subscriptions - negative_votes) + + + +#end votes + +class ChannelCastDBHandler(BasicDBHandler): + """ """ + __single = None # used for multithreaded singletons pattern + lock = threading.Lock() + + def getInstance(*args, **kw): + if ChannelCastDBHandler.__single is None: + ChannelCastDBHandler.lock.acquire() + try: + if ChannelCastDBHandler.__single is None: + ChannelCastDBHandler(*args, **kw) + finally: + ChannelCastDBHandler.lock.release() + return ChannelCastDBHandler.__single + + getInstance = staticmethod(getInstance) + + def __init__(self): + ChannelCastDBHandler.__single = self + try: + db = SQLiteCacheDB.getInstance() + BasicDBHandler.__init__(self,db,'ChannelCast') + print >> sys.stderr, "ChannelCast: DB made" + except: + print >> sys.stderr, "ChannelCast: couldn't make the table" + + self.peer_db = PeerDBHandler.getInstance() + self.firstQueryMySubscriptions=True + self.allRecordsMySubscriptions=None + self.firstQueryPopularChannels=True + self.allRecordsPopularChannels=None + + if DEBUG: + print >> sys.stderr, "ChannelCast: " + + self.value_name = ['publisher_id','publisher_name','infohash','torrenthash','torrentname','time_stamp','signature'] ## + + def registerSession(self, session): + self.session = session + self.my_permid = session.get_permid() + self.getMySubscribedChannels() + self.getMostPopularUnsubscribedChannels() + self.ensureRecentNames() + if DEBUG: + print >> sys.stderr, "ChannelCast: My permid is",`self.my_permid` + + def _sign(self, record): + assert record is not None + # Nitin on Feb 5, 2010: Signature is generated using binary forms of permid, infohash, torrenthash fields + r = (str2bin(record[0]),str2bin(record[2]),str2bin(record[3]),record[5]) + bencoding = bencode(r) + signature = bin2str(sign_data(bencoding, self.session.keypair)) + record.append(signature) + + def ensureRecentNames(self): + sql = "select distinct publisher_id from ChannelCast" + publisher_ids = self._db.fetchall(sql) + for publisher_id in publisher_ids: + sql = "select publisher_name from ChannelCast where publisher_id==? order by time_stamp desc limit 1" + latest_publisher_name = self._db.fetchone(sql,(publisher_id[0],)) + sql = "update ChannelCast set publisher_name==? where publisher_id==?" + self._db.execute_write(sql,(latest_publisher_name,publisher_id[0],)) + + def addOwnTorrent(self, torrentdef): #infohash, torrentdata): + assert isinstance(torrentdef, TorrentDef), "TORRENTDEF has invalid type: %s" % type(torrentdef) + flag = False + publisher_id = bin2str(self.my_permid) + infohash = bin2str(torrentdef.get_infohash()) + sql = "select count(*) from ChannelCast where publisher_id==? and infohash==?" + num_records = self._db.fetchone(sql, (publisher_id, infohash,)) + if num_records==0: + torrenthash = bin2str(sha(bencode(torrentdef.get_metainfo())).digest()) + # Arno, 2010-01-27: sqlite don't like binary encoded names + unickname = self.session.get_nickname() + utorrentname = torrentdef.get_name_as_unicode() + record = [publisher_id,unickname,infohash,torrenthash,utorrentname,now()] + self._sign(record) + sql = "insert into ChannelCast Values(?,?,?,?,?,?,?)" + self._db.execute_write(sql,(record[0], record[1], record[2], record[3], record[4], record[5], record[6]), commit=True) + flag = True + sql = "select publisher_name from ChannelCast where publisher_id==? order by time_stamp desc limit 1" + latest_publisher_name = self._db.fetchone(sql,(publisher_id,)) + sql = "update ChannelCast set publisher_name==? where publisher_id==?" + self._db.execute_write(sql,(latest_publisher_name,publisher_id,)) + return flag + + + def deleteOwnTorrent(self, infohash): ## + sql = 'Delete From ChannelCast where infohash=? and publisher_id=?' + self._db.execute_write(sql,(bin2str(infohash),bin2str(self.my_permid),)) + + + def deleteTorrentsFromPublisherId(self, permid): ## + sql = "Delete From ChannelCast where publisher_id==?" + self._db.execute_write(sql,(bin2str(permid),)) + + + def updateMyChannelName(self, name): ## + sql = "update ChannelCast set publisher_name==? where publisher_id==?" + self._db.execute_write(sql,(name,bin2str(self.my_permid),)) + + + def addTorrent(self,record): + if __debug__: + assert len(record) == 7, "RECORD has invalid length: %d" % len(record) + publisher_id, publisher_name, infohash, torrenthash, torrentname, timestamp, signature = record + assert isinstance(publisher_id, str), "PUBLISHER_ID has invalid type: %s" % type(publisher_id) + assert isinstance(publisher_name, unicode), "PUBLISHER_NAME has invalid type: %s" % type(publisher_name) + assert isinstance(infohash, str), "INFOHASH has invalid type: %s" % type(infohash) + assert isinstance(torrentname, unicode), "TORRENTNAME has invalid type: %s" % type(torrentname) + assert isinstance(timestamp, int), "TIMESTAMP has invalid type: %s" % type(timestamp) + assert isinstance(signature, str), "SIGNATURE has invalid type: %s" % type(signature) + flag = False + sql = "select count(*) from ChannelCast where publisher_id='" + record[0] + "' and infohash='" + record[2] + "'" + num_records = self._db.fetchone(sql) + if num_records==0: + sql = "insert into ChannelCast (publisher_id, publisher_name, infohash, torrenthash, torrentname, time_stamp, signature) Values(?,?,?,?,?,?,?)" + self._db.execute_write(sql,(record[0], record[1], record[2], record[3], record[4], record[5], record[6]), commit=True) + flag = True + sql = "select publisher_name from ChannelCast where publisher_id==? order by time_stamp desc limit 1" + latest_publisher_name = self._db.fetchone(sql,(record[0],)) + sql = "update ChannelCast set publisher_name==? where publisher_id==?" + self._db.execute_write(sql,(latest_publisher_name,record[0],)) + return flag + + def existsTorrent(self, infohash): + sql = "select count(*) from Torrent where infohash==? and name<>''" + num_records = self._db.fetchone(sql, (bin2str(infohash),)) + if num_records > 0: + return True + return False + + def getRecentAndRandomTorrents(self,NUM_OWN_RECENT_TORRENTS=15,NUM_OWN_RANDOM_TORRENTS=10,NUM_OTHERS_RECENT_TORRENTS=15,NUM_OTHERS_RANDOM_TORRENTS=10): + allrecords = [] + + sql = "select * from ChannelCast where publisher_id==? order by time_stamp desc limit ?" + myrecenttorrents = self._db.fetchall(sql,(permid_for_user(self.my_permid),NUM_OWN_RECENT_TORRENTS,)) + allrecords.extend(myrecenttorrents) + + if myrecenttorrents is not None and len(myrecenttorrents)>=NUM_OWN_RECENT_TORRENTS: + t = myrecenttorrents[len(myrecenttorrents)-1][5] + sql = "select * from ChannelCast where publisher_id==? and time_stamp0: + allrecords.extend(othersrecenttorrents) + + if othersrecenttorrents is not None and len(othersrecenttorrents)>=NUM_OTHERS_RECENT_TORRENTS: + t = othersrecenttorrents[len(othersrecenttorrents)-1][5] + sql = "select * from ChannelCast where publisher_id in (select mod_id from VoteCast where voter_id=? and vote=2) and time_stamp'' " + torrent_results = self._db.fetchall(sql,(publisher_id,)) + results=[] + + # convert infohashes to binary + for torrent in torrent_results: + t_name = torrent[2] + + # check if torrent name contains only white spaces + index=0 + while index <= len(t_name) - 1 and t_name[index] == ' ': + index += 1 + if index == len(t_name): + continue + + t_list = list(torrent) + infohash = str2bin(t_list[1]) + t_list[1] = infohash + t_tuple = tuple(t_list) + results.append(t_tuple) + return results + + def searchChannels(self,query): + # query would be of the form: "k barack obama" or "p 4fw342d23re2we2w3e23d2334d" permid + value_name = deepcopy(self.value_name) ## + if query[0] == 'k': + # search torrents based on keywords + + kwlist = split_into_keywords(query[2:]) + sql = "select publisher_id, publisher_name from ChannelCast where " + count = 0 + for kw in kwlist: + count += 1 + if kw is None or len(kw)==0: + continue + sql += " publisher_name like '%" + kw + "%' " + if count>sys.stderr, "channel:", repr(channel) + # now, retrieve the last 20 of each of these channels' torrents + s = "select * from ChannelCast where publisher_id==? order by time_stamp desc limit 20" + record = self._db.fetchall(s,(channel[0],)) + if record is not None and len(record)>0: + allrecords.extend(record) + + records = [] + for record in allrecords: + records.append((str2bin(record[0]), record[1], str2bin(record[2]), str2bin(record[3]), record[4], record[5], str2bin(record[6]))) + return records + elif query[0] == 'p': + # search channel's torrents based on permid + q = query[2:] + #print>>sys.stderr, "ChannelCastDB: searchChannels: This is a permid-based search:", `q` + s = "select * from ChannelCast where publisher_id==? order by time_stamp desc limit 20" + allrecords = self._db.fetchall(s,(q,)) ## before records = {'torrents':self._db.fetchall(s)} + #channelList = self.valuelist2channellist(records,value_name) + records = [] + for record in allrecords: + records.append((str2bin(record[0]), record[1], str2bin(record[2]), str2bin(record[3]), record[4], record[5], str2bin(record[6]))) + + return records #channelList # + else: + # Query is invalid: hence, it should not even come here + return None + + def getTorrents(self, publisher_id): + sql = "select * from Torrent where infohash in (select infohash from ChannelCast where publisher_id==?)" + return self._db.fetchall(sql,(publisher_id,)) + + def getInfohashesForChannel(self, publisher_id): + sql = "select infohash from ChannelCast where publisher_id==? ;" + return self._db.fetchall(sql,(publisher_id,)) + + def isItemInChannel(self,publisher_id,infohash): + sql = "select count(*) from ChannelCast where publisher_id=? and infohash=? ;" + + isAvailable = self._db.fetchone(sql,(publisher_id,infohash)) + if isAvailable: + return True + else: + return False + + def valuelist2channellist(self,res_list,value_name): ## + + channel_list = [] + for item in res_list: + channel = dict(zip(value_name, item)) + + channel['infohash'] = str2bin(channel['infohash']) + channel['torrenthash'] = str2bin(channel['torrenthash']) + + channel_list.append(channel) + return channel_list + + def getMostPopularChannels(self): + """return a list of tuples: [(permid,channel_name,#subscriptions)]""" + records = [] + votecastdb = VoteCastDBHandler.getInstance() + # Inner query: First, identify the publishers you are subscribed to + # Outer query: Get all publishers that are not in your publishers' list, along with the number of subscriptions + ## sql = "select mod_id, count(*) from VoteCast where mod_id not in (select mod_id from VoteCast where voter_id='"+ bin2str(self.my_permid)+"' and vote=2) and mod_id<>'"+bin2str(self.my_permid)+"' group by mod_id order by 2 desc" + sql = "select mod_id, count(*) from VoteCast where mod_id<>? group by mod_id order by 2 desc" ## Richard : for now popular channels can contain channels i am subscribed to + votes = self._db.fetchall(sql,(bin2str(self.my_permid),)) + for vote in votes: + sql = "select publisher_name, time_stamp from ChannelCast where publisher_id==? order by 2 desc" + record = self._db.fetchone(sql, (vote[0],)) + if not record is None: + mod_name = record[0] + records.append((vote[0],mod_name,vote[1], {})) + return records + + + + def getMostPopularUnsubscribedChannels(self,from_channelcast=False): ## + """return a list of tuples: [(permid,channel_name,#votes)]""" + + #if not self.firstQueryPopularChannels and not from_channelcast: + # self.firstQueryPopularChannels=True + # return self.allRecordsPopularChannels + + votecastdb = VoteCastDBHandler.getInstance() + allrecords = [] + + + sql = "select distinct publisher_id, publisher_name from ChannelCast" + channel_records = self._db.fetchall(sql) + +# sql = "select mod_id, (2*sum(vote)-count(*))/3 from VoteCast group by mod_id order by 2 desc" + sql = "select mod_id, sum(vote-1) from VoteCast group by mod_id order by 2 desc" # only subscriptions, not spam votes + + votecast_records = self._db.fetchall(sql) + + sql = "select distinct mod_id from VoteCast where voter_id==? and vote=2" + subscribed_channels = self._db.fetchall(sql,(bin2str(self.my_permid),)) + + subscribers = {} + for record in subscribed_channels: + subscribers[record[0]]="12" + + publishers = {} + for publisher_id, publisher_name in channel_records: + if publisher_id not in publishers and publisher_id!=bin2str(self.my_permid): + publishers[publisher_id]=[publisher_name, 0] + + for mod_id, vote in votecast_records: + if vote < -5: # it is considered SPAM + if mod_id in publishers: + del publishers[mod_id] + continue + if mod_id in publishers: + if mod_id not in subscribers: + publishers[mod_id][1] = vote + else: + del publishers[mod_id] + for k, v in publishers.items(): + if votecastdb.getVote(k, bin2str(self.my_permid)) != -1: + allrecords.append((k, v[0], v[1], {})) + def compare(a,b): + if a[2]>b[2] : return -1 + if a[2]> sys.stderr, "getMostPopularUnsubscribedChannels: execution times %.3f, %.3f, %.3f" %(t2-t1, t3-t2, time()-t3) + + + #if not from_channelcast: + # if self.allRecordsPopularChannels is None: + # self.firstQueryPopularChannels=False + # self.allRecordsPopularChannels=allrecords + return allrecords + + + def getMyChannel(self): + mychannel = [] + votecastdb = VoteCastDBHandler.getInstance() + sql = "select publisher_id, publisher_name from ChannelCast where publisher_id==? group by publisher_id" + res = self._db.fetchall(sql,(bin2str(self.my_permid),)) + if res is not None: + # mychannel.append((self.my_permid,"MyChannel" , votecastdb.getNumSubscriptions(bin2str(self.my_permid)) - votecastdb.getNegVotes(bin2str(self.my_permid)), {})) + # for now only count subscriptions, not negative votes + mychannel.append((self.my_permid,"MyChannel" , votecastdb.getNumSubscriptions(bin2str(self.my_permid)),{})) + + + else: + mychannel.append((self.my_permid,"MyChannel" , 0, {})) + return mychannel + + + + def getSubscribersCount(self,permid): + """returns the number of subscribers in integer format""" + sql = "select count(*) from VoteCast where mod_id==? and vote=2" + numrecords = self._db.fetchone(sql, (permid,)) + return numrecords + + def getMyNumberSubscriptions(self): ## + """returns the number of subscribers in integer format""" + sql = "select count(*) from VoteCast where voter_id==? and vote=2" + numrecords = self._db.fetchone(sql, (bin2str(self.my_permid),)) + return numrecords + + + def getOtherChannels(self): ## + """Returns all the channels different from my channel + Returns a list of tuples: [(permid,channel_name,#votes)] + """ + records = [] + votecastdb = VoteCastDBHandler.getInstance() + sql = "select distinct publisher_id, publisher_name from ChannelCast" + channels = self._db.fetchall(sql) + for channel in channels: + if channel[0] != bin2str(self.my_permid): + num_votes = self.getSubscribersCount(channel[0]) + records.append((channel[0], channel[1], num_votes, {})) + if DEBUG: print >> sys.stderr , "records" , records + return records + + + + def getMySubscribedChannels(self, from_channelcast=False): + """return a list of tuples: [(permid,channel_name,#votes)]""" +# records = [] +# votecastdb = VoteCastDBHandler.getInstance() + #sql = "select mod_id, count(*) from VoteCast where mod_id in (select mod_id from VoteCast where voter_id='"+ bin2str(self.my_permid)+"' and vote=2) and mod_id<>'"+bin2str(self.my_permid)+"' group by mod_id order by 2 desc" + +# t1 = time() +# sql = "select mod_id, count(*) from VoteCast where mod_id <>'"+bin2str(self.my_permid)+"'" + " and vote=2 and voter_id='" + bin2str(self.my_permid) + "'" + " group by mod_id order by 2 desc" +# votes = self._db.fetchall(sql) +# for vote in votes: +# sql = "select publisher_name, time_stamp from ChannelCast where publisher_id='"+vote[0]+"' order by 2 desc" +# record = self._db.fetchone(sql) +# mod_name = record[0] +# records.append((vote[0],mod_name,vote[1])) +# t2 = time() +# print >> sys.stderr , "subscribed" , t2 - t1 + +# return records + + if DEBUG and from_channelcast: + print >> sys.stderr , "FROM CHANNELCAST" + + if not self.firstQueryMySubscriptions and not from_channelcast: + self.firstQueryMySubscriptions=True + return self.allRecordsMySubscriptions + + + + if DEBUG: + print >> sys.stderr , "getMySubscribedChannels" + allrecords = [] + + sql = "select distinct publisher_id, publisher_name from ChannelCast" + channel_records = self._db.fetchall(sql) + +# sql = "select mod_id, (2*sum(vote)-count(*))/3 from VoteCast group by mod_id order by 2 desc" + sql = "select mod_id, sum(vote-1) from VoteCast group by mod_id order by 2 desc" # only subscriptions, not spam votes + votecast_records = self._db.fetchall(sql) + + sql = "select distinct mod_id from VoteCast where voter_id==? and vote=2" + subscribed_channels = self._db.fetchall(sql,(bin2str(self.my_permid),)) + + + + subscribers = {} + for record in subscribed_channels: + subscribers[record[0]]="12" + + publishers = {} + for publisher_id, publisher_name in channel_records: + if publisher_id not in publishers and publisher_id in subscribers and publisher_id!=bin2str(self.my_permid): + publishers[publisher_id]=[publisher_name, 0] + + for mod_id, vote in votecast_records: + if mod_id in publishers: + publishers[mod_id][1] = vote + + for k, v in publishers.items(): + allrecords.append((k, v[0], v[1], {})) + def compare(a,b): + if a[2]>b[2] : return -1 + if a[2] maxvote: + publisher_id = publisher_item[0] + publisher_name = publisher_item[1] + maxvote = num_subscribers + channel = (publisher_id, publisher_name, maxvote, {}) + return channel + + + +class GUIDBHandler: + """ All the functions of this class are only (or mostly) used by GUI. + It is not associated with any db table, but will use any of them + """ + + __single = None # used for multithreaded singletons pattern + lock = threading.Lock() + + def getInstance(*args, **kw): + # Singleton pattern with double-checking + if GUIDBHandler.__single is None: + GUIDBHandler.lock.acquire() + try: + if GUIDBHandler.__single is None: + GUIDBHandler(*args, **kw) + finally: + GUIDBHandler.lock.release() + return GUIDBHandler.__single + + getInstance = staticmethod(getInstance) + + def __init__(self): + if GUIDBHandler.__single is not None: + raise RuntimeError, "GUIDBHandler is singleton" + self._db = SQLiteCacheDB.getInstance() + self.notifier = Notifier.getInstance() + GUIDBHandler.__single = self + + def getCommonFiles(self, permid): + peer_id = self._db.getPeerID(permid) + if peer_id is None: + return [] + + sql_get_common_files = """select name from CollectedTorrent where torrent_id in ( + select torrent_id from Preference where peer_id=? + and torrent_id in (select torrent_id from MyPreference) + ) and status_id <> 2 + """ + self.get_family_filter_sql() + res = self._db.fetchall(sql_get_common_files, (peer_id,)) + return [t[0] for t in res] + + def getOtherFiles(self, permid): + peer_id = self._db.getPeerID(permid) + if peer_id is None: + return [] + + sql_get_other_files = """select infohash,name from CollectedTorrent where torrent_id in ( + select torrent_id from Preference where peer_id=? + and torrent_id not in (select torrent_id from MyPreference) + ) and status_id <> 2 + """ + self.get_family_filter_sql() + res = self._db.fetchall(sql_get_other_files, (peer_id,)) + return [(str2bin(t[0]),t[1]) for t in res] + + def getSimItems(self, infohash, limit): + # recommendation based on collaborative filtering + torrent_id = self._db.getTorrentID(infohash) + if torrent_id is None: + return [] + + sql_get_sim_files = """ + select infohash, name, status_id, count(P2.torrent_id) c + from Preference as P1, Preference as P2, CollectedTorrent as T + where P1.peer_id=P2.peer_id and T.torrent_id=P2.torrent_id + and P2.torrent_id <> P1.torrent_id + and P1.torrent_id=? + and P2.torrent_id not in (select torrent_id from MyPreference) + %s + group by P2.torrent_id + order by c desc + limit ? + """ % self.get_family_filter_sql('T') + + res = self._db.fetchall(sql_get_sim_files, (torrent_id,limit)) + return [(str2bin(t[0]),t[1], t[2], t[3]) for t in res] + + def getSimilarTitles(self, name, limit, infohash, prefix_len=5): + # recommendation based on similar titles + name = name.replace("'","`") + sql_get_sim_files = """ + select infohash, name, status_id from Torrent + where name like '%s%%' + and infohash <> '%s' + and torrent_id not in (select torrent_id from MyPreference) + %s + order by name + limit ? + """ % (name[:prefix_len], bin2str(infohash), self.get_family_filter_sql()) + + res = self._db.fetchall(sql_get_sim_files, (limit,)) + return [(str2bin(t[0]),t[1], t[2]) for t in res] + + def _how_many_prefix(self): + """ test how long the prefix is enough to find similar titles """ + # Jie: I found 5 is the best value. + + sql = "select name from Torrent where name is not NULL order by name" + names = self._db.fetchall(sql) + + for top in range(3, 10): + sta = {} + for line in names: + prefix = line[0][:top] + if prefix not in sta: + sta[prefix] = 1 + else: + sta[prefix] += 1 + + res = [(v,k) for k,v in sta.items()] + res.sort() + res.reverse() + + print >> sys.stderr, '------------', top, '-------------' + for k in res[:10]: + print >> sys.stderr, k + + def get_family_filter_sql(self, table_name=''): + torrent_db_handler = TorrentDBHandler.getInstance() + return torrent_db_handler.category.get_family_filter_sql(torrent_db_handler._getCategoryID, table_name=table_name) + + +class PopularityDBHandler(BasicDBHandler): + ''' + @author: Rahim 04-2009 + This class handles access to Popularity tables that is used for + keeping swarm size info, received through BuddyCast messages. + ''' + __single = None # used for multithreaded singletons pattern + lock = threading.Lock() + + def getInstance(*args, **kw): + # Singleton pattern with double-checking + if PopularityDBHandler.__single is None: + PopularityDBHandler.lock.acquire() + try: + if PopularityDBHandler.__single is None: + PopularityDBHandler(*args, **kw) + finally: + PopularityDBHandler.lock.release() + return PopularityDBHandler.__single + getInstance = staticmethod(getInstance) + + def __init__(self): + if PopularityDBHandler.__single is not None: + raise RuntimeError, "PopularityDBHandler is singleton" + PopularityDBHandler.__single = self + db = SQLiteCacheDB.getInstance() + BasicDBHandler.__init__(self,db, 'Popularity') + + # define local handlers to access Peer and Torrent tables. + self.peer_db = PeerDBHandler.getInstance() + self.torrent_db = TorrentDBHandler.getInstance() + + ###-------------------------------------------------------------------------------------------------------------------------- + + def calculateSwarmSize(self, torrentList, content, toBC=True): + """ + This method gets a list of torrent_ids and then calculat the size of the swarm for those torrents. + @author: Rahim + @param torrentList: A list of torrent_id. + @param content: If it be 'infohash' , the content of the torrentList is infohsh of torrents. If it be 'torrentIds', the content is a list + of torrent_id. + @param toBc: This flag determines that whether the returned result will be used to create a new BC message or not. The difference is that nodes + just send first hand information to each other. The prevent speard of contamination if one of the nodes receive incorrent value from sender. + The main difference in the flow of the process is that, if toBC be set to False, this fucntion will look for the most recenct report inside + both Popularity and Torrent table, otherwise it will just use torrent table. + @return: returns a list with the same size az input and each items is composed of below items: + (torrent_id, num_seeders, num_leechers, num_of_sources) + """ + if content=='Infohash': + torrentList = [self.torrent_db.getTorrentID(infohash) for infohash in torrentList ] + elif content=='TorrentIds': + pass + else: + return [] + + trackerSizeList =[] + popularityList=[] + for torrentId in torrentList: + trackerSizeList.append(self.torrent_db.getSwarmInfo(torrentId)) + if not toBC: + popularityList.append(self.getPopularityList(torrent_id=torrentId)) + result =[] + timeNow=int(time()) + + averagePeerUpTime = 2 * 60 * 60 # we suppose that the average uptime is roughly two hours. + listIndex = 0 + for id in torrentList: + result.append([id, -1, -1, -1, -1]) # (torrent_id, calc_age, num_seeders, num_leechers, num_sources_seen) + if not toBC and len(popularityList[listIndex]) > 0 : + #if popularityList[listIndex][0] is not None: + latest = self.getLatestPopularityReport(popularityList[listIndex], timeNow) + result[listIndex][1] = latest[4] # num_seeders + result[listIndex][2] = latest[5] # num_leechers + result[listIndex][3] = timeNow - latest[2]+latest[3] # age of the report + result[listIndex][4] = latest[6] # num_sources + # print latest + if len(trackerSizeList[listIndex]) > 0 and len(trackerSizeList[listIndex][0]) > 0: + #if trackerSizeList[listIndex][0] is not None: + temp=trackerSizeList[listIndex][0] + tempAge = timeNow - temp[3] + if tempAge < result[listIndex][3]: + result[listIndex][1] = temp[1] #num_seeders + result[listIndex][2] = temp[2] #num_leechers + result[listIndex][3] = tempAge # Age of the tracker asking + othersSeenSources = self.torrent_db.getLargestSourcesSeen(id, timeNow, averagePeerUpTime) + result[listIndex][4] = max(temp[4], othersSeenSources) # num_sources + + elif len(trackerSizeList[listIndex]) > 0 and len(trackerSizeList[listIndex][0]) > 0: + #if trackerSizeList[listIndex][0] is not None: + temp=trackerSizeList[listIndex][0] + result[listIndex][1] = temp[1] #num seeders + result[listIndex][2] = temp[2] #num leechers + result[listIndex][3] = timeNow - temp[3] # age of check + result[listIndex][4] = temp[4] # num_sources + listIndex +=1 + + return result + + def getLatestPopularityReport(self, reportList, timeNow): + + """ + gets a list of list and then returns on of the them that has highest value in the specified index. + @author: Rahim + @param reportList: A list that contains popularity report for specified torrent. The first item contains torrent_id. + @param index: The index of item that comparision is done based on it. + @param timeNow: Indictes local time of the node that runs this process. + + """ + if len(reportList) ==0: + return [] + + result=reportList.pop(0) + listLength = len(reportList) + + for i in range(0,listLength): + if (timeNow - reportList[i][2] + reportList[i][3]) < (timeNow - result[2] + result[3]): #it selects the youngest report + result = reportList[i] + + return result + + + ###-------------------------------------------------------------------------------------------------------------------------- + def checkPeerValidity(self, peer_id): + ''' + checks whether the peer_id is valid or not, in other word it is in the Peer table or not? + @param peer_id: the id of the peer to be checked. + @return: True if the peer_is is valid, False if not. + ''' + if self.peer_db.getPermid(peer_id) is None: + return False + else: + return True + ###-------------------------------------------------------------------------------------------------------------------------- + def checkTorrentValidity(self, torrent_id): + ''' + checks whether the torrent_id is valid or not, in other word it is in the Torrent table or not? + @param torrent_id: the id of the torrent to be checked. + @return: True if the torrent_is is valid, False if not. + ''' + if self.torrent_db.getInfohash(torrent_id) is None: + return False + else: + return True + ###-------------------------------------------------------------------------------------------------------------------------- + def addPopularity(self, torrent_id, peer_id, recv_time, calc_age=sys.maxint, num_seeders=-1, num_leechers=-1, num_sources=-1, validatePeerId=False, validateTorrentId=False, + checkNumRecConstraint=True, commit=True): + ''' + Addes a new popularity record to the popularity table. + @param torrent_id: The id of the torrent that is added to the table. + @param peer_id: The id of the peer that is added to the table. + @param recv_time: The time that peer has received the message. + @param num_seeders: Number of seeders reportd by the remote peer. + @param num_leechers: Number of leechers reported by the remote peer. + @param num_sources: Number of the Tribler sources that have seen this torrent, reported by the remote peer. + @param calc_age: The time that the remote peer has calculated( or message send time) the swarm size. + @param validateTorrent: If set to True check validity of the Torrent otherwise no. + @param validatePeer: If set to True check validity of the Peer otherwise no. + ''' + if validatePeerId: # checks whether the peer is valid or not + if not self.checkPeerValidity(peer_id): + return None + if validateTorrentId: #checks whether the torrent is valid or not + if not self.checkTorrentValidity(torrent_id): + return None + + sql_delete_already_existing_record = u"""DELETE FROM Popularity WHERE torrent_id=? AND peer_id=? AND msg_receive_time=?""" + self._db.execute_write(sql_delete_already_existing_record, (torrent_id, peer_id, recv_time), commit=commit) + + + sql_insert_new_populairty = u"""INSERT INTO Popularity (torrent_id, peer_id, msg_receive_time, size_calc_age, num_seeders, + num_leechers, num_of_sources) VALUES (?,?,?,?,?,?,?)""" + try: + self._db.execute_write(sql_insert_new_populairty, (torrent_id, peer_id, recv_time, calc_age, num_seeders, num_leechers, num_sources), commit=commit) + except Exception, msg: + print_exc() + + timeNow = int(time()) + if checkNumRecConstraint: # Removes old records. The number of records should not exceed defined limitations. + + availableRecsT = self.countTorrentPopularityRec(torrent_id, timeNow) + if availableRecsT[0] > MAX_POPULARITY_REC_PER_TORRENT: + self.deleteOldTorrentRecords(torrent_id, availableRecsT[0] - MAX_POPULARITY_REC_PER_TORRENT, timeNow, commit=commit) + + + availableRecsTP = self.countTorrentPeerPopularityRec(torrent_id, peer_id, timeNow) + if availableRecsTP[0] > MAX_POPULARITY_REC_PER_TORRENT_PEER: + self.deleteOldTorrentPeerRecords(torrent_id,peer_id, availableRecsTP[0] - MAX_POPULARITY_REC_PER_TORRENT_PEER, timeNow, commit=commit) + + ###-------------------------------------------------------------------------------------------------------------------------- + def storePeerPopularity(self, peer_id, popularityList, validatePeerId=False, validateTorrentId=False, commit=True): + ''' + Insert all popularity info received through BuddyCast message. popularityList is a tuple of + @param peer_id: The id of the popularity info sender. + @param popularityList: A list of tuple (torrent_id, recv_time, calc_age, num_seeders, num_leechers, num_sources), usually received through BuddyCast message. + ''' + if validatePeerId: + if not self.checkPeerValidity(peer_id): + return None + + for item in popularityList[:-1]: + self.addPopularity(item[0], peer_id, item[1], item[2], item[3], item[4], item[5], validateTorrentId=validateTorrentId, commit=False) + + if len(popularityList)>0: + item = popularityList[-1] + self.addPopularity(item[0], peer_id, item[1], item[2], item[3], item[4], item[5], validateTorrentId=validateTorrentId, commit=commit) + ###-------------------------------------------------------------------------------------------------------------------------- + def countTorrentPopularityRec(self, torrent_id, timeNow): + ''' + This method counts the number of logged popularity for the input torrrent. + @param torrent_id: the id of the torrent + @return: (number_of_records, oldest_record_time) + ''' + + count_sql = "SELECT count(*) FROM Popularity WHERE torrent_id=?" + num_of_popularity = self._db.fetchone(count_sql,(torrent_id, )) + + if num_of_popularity > 0: + sql_oldest_record = "SELECT size_calc_age FROM Popularity WHERE torrent_id=? ORDER BY ( ? - msg_receive_time+size_calc_age) DESC LIMIT ?" + oldest_record_age = self._db.fetchone(sql_oldest_record, (torrent_id, timeNow, 1)) + return (num_of_popularity, oldest_record_age) + else: + if DEBUG: + print >> sys.stderr, "The torrent with the id ", torrent_id, " does not have any popularity record." + return (0 , sys.maxint) + ###-------------------------------------------------------------------------------------------------------------------------- + def countTorrentPeerPopularityRec(self, torrent_id, peer_id, timeNow): + ''' + counts the number of popularity records done for the input torrent_id by the input peer_id. + @param torrent_id: the id of the torrent. + @param peer_id: the id of the peer. + @return: (number_of_records, oldest_record_time) with same torrent_id and peer_id as input. + ''' + count_sql = "SELECT count(*) FROM Popularity WHERE torrent_id=? AND peer_id=?" + num_of_popularity = self._db.fetchone(count_sql,(torrent_id, peer_id)) + + if num_of_popularity > 0: + sql_oldest_record = "SELECT size_calc_age FROM Popularity WHERE torrent_id=? AND peer_id=? ORDER BY ( ? - msg_receive_time+size_calc_age) DESC LIMIT ?" + oldest_record_age = self._db.fetchone(sql_oldest_record, (torrent_id, peer_id, timeNow, 1)) + return (num_of_popularity, oldest_record_age) + else: + if DEBUG: + print >> sys.stderr, "The peer with the id ", peer_id, "has not reported any thing about the torrent: ", torrent_id + return (0 , sys.maxint) + ###-------------------------------------------------------------------------------------------------------------------------- + def deleteOldTorrentRecords(self, torrent_id, num_rec_to_delete, timeNow, commit=True): + ''' + Deletes the oldest num_rec_to_del popularity records about the torrect_id from popularity table. + @param torrent_id: the id of the torrent. + @param num_rec_to_delete: Number of the oldest records that should be removed from the table. + ''' + + sql_delete = u""" DELETE FROM Popularity WHERE torrent_id=? AND size_calc_age IN + (SELECT size_calc_age FROM Popularity WHERE torrent_id=? + ORDER BY (? - msg_receive_time+size_calc_age) DESC LIMIT ?)""" + + self._db.execute_write(sql_delete, (torrent_id, torrent_id, timeNow, num_rec_to_delete), commit=commit) + + ###-------------------------------------------------------------------------------------------------------------------------- + def deleteOldTorrentPeerRecords(self, torrent_id, peer_id, num_rec_to_delete, timeNow, commit=True): + ''' + Deletes the oldest num_rec_to_del popularity records about the torrect_id repported by peer_id from popularity table. + @param torrent_id: the id of the torrent. + @param peer_id: the id of the popularity sender. + @param num_rec_to_delete: Number of the oldest records that should be removed from the table. + ''' + + sql_delete = u""" DELETE FROM Popularity where torrent_id=? AND peer_id=? AND size_calc_age IN + (SELECT size_calc_age FROM popularity WHERE torrent_id=? AND peer_id=? + ORDER BY (? - msg_receive_time+size_calc_age) DESC LIMIT ?)""" + + self._db.execute_write(sql_delete, (torrent_id, peer_id,torrent_id, peer_id,timeNow, num_rec_to_delete), commit=commit) + + ###-------------------------------------------------------------------------------------------------------------------------- + def getPopularityList(self, torrent_id=None, peer_id=None , recv_time_lbound=0, recv_time_ubound=sys.maxint): + ''' + Returns a list of the records from the Popularity table, by using input parameters. + @param torremt_id: The id of the torrent. + @param peer_id: The id of the peer. + @param recv_time_lbound: Lower bound for the message receive time. Default value is 0. + @param recv_time_ubound: Upper bound for the message receive time. Default value is 0x10000000L + @return: A list of tuple (torrent_id, recv_time, calc_age, num_seeders, num_leechers, num_sources) + ''' + sql_getPopList=" SELECT * FROM Popularity" + + if (torrent_id is not None) or (peer_id is not None) or (not recv_time_lbound==0) or (not recv_time_ubound==sys.maxint): + sql_getPopList += " WHERE " + + if torrent_id is not None: + sql_getPopList += "torrent_id = %s" % torrent_id + if (peer_id is not None) or (not recv_time_lbound==0) or (not recv_time_ubound==sys.maxint): + sql_getPopList += " AND " + + if peer_id is not None: + sql_getPopList += "peer_id = %d" % peer_id + if (not recv_time_lbound==0) or (not recv_time_ubound==sys.maxint): + sql_getPopList += " AND " + + if not recv_time_lbound==0: + sql_getPopList += "msg_receive_time >= %d" % recv_time_lbound + if not recv_time_ubound==sys.maxint: + sql_getPopList += " AND " + + if not recv_time_ubound==sys.maxint: + sql_getPopList += "msg_receive_time <= %d" % recv_time_ubound + + print sql_getPopList + popularityList = self._db.fetchall(sql_getPopList) + + return popularityList + + ###---------------------------------------------------------------------------------------------------------------------- + + def addPopularityRecord(self, peer_permid, pops, selversion, recvTime, is_torrent_id=False, commit=True): + """ + """ + + peer_id = self._db.getPeerID(peer_permid) + if peer_id is None: + print >> sys.stderr, 'PopularityDBHandler: update received popularity list from a peer that is not existed in Peer table', `peer_permid` + return + + pops = [type(pop) is str and {"infohash":pop} or pop + for pop + in pops] + + if __debug__: + for pop in pops: + assert isinstance(pop["infohash"], str), "INFOHASH has invalid type: %s" % type(pop["infohash"]) + assert len(pop["infohash"]) == INFOHASH_LENGTH, "INFOHASH has invalid length: %d" % len(pop["infohash"]) + + if is_torrent_id: + #Rahim : Since overlay version 11 swarm size information is + # appended and should be added to the database . The codes below + # does this. torrent_id, recv_time, calc_age, num_seeders, + # num_leechers, num_sources + # + torrent_id_swarm_size =[] + for pop in pops: + if pop is not None: + tempAge = pop.get('calc_age') + tempSeeders = pop.get('num_seeders') + tempLeechers = pop.get('num_leechers') + if tempAge > 0 and tempSeeders >= 0 and tempLeechers >= 0: + torrent_id_swarm_size.append( [pop['torrent_id'], + recvTime, + tempAge, + tempSeeders, + tempLeechers, + pop.get('num_sources_seen', -1)]# -1 means invalud value + ) + else: + torrent_id_swarm_size = [] + for pop in pops: + if type(pop)==dict: + infohash = pop["infohash"] + else: + # Nicolas: from wherever this might come, we even handle + # old list of infohashes style + infohash = pop + torrent_id = self._db.getTorrentID(infohash) + if not torrent_id: + self._db.insertInfohash(infohash) + torrent_id = self._db.getTorrentID(infohash) + #Rahim: Amended for handling and adding swarm size info. + #torrent_id_swarm_size.append((torrent_id, timeNow,0, -1, -1, -1)) + if len(torrent_id_swarm_size) > 0: + try: + #popularity_db = PopularityDBHandler.getInstance() + #popularity_db.storePeerPopularity(peer_id, torrent_id_swarm_size, commit=commit) + self.storePeerPopularity(peer_id, torrent_id_swarm_size, commit=commit) + except Exception, msg: + print_exc() + print >> sys.stderr, 'dbhandler: updatePopularity:', Exception, msg + +class TermDBHandler(BasicDBHandler): + + __single = None # used for multithreaded singletons pattern + lock = threading.Lock() + + def getInstance(*args, **kw): + # Singleton pattern with double-checking + if TermDBHandler.__single is None: + TermDBHandler.lock.acquire() + try: + if TermDBHandler.__single is None: + TermDBHandler(*args, **kw) + finally: + TermDBHandler.lock.release() + return TermDBHandler.__single + getInstance = staticmethod(getInstance) + + def __init__(self): + if TermDBHandler.__single is not None: + raise RuntimeError, "TermDBHandler is singleton" + TermDBHandler.__single = self + db = SQLiteCacheDB.getInstance() + BasicDBHandler.__init__(self,db, 'ClicklogTerm') + + + def getNumTerms(self): + """returns number of terms stored""" + return self.getOne("count(*)") + + + + def bulkInsertTerms(self, terms, commit=True): + for term in terms: + term_id = self.getTermIDNoInsert(term) + if not term_id: + self.insertTerm(term, commit=False) # this HAS to commit, otherwise last_insert_row_id() won't work. + # if you want to avoid committing too often, use bulkInsertTerm + if commit: + self.commit() + + def getTermIDNoInsert(self, term): + return self.getOne('term_id', term=term[:MAX_KEYWORD_LENGTH].lower()) + + def getTermID(self, term): + """returns the ID of term in table ClicklogTerm; creates a new entry if necessary""" + term_id = self.getTermIDNoInsert(term) + if term_id: + return term_id + else: + self.insertTerm(term, commit=True) # this HAS to commit, otherwise last_insert_row_id() won't work. + return self.getOne("last_insert_rowid()") + + def insertTerm(self, term, commit=True): + """creates a new entry for term in table Term""" + self._db.insert(self.table_name, commit=commit, term=term[:MAX_KEYWORD_LENGTH]) + + def getTerm(self, term_id): + """returns the term for a given term_id""" + return self.getOne("term", term_id=term_id) + # if term_id==-1: + # return "" + # term = self.getOne('term', term_id=term_id) + # try: + # return str2bin(term) + # except: + # return term + + def getTermsStartingWith(self, beginning, num=10): + """returns num most frequently encountered terms starting with beginning""" + + # request twice the amount of hits because we need to apply + # the familiy filter... + terms = self.getAll('term', + term=("like", u"%s%%" % beginning), + order_by="times_seen DESC", + limit=num * 2) + + if terms: + # terms is a list containing lists. We only want the first + # item of the inner lists. + terms = [term for (term,) in terms] + + catobj = Category.getInstance() + if catobj.family_filter_enabled(): + return filter(lambda term: not catobj.xxx_filter.foundXXXTerm(term), terms)[:num] + else: + return terms[:num] + + else: + return [] + + def getAllEntries(self): + """use with caution,- for testing purposes""" + return self.getAll("term_id, term", order_by="term_id") + +class SimilarityDBHandler: + __single = None # used for multithreaded singletons pattern + lock = threading.Lock() + + def getInstance(*args, **kw): + # Singleton pattern with double-checking + if SimilarityDBHandler.__single is None: + SimilarityDBHandler.lock.acquire() + try: + if SimilarityDBHandler.__single is None: + SimilarityDBHandler(*args, **kw) + finally: + SimilarityDBHandler.lock.release() + return SimilarityDBHandler.__single + getInstance = staticmethod(getInstance) + + def __init__(self): + if SimilarityDBHandler.__single is not None: + raise RuntimeError, "SimilarityDBHandler is singleton" + SimilarityDBHandler.__single = self + self._db = SQLiteCacheDB.getInstance() + + def getOverlapWithPeer(self, peer_id, myprefs): + sql_get_overlap_with_peer = """SELECT Peer.peer_id, num_prefs, COUNT(torrent_id) FROM Peer + JOIN Preference ON Peer.peer_id = Preference.peer_id + WHERE torrent_id IN("""+','.join(map(str,myprefs))+""") + AND Peer.peer_id = ? GROUP BY Peer.peer_id""" + row = self._db.fetchone(sql_get_overlap_with_peer, (peer_id,)) + return row + + def getPeersWithOverlap(self, not_peer_id, myprefs): + sql_get_peers_with_overlap = """SELECT Peer.peer_id, num_prefs, COUNT(torrent_id) FROM Peer + JOIN Preference ON Peer.peer_id = Preference.peer_id + WHERE torrent_id IN("""+','.join(map(str,myprefs))+""") + AND Peer.peer_id <> ? GROUP BY Peer.peer_id""" + row = self._db.fetchall(sql_get_peers_with_overlap, (not_peer_id,)) + return row + + def getTorrentsWithSimilarity(self, myprefs, top_x): + sql_get_torrents_with_similarity = """SELECT similarity, torrent_id FROM Peer + JOIN Preference ON Peer.peer_id = Preference.peer_id + WHERE Peer.peer_id IN(Select peer_id from Peer WHERE similarity > 0 ORDER By similarity DESC Limit ?) + AND torrent_id NOT IN(""" + ','.join(map(str,myprefs))+""")""" + row = self._db.fetchall(sql_get_torrents_with_similarity, (top_x,)) + return row + + + + +class SearchDBHandler(BasicDBHandler): + + __single = None # used for multithreaded singletons pattern + lock = threading.Lock() + + def getInstance(*args, **kw): + # Singleton pattern with double-checking + if SearchDBHandler.__single is None: + SearchDBHandler.lock.acquire() + try: + if SearchDBHandler.__single is None: + SearchDBHandler(*args, **kw) + finally: + SearchDBHandler.lock.release() + return SearchDBHandler.__single + getInstance = staticmethod(getInstance) + + def __init__(self): + if SearchDBHandler.__single is not None: + raise RuntimeError, "SearchDBHandler is singleton" + SearchDBHandler.__single = self + db = SQLiteCacheDB.getInstance() + BasicDBHandler.__init__(self,db, 'ClicklogSearch') ## self,db,'Search' + + + ### write methods + + def storeKeywordsByID(self, peer_id, torrent_id, term_ids, commit=True): + sql_insert_search = u"INSERT INTO ClicklogSearch (peer_id, torrent_id, term_id, term_order) values (?, ?, ?, ?)" + + if len(term_ids)>MAX_KEYWORDS_STORED: + term_ids= term_ids[0:MAX_KEYWORDS_STORED] + + # TODO before we insert, we should delete all potentially existing entries + # with these exact values + # otherwise, some strange attacks might become possible + # and again we cannot assume that user/torrent/term only occurs once + + # create insert data + values = [(peer_id, torrent_id, term_id, term_order) + for (term_id, term_order) + in zip(term_ids, range(len(term_ids)))] + self._db.executemany(sql_insert_search, values, commit=commit) + + # update term popularity + sql_update_term_popularity= u"UPDATE ClicklogTerm SET times_seen = times_seen+1 WHERE term_id=?" + self._db.executemany(sql_update_term_popularity, [[term_id] for term_id in term_ids], commit=commit) + + def storeKeywords(self, peer_id, torrent_id, terms, commit=True): + """creates a single entry in Search with peer_id and torrent_id for every term in terms""" + terms = [term.strip() for term in terms if len(term.strip())>0] + term_db = TermDBHandler.getInstance() + term_ids = [term_db.getTermID(term) for term in terms] + self.storeKeywordsByID(peer_id, torrent_id, term_ids, commit) + + def getAllEntries(self): + """use with caution,- for testing purposes""" + return self.getAll("rowid, peer_id, torrent_id, term_id, term_order ", order_by="rowid") + + def getAllOwnEntries(self): + """use with caution,- for testing purposes""" + return self.getAll("rowid, peer_id, torrent_id, term_id, term_order ", where="peer_id=0", order_by="rowid") + + + + ### read methods + + def getNumTermsPerTorrent(self, torrent_id): + """returns the number of terms associated with a given torrent""" + return self.getOne("COUNT (DISTINCT term_id)", torrent_id=torrent_id) + + def getNumTorrentsPerTerm(self, term_id): + """returns the number of torrents stored with a given term.""" + return self.getOne("COUNT (DISTINCT torrent_id)", term_id=term_id) + + def getNumTorrentTermCooccurrences(self, term_id, torrent_id): + """returns the number of times a torrent has been associated with a term""" + return self.getOne("COUNT (*)", term_id=term_id, torrent_id=torrent_id) + + def getRelativeTermFrequency(self, term_id, torrent_id): + """returns the relative importance of a term for a torrent + This is basically tf/idf + term frequency tf = # keyword used per torrent/# keywords used with torrent at all + inverse document frequency = # of torrents associated with term at all + + normalization in tf ensures that a torrent cannot get most important for all keywords just + by, e.g., poisoning the db with a lot of keywords for this torrent + idf normalization ensures that returned values are meaningful across several keywords + """ + + terms_per_torrent = self.getNumTermsPerTorrent(torrent_id) + if terms_per_torrent==0: + return 0 + + torrents_per_term = self.getNumTorrentsPerTerm(term_id) + if torrents_per_term == 0: + return 0 + + coocc = self.getNumTorrentTermCooccurrences(term_id, torrent_id) + + tf = coocc/float(terms_per_torrent) + idf = 1.0/math.log(torrents_per_term+1) + + return tf*idf + + + def getTorrentSearchTerms(self, torrent_id, peer_id): + return self.getAll("term_id", "torrent_id=%d AND peer_id=%s" % (torrent_id, peer_id), order_by="term_order") + + def getMyTorrentSearchTerms(self, torrent_id): + return [x[0] for x in self.getTorrentSearchTerms(torrent_id, peer_id=0)] + + + ### currently unused + + def numSearchesWithTerm(self, term_id): + """returns the number of searches stored with a given term. + I feel like I might miss something, but this should simply be the number of rows containing + the term""" + return self.getOne("COUNT (*)", term_id=term_id) + + def getNumTorrentPeers(self, torrent_id): + """returns the number of users for a given torrent. if this should be used + extensively, an index on torrent_id might be in order""" + return self.getOne("COUNT (DISTINCT peer_id)", torrent_id=torrent_id) + + def removeKeywords(self, peer_id, torrent_id, commit=True): + """removes records of keywords used by peer_id to find torrent_id""" + # TODO + # would need to be called by deletePreference + pass + + + + +def doPeerSearchNames(self,dbname,kws): + """ Get all peers that have the specified keywords in their name. + Return a list of dictionaries. Each dict is in the NEWDBSTANDARD format. + """ + if dbname == 'Peer': + where = '(Peer.last_connected>0 or Peer.friend=1) and ' + elif dbname == 'Friend': + where = '' + else: + raise Exception('unknown dbname: %s' % dbname) + + # Must come before query + ranks = self.getRanks() + + for i in range(len(kws)): + kw = kws[i] + where += ' name like "%'+kw+'%"' + if (i+1) != len(kws): + where += ' and' + + # See getGUIPeers() + value_name = PeerDBHandler.gui_value_name + + #print >>sys.stderr,"peer_db: searchNames: sql",where + res_list = self._db.getAll(dbname, value_name, where) + #print >>sys.stderr,"peer_db: searchNames: res",res_list + + peer_list = [] + for item in res_list: + #print >>sys.stderr,"peer_db: searchNames: Got Record",`item` + peer = dict(zip(value_name, item)) + peer['name'] = dunno2unicode(peer['name']) + peer['simRank'] = ranksfind(ranks,peer['permid']) + peer['permid'] = str2bin(peer['permid']) + peer_list.append(peer) + return peer_list + +def ranksfind(ranks,key): + if ranks is None: + return -1 + try: + return ranks.index(key)+1 + except: + return -1 + diff --git a/instrumentation/next-share/BaseLib/Core/CacheDB/SqliteFriendshipStatsCacheDB.py b/instrumentation/next-share/BaseLib/Core/CacheDB/SqliteFriendshipStatsCacheDB.py new file mode 100644 index 0000000..a368148 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/CacheDB/SqliteFriendshipStatsCacheDB.py @@ -0,0 +1,201 @@ +# Written by Ali Abbas +# see LICENSE.txt for license information + +import sys +import os +import threading + +from BaseLib.__init__ import LIBRARYNAME +from BaseLib.Core.CacheDB.sqlitecachedb import * +from BaseLib.Core.CacheDB.SqliteCacheDBHandler import BasicDBHandler + +CREATE_FRIENDSHIP_STATS_SQL_FILE = None +CREATE_FRIENDSHIP_STATS_SQL_FILE_POSTFIX = os.path.join(LIBRARYNAME, 'Core', 'Statistics', 'tribler_friendship_stats_sdb.sql') +DB_FILE_NAME = 'tribler_friendship_stats.sdb' +DB_DIR_NAME = 'sqlite' # db file path = DB_DIR_NAME/DB_FILE_NAME +CURRENT_DB_VERSION = 2 + +DEBUG = False + +def init_friendship_stats(config, db_exception_handler = None): + """ create friendship statistics database """ + global CREATE_FRIENDSHIP_STATS_SQL_FILE + config_dir = config['state_dir'] + install_dir = config['install_dir'] + CREATE_FRIENDSHIP_STATS_SQL_FILE = os.path.join(install_dir,CREATE_FRIENDSHIP_STATS_SQL_FILE_POSTFIX) + sqlitedb = SQLiteFriendshipStatsCacheDB.getInstance(db_exception_handler) + sqlite_db_path = os.path.join(config_dir, DB_DIR_NAME, DB_FILE_NAME) + sqlitedb.initDB(sqlite_db_path, CREATE_FRIENDSHIP_STATS_SQL_FILE,current_db_version=CURRENT_DB_VERSION) # the first place to create db in Tribler + return sqlitedb + + +class FSCacheDBBaseV2(SQLiteCacheDBBase): + """ See Tribler/Core/Statistics/tribler_friendship_stats_sdb.sql + for a description of the various versions + """ + + def updateDB(self,fromver,tover): + if DEBUG: + print >>sys.stderr,"fscachedb2: Upgrading",fromver,tover + if fromver == 1 and tover == 2: + # Do ALTER TABLE stuff to add crawler_permid field. + sql = "ALTER TABLE FriendshipStatistics ADD COLUMN crawled_permid TEXT DEFAULT client NOT NULL;" + self.execute_write(sql, commit=False) + # updating version stepwise so if this works, we store it + # regardless of later, potentially failing updates + self.writeDBVersion(2, commit=False) + self.commit() + + +class SQLiteFriendshipStatsCacheDB(FSCacheDBBaseV2): + __single = None # used for multithreaded singletons pattern + lock = threading.RLock() + + @classmethod + def getInstance(cls, *args, **kw): + # Singleton pattern with double-checking to ensure that it can only create one object + if cls.__single is None: + cls.lock.acquire() + try: + if cls.__single is None: + cls.__single = cls(*args, **kw) + finally: + cls.lock.release() + return cls.__single + + def __init__(self, *args, **kw): + # always use getInstance() to create this object + if self.__single != None: + raise RuntimeError, "SQLiteFriendshipStatsCacheDB is singleton" + + FSCacheDBBaseV2.__init__(self, *args, **kw) + + + +class FriendshipStatisticsDBHandler(BasicDBHandler): + + __single = None # used for multi-threaded singletons pattern + lock = threading.Lock() + + def getInstance(*args, **kw): + # Singleton pattern with double-checking + if FriendshipStatisticsDBHandler.__single is None: + FriendshipStatisticsDBHandler.lock.acquire() + try: + if FriendshipStatisticsDBHandler.__single is None: + FriendshipStatisticsDBHandler(*args, **kw) + finally: + FriendshipStatisticsDBHandler.lock.release() + return FriendshipStatisticsDBHandler.__single + + getInstance = staticmethod(getInstance) + + def __init__(self): + if FriendshipStatisticsDBHandler.__single is not None: + raise RuntimeError, "FriendshipStatisticsDBHandler is singleton" + FriendshipStatisticsDBHandler.__single = self + db = SQLiteFriendshipStatsCacheDB.getInstance() + BasicDBHandler.__init__(self, db, 'FriendshipStatistics') + #BasicDBHandler.__init__(self, 'Peer') + #self.tableName = 'FriendshipStatistics' + + + def getAllFriendshipStatistics(self, permid, last_update_time = None, range = None, sort = None, reverse = False): + + """ + db keys: 'source_permid', 'target_permid', 'isForwarder', 'request_time', 'response_time', + 'no_of_attempts', 'no_of_helpers' + + @in: get_online: boolean: if true, give peers a key 'online' if there is a connection now + """ + + value_name = ('source_permid', 'target_permid', 'isForwarder', 'request_time', 'response_time', 'no_of_attempts', + 'no_of_helpers', 'modified_on') + where = 'request_time > '+str(last_update_time) # source_permid done below + + if range: + offset= range[0] + limit = range[1] - range[0] + else: + limit = offset = None + if sort: + desc = (not reverse) and 'desc' or '' + if sort in ('name'): + order_by = ' lower(%s) %s' % (sort, desc) + else: + order_by = ' %s %s' % (sort, desc) + else: + order_by = None + + permidstr = bin2str(permid) + res_list = self.getAll(value_name, where=where, offset= offset, limit=limit, order_by=order_by, source_permid=permidstr) + + if DEBUG: + print >>sys.stderr,"FriendshipStatisticsDBHandler: getAll: result is",res_list + + return res_list + + def saveFriendshipStatisticData (self, data): + + self._db.insertMany('FriendshipStatistics', data) + + def insertFriendshipStatistics(self, my_permid, target_permid, current_time, isForwarder = 0, no_of_attempts = 0, no_of_helpers = 0, commit = True): + +# db keys: 'source_permid', 'target_permid', 'isForwarder', 'request_time', 'response_time', +# 'no_of_attempts', 'no_of_helpers' +# self._db.insert(self.table_name, entry=key, value=value) + + sql_insert_friendstatistics = "INSERT INTO FriendshipStatistics (source_permid, target_permid, isForwarder, request_time, response_time, no_of_attempts, no_of_helpers, modified_on) VALUES ('"+my_permid+"','"+target_permid+"',"+str(isForwarder)+","+str(current_time)+", 0 , "+str(no_of_attempts)+","+str(no_of_helpers)+","+str(current_time)+")" + + self._db.execute_write(sql_insert_friendstatistics,commit=commit) + + def updateFriendshipStatistics(self, my_permid, target_permid, current_time, isForwarder = 0, no_of_attempts = 0, no_of_helpers = 0, commit = True): + + sql_insert_friendstatistics = "UPDATE FriendshipStatistics SET request_time = "+str(current_time) +", no_of_attempts = "+str(no_of_attempts)+", no_of_helpers = "+str(no_of_helpers)+", modified_on = "+str(current_time)+" where source_permid = '"+my_permid+"' and target_permid = '"+target_permid+"'" + + self._db.execute_write(sql_insert_friendstatistics,commit=commit) + + def updateFriendshipResponseTime(self, my_permid, target_permid, current_time, commit = True): + + + sql_insert_friendstatistics = "UPDATE FriendshipStatistics SET response_time = "+str(current_time)+ ", modified_on = "+str(current_time)+" where source_permid = '"+my_permid+"' and target_permid = '"+target_permid+"'" + + if DEBUG: + print >> sys.stderr, sql_insert_friendstatistics + + self._db.execute_write(sql_insert_friendstatistics,commit=commit) + + def insertOrUpdateFriendshipStatistics(self, my_permid, target_permid, current_time, isForwarder = 0, no_of_attempts = 0, no_of_helpers = 0, commit = True): + +# sql_entry_exists_of_the_peer = "SELECT souce_permid FROM FriendshipStatistics where source_permid = " + my_permid + if DEBUG: + print >> sys.stderr, 'Friendship record being inserted of permid' + print >> sys.stderr, target_permid + res = self._db.getOne('FriendshipStatistics', 'target_permid', target_permid=target_permid) + + if not res: + sql_insert_friendstatistics = "INSERT INTO FriendshipStatistics (source_permid, target_permid, isForwarder, request_time, response_time, no_of_attempts, no_of_helpers, modified_on) VALUES ('"+my_permid+"','"+target_permid+"',"+str(isForwarder)+","+str(current_time)+", 0 , "+str(no_of_attempts)+","+str(no_of_helpers)+","+str(current_time)+")" + else: + sql_insert_friendstatistics = "UPDATE FriendshipStatistics SET no_of_attempts = "+str(no_of_attempts)+", no_of_helpers = "+str(no_of_helpers)+", modified_on = "+str(current_time)+" where source_permid = '"+my_permid+"' and target_permid = '"+target_permid+"'" + + if DEBUG: + print >> sys.stderr, 'result is ', res + print >> sys.stderr, sql_insert_friendstatistics + + try: + self._db.execute_write(sql_insert_friendstatistics,commit=commit) + except: + print >> sys.stderr + + + def getLastUpdateTimeOfThePeer(self, permid): + + res = self._db.getAll('FriendshipStatistics', 'source_permid', order_by='modified_on desc', limit=1) + + if not res: + return 0 + else: + # todo! + return 0 # bug??? res['modified_on'] + + diff --git a/instrumentation/next-share/BaseLib/Core/CacheDB/SqliteSeedingStatsCacheDB.py b/instrumentation/next-share/BaseLib/Core/CacheDB/SqliteSeedingStatsCacheDB.py new file mode 100644 index 0000000..06adfcc --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/CacheDB/SqliteSeedingStatsCacheDB.py @@ -0,0 +1,202 @@ +# Written by Boxun Zhang +# see LICENSE.txt for license information + +import os +from time import time +import threading +from traceback import print_exc + +from BaseLib.__init__ import LIBRARYNAME +from BaseLib.Core.CacheDB.sqlitecachedb import * +from BaseLib.Core.CacheDB.SqliteCacheDBHandler import BasicDBHandler +from BaseLib.Core.simpledefs import * + +CREATE_SEEDINGSTATS_SQL_FILE = None +CREATE_SEEDINGSTATS_SQL_FILE_POSTFIX = os.path.join(LIBRARYNAME, 'Core', 'Statistics', 'tribler_seedingstats_sdb.sql') +DB_FILE_NAME = 'tribler_seedingstats.sdb' +DB_DIR_NAME = 'sqlite' # db file path = DB_DIR_NAME/DB_FILE_NAME +CURRENT_DB_VERSION = 1 +DEFAULT_BUSY_TIMEOUT = 10000 +MAX_SQL_BATCHED_TO_TRANSACTION = 1000 # don't change it unless carefully tested. A transaction with 1000 batched updates took 1.5 seconds +SHOW_ALL_EXECUTE = False +costs = [] +cost_reads = [] + +DEBUG = False + +def init_seeding_stats(config, db_exception_handler = None): + """ create SeedingStats database """ + global CREATE_SEEDINGSTATS_SQL_FILE + config_dir = config['state_dir'] + install_dir = config['install_dir'] + CREATE_SEEDINGSTATS_SQL_FILE = os.path.join(install_dir,CREATE_SEEDINGSTATS_SQL_FILE_POSTFIX) + sqlitedb = SQLiteSeedingStatsCacheDB.getInstance(db_exception_handler) + sqlite_db_path = os.path.join(config_dir, DB_DIR_NAME, DB_FILE_NAME) + sqlitedb.initDB(sqlite_db_path, CREATE_SEEDINGSTATS_SQL_FILE) # the first place to create db in Tribler + return sqlitedb + +class SQLiteSeedingStatsCacheDB(SQLiteCacheDBBase): + __single = None # used for multithreaded singletons pattern + lock = threading.RLock() + + @classmethod + def getInstance(cls, *args, **kw): + # Singleton pattern with double-checking to ensure that it can only create one object + if cls.__single is None: + cls.lock.acquire() + try: + if cls.__single is None: + cls.__single = cls(*args, **kw) + finally: + cls.lock.release() + return cls.__single + + def __init__(self, *args, **kw): + # always use getInstance() to create this object + if self.__single != None: + raise RuntimeError, "SQLiteSeedingStatsCacheDB is singleton" + + SQLiteCacheDBBase.__init__(self, *args, **kw) + + +class SeedingStatsDBHandler(BasicDBHandler): + + __single = None # used for multithreaded singletons pattern + lock = threading.Lock() + + def getInstance(*args, **kw): + # Singleton pattern with double-checking + if SeedingStatsDBHandler.__single is None: + SeedingStatsDBHandler.lock.acquire() + try: + if SeedingStatsDBHandler.__single is None: + SeedingStatsDBHandler(*args, **kw) + finally: + SeedingStatsDBHandler.lock.release() + return SeedingStatsDBHandler.__single + + getInstance = staticmethod(getInstance) + + def __init__(self): + if SeedingStatsDBHandler.__single is not None: + raise RuntimeError, "SeedingStatDBHandler is singleton" + SeedingStatsDBHandler.__single = self + db = SQLiteSeedingStatsCacheDB.getInstance() + BasicDBHandler.__init__(self, db, 'SeedingStats') + + def updateSeedingStats(self, permID, reputation, dslist, interval): + permID = bin2str(permID) + + seedings = [] + + for item in dslist: + if item.get_status() == DLSTATUS_SEEDING: + seedings.append(item) + + commit = False + for i in range(0, len(seedings)): + ds = seedings[i] + + infohash = bin2str(ds.get_download().get_def().get_infohash()) + + stats = ds.stats['stats'] + ul = stats.upTotal + + if i == len(seedings)-1: + commit = True + + res = self.existedInfoHash(infohash) + + if res is not None: + # res is list of ONE tuple + #self.updateSeedingStat(infohash, reputation, res[0][0], interval, commit) + + # NAT/Firewall & Seeding Behavior + # Store upload amount instead peer reputation + self.updateSeedingStat(infohash, ul, res[0][0], interval, commit) + else: + # Insert new record + #self.insertSeedingStat(infohash, permID, reputation, interval, commit) + + # NAT/Firewall & Seeding Behavior + # Store upload amount instead peer reputation + self.insertSeedingStat(infohash, permID, ul, interval, commit) + + + def existedInfoHash(self, infohash): + + sql = "SELECT seeding_time FROM SeedingStats WHERE info_hash='%s' and crawled=0"%infohash + + try: + cursor = self._db.execute_read(sql) + if cursor: + res = list(cursor) + + if len(res) > 0: + return res + else: + return None + else: + # something wrong, throw an exception? + return None + except: + return None + + def updateSeedingStat(self, infohash, reputation, seedingtime, interval, commit): + try: + sql_update = "UPDATE SeedingStats SET seeding_time=%s, reputation=%s WHERE info_hash='%s' AND crawled=0"%(seedingtime + interval, reputation, infohash) + self._db.execute_write(sql_update, None, commit) + except: + print_exc() + + def insertSeedingStat(self, infohash, permID, reputation, interval, commit): + try: + sql_insert = "INSERT INTO SeedingStats VALUES(%s, '%s', '%s', %s, %s, %s)"%(time(), permID, infohash, interval, reputation, 0) + self._db.execute_write(sql_insert, None, commit) + except: + print_exc() + + +class SeedingStatsSettingsDBHandler(BasicDBHandler): + + __single = None # used for multithreaded singletons pattern + lock = threading.Lock() + + def getInstance(*args, **kw): + # Singleton pattern with double-checking + if SeedingStatsSettingsDBHandler.__single is None: + SeedingStatsSettingsDBHandler.lock.acquire() + try: + if SeedingStatsSettingsDBHandler.__single is None: + SeedingStatsSettingsDBHandler(*args, **kw) + finally: + SeedingStatsSettingsDBHandler.lock.release() + return SeedingStatsSettingsDBHandler.__single + + getInstance = staticmethod(getInstance) + + def __init__(self): + if SeedingStatsSettingsDBHandler.__single is not None: + raise RuntimeError, "SeedingStatDBHandler is singleton" + SeedingStatsSettingsDBHandler.__single = self + db = SQLiteSeedingStatsCacheDB.getInstance() + BasicDBHandler.__init__(self, db, 'CrawlingSettings') + + def loadCrawlingSettings(self): + try: + sql_query = "SELECT * FROM SeedingStatsSettings" + cursor = self._db.execute_read(sql_query) + + if cursor: + return list(cursor) + else: + return None + except: + print_exc() + + def updateCrawlingSettings(self, args): + try: + sql_update = "UPDATE SeedingStatsSettings SET crawling_interval=%s, crawling_enabled=%s WHERE version=1"%(args[0], args[1]) + cursor = self._db.execute_write(sql_update) + except: + print_exc() diff --git a/instrumentation/next-share/BaseLib/Core/CacheDB/SqliteVideoPlaybackStatsCacheDB.py b/instrumentation/next-share/BaseLib/Core/CacheDB/SqliteVideoPlaybackStatsCacheDB.py new file mode 100644 index 0000000..0eb7c85 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/CacheDB/SqliteVideoPlaybackStatsCacheDB.py @@ -0,0 +1,154 @@ +# Written by Boudewijn Schoon +# see LICENSE.txt for license information + +""" +Database wrapper to add and retrieve Video playback statistics +""" + +import sys +import os +import thread +from base64 import b64encode +from time import time + +from BaseLib.__init__ import LIBRARYNAME +from BaseLib.Core.CacheDB.sqlitecachedb import SQLiteCacheDBBase +from BaseLib.Core.CacheDB.SqliteCacheDBHandler import BasicDBHandler + +CREATE_VIDEOPLAYBACK_STATS_SQL_FILE = None +CREATE_VIDEOPLAYBACK_STATS_SQL_FILE_POSTFIX = os.path.join(LIBRARYNAME, 'Core', 'Statistics', "tribler_videoplayback_stats.sql") +DB_FILE_NAME = 'tribler_videoplayback_stats.sdb' +DB_DIR_NAME = 'sqlite' # db file path = DB_DIR_NAME/DB_FILE_NAME +CURRENT_DB_VERSION = 2 + +ENABLE_LOGGER = False +DEBUG = False + +def init_videoplayback_stats(config, db_exception_handler = None): + """ create VideoPlayback statistics database """ + global CREATE_VIDEOPLAYBACK_STATS_SQL_FILE + config_dir = config['state_dir'] + install_dir = config['install_dir'] + CREATE_VIDEOPLAYBACK_STATS_SQL_FILE = os.path.join(install_dir,CREATE_VIDEOPLAYBACK_STATS_SQL_FILE_POSTFIX) + sqlitedb = SQLiteVideoPlaybackStatsCacheDB.get_instance(db_exception_handler) + sqlite_db_path = os.path.join(config_dir, DB_DIR_NAME, DB_FILE_NAME) + sqlitedb.initDB(sqlite_db_path, CREATE_VIDEOPLAYBACK_STATS_SQL_FILE,current_db_version=CURRENT_DB_VERSION) # the first place to create db in Tribler + return sqlitedb + +class SQLiteVideoPlaybackStatsCacheDBV2(SQLiteCacheDBBase): + def updateDB(self, fromver, tover): + # convert database version 1 --> 2 + if fromver < 2: + sql = """ +-- Simplify the database. All info is now an event. + +DROP TABLE IF EXISTS playback_info; +DROP INDEX IF EXISTS playback_info_idx; + +-- Simplify the database. Events are simplified to key/value +-- pairs. Because sqlite is unable to remove a column, we are forced +-- to DROP and re-CREATE the event table. +-- +-- Note that this will erase previous statistics... + +DROP TABLE IF EXISTS playback_event; +DROP INDEX IF EXISTS playback_event_idx; + +CREATE TABLE playback_event ( + key text NOT NULL, + timestamp real NOT NULL, + event text NOT NULL +); + +CREATE INDEX playback_event_idx + ON playback_event (key, timestamp); +""" + + self.execute_write(sql, commit=False) + + # updating version stepwise so if this works, we store it + # regardless of later, potentially failing updates + self.writeDBVersion(CURRENT_DB_VERSION, commit=False) + self.commit() + +class SQLiteVideoPlaybackStatsCacheDB(SQLiteVideoPlaybackStatsCacheDBV2): + """ + Wrapper around Database engine. Used to perform raw SQL queries + and ensure that Database schema is correct. + """ + + __single = None # used for multithreaded singletons pattern + lock = thread.allocate_lock() + + @classmethod + def get_instance(cls, *args, **kw): + # Singleton pattern with double-checking to ensure that it can only create one object + if cls.__single is None: + cls.lock.acquire() + try: + if cls.__single is None: + cls.__single = cls(*args, **kw) + finally: + cls.lock.release() + return cls.__single + + def __init__(self, *args, **kw): + # always use get_instance() to create this object + if self.__single != None: + raise RuntimeError, "SQLiteVideoPlaybackStatsCacheDB is singleton" + SQLiteCacheDBBase.__init__(self, *args, **kw) + +class VideoPlaybackDBHandler(BasicDBHandler): + """ + Interface to add and retrieve events from the database. + + Manages the playback_event table. This table may contain several + entries for events that occur during playback such as when it was + started and when it was paused. + + The interface of this class should match that of + VideoPlaybackReporter in Tribler.Player.Reporter which is used to + report the same information through HTTP callbacks when there is + no overlay network + """ + + __single = None # used for multi-threaded singletons pattern + lock = thread.allocate_lock() + + @classmethod + def get_instance(cls, *args, **kw): + # Singleton pattern with double-checking to ensure that it can only create one object + if cls.__single is None: + cls.lock.acquire() + try: + if cls.__single is None: + cls.__single = cls(*args, **kw) + finally: + cls.lock.release() + return cls.__single + + def __init__(self): + if VideoPlaybackDBHandler.__single is not None: + raise RuntimeError, "VideoPlaybackDBHandler is singleton" + BasicDBHandler.__init__(self, SQLiteVideoPlaybackStatsCacheDB.get_instance(), 'playback_event') + + def add_event(self, key, event): + if ENABLE_LOGGER: + assert type(key) in (str, unicode) + # assert not "'" in key # TODO: This assert is unnecessary and breaks for certain infohashes? (Raynor Vliegendhart) + assert type(event) in (str, unicode) + assert not "'" in event + + # because the key usually an infohash, and because this is + # usually (and incorrectly) stored in a string instead of a + # unicode string, this will crash the database wrapper. + key = b64encode(key) + + if DEBUG: print >>sys.stderr, "VideoPlaybackDBHandler add_event", key, event + self._db.execute_write("INSERT INTO %s (key, timestamp, event) VALUES ('%s', %s, '%s')" % (self.table_name, key, time(), event)) + + def flush(self): + """ + Flush the statistics. This is not used for database-based logging + """ + pass diff --git a/instrumentation/next-share/BaseLib/Core/CacheDB/__init__.py b/instrumentation/next-share/BaseLib/Core/CacheDB/__init__.py new file mode 100644 index 0000000..57fd4af --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/CacheDB/__init__.py @@ -0,0 +1,2 @@ +# Written by Jie Yang +# see LICENSE.txt for license information diff --git a/instrumentation/next-share/BaseLib/Core/CacheDB/cachedb.py b/instrumentation/next-share/BaseLib/Core/CacheDB/cachedb.py new file mode 100644 index 0000000..ff18cf4 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/CacheDB/cachedb.py @@ -0,0 +1,7 @@ +# Written by Jie Yang +# see LICENSE.txt for license information + +from sqlitecachedb import * +from SqliteSeedingStatsCacheDB import * +from SqliteFriendshipStatsCacheDB import * +from SqliteVideoPlaybackStatsCacheDB import * diff --git a/instrumentation/next-share/BaseLib/Core/CacheDB/friends.py b/instrumentation/next-share/BaseLib/Core/CacheDB/friends.py new file mode 100644 index 0000000..dc8a5a3 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/CacheDB/friends.py @@ -0,0 +1,142 @@ +# Written by Jie Yang +# see LICENSE.txt for license information +import sys +from time import time +import os +import base64 +from traceback import print_exc + +from BaseLib.Core.Utilities.utilities import validIP, validPort, validPermid, validName, show_permid +from CacheDBHandler import FriendDBHandler +from BaseLib.Core.simpledefs import NTFY_FRIENDS,NTFY_PEERS + +default_friend_file = 'friends.txt' + +DEBUG = False + +def init(session): + friend_db = session.open_dbhandler(NTFY_FRIENDS) + peer_db = session.open_dbhandler(NTFY_PEERS) + filename = make_filename(session.get_state_dir(), default_friend_file) + ExternalFriendList(friend_db,peer_db,filename).updateFriendList() + +def done(session): + friend_db = session.open_dbhandler(NTFY_FRIENDS) + peer_db = session.open_dbhandler(NTFY_PEERS) + filename = make_filename(session.get_state_dir(), default_friend_file) + ExternalFriendList(friend_db,peer_db,filename).writeFriendList() + +def make_filename(config_dir,filename): + if config_dir is None: + return filename + else: + return os.path.join(config_dir,filename) + +class ExternalFriendList: + def __init__(self,friend_db,peer_db,friend_file=default_friend_file): + self.friend_file = friend_file + self.friend_db = friend_db + self.peer_db = peer_db + + def clean(self): # delete friend file + try: + os.remove(self.friend_file) + except Exception: + pass + + def updateFriendList(self, friend_file=''): + if not friend_file: + friend_file = self.friend_file + self.friend_list = self.readFriendList(friend_file) + self.updateDB(self.friend_list) + #self.clean() + + def updateDB(self, friend_list): + if not friend_list: + return + for friend in friend_list: + self.friend_db.addExternalFriend(friend) + + def getFriends(self): + friends = [] + permids = self.friend_db.getFriends() + for permid in permids: + friend = self.peer_db.getPeer(permid) + friends.append(friend) + return friends + + def deleteFriend(self, permid): + self.friend_db.deleteFriend(permid) + + def readFriendList(self, filename=''): + """ read (name, permid, friend_ip, friend_port) lines from a text file """ + + if not filename: + filename = self.friend_file + try: + file = open(filename, "r") + friends = file.readlines() + file.close() + except IOError: # create a new file + file = open(filename, "w") + file.close() + return [] + + friends_info = [] + for friend in friends: + if friend.strip().startswith("#"): # skip commended lines + continue + friend_line = friend.split(',') + friend_info = [] + for i in range(len(friend_line)): + friend_info.append(friend_line[i].strip()) + try: + friend_info[1] = base64.decodestring( friend_info[1]+'\n' ) + except: + continue + if self.validFriendList(friend_info): + friend = {'name':friend_info[0], 'permid':friend_info[1], + 'ip':friend_info[2], 'port':int(friend_info[3])} + friends_info.append(friend) + return friends_info + + def validFriendList(self, friend_info): + try: + if len(friend_info) < 4: + raise RuntimeError, "one line in friends.txt can only contain at least 4 elements" + validName(friend_info[0]) + validPermid(friend_info[1]) + validIP(friend_info[2]) + validPort(int(friend_info[3])) + except Exception, msg: + if DEBUG: + print "======== reading friend list error ========" + print friend_info + print msg + print "===========================================" + return False + else: + return True + + def writeFriendList(self, filename=''): + if not filename: + filename = self.friend_file + try: + file = open(filename, "w") + except IOError: + print_exc() + return + + friends = self.getFriends() + friends_to_write = self.formatForText(friends) + file.writelines(friends_to_write) + file.close() + + def formatForText(self, friends): + lines = [] + for friend in friends: + permid = show_permid(friend['permid']) + line = ', '.join([friend['name'], permid, friend['ip'], str(friend['port'])]) + line += '\n' + lines.append(line) + return lines \ No newline at end of file diff --git a/instrumentation/next-share/BaseLib/Core/CacheDB/maxflow.py b/instrumentation/next-share/BaseLib/Core/CacheDB/maxflow.py new file mode 100644 index 0000000..15afa91 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/CacheDB/maxflow.py @@ -0,0 +1,162 @@ +import sets + +# Computes maximal flow in a graph +# Adam Langley http://www.imperialviolet.org +# Creative Commons http://creativecommons.org/licenses/by-sa/2.0/ + +# Adapted for Tribler +from copy import deepcopy + +class Network(object): + """This class can be used to calculate the maximal flow between two points in a network/graph. + A network consists of nodes and arcs (egdes) that link them. Each arc has a capacity (the maximum flow down that arc). + The iterative algorithm is described at http://carbon.cudenver.edu/~hgreenbe/glossary/notes/maxflow-FF.pdf""" + + __slots__ = ['arcs', 'backarcs', 'nodes', 'labels'] + + def __init__ (self, arcs): + + self.nodes = [] + self.labels = {} + + self.arcs = arcs + self.backarcs = {} + + for source in arcs: + + if not source in self.nodes: + self.nodes.append(source) + + if not source in self.backarcs: + self.backarcs[source] = {} + + for dest in arcs[source]: + + if not dest in self.nodes: + self.nodes.append(dest) + + if not dest in self.backarcs: + self.backarcs[dest] = {} + + self.backarcs[dest][source] = {'cap' : arcs[source][dest]['cap'], 'flow' : 0} + + + def min (a, b): + """private function""" + if (a == -1): + return b + if (b == -1): + return a + return min (a, b) + + min = staticmethod (min) + + def maxflow (self, source, sink, max_distance = 10000): + """Return the maximum flow from the source to the sink""" + + if not source in self.nodes or not sink in self.nodes: + return 0.0 + + arcscopy = deepcopy(self.arcs) + backarcscopy = deepcopy(self.backarcs) + + DEBUG = False + + while 1: + labels = {} + labels[source] = ((0, 0), -1) + + unscanned = {source: 0} # sets.Set ([source]) + scanned = sets.Set() + + while 1: + # Select any node, x, that is labeled and unscanned + + for node in unscanned: + + if DEBUG: + print "Unscanned: " + str(node) + + # To all unlabeled succ nodes + for outnode in arcscopy[node]: + + if DEBUG: + print "to ", outnode + + if (outnode in unscanned or outnode in scanned): + continue + arc = arcscopy[node][outnode] + if (arc['flow'] >= arc['cap']) or (unscanned[node] + 1) > max_distance: + continue + + labels[outnode] = ((node, 1), Network.min(labels[node][1], arc['cap'] - arc['flow'])) + + if DEBUG: + print labels[outnode] + + unscanned[outnode] = unscanned[node] + 1 + #unscanned.add(outnode) + + # To all predecessor nodes + for innode in backarcscopy[node]: + + if DEBUG: + print "from ", innode + + if (innode in unscanned or innode in scanned): + continue + arc = arcscopy[innode][node] + if (arc['flow'] == 0) or (unscanned[node] + 1) > max_distance: + continue + labels[innode] = ((node, -1), Network.min(labels[node][1], arc['flow'])) + + if DEBUG: + print labels[innode] + + unscanned[innode] = unscanned[node] + 1 + #unscanned.add(innode) + + del unscanned[node] + #unscanned.remove(node) + + scanned.add(node) + + # print labels + break; + + else: + # no labels could be assigned + # total the incoming flows to the sink + sum = 0 + for innode in backarcscopy[sink]: + sum += arcscopy[innode][sink]['flow'] + return sum + + if (sink in unscanned): + # sink is labeled and unscanned + break; + + # Routine B + s = sink + ((node, sense), et) = labels[s] + # print "et: " + str (et) + while 1: + if (s == source): + break + ((node, sense), epi) = labels[s] + # If the first part of the label is y+ + if (sense == 1): + # print " add " + str(node) + " " + str(s) + arcscopy[node][s]['flow'] += et + else: + # print " rm " + str(s) + " " + str(node) + arcscopy[s][node]['flow'] -= et + s = node + ##print self.arcs + +if (__name__ == "__main__"): + n = Network ({'s' : {'a': {'cap': 20, 'flow': 0}, 'x' : {'cap' : 1, 'flow' : 0}, 'y' : {'cap' : 3, 'flow' : 0}}, 'x' : {'y' : {'cap' : 1, 'flow' : 0}, 't' : {'cap' : 3, 'flow' : 0}}, 'y' : {'x' : {'cap' : 1, 'flow' : 0}, 't' : {'cap' : 1, 'flow' : 0}}, 'a': {'b': {'cap': 20, 'flow': 0}}, 'b': {'c': {'cap': 20, 'flow': 0}}, 'c': {'t': {'cap': 20, 'flow': 0}}}) + + print n.nodes + print n.maxflow ('s', 'q', max_distance = 2) + diff --git a/instrumentation/next-share/BaseLib/Core/CacheDB/sqlitecachedb.py b/instrumentation/next-share/BaseLib/Core/CacheDB/sqlitecachedb.py new file mode 100644 index 0000000..b9103f4 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/CacheDB/sqlitecachedb.py @@ -0,0 +1,1221 @@ +# Written by Jie Yang +# see LICENSE.txt for license information + +import sys +import os +from time import sleep +from base64 import encodestring, decodestring +import threading +from traceback import print_exc, print_stack + +from BaseLib.Core.simpledefs import INFOHASH_LENGTH +from BaseLib.__init__ import LIBRARYNAME +from BaseLib.Core.Utilities.unicode import dunno2unicode + +# ONLY USE APSW >= 3.5.9-r1 +import apsw +#support_version = (3,5,9) +#support_version = (3,3,13) +#apsw_version = tuple([int(r) for r in apsw.apswversion().split('-')[0].split('.')]) +##print apsw_version +#assert apsw_version >= support_version, "Required APSW Version >= %d.%d.%d."%support_version + " But your version is %d.%d.%d.\n"%apsw_version + \ +# "Please download and install it from http://code.google.com/p/apsw/" + +##Changed from 4 to 5 by andrea for subtitles support +CURRENT_MAIN_DB_VERSION = 5 + +TEST_SQLITECACHEDB_UPGRADE = False +CREATE_SQL_FILE = None +CREATE_SQL_FILE_POSTFIX = os.path.join(LIBRARYNAME, 'schema_sdb_v'+str(CURRENT_MAIN_DB_VERSION)+'.sql') +DB_FILE_NAME = 'tribler.sdb' +DB_DIR_NAME = 'sqlite' # db file path = DB_DIR_NAME/DB_FILE_NAME +DEFAULT_BUSY_TIMEOUT = 10000 +MAX_SQL_BATCHED_TO_TRANSACTION = 1000 # don't change it unless carefully tested. A transaction with 1000 batched updates took 1.5 seconds +NULL = None +icon_dir = None +SHOW_ALL_EXECUTE = False +costs = [] +cost_reads = [] +torrent_dir = None +config_dir = None +TEST_OVERRIDE = False + + +DEBUG = False + +class Warning(Exception): + pass + +def init(config, db_exception_handler = None): + """ create sqlite database """ + global CREATE_SQL_FILE + global icon_dir + global torrent_dir + global config_dir + torrent_dir = os.path.abspath(config['torrent_collecting_dir']) + config_dir = config['state_dir'] + install_dir = config['install_dir'] + CREATE_SQL_FILE = os.path.join(install_dir,CREATE_SQL_FILE_POSTFIX) + sqlitedb = SQLiteCacheDB.getInstance(db_exception_handler) + + if config['superpeer']: + sqlite_db_path = ':memory:' + else: + sqlite_db_path = os.path.join(config_dir, DB_DIR_NAME, DB_FILE_NAME) + print >>sys.stderr,"cachedb: init: SQL FILE",sqlite_db_path + + icon_dir = os.path.abspath(config['peer_icon_path']) + + sqlitedb.initDB(sqlite_db_path, CREATE_SQL_FILE) # the first place to create db in Tribler + return sqlitedb + +def done(config_dir): + SQLiteCacheDB.getInstance().close() + +def make_filename(config_dir,filename): + if config_dir is None: + return filename + else: + return os.path.join(config_dir,filename) + +def bin2str(bin): + # Full BASE64-encoded + return encodestring(bin).replace("\n","") + +def str2bin(str): + return decodestring(str) + +def print_exc_plus(): + """ + Print the usual traceback information, followed by a listing of all the + local variables in each frame. + http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52215 + http://initd.org/pub/software/pysqlite/apsw/3.3.13-r1/apsw.html#augmentedstacktraces + """ + + tb = sys.exc_info()[2] + stack = [] + + while tb: + stack.append(tb.tb_frame) + tb = tb.tb_next + + print_exc() + print >> sys.stderr, "Locals by frame, innermost last" + + for frame in stack: + print >> sys.stderr + print >> sys.stderr, "Frame %s in %s at line %s" % (frame.f_code.co_name, + frame.f_code.co_filename, + frame.f_lineno) + for key, value in frame.f_locals.items(): + print >> sys.stderr, "\t%20s = " % key, + #We have to be careful not to cause a new error in our error + #printer! Calling str() on an unknown object could cause an + #error we don't want. + try: + print >> sys.stderr, value + except: + print >> sys.stderr, "" + +class safe_dict(dict): + def __init__(self, *args, **kw): + self.lock = threading.RLock() + dict.__init__(self, *args, **kw) + + def __getitem__(self, key): + self.lock.acquire() + try: + return dict.__getitem__(self, key) + finally: + self.lock.release() + + def __setitem__(self, key, value): + self.lock.acquire() + try: + dict.__setitem__(self, key, value) + finally: + self.lock.release() + + def __delitem__(self, key): + self.lock.acquire() + try: + dict.__delitem__(self, key) + finally: + self.lock.release() + + def __contains__(self, key): + self.lock.acquire() + try: + return dict.__contains__(self, key) + finally: + self.lock.release() + + def values(self): + self.lock.acquire() + try: + return dict.values(self) + finally: + self.lock.release() + +class SQLiteCacheDBBase: + lock = threading.RLock() + + def __init__(self,db_exception_handler=None): + self.exception_handler = db_exception_handler + self.cursor_table = safe_dict() # {thread_name:cur} + self.cache_transaction_table = safe_dict() # {thread_name:[sql] + self.class_variables = safe_dict({'db_path':None,'busytimeout':None}) # busytimeout is in milliseconds + + self.permid_id = safe_dict() + self.infohash_id = safe_dict() + self.show_execute = False + + #TODO: All global variables must be protected to be thread safe? + self.status_table = None + self.category_table = None + self.src_table = None + self.applied_pragma_sync_norm = False + + def __del__(self): + self.close() + + def close(self, clean=False): + # only close the connection object in this thread, don't close other thread's connection object + thread_name = threading.currentThread().getName() + cur = self.getCursor(create=False) + + if cur: + con = cur.getconnection() + cur.close() + con.close() + con = None + del self.cursor_table[thread_name] + # Arno, 2010-01-25: Remove entry in cache_transaction_table for this thread + try: + if thread_name in self.cache_transaction_table.keys(): + del self.cache_transaction_table[thread_name] + except: + print_exc() + if clean: # used for test suite + self.permid_id = safe_dict() + self.infohash_id = safe_dict() + self.exception_handler = None + self.class_variables = safe_dict({'db_path':None,'busytimeout':None}) + self.cursor_table = safe_dict() + self.cache_transaction_table = safe_dict() + + + # --------- static functions -------- + def getCursor(self, create=True): + thread_name = threading.currentThread().getName() + curs = self.cursor_table + cur = curs.get(thread_name, None) # return [cur, cur, lib] or None + #print >> sys.stderr, '-------------- getCursor::', len(curs), time(), curs.keys() + if cur is None and create: + self.openDB(self.class_variables['db_path'], self.class_variables['busytimeout']) # create a new db obj for this thread + cur = curs.get(thread_name) + + return cur + + def openDB(self, dbfile_path=None, busytimeout=DEFAULT_BUSY_TIMEOUT): + """ + Open a SQLite database. Only one and the same database can be opened. + @dbfile_path The path to store the database file. + Set dbfile_path=':memory:' to create a db in memory. + @busytimeout Set the maximum time, in milliseconds, that SQLite will wait if the database is locked. + """ + + # already opened a db in this thread, reuse it + thread_name = threading.currentThread().getName() + #print >>sys.stderr,"sqlcachedb: openDB",dbfile_path,thread_name + if thread_name in self.cursor_table: + #assert dbfile_path == None or self.class_variables['db_path'] == dbfile_path + return self.cursor_table[thread_name] + + assert dbfile_path, "You must specify the path of database file" + + if dbfile_path.lower() != ':memory:': + db_dir,db_filename = os.path.split(dbfile_path) + if db_dir and not os.path.isdir(db_dir): + os.makedirs(db_dir) + + con = apsw.Connection(dbfile_path) + con.setbusytimeout(busytimeout) + + cur = con.cursor() + self.cursor_table[thread_name] = cur + + if not self.applied_pragma_sync_norm: + # http://www.sqlite.org/pragma.html + # When synchronous is NORMAL, the SQLite database engine will still + # pause at the most critical moments, but less often than in FULL + # mode. There is a very small (though non-zero) chance that a power + # failure at just the wrong time could corrupt the database in + # NORMAL mode. But in practice, you are more likely to suffer a + # catastrophic disk failure or some other unrecoverable hardware + # fault. + # + self.applied_pragma_sync_norm = True + cur.execute("PRAGMA synchronous = NORMAL;") + + return cur + + def createDBTable(self, sql_create_table, dbfile_path, busytimeout=DEFAULT_BUSY_TIMEOUT): + """ + Create a SQLite database. + @sql_create_table The sql statements to create tables in the database. + Every statement must end with a ';'. + @dbfile_path The path to store the database file. Set dbfile_path=':memory:' to creates a db in memory. + @busytimeout Set the maximum time, in milliseconds, that SQLite will wait if the database is locked. + Default = 10000 milliseconds + """ + cur = self.openDB(dbfile_path, busytimeout) + print dbfile_path + cur.execute(sql_create_table) # it is suggested to include begin & commit in the script + + def initDB(self, sqlite_filepath, + create_sql_filename = None, + busytimeout = DEFAULT_BUSY_TIMEOUT, + check_version = True, + current_db_version = CURRENT_MAIN_DB_VERSION): + """ + Create and initialize a SQLite database given a sql script. + Only one db can be opened. If the given dbfile_path is different with the opened DB file, warn and exit + @configure_dir The directory containing 'bsddb' directory + @sql_filename The path of sql script to create the tables in the database + Every statement must end with a ';'. + @busytimeout Set the maximum time, in milliseconds, to wait and retry + if failed to acquire a lock. Default = 5000 milliseconds + """ + if create_sql_filename is None: + create_sql_filename=CREATE_SQL_FILE + try: + self.lock.acquire() + + # verify db path identity + class_db_path = self.class_variables['db_path'] + if sqlite_filepath is None: # reuse the opened db file? + if class_db_path is not None: # yes, reuse it + # reuse the busytimeout + return self.openDB(class_db_path, self.class_variables['busytimeout']) + else: # no db file opened + raise Exception, "You must specify the path of database file when open it at the first time" + else: + if class_db_path is None: # the first time to open db path, store it + + #print 'quit now' + #sys.exit(0) + # open the db if it exists (by converting from bsd) and is not broken, otherwise create a new one + # it will update the db if necessary by checking the version number + self.safelyOpenTriblerDB(sqlite_filepath, create_sql_filename, busytimeout, check_version=check_version, current_db_version=current_db_version) + + self.class_variables = {'db_path': sqlite_filepath, 'busytimeout': int(busytimeout)} + + return self.openDB() # return the cursor, won't reopen the db + + elif sqlite_filepath != class_db_path: # not the first time to open db path, check if it is the same + raise Exception, "Only one database file can be opened. You have opened %s and are trying to open %s." % (class_db_path, sqlite_filepath) + + finally: + self.lock.release() + + def safelyOpenTriblerDB(self, dbfile_path, sql_create, busytimeout=DEFAULT_BUSY_TIMEOUT, check_version=False, current_db_version=None): + """ + open the db if possible, otherwise create a new one + update the db if necessary by checking the version number + + safeOpenDB(): + try: + if sqlite db doesn't exist: + raise Error + open sqlite db + read sqlite_db_version + if sqlite_db_version dosen't exist: + raise Error + except: + close and delete sqlite db if possible + create new sqlite db file without sqlite_db_version + write sqlite_db_version at last + commit + open sqlite db + read sqlite_db_version + # must ensure these steps after except will not fail, otherwise force to exit + + if sqlite_db_version < current_db_version: + updateDB(sqlite_db_version, current_db_version) + commit + update sqlite_db_version at last + commit + """ + try: + if not os.path.isfile(dbfile_path): + raise Warning("No existing database found. Attempting to creating a new database %s" % repr(dbfile_path)) + + cur = self.openDB(dbfile_path, busytimeout) + if check_version: + sqlite_db_version = self.readDBVersion() + if sqlite_db_version == NULL or int(sqlite_db_version)<1: + raise NotImplementedError + except Exception, exception: + if isinstance(exception, Warning): + # user friendly warning to log the creation of a new database + print >>sys.stderr, exception + + else: + # user unfriendly exception message because something went wrong + print_exc() + + if os.path.isfile(dbfile_path): + self.close(clean=True) + os.remove(dbfile_path) + + if os.path.isfile(sql_create): + f = open(sql_create) + sql_create_tables = f.read() + f.close() + else: + raise Exception, "Cannot open sql script at %s" % os.path.realpath(sql_create) + + self.createDBTable(sql_create_tables, dbfile_path, busytimeout) + if check_version: + sqlite_db_version = self.readDBVersion() + + if check_version: + self.checkDB(sqlite_db_version, current_db_version) + + def checkDB(self, db_ver, curr_ver): + # read MyDB and check the version number. + if not db_ver or not curr_ver: + self.updateDB(db_ver,curr_ver) + return + db_ver = int(db_ver) + curr_ver = int(curr_ver) + #print "check db", db_ver, curr_ver + if db_ver != curr_ver or \ + (not config_dir is None and os.path.exists(os.path.join(config_dir, "upgradingdb.txt"))): + self.updateDB(db_ver,curr_ver) + + def updateDB(self,db_ver,curr_ver): + pass #TODO + + def readDBVersion(self): + cur = self.getCursor() + sql = u"select value from MyInfo where entry='version'" + res = self.fetchone(sql) + if res: + find = list(res) + return find[0] # throw error if something wrong + else: + return None + + def writeDBVersion(self, version, commit=True): + sql = u"UPDATE MyInfo SET value=? WHERE entry='version'" + self.execute_write(sql, [version], commit=commit) + + def show_sql(self, switch): + # temporary show the sql executed + self.show_execute = switch + + # --------- generic functions ------------- + + def commit(self): + self.transaction() + + def _execute(self, sql, args=None): + cur = self.getCursor() + + if SHOW_ALL_EXECUTE or self.show_execute: + thread_name = threading.currentThread().getName() + print >> sys.stderr, '===', thread_name, '===\n', sql, '\n-----\n', args, '\n======\n' + try: + if args is None: + return cur.execute(sql) + else: + return cur.execute(sql, args) + except Exception, msg: + if True: + print_exc() + print_stack() + print >> sys.stderr, "cachedb: execute error:", Exception, msg + thread_name = threading.currentThread().getName() + print >> sys.stderr, '===', thread_name, '===\nSQL Type:', type(sql), '\n-----\n', sql, '\n-----\n', args, '\n======\n' + #return None + # ARNODB: this is incorrect, it should reraise the exception + # such that _transaction can rollback or recommit. + # This bug already reported by Johan + raise msg + + + def execute_read(self, sql, args=None): + # this is only called for reading. If you want to write the db, always use execute_write or executemany + return self._execute(sql, args) + + def execute_write(self, sql, args=None, commit=True): + self.cache_transaction(sql, args) + if commit: + self.commit() + + def executemany(self, sql, args, commit=True): + + thread_name = threading.currentThread().getName() + if thread_name not in self.cache_transaction_table: + self.cache_transaction_table[thread_name] = [] + all = [(sql, arg) for arg in args] + self.cache_transaction_table[thread_name].extend(all) + + if commit: + self.commit() + + def cache_transaction(self, sql, args=None): + thread_name = threading.currentThread().getName() + if thread_name not in self.cache_transaction_table: + self.cache_transaction_table[thread_name] = [] + self.cache_transaction_table[thread_name].append((sql, args)) + + def transaction(self, sql=None, args=None): + if sql: + self.cache_transaction(sql, args) + + thread_name = threading.currentThread().getName() + + n = 0 + sql_full = '' + arg_list = [] + sql_queue = self.cache_transaction_table.get(thread_name,None) + if sql_queue: + while True: + try: + _sql,_args = sql_queue.pop(0) + except IndexError: + break + + _sql = _sql.strip() + if not _sql: + continue + if not _sql.endswith(';'): + _sql += ';' + sql_full += _sql + '\n' + if _args != None: + arg_list += list(_args) + n += 1 + + # if too many sql in cache, split them into batches to prevent processing and locking DB for a long time + # TODO: optimize the value of MAX_SQL_BATCHED_TO_TRANSACTION + if n % MAX_SQL_BATCHED_TO_TRANSACTION == 0: + self._transaction(sql_full, arg_list) + sql_full = '' + arg_list = [] + + self._transaction(sql_full, arg_list) + + def _transaction(self, sql, args=None): + if sql: + sql = 'BEGIN TRANSACTION; \n' + sql + 'COMMIT TRANSACTION;' + try: + self._execute(sql, args) + except Exception,e: + self.commit_retry_if_busy_or_rollback(e,0,sql=sql) + + def commit_retry_if_busy_or_rollback(self,e,tries,sql=None): + """ + Arno: + SQL_BUSY errors happen at the beginning of the experiment, + very quickly after startup (e.g. 0.001 s), so the busy timeout + is not honoured for some reason. After the initial errors, + they no longer occur. + """ + print >>sys.stderr,"sqlcachedb: commit_retry: after",str(e),repr(sql) + + if str(e).startswith("BusyError"): + try: + self._execute("COMMIT") + except Exception,e2: + if tries < 5: #self.max_commit_retries + # Spec is unclear whether next commit will also has + # 'busytimeout' seconds to try to get a write lock. + sleep(pow(2.0,tries+2)/100.0) + self.commit_retry_if_busy_or_rollback(e2,tries+1) + else: + self.rollback(tries) + raise Exception,e2 + else: + self.rollback(tries) + m = "cachedb: TRANSACTION ERROR "+threading.currentThread().getName()+' '+str(e) + raise Exception, m + + + def rollback(self, tries): + print_exc() + try: + self._execute("ROLLBACK") + except Exception, e: + # May be harmless, see above. Unfortunately they don't specify + # what the error is when an attempt is made to roll back + # an automatically rolled back transaction. + m = "cachedb: ROLLBACK ERROR "+threading.currentThread().getName()+' '+str(e) + #print >> sys.stderr, 'SQLite Database', m + raise Exception, m + + + # -------- Write Operations -------- + def insert(self, table_name, commit=True, **argv): + if len(argv) == 1: + sql = 'INSERT INTO %s (%s) VALUES (?);'%(table_name, argv.keys()[0]) + else: + questions = '?,'*len(argv) + sql = 'INSERT INTO %s %s VALUES (%s);'%(table_name, tuple(argv.keys()), questions[:-1]) + self.execute_write(sql, argv.values(), commit) + + def insertMany(self, table_name, values, keys=None, commit=True): + """ values must be a list of tuples """ + + questions = u'?,'*len(values[0]) + if keys is None: + sql = u'INSERT INTO %s VALUES (%s);'%(table_name, questions[:-1]) + else: + sql = u'INSERT INTO %s %s VALUES (%s);'%(table_name, tuple(keys), questions[:-1]) + self.executemany(sql, values, commit=commit) + + def update(self, table_name, where=None, commit=True, **argv): + sql = u'UPDATE %s SET '%table_name + arg = [] + for k,v in argv.iteritems(): + if type(v) is tuple: + sql += u'%s %s ?,' % (k, v[0]) + arg.append(v[1]) + else: + sql += u'%s=?,' % k + arg.append(v) + sql = sql[:-1] + if where != None: + sql += u' where %s'%where + self.execute_write(sql, arg, commit) + + def delete(self, table_name, commit=True, **argv): + sql = u'DELETE FROM %s WHERE '%table_name + arg = [] + for k,v in argv.iteritems(): + if type(v) is tuple: + sql += u'%s %s ? AND ' % (k, v[0]) + arg.append(v[1]) + else: + sql += u'%s=? AND ' % k + arg.append(v) + sql = sql[:-5] + self.execute_write(sql, argv.values(), commit) + + # -------- Read Operations -------- + def size(self, table_name): + num_rec_sql = u"SELECT count(*) FROM %s;"%table_name + result = self.fetchone(num_rec_sql) + return result + + def fetchone(self, sql, args=None): + # returns NULL: if the result is null + # return None: if it doesn't found any match results + find = self.execute_read(sql, args) + if not find: + return NULL + else: + find = list(find) + if len(find) > 0: + find = find[0] + else: + return NULL + if len(find)>1: + return find + else: + return find[0] + + def fetchall(self, sql, args=None, retry=0): + res = self.execute_read(sql, args) + if res != None: + find = list(res) + return find + else: + return [] # should it return None? + + def getOne(self, table_name, value_name, where=None, conj='and', **kw): + """ value_name could be a string, a tuple of strings, or '*' + """ + + if isinstance(value_name, tuple): + value_names = u",".join(value_name) + elif isinstance(value_name, list): + value_names = u",".join(value_name) + else: + value_names = value_name + + if isinstance(table_name, tuple): + table_names = u",".join(table_name) + elif isinstance(table_name, list): + table_names = u",".join(table_name) + else: + table_names = table_name + + sql = u'select %s from %s'%(value_names, table_names) + + if where or kw: + sql += u' where ' + if where: + sql += where + if kw: + sql += u' %s '%conj + if kw: + arg = [] + for k,v in kw.iteritems(): + if type(v) is tuple: + operator = v[0] + arg.append(v[1]) + else: + operator = "=" + arg.append(v) + sql += u' %s %s ? ' % (k, operator) + sql += conj + sql = sql[:-len(conj)] + else: + arg = None + + # print >> sys.stderr, 'SQL: %s %s' % (sql, arg) + return self.fetchone(sql,arg) + + def getAll(self, table_name, value_name, where=None, group_by=None, having=None, order_by=None, limit=None, offset=None, conj='and', **kw): + """ value_name could be a string, or a tuple of strings + order by is represented as order_by + group by is represented as group_by + """ + if isinstance(value_name, tuple): + value_names = u",".join(value_name) + elif isinstance(value_name, list): + value_names = u",".join(value_name) + else: + value_names = value_name + + if isinstance(table_name, tuple): + table_names = u",".join(table_name) + elif isinstance(table_name, list): + table_names = u",".join(table_name) + else: + table_names = table_name + + sql = u'select %s from %s'%(value_names, table_names) + + if where or kw: + sql += u' where ' + if where: + sql += where + if kw: + sql += u' %s '%conj + if kw: + arg = [] + for k,v in kw.iteritems(): + if type(v) is tuple: + operator = v[0] + arg.append(v[1]) + else: + operator = "=" + arg.append(v) + + sql += u' %s %s ?' % (k, operator) + sql += conj + sql = sql[:-len(conj)] + else: + arg = None + + if group_by != None: + sql += u' group by ' + group_by + if having != None: + sql += u' having ' + having + if order_by != None: + sql += u' order by ' + order_by # you should add desc after order_by to reversely sort, i.e, 'last_seen desc' as order_by + if limit != None: + sql += u' limit %d'%limit + if offset != None: + sql += u' offset %d'%offset + + try: + return self.fetchall(sql, arg) or [] + except Exception, msg: + print >> sys.stderr, "sqldb: Wrong getAll sql statement:", sql + raise Exception, msg + + # ----- Tribler DB operations ---- + + #------------- useful functions for multiple handlers ---------- + def insertPeer(self, permid, update=True, commit=True, **argv): + """ Insert a peer. permid is the binary permid. + If the peer is already in db and update is True, update the peer. + """ + peer_id = self.getPeerID(permid) + peer_existed = False + if 'name' in argv: + argv['name'] = dunno2unicode(argv['name']) + if peer_id != None: + peer_existed = True + if update: + where=u'peer_id=%d'%peer_id + self.update('Peer', where, commit=commit, **argv) + else: + self.insert('Peer', permid=bin2str(permid), commit=commit, **argv) + return peer_existed + + def deletePeer(self, permid=None, peer_id=None, force=True, commit=True): + if peer_id is None: + peer_id = self.getPeerID(permid) + + deleted = False + if peer_id != None: + if force: + self.delete('Peer', peer_id=peer_id, commit=commit) + else: + self.delete('Peer', peer_id=peer_id, friend=0, superpeer=0, commit=commit) + deleted = not self.hasPeer(permid, check_db=True) + if deleted and permid in self.permid_id: + self.permid_id.pop(permid) + + return deleted + + def getPeerID(self, permid): + assert isinstance(permid, str), permid + # permid must be binary + if permid in self.permid_id: + return self.permid_id[permid] + + sql_get_peer_id = "SELECT peer_id FROM Peer WHERE permid==?" + peer_id = self.fetchone(sql_get_peer_id, (bin2str(permid),)) + if peer_id != None: + self.permid_id[permid] = peer_id + + return peer_id + + def hasPeer(self, permid, check_db=False): + if not check_db: + return bool(self.getPeerID(permid)) + else: + permid_str = bin2str(permid) + sql_get_peer_id = "SELECT peer_id FROM Peer WHERE permid==?" + peer_id = self.fetchone(sql_get_peer_id, (permid_str,)) + if peer_id is None: + return False + else: + return True + + def insertInfohash(self, infohash, check_dup=False, commit=True): + """ Insert an infohash. infohash is binary """ + assert isinstance(infohash, str), "INFOHASH has invalid type: %s" % type(infohash) + assert len(infohash) == INFOHASH_LENGTH, "INFOHASH has invalid length: %d" % len(infohash) + if infohash in self.infohash_id: + if check_dup: + print >> sys.stderr, 'sqldb: infohash to insert already exists', `infohash` + return + + infohash_str = bin2str(infohash) + sql_insert_torrent = "INSERT INTO Torrent (infohash) VALUES (?)" + self.execute_write(sql_insert_torrent, (infohash_str,), commit) + + def deleteInfohash(self, infohash=None, torrent_id=None, commit=True): + assert infohash is None or isinstance(infohash, str), "INFOHASH has invalid type: %s" % type(infohash) + assert infohash is None or len(infohash) == INFOHASH_LENGTH, "INFOHASH has invalid length: %d" % len(infohash) + if torrent_id is None: + torrent_id = self.getTorrentID(infohash) + + if torrent_id != None: + self.delete('Torrent', torrent_id=torrent_id, commit=commit) + if infohash in self.infohash_id: + self.infohash_id.pop(infohash) + + def getTorrentID(self, infohash): + assert isinstance(infohash, str), "INFOHASH has invalid type: %s" % type(infohash) + assert len(infohash) == INFOHASH_LENGTH, "INFOHASH has invalid length: %d" % len(infohash) + if infohash in self.infohash_id: + return self.infohash_id[infohash] + + sql_get_torrent_id = "SELECT torrent_id FROM Torrent WHERE infohash==?" + tid = self.fetchone(sql_get_torrent_id, (bin2str(infohash),)) + if tid != None: + self.infohash_id[infohash] = tid + return tid + + def getInfohash(self, torrent_id): + sql_get_infohash = "SELECT infohash FROM Torrent WHERE torrent_id==?" + arg = (torrent_id,) + ret = self.fetchone(sql_get_infohash, arg) + ret = str2bin(ret) + return ret + + def getTorrentStatusTable(self): + if self.status_table is None: + st = self.getAll('TorrentStatus', ('lower(name)', 'status_id')) + self.status_table = dict(st) + return self.status_table + + def getTorrentCategoryTable(self): + # The key is in lower case + if self.category_table is None: + ct = self.getAll('Category', ('lower(name)', 'category_id')) + self.category_table = dict(ct) + return self.category_table + + def getTorrentSourceTable(self): + # Don't use lower case because some URLs are case sensitive + if self.src_table is None: + st = self.getAll('TorrentSource', ('name', 'source_id')) + self.src_table = dict(st) + return self.src_table + + def test(self): + res1 = self.getAll('Category', '*') + res2 = len(self.getAll('Peer', 'name', 'name is not NULL')) + return (res1, res2) + + +class SQLiteCacheDBV5(SQLiteCacheDBBase): + def updateDB(self, fromver, tover): + + # bring database up to version 2, if necessary + if fromver < 2: + sql = """ + +-- Patch for BuddyCast 4 + +ALTER TABLE MyPreference ADD COLUMN click_position INTEGER DEFAULT -1; +ALTER TABLE MyPreference ADD COLUMN reranking_strategy INTEGER DEFAULT -1; +ALTER TABLE Preference ADD COLUMN click_position INTEGER DEFAULT -1; +ALTER TABLE Preference ADD COLUMN reranking_strategy INTEGER DEFAULT -1; +CREATE TABLE ClicklogSearch ( + peer_id INTEGER DEFAULT 0, + torrent_id INTEGER DEFAULT 0, + term_id INTEGER DEFAULT 0, + term_order INTEGER DEFAULT 0 + ); +CREATE INDEX idx_search_term ON ClicklogSearch (term_id); +CREATE INDEX idx_search_torrent ON ClicklogSearch (torrent_id); + + +CREATE TABLE ClicklogTerm ( + term_id INTEGER PRIMARY KEY AUTOINCREMENT DEFAULT 0, + term VARCHAR(255) NOT NULL, + times_seen INTEGER DEFAULT 0 NOT NULL + ); +CREATE INDEX idx_terms_term ON ClicklogTerm(term); + +""" + + self.execute_write(sql, commit=False) + + + if fromver < 3: + sql = """ +-- Patch for Local Peer Discovery + +ALTER TABLE Peer ADD COLUMN is_local integer DEFAULT 0; +""" + self.execute_write(sql, commit=False) + + if fromver < 4: + sql=""" +-- V2: Patch for VoteCast + +DROP TABLE IF EXISTS ModerationCast; +DROP INDEX IF EXISTS moderationcast_idx; + +DROP TABLE IF EXISTS Moderators; +DROP INDEX IF EXISTS moderators_idx; + +DROP TABLE IF EXISTS VoteCast; +DROP INDEX IF EXISTS votecast_idx; + +CREATE TABLE VoteCast ( +mod_id text, +voter_id text, +vote integer, +time_stamp integer +); + +CREATE INDEX mod_id_idx +on VoteCast +(mod_id); + +CREATE INDEX voter_id_idx +on VoteCast +(voter_id); + +CREATE UNIQUE INDEX votecast_idx +ON VoteCast +(mod_id, voter_id); + +--- patch for BuddyCast 5 : Creation of Popularity table and relevant stuff + +CREATE TABLE Popularity ( + torrent_id INTEGER, + peer_id INTEGER, + msg_receive_time NUMERIC, + size_calc_age NUMERIC, + num_seeders INTEGER DEFAULT 0, + num_leechers INTEGER DEFAULT 0, + num_of_sources INTEGER DEFAULT 0 + ); + +CREATE INDEX Message_receive_time_idx + ON Popularity + (msg_receive_time); + +CREATE INDEX Size_calc_age_idx + ON Popularity + (size_calc_age); + +CREATE INDEX Number_of_seeders_idx + ON Popularity + (num_seeders); + +CREATE INDEX Number_of_leechers_idx + ON Popularity + (num_leechers); + +CREATE UNIQUE INDEX Popularity_idx + ON Popularity + (torrent_id, peer_id, msg_receive_time); + +-- v4: Patch for ChannelCast, Search + +CREATE TABLE ChannelCast ( +publisher_id text, +publisher_name text, +infohash text, +torrenthash text, +torrentname text, +time_stamp integer, +signature text +); + +CREATE INDEX pub_id_idx +on ChannelCast +(publisher_id); + +CREATE INDEX pub_name_idx +on ChannelCast +(publisher_name); + +CREATE INDEX infohash_ch_idx +on ChannelCast +(infohash); + +---------------------------------------- + +CREATE TABLE InvertedIndex ( +word text NOT NULL, +torrent_id integer +); + +CREATE INDEX word_idx +on InvertedIndex +(word); + +CREATE UNIQUE INDEX invertedindex_idx +on InvertedIndex +(word,torrent_id); + +---------------------------------------- + +-- Set all similarity to zero because we are using a new similarity +-- function and the old values no longer correspond to the new ones +UPDATE Peer SET similarity = 0; +UPDATE Torrent SET relevance = 0; + +""" + self.execute_write(sql, commit=False) + if fromver < 5: + sql=\ +""" +-------------------------------------- +-- Creating Subtitles (future RichMetadata) DB +---------------------------------- +CREATE TABLE Metadata ( + metadata_id integer PRIMARY KEY ASC AUTOINCREMENT NOT NULL, + publisher_id text NOT NULL, + infohash text NOT NULL, + description text, + timestamp integer NOT NULL, + signature text NOT NULL, + UNIQUE (publisher_id, infohash), + FOREIGN KEY (publisher_id, infohash) + REFERENCES ChannelCast(publisher_id, infohash) + ON DELETE CASCADE -- the fk constraint is not enforced by sqlite +); + +CREATE INDEX infohash_md_idx +on Metadata(infohash); + +CREATE INDEX pub_md_idx +on Metadata(publisher_id); + + +CREATE TABLE Subtitles ( + metadata_id_fk integer, + subtitle_lang text NOT NULL, + subtitle_location text, + checksum text NOT NULL, + UNIQUE (metadata_id_fk,subtitle_lang), + FOREIGN KEY (metadata_id_fk) + REFERENCES Metadata(metadata_id) + ON DELETE CASCADE, -- the fk constraint is not enforced by sqlite + + -- ISO639-2 uses 3 characters for lang codes + CONSTRAINT lang_code_length + CHECK ( length(subtitle_lang) == 3 ) +); + + +CREATE INDEX metadata_sub_idx +on Subtitles(metadata_id_fk); + +-- Stores the subtitles that peers have as an integer bitmask + CREATE TABLE SubtitlesHave ( + metadata_id_fk integer, + peer_id text NOT NULL, + have_mask integer NOT NULL, + received_ts integer NOT NULL, --timestamp indicating when the mask was received + UNIQUE (metadata_id_fk, peer_id), + FOREIGN KEY (metadata_id_fk) + REFERENCES Metadata(metadata_id) + ON DELETE CASCADE, -- the fk constraint is not enforced by sqlite + + -- 32 bit unsigned integer + CONSTRAINT have_mask_length + CHECK (have_mask >= 0 AND have_mask < 4294967296) +); + +CREATE INDEX subtitles_have_idx +on SubtitlesHave(metadata_id_fk); + +-- this index can boost queries +-- ordered by timestamp on the SubtitlesHave DB +CREATE INDEX subtitles_have_ts +on SubtitlesHave(received_ts); + +""" + self.execute_write(sql, commit=False) + + # updating version stepwise so if this works, we store it + # regardless of later, potentially failing updates + self.writeDBVersion(CURRENT_MAIN_DB_VERSION, commit=False) + self.commit() + + # now the start the process of parsing the torrents to insert into + # InvertedIndex table. + if TEST_SQLITECACHEDB_UPGRADE: + state_dir = "." + else: + from BaseLib.Core.Session import Session + session = Session.get_instance() + state_dir = session.get_state_dir() + tmpfilename = os.path.join(state_dir,"upgradingdb.txt") + if fromver < 4 or os.path.exists(tmpfilename): + def upgradeTorrents(): + # fetch some un-inserted torrents to put into the InvertedIndex + sql = """ + SELECT torrent_id, name, torrent_file_name + FROM Torrent + WHERE torrent_id NOT IN (SELECT DISTINCT torrent_id FROM InvertedIndex) + AND torrent_file_name IS NOT NULL + LIMIT 20""" + records = self.fetchall(sql) + + if len(records) == 0: + # upgradation is complete and hence delete the temp file + os.remove(tmpfilename) + if DEBUG: print >> sys.stderr, "DB Upgradation: temp-file deleted", tmpfilename + return + + for torrent_id, name, torrent_file_name in records: + try: + abs_filename = os.path.join(session.get_torrent_collecting_dir(), torrent_file_name) + if not os.path.exists(abs_filename): + raise RuntimeError(".torrent file not found. Use fallback.") + torrentdef = TorrentDef.load(abs_filename) + torrent_name = torrentdef.get_name_as_unicode() + keywords = Set(split_into_keywords(torrent_name)) + for filename in torrentdef.get_files_as_unicode(): + keywords.update(split_into_keywords(filename)) + + except: + # failure... most likely the .torrent file + # is invalid + + # use keywords from the torrent name + # stored in the database + torrent_name = dunno2unicode(name) + keywords = Set(split_into_keywords(torrent_name)) + + # store the keywords in the InvertedIndex + # table in the database + if len(keywords) > 0: + values = [(keyword, torrent_id) for keyword in keywords] + self.executemany(u"INSERT OR REPLACE INTO InvertedIndex VALUES(?, ?)", values, commit=False) + if DEBUG: + print >> sys.stderr, "DB Upgradation: Extending the InvertedIndex table with", len(values), "new keywords for", torrent_name + + # now commit, after parsing the batch of torrents + self.commit() + + # upgradation not yet complete; comeback after 5 sec + tqueue.add_task(upgradeTorrents, 5) + + + # Create an empty file to mark the process of upgradation. + # In case this process is terminated before completion of upgradation, + # this file remains even though fromver >= 4 and hence indicating that + # rest of the torrents need to be inserted into the InvertedIndex! + + # ensure the temp-file is created, if it is not already + try: + open(tmpfilename, "w") + if DEBUG: print >> sys.stderr, "DB Upgradation: temp-file successfully created" + except: + if DEBUG: print >> sys.stderr, "DB Upgradation: failed to create temp-file" + + if DEBUG: print >> sys.stderr, "Upgrading DB .. inserting into InvertedIndex" + from BaseLib.Utilities.TimedTaskQueue import TimedTaskQueue + from sets import Set + from BaseLib.Core.Search.SearchManager import split_into_keywords + from BaseLib.Core.TorrentDef import TorrentDef + + # start the upgradation after 10 seconds + tqueue = TimedTaskQueue("UpgradeDB") + tqueue.add_task(upgradeTorrents, 10) + +class SQLiteCacheDB(SQLiteCacheDBV5): + __single = None # used for multithreaded singletons pattern + + @classmethod + def getInstance(cls, *args, **kw): + # Singleton pattern with double-checking to ensure that it can only create one object + if cls.__single is None: + cls.lock.acquire() + try: + if cls.__single is None: + cls.__single = cls(*args, **kw) + #print >>sys.stderr,"SqliteCacheDB: getInstance: created is",cls,cls.__single + finally: + cls.lock.release() + return cls.__single + + def __init__(self, *args, **kargs): + # always use getInstance() to create this object + + # ARNOCOMMENT: why isn't the lock used on this read?! + + if self.__single != None: + raise RuntimeError, "SQLiteCacheDB is singleton" + SQLiteCacheDBBase.__init__(self, *args, **kargs) + +if __name__ == '__main__': + configure_dir = sys.argv[1] + config = {} + config['state_dir'] = configure_dir + config['install_dir'] = u'.' + config['peer_icon_path'] = u'.' + sqlite_test = init(config) + sqlite_test.test() + diff --git a/instrumentation/next-share/BaseLib/Core/ClosedSwarm/ClosedSwarm.py b/instrumentation/next-share/BaseLib/Core/ClosedSwarm/ClosedSwarm.py new file mode 100644 index 0000000..614e954 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/ClosedSwarm/ClosedSwarm.py @@ -0,0 +1,605 @@ +# Written by Njaal Borch +# see LICENSE.txt for license information + +import time +import os.path + +from base64 import encodestring, decodestring +from M2Crypto.EC import pub_key_from_der + +from BaseLib.Core.Overlay import permid +from BaseLib.Core.BitTornado.bencode import bencode, bdecode + +from BaseLib.Core.BitTornado.BT1.MessageID import * + + +# Constants to be put into BaseLib.Core.BitTornado.BT1.MessageID.py +# Also update all the protocol stuff there (flag the extension) + + +# Parent exception - all exceptions thrown by the ClosedSwarm class +# are children of this class +class ClosedSwarmException(Exception): + """ + Base class for closed swarm related exceptions + """ + pass + +# Specialized exceptions +class MissingKeyException(ClosedSwarmException): + """ + No key was available for the operation + """ + pass + +class MissingCertificateException(ClosedSwarmException): + """ + A required certificate was not found + """ + pass + +class BadMessageException(ClosedSwarmException): + """ + A bad closed swarm message was received + """ + pass + +class WrongSwarmException(ClosedSwarmException): + """ + The wrong swarm was specified + """ + pass + +class InvalidSignatureException(ClosedSwarmException): + """ + Invalid signature + """ + pass + +class InvalidPOAException(ClosedSwarmException): + """ + The Proof-of-Access was invalid + """ + pass + +class POAExpiredException(ClosedSwarmException): + """ + The Proof-of-Access has timed out + """ + pass + +# Some helper functions + +def pubkey_from_der(der_key): + """ + Return a public key object from a DER encoded key + """ + return pub_key_from_der(decodestring(der_key)) + +def generate_cs_keypair(keypair_filename=None, pubkey_filename=None): + """ + Generate a keypair suitable for a Closed Swarm + + Saves to the given files if specified, returns keypair, pubkey + """ + keypair = permid.generate_keypair() + if keypair_filename: + permid.save_keypair(keypair, keypair_filename) + + pubkey = encodestring(str(keypair.pub().get_der())).replace("\n","") + if pubkey_filename: + permid.save_pub_key(keypair, pubkey_filename) + + return keypair, pubkey + +def read_cs_keypair(keypair_filename): + """ + Read and return a CS keypair from a file + """ + return permid.read_keypair(keypair_filename) + +def save_cs_keypair(keypair, keypairfilename): + """ + Save a CS keypair to a file + """ + return keypair.save_key(keypairfilename, None) + +def read_cs_pubkey(pubkey_filename): + """ + Read and return the public key of a torrent from a file + """ + return open(pubkey_filename,"r").read() + +def write_poa_to_file(filename, poa): + """ + Dump the POA to the given file in serialized form + """ + target = open(filename,"wb") + target.write(poa.serialize()) + return filename + +def read_poa_from_file(filename): + """ + Read and return a POA object from a file. Throws exception if + the file was not found or the POA could not be deserialized + """ + if not os.path.exists(filename): + raise Exception("File '%s' not found"%filename) + + data = open(filename,"rb").read() + return POA.deserialize(data) + +# Some POA helpers +def trivial_get_poa(path, perm_id, swarm_id): + """ + Look for a POA file for the given permid,swarm_id + """ + filename = encodestring(perm_id).replace("\n","") + filename = filename.replace("/","") + filename = filename.replace("\\","") + + t_id = encodestring(swarm_id).replace("\n","") + t_id = t_id.replace("/","") + t_id = t_id.replace("/","") + + poa_path = os.path.join(path, filename + "." + t_id + ".poa") + + return read_poa_from_file(poa_path) + +def trivial_save_poa(path, perm_id, swarm_id, poa): + """ + Save POA + """ + filename = encodestring(perm_id).replace("\n","") + filename = filename.replace("/","") + filename = filename.replace("\\","") + + t_id = encodestring(swarm_id).replace("\n","") + t_id = t_id.replace("/","") + t_id = t_id.replace("/","") + + # if the path does not exist, try to create it + if not os.path.exists(path): + os.makedirs(path) + + poa_path = os.path.join(path, filename + "." + t_id + ".poa") + + return write_poa_to_file(poa_path, poa) + + +class POA: + """ + Proof of access wrapper + """ + + def __init__(self, torrent_id, torrent_pub_key, node_pub_key, + signature="", expire_time=0): + """ + Create a new POA for this torrent + """ + self.torrent_id = torrent_id + self.torrent_pub_key = torrent_pub_key + self.node_pub_key = node_pub_key + self.signature = signature + self.expire_time = expire_time + + def serialize_to_list(self): + """ + Serialize to a list of entries + """ + return [self.torrent_id, + self.torrent_pub_key, + self.node_pub_key, + self.expire_time, + self.signature] + + def deserialize_from_list(lst): + """ + Deserialize a POA from a list of elements. + + The POA object should be verified after deserializing + """ + if not lst or len(lst) < 5: + raise InvalidPOAException("Bad list") + + torrent_id = lst[0] + torrent_pub_key = lst[1] + node_pub_key = lst[2] + expire_time = lst[3] + signature = lst[4] + return POA(torrent_id, torrent_pub_key, + node_pub_key, signature, expire_time) + + deserialize_from_list = staticmethod(deserialize_from_list) + + def serialize(self): + """ + Return a bencoded, serialized POA + """ + lst = [self.torrent_id, + self.torrent_pub_key, + self.node_pub_key, + self.expire_time, + self.signature] + return bencode(lst) + + def deserialize(encoded): + """ + De-serialize a serialized POA. Returns a POA object, or raises + InvalidPOAException if the POA was bad + """ + if not encoded: + raise InvalidPOAException("Cannot deserialize nothing") + + try: + lst = bdecode(encoded) + if len(lst) < 5: + raise InvalidPOAException("Too few entries (got %d, " + "expected 5)"%len(lst)) + return POA(lst[0], lst[1], + lst[2], expire_time=lst[3], signature=lst[4]) + except Exception, e: + raise InvalidPOAException("De-serialization failed (%s)"%e) + + + deserialize = staticmethod(deserialize) + + + def get_torrent_pub_key(self): + """ + Return the base64 encoded torrent pub key for this POA + """ + return encodestring(self.torrent_pub_key).replace("\n","") + + def verify(self): + """ + Throws an exception if the POA does not hold or has expired + """ + + if self.expire_time and \ + self.expire_time < time.mktime(time.gmtime()): + raise POAExpiredException("Expired") + + try: + lst = [self.torrent_id, + self.torrent_pub_key, + self.node_pub_key] + b_list = bencode(lst) + digest = permid.sha(b_list).digest() + pub = pub_key_from_der(self.torrent_pub_key) + if not pub.verify_dsa_asn1(digest, self.signature): + raise InvalidPOAException("Proof of access verification failed") + except Exception, e: + raise InvalidPOAException("Bad POA: %s"%e) + + def sign(self, torrent_key_pair): + """ + Sign the POA + """ + + lst = [self.torrent_id, + self.torrent_pub_key, + self.node_pub_key] + b_list = bencode(lst) + digest = permid.sha(b_list).digest() + + self.signature = torrent_key_pair.sign_dsa_asn1(digest) + + def save(self, filename): + target = open(filename,"wb") + target.write(self.serialize()) + target.close() + return filename + + def load(filename): + """ + Read and return a POA object from a file. Throws exception if + the file was not found or the POA could not be deserialized + """ + if not os.path.exists(filename): + raise Exception("File '%s' not found"%filename) + + data = open(filename,"rb").read() + return POA.deserialize(data) + + load = staticmethod(load) + + + +def create_poa(torrent_id, torrent_keypair, pub_permid, expire_time=0): + """ + Create and return a certificate 'proof of access' to the given node. + Notice that this function reuire the full keypair of the torrent + """ + poa = POA(torrent_id, + str(torrent_keypair.pub().get_der()), + pub_permid, + expire_time=expire_time) + poa.sign(torrent_keypair) + return poa + + + +class ClosedSwarm: + """ + This is a class that can authenticate two peers to participate in + a closed swarm. + The certificate given must be the "proof of access" to the torrent + in question + + How to use: + For the initiator: + cs = ClosedSwarm(my_keypair, torrent_id, torrent_pubkey, poa) + + node a (initiator) | node b + msg1 = cs_a.a_create_challenge() | + send(msg1) | msg1 = recv() + | msg2 = cs.b_create_challenge(msg1) + msg2 = recv() | send(msg2) + msg3 = cs.a_provide_poa_message(msg2)| + send(msg3) | msg3 = recv() + | msg4 = b_provide_poa_message(msg3) + msg4 = recv() | send(msg4) + cs.a_check_poa_message(msg4) | + if cs.is_remote_node_authorized():| if cs.is_remote_node_authorized(): + ... + + The protocol is allowed to stop after msg3, which means that node b + will not be authorized. This will happen if node b is seeding, or is + not authorized. + + If something bad happens (bad messages, invalid signatures or keys etc), + exceptions are thrown. + + All exceptions thrown are children of ClosedSwarmException + + """ + IDLE = 0 + EXPECTING_RETURN_CHALLENGE = 1 # A sent challenge to B, expects challenge + EXPECTING_INITIATOR_RESPONSE = 2 # B sent challenge to A, expects POA + SEND_INITIATOR_RESPONSE = 3 # A sent POA to B, expects POA + COMPLETED = 4 # Nothing more expected + + def __init__(self, my_keypair, + torrent_id, torrent_pubkeys, + poa): + + if poa: + if not poa.__class__ == POA: + raise Exception("POA is not of class POA, but of class %s"%poa.__class__) + + assert torrent_pubkeys.__class__ == list + + self.state = self.IDLE + + self.my_keypair = my_keypair + self.pub_permid = str(my_keypair.pub().get_der()) + + self.torrent_id = torrent_id + self.torrent_pubkeys = torrent_pubkeys + self.poa = poa + self.remote_node_authorized = False + + self.nonce_a = None + self.nonce_b = None + self.remote_nonce = None # Shortcut + self.my_nonce = None # Shortcut + + if self.poa: # Allow nodes to support CS but not have a POA (e.g. if they are seeding) + if self.poa.get_torrent_pub_key() not in self.torrent_pubkeys: + import sys + print >> sys.stderr, "Bad POA for this torrent (wrong torrent key!)" + self.poa = None + + def is_remote_node_authorized(self): + """ + Returns True iff the remote node is authorized to receive traffic + """ + return self.remote_node_authorized + + def set_poa(self, poa): + """ + Set the POA of the closed swarm run + """ + assert poa.__class__ == POA + self.poa = poa + + def give_up(self): + """ + Give up the protocol - set to completed + """ + self.state = self.COMPLETED + + def is_incomplete(self): + """ + Not completed the CS exchange yet + """ + return self.state != self.COMPLETED + + def _create_challenge_msg(self, msg_id): + """ + Create the first message (from both nodes) + """ + [self.my_nonce, my_nonce_bencoded] = permid.generate_challenge() + # Serialize this message + return [msg_id, + self.torrent_id, + self.my_nonce] + + + def a_create_challenge(self): + """ + 1st message + Initiate a challenge, returns a list + """ + assert self.state == self.IDLE + self.state = self.EXPECTING_RETURN_CHALLENGE + return self._create_challenge_msg(CS_CHALLENGE_A) + + def b_create_challenge(self, lst): + """ + 2nd message + Return a message that can be sent in reply to the given challenge. + Throws exception on bad message or if this cannot be done + BadMessageException - Message was bad + MissingKeyException - Don't have the necessary keys + MissingCertificateException - Don't have a certificate + """ + assert self.state == self.IDLE + self.state = self.EXPECTING_INITIATOR_RESPONSE + + # List should be [INITIAL_CHALLENGE, torrent_id, nonce] + if len(lst) != 3: + raise BadMessageException("Bad number of elements in message, expected 2, got %d"%len(lst)) + if lst[0] != CS_CHALLENGE_A: + raise BadMessageException("Expected initial challenge, got something else") + [torrent_id, nonce_a] = lst[1:] + + # Now we generate the response + if self.torrent_id != torrent_id: + raise WrongSwarmException("Expected %s, got %s"%(self.torrent_id, + torrent_id)) + + # Save the remote nonce too + self.remote_nonce = nonce_a + + # We got a correct challenge for the correct torrent, make our message + return self._create_challenge_msg(CS_CHALLENGE_B) + + def _create_poa_message(self, msg_id, nonce_a, nonce_b): + """ + Create the POA exchange message (messages 3 and 4). + """ + assert msg_id + assert nonce_a + assert nonce_b + assert self.poa + + # Provide the certificate + if not self.poa: + raise MissingCertificateException("Missing certificate") + + msg = [msg_id] + self.poa.serialize_to_list() + + # Add signature + lst = [nonce_a, + nonce_b, + self.poa.serialize()] + + b_list = bencode(lst) + digest = permid.sha(b_list).digest() + sig = self.my_keypair.sign_dsa_asn1(digest) + msg.append(sig) + + return msg + + def _validate_poa_message(self, lst, nonce_a, nonce_b): + """ + Validate an incoming POA message - throw exception if bad. + Returns the POA if successful + """ + assert nonce_a + assert nonce_b + + if len(lst) != 7: + raise BadMessageException("Require 7 elements, got %d"%len(lst)) + + poa = POA.deserialize_from_list(lst[1:-1]) + sig = lst[-1] + assert poa.node_pub_key + + if poa.torrent_id != self.torrent_id: + raise WrongSwarmException("Wrong swarm") + + if poa.get_torrent_pub_key() not in self.torrent_pubkeys: + raise InvalidPOAException("Bad POA for this torrent") + + # Check the signature + lst = [nonce_a, + nonce_b, + poa.serialize()] + import sys + b_list = bencode(lst) + digest = permid.sha(b_list).digest() + try: + pub = pub_key_from_der(poa.node_pub_key) + except: + print >> sys.stderr, "The node_pub_key is no good" + print >> sys.stderr, poa.node_pub_key + raise Exception("Node's public key is no good...") + + if not pub.verify_dsa_asn1(digest, sig): + raise InvalidSignatureException("Freshness test failed") + + # Passed the freshness test, now check the certificate + poa.verify() # Throws exception if bad + return poa + + + def a_provide_poa_message(self, lst): + """ + 3rd message + Got a reply to an initial challenge. Returns + the challenge sent by the remote node + """ + assert self.state == self.EXPECTING_RETURN_CHALLENGE + #self.state = self.SEND_INITIATOR_RESPONSE + self.state = self.COMPLETED # Not sure we get a POA from the remote node + if len(lst) != 3: + raise BadMessageException("Require 3 elements, got %d"%\ + len(lst)) + if lst[0] != CS_CHALLENGE_B: + raise BadMessageException("Expected RETURN_CHALLENGE, got '%s'"%lst[0]) + if lst[1] != self.torrent_id: + raise WrongSwarmException("POA for wrong swarm") + + self.remote_nonce = lst[2] + msg = self._create_poa_message(CS_POA_EXCHANGE_A, self.my_nonce, self.remote_nonce) + return msg + + + def b_provide_poa_message(self, lst, i_am_seeding=False): + """ + 4rd message + Got a reply to an initial challenge. Returns + the challenge sent by the remote node or None if i_am_seeding is true + """ + assert self.state == self.EXPECTING_INITIATOR_RESPONSE + self.state = self.COMPLETED + + if lst[0] != CS_POA_EXCHANGE_A: + import sys + print >> sys.stderr, "Not CS_POA_EXCHANGE_A" + raise BadMessageException("Expected POA EXCHANGE") + + try: + self._validate_poa_message(lst, self.remote_nonce, + self.my_nonce) + self.remote_node_authorized = True + except Exception, e: + self.remote_node_authorized = False + import sys + print >> sys.stderr, "POA could not be validated:",e + #raise e // The remote node failed, but we can still make it! + + if i_am_seeding: + return None + + msg = self._create_poa_message(CS_POA_EXCHANGE_B, self.remote_nonce, self.my_nonce) + return msg + + def a_check_poa_message(self, lst): + """ + Verify receiption of 4th message + """ + assert self.state == self.SEND_INITIATOR_RESPONSE + self.state = self.COMPLETED + + if lst[0] != CS_POA_EXCHANGE_B: + raise BadMessageException("Expected POA EXCHANGE") + + self._validate_poa_message(lst, self.my_nonce, self.remote_nonce) + + # Remote node authorized! + self.remote_node_authorized = True + diff --git a/instrumentation/next-share/BaseLib/Core/ClosedSwarm/PaymentIntegration.py b/instrumentation/next-share/BaseLib/Core/ClosedSwarm/PaymentIntegration.py new file mode 100644 index 0000000..89132a0 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/ClosedSwarm/PaymentIntegration.py @@ -0,0 +1,267 @@ +# Written by Njaal Borch +# see LICENSE.txt for license information +# +# Arno TODO: move this to ../Tools/, wx not allowed in ../Core +# + +import wx +import sys +import urllib +import re + +import xmlrpclib # Not needed for proper payment system integration +from base64 import encodestring,decodestring + +from BaseLib.Core.ClosedSwarm import ClosedSwarm + + +class PaymentSystem: + + def __init__(self, perm_id, swarm_id, mobile_number=None): + self.mobile_number = mobile_number + self.perm_id = perm_id + self.swarm_id = swarm_id + self.request_sent = None + + def set_phone_number(self, mobile_number): + self.mobile_number = mobile_number + + + def request_code(self): + if self.request_sent == self.mobile_number: + import sys + print >> sys.stderr, "Refusing to send new request to same number" + + data = urllib.urlencode({"mobile": self.mobile_number, "request": "code", "swarm_id":self.swarm_id, "nosend": "off", "debug": "off"}) + + f = urllib.urlopen("http://daccer.for-the.biz/smps.php", data) + s = f.read() + f.close() + p = re.compile(r"error=(\S+)", re.MULTILINE) + m = p.search(s) + error = m.group(1) + self.request_sent = self.mobile_number + + # TODO: Check for errors and throw exceptions + return error + + def verify_code(self, code): + import sys + print >> sys.stderr, {"request": "validate", "code": code, "mobile": self.mobile_number, "perm_id": self.perm_id, "swarm_id": self.swarm_id} + + data = urllib.urlencode({"request": "validate", "code": code, "mobile": self.mobile_number, "perm_id": self.perm_id, "swarm_id": self.swarm_id}) + f = urllib.urlopen("http://daccer.for-the.biz/smps.php", data) + s = f.read() + f.close() + p = re.compile(r"code=(\w+)", re.MULTILINE) + m = p.search(s) + if m != None: + validation = m.group(1) + else: + validation = None + p = re.compile(r"poa=(.*)..error=", re.DOTALL) + m = p.search(s) + if m != None: + poa = m.group(1) + else: + poa = None + p = re.compile(r"error=(\S+)", re.MULTILINE) + m = p.search(s) + if m != None: + error = m.group(1) + else: + error = ",no_error_return" + + print >>sys.stderr,"Verify Code returned ",s,"with error:",error + + # TODO: Check for errors and throw exceptions + return [validation, poa, error] + + + + +class PaymentDialog(wx.Dialog): + """ + The dialog to interact with the payment service + TODO: Do some design here... :) + """ + def __init__(self, swarm_title, payment_url, swarm_id, node_id): + kwds = {"style":wx.DEFAULT_DIALOG_STYLE} + wx.Dialog.__init__(self, None) + + self.payment_url = payment_url + self.swarm_id = swarm_id + self.node_id = node_id + self.phone_number = None + self.poa = None + + self.label_0 = wx.StaticText(self, -1, "\nRegister your phone number (with country code) to get \nhigh speed access to the resource '" + swarm_title + "'\n") + + self.label_1 = wx.StaticText(self, -1, "Phone number") + self.txt_phone_number = wx.TextCtrl(self, -1, "") + self.btn_request_code = wx.Button(self, -1, "Request code") + + self.label_2 = wx.StaticText(self, -1, "Code") + self.txt_code = wx.TextCtrl(self, -1, "") + self.btn_send_code = wx.Button(self, -1, "Send code") + + self.status = wx.StatusBar(self, -1) + + self.__set_properties() + self.__do_layout() + + self.Bind(wx.EVT_BUTTON, self._request_code, self.btn_request_code) + self.Bind(wx.EVT_BUTTON, self._request_token, self.btn_send_code) + + self.status.SetStatusText("Please enter your phone number") + + self._payment_service = PaymentSystem(node_id, swarm_id) + + def __set_properties(self): + self.SetTitle("NextShare payment test") + + self.txt_code.Enable(False) + self.btn_send_code.Enable(False) + + def __do_layout(self): + + # begin wxGlade: MyDialog.__do_layout + sizer_1 = wx.BoxSizer(wx.VERTICAL) + grid_sizer_1 = wx.GridSizer(2, 3, 0, 0) + sizer_1.Add(self.label_0, 0, 0, 0) + grid_sizer_1.Add(self.label_1, 0, wx.ALIGN_CENTER_VERTICAL, 0) + grid_sizer_1.Add(self.txt_phone_number, 0, wx.EXPAND, 0) + grid_sizer_1.Add(self.btn_request_code, 0, wx.EXPAND, 0) + grid_sizer_1.Add(self.label_2, 0, wx.ALIGN_CENTER_VERTICAL, 0) + grid_sizer_1.Add(self.txt_code, 0, wx.EXPAND, 0) + grid_sizer_1.Add(self.btn_send_code, 0, wx.EXPAND, 0) + sizer_1.Add(grid_sizer_1, 1, wx.EXPAND, 0) + sizer_1.Add(self.status, 0, 0, 0) + self.SetSizer(sizer_1) + sizer_1.Fit(self) + self.Layout() + + + + def _request_code(self, event): + + num = self.txt_phone_number.Value + if not num: + # TODO: Error handling + return + try: + self._payment_service.set_phone_number(num.strip()) + self.status.SetStatusText("Requesting code...") + error = self._payment_service.request_code() + + if error != "0": + if error.count("mobile_number_wrong") > 0: + txt = "Bad mobile number" + elif error.count("swarm_id_unavailable") > 0: + txt = "Unknown resource" + else: + txt = "Unknown error: " + error + + self.status.SetStatusText("Got errors:" + txt) + return + + except Exception,e: + print >>sys.stderr,"Error contacting payment system:",e + # TODO: Handle errors properly + return + + # TODO: Figure out why the payment system doesn't want the swarm ID + # TODO: to figure out the price/availability etc. + + #s = xmlrpclib.ServerProxy(self.payment_url) + #s.initialize_payment(num, self.swarm_id) + + # Enable code field and button + self.phone_number = num + self.txt_code.Enable(True) + self.btn_send_code.Enable(True) + + self.status.SetStatusText("Please enter the code") + + def _request_token(self, event): + + code = self.txt_code.Value + if not code: + # TODO: Error handling + return + + [validation, poa, error] = self._payment_service.verify_code(code) + + if error != "0": + if error.count("no_such_code"): + txt = "Bad code" + elif error.count("code_to_old"): + txt = "Code has expired" + elif error.count("code_already_consumed"): + txt = "This code has already been used" + elif error.count("mobile_number_different"): + txt = "INTERNAL: phone number has changed..." + elif error.count("invalid_request"): + txt = "The request vas invalid" + else: + txt = "Unknown error: " + error + self.status.SetStatusText("Got error: " + txt) + return + + self.poa = poa + self.EndModal(0) + + def get_poa(self): + return self.poa + + +def wx_get_poa(url, swarm_id, perm_id, root_window=None, swarm_title="Unknown content"): + """ + Pop up a WX interface for the payment system + """ + + d = PaymentDialog(swarm_title, + url, + swarm_id, + perm_id) + + retval = d.ShowModal() + try: + poa_b64 = d.get_poa() + poa_serialized = decodestring(poa_b64) + poa = ClosedSwarm.POA.deserialize(poa_serialized) + poa.verify() + return poa + except: + return None + + + +# Test +if __name__ == "__main__": + + app = wx.PySimpleApp() + import threading + t = threading.Thread(target=app.MainLoop) + t.start() + + + + d = PaymentDialog("Test file", + "http://127.0.0.1:9090", + "1234", + "myid") + + retval = d.ShowModal() + print "Modal returned" + poa_b64 = d.get_poa() + if poa_b64: + poa_serialized = decodestring(poa_b64) + from BaseLib.Core.ClosedSwarm import ClosedSwarm + poa = ClosedSwarm.POA.deserialize(poa_serialized) + poa.verify() + print "Got valid poa" + + else: + print "No POA for us..." + diff --git a/instrumentation/next-share/BaseLib/Core/ClosedSwarm/Tools.py b/instrumentation/next-share/BaseLib/Core/ClosedSwarm/Tools.py new file mode 100644 index 0000000..377028d --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/ClosedSwarm/Tools.py @@ -0,0 +1,166 @@ +# Written by Njaal Borch +# see LICENSE.txt for license information +# +# Arno TODO: move this to ../Tools/, wx not allowed in ../Core +# + +import re +import urllib +import urllib2 + +from ClosedSwarm import read_poa_from_file + +def wx_get_poa(root_window=None): + """ + Pop up a graphical file selector + """ + import wx + import sys + print >>sys.stderr, "Using GUI poa browser" + fd = wx.FileDialog(root_window, "Select Proof of Access", wildcard="*.poa", style=wx.OPEN) + if fd.ShowModal() == wx.ID_OK: + return read_poa_from_file(fd.GetPath()) + raise Exception("User aborted") + + +def wx_get_http_poa(url, swarm_id, perm_id, root_window=None): + """ + Pop up a graphical authorization thingy if required by the + web server + """ + + def auth_handler(realm): + """ + As for username,password + """ + import wx + import sys + print >>sys.stderr, "Using GUI poa browser" + + pw = wx.Dialog(root_window, -1, "Authenticate") + + vert = wx.BoxSizer(wx.VERTICAL) + label_1 = wx.StaticText(pw, -1, "Authentication for %s reqired"%realm) + vert.Add(label_1, 0, wx.EXPAND | wx.LEFT, 10) + + horiz = wx.BoxSizer(wx.HORIZONTAL) + vert.Add(horiz, 0, 0, 0) + label_2 = wx.StaticText(pw, -1, "Username") + label_2.SetMinSize((70,15)) + horiz.Add(label_2, 0, wx.LEFT, 0) + pw.username = wx.TextCtrl(pw, -1, "") + horiz.Add(pw.username, 0, wx.LEFT, 0) + + horiz = wx.BoxSizer(wx.HORIZONTAL) + vert.Add(horiz, 0, 0, 0) + pw.pwd = wx.TextCtrl(pw, -1, "", style=wx.TE_PASSWORD) + label_3 = wx.StaticText(pw, -1, "Password") + label_3.SetMinSize((70,15)) + horiz.Add(label_3, 0, wx.LEFT, 0) + horiz.Add(pw.pwd, 0, wx.LEFT, 0) + + horiz = wx.BoxSizer(wx.HORIZONTAL) + vert.Add(horiz, 0, wx.LEFT, 0) + + horiz.Add(wx.Button(pw, wx.ID_CANCEL), 0,0,0) + ok = wx.Button(pw, wx.ID_OK) + ok.SetDefault() + horiz.Add(ok, 0,0,0) + + pw.username.SetFocus() + order = (pw.username, pw.pwd, ok) + for i in xrange(len(order) - 1): + order[i+1].MoveAfterInTabOrder(order[i]) + + pw.SetSizer(vert) + vert.Fit(pw) + pw.Layout() + + try: + if pw.ShowModal() == wx.ID_OK: + return (pw.username.GetValue(), pw.pwd.GetValue()) + finally: + pw.Destroy() + + raise Exception("User aborted") + + w = web_get_poa(url, swarm_id, perm_id, auth_handler) + return w.get_poa() + + +class web_get_poa: + """ + Class that will call the auth_handler if authentication + is required + """ + + def __init__(self, url, swarm_id, perm_id, auth_handler=None): + + self.url = url + self.swarm_id = swarm_id + self.perm_id = perm_id + self.auth_handler = auth_handler + + + def get_poa(self, credentials=None): + """ + Try to fetch a POA + """ + + if credentials and len(credentials) == 4: + (protocol, realm, name, password) = credentials + password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm() + password_mgr.add_password(realm, self.url, name, password) + if protocol.lower() == "digest": + handler = urllib2.HTTPDigestAuthHandler(password_mgr) + elif protocol.lower() == "basic": + handler = urllib2.HTTPBasicAuthHandler(password_mgr) + else: + raise Exception("Unknown authorization protocol: '%s'"%protocol) + + opener = urllib2.build_opener(handler) + urllib2.install_opener(opener) + + values = {'swarm_id':self.swarm_id, + 'perm_id':self.perm_id} + + try: + data = urllib.urlencode(values) + req = urllib2.Request(self.url, data) + response = urllib2.urlopen(req) + except urllib2.HTTPError,e: + # Need authorization? + if e.code == 401 and not credentials: + try: + type, realm = e.headers["WWW-Authenticate"].split() + m = re.match('realm="(.*)"', realm) + if m: + realm = m.groups()[0] + else: + raise Exception("Bad www-authenticate reponse") + except Exception,e: + raise Exception("Authentication failed: %s"%e) + + if self.auth_handler: + name, passwd = self.auth_handler(realm) + if name and passwd: + credentials = (type, realm, name, passwd) + return self.get_poa(credentials) + + raise Exception("Could not get POA: %s"%e) + except urllib2.URLError,e: + raise Exception("Could not get POA: %s"%e.reason) + + + # Connected ok, now get the POA + try: + poa_str = response.read() + from BaseLib.Core.ClosedSwarm import ClosedSwarm + return ClosedSwarm.POA.deserialize(poa_str) + except Exception,e: + raise Exception("Could not fetch POA: %s"%e) + + raise Exception("Could not get POA: Unknown reason") + + + diff --git a/instrumentation/next-share/BaseLib/Core/ClosedSwarm/__init__.py b/instrumentation/next-share/BaseLib/Core/ClosedSwarm/__init__.py new file mode 100644 index 0000000..94c71b3 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/ClosedSwarm/__init__.py @@ -0,0 +1,3 @@ +# Written by Njaal Borch +# see LICENSE.txt for license information + diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/MagnetLink/MagnetLink.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/MagnetLink/MagnetLink.py new file mode 100644 index 0000000..a111994 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/MagnetLink/MagnetLink.py @@ -0,0 +1,208 @@ +# Written by Boudewijn Schoon +# see LICENSE.txt for license information +""" +The MagnetLink module handles the retrieval of the 'info' part of a +.torrent file given a magnet link. + +Ideally we should use the regular BitTorrent connection classes to +make connection to peers, but all these classes assume that the +.torrent information is already available. + +Hence, this module will make BitTorrent connection for the sole +purpose of retrieving the .torrent info part. After retrieval has +finished all connections are closed and a regular download will begin. +""" +import sys +from binascii import unhexlify +from urlparse import urlsplit +from traceback import print_exc +from threading import Lock + +try: + # parse_sql requires python 2.6 or higher + from urlparse import parse_qsl +except ImportError: + from urllib import unquote_plus + def parse_qsl(query): + """ + 'foo=bar&moo=milk' --> [('foo', 'bar'), ('moo', 'milk')] + """ + query = unquote_plus(query) + for part in query.split("&"): + if "=" in part: + yield part.split("=", 1) + +from BaseLib.Core.DecentralizedTracking.kadtracker.identifier import Id, IdError +from BaseLib.Core.DecentralizedTracking.MagnetLink.MiniBitTorrent import MiniSwarm, MiniTracker +import BaseLib.Core.DecentralizedTracking.mainlineDHT as mainlineDHT + +DEBUG = False + +class Singleton: + _singleton_lock = Lock() + + @classmethod + def get_instance(cls, *args, **kargs): + if hasattr(cls, "_singleton_instance"): + return getattr(cls, "_singleton_instance") + + else: + cls._singleton_lock.acquire() + try: + if not hasattr(cls, "_singleton_instance"): + setattr(cls, "_singleton_instance", cls(*args, **kargs)) + return getattr(cls, "_singleton_instance") + + finally: + cls._singleton_lock.release() + +class MagnetHandler(Singleton): + def __init__(self, raw_server): + self._raw_server = raw_server + self._magnets = [] + + def get_raw_server(self): + return self._raw_server + + def add_magnet(self, magnet_link): + self._magnets.append(magnet_link) + + def remove_magnet(self, magnet_link): + self._magnets.remove(magnet_link) + + def get_magnets(self): + return self._magnets + +class MagnetLink: + def __init__(self, url, callback): + """ + If the URL conforms to a magnet link, the .torrent info is + downloaded and returned to CALLBACK. + """ + # _callback is called when the metadata is retrieved. + self._callback = callback + + dn, xt, tr = self._parse_url(url) + + # _name is the unicode name suggested for the swarm. + assert dn is None or isinstance(dn, unicode), "DN has invalid type: %s" % type(dn) + self._name = dn + + # _info_hash is the 20 byte binary info hash that identifies + # the swarm. + assert isinstance(xt, str), "XT has invalid type: %s" % type(xt) + assert len(xt) == 20, "XT has invalid length: %d" % len(xt) + self._info_hash = xt + + # _tracker is an optional tracker address. + self._tracker = tr + + # _swarm is a MiniBitTorrent.MiniSwarm instance that connects + # to peers to retrieve the metadata. + magnet_handler = MagnetHandler.get_instance() + magnet_handler.add_magnet(self) + self._swarm = MiniSwarm(self._info_hash, magnet_handler.get_raw_server(), self.metainfo_retrieved) + + def get_infohash(self): + return self._info_hash + + def get_name(self): + return self._name + + def retrieve(self): + """ + Start retrieving the metainfo + + Returns True when attempting to obtain the metainfo, in this + case CALLBACK will always be called. Otherwise False is + returned, in this case CALLBACK will not be called. + """ + if self._info_hash: + # todo: catch the result from get_peers and call its stop + # method. note that this object does not yet contain a + # stop method... + dht = mainlineDHT.dht + dht.get_peers(Id(self._info_hash), self._swarm.add_potential_peers) + + try: + if self._tracker: + MiniTracker(self._swarm, self._tracker) + except: + print_exc() + + return True + else: + return False + + def metainfo_retrieved(self, metainfo, peers=[]): + """ + Called when info part for metadata is retrieved. If we have + more metadata, we will add it at this point. + + PEERS optionally contains a list of valid BitTorrent peers, + found during metadata download, to help bootstrap the + download. + """ + assert isinstance(metainfo, dict) + assert isinstance(peers, list) + if __debug__: + for address in peers: + assert isinstance(address[0], str) + assert isinstance(address[1], int) + + # create metadata + metadata = {"info":metainfo} + if self._tracker: + metadata["announce"] = self._tracker + else: + metadata["nodes"] = [] + if peers: + metadata["initial peers"] = peers + + self._callback(metadata) + self.close() + + def close(self): + magnet_handler = MagnetHandler.get_instance() + magnet_handler.remove_magnet(self) + + # close all MiniBitTorrent activities + self._swarm.close() + + @staticmethod + def _parse_url(url): + # url must be a magnet link + dn = None + xt = None + tr = None + + if DEBUG: print >> sys.stderr, "Magnet._parse_url()", url + + schema, netloc, path, query, fragment = urlsplit(url) + if schema == "magnet": + # magnet url's do not conform to regular url syntax (they + # do not have a netloc.) This causes path to contain the + # query part. + if "?" in path: + pre, post = path.split("?", 1) + if query: + query = "&".join((post, query)) + else: + query = post + + for key, value in parse_qsl(query): + if key == "dn": + # convert to unicode + dn = value.decode() + + elif key == "xt" and value.startswith("urn:btih:"): + xt = unhexlify(value[9:49]) + + elif key == "tr": + tr = value + + if DEBUG: print >> sys.stderr, "Magnet._parse_url() NAME:", dn + if DEBUG: print >> sys.stderr, "Magnet._parse_url() HASH:", xt + if DEBUG: print >> sys.stderr, "Magnet._parse_url() TRAC:", tr + + return (dn, xt, tr) diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/MagnetLink/MiniBitTorrent.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/MagnetLink/MiniBitTorrent.py new file mode 100644 index 0000000..38420de --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/MagnetLink/MiniBitTorrent.py @@ -0,0 +1,560 @@ +# Written by Boudewijn Schoon +# see LICENSE.txt for license information + +""" +The MiniBitTorrent module sets up connections to BitTorrent peers with +the sole purpose of obtaining the .Torrent metadata. + +The peers are obtained though either the tracker, PEX, or the DHT +provided in the MagnetLink. All connections will be closed once the +metadata is obtained. +""" + +from cStringIO import StringIO +from random import getrandbits +from threading import Lock, Event, Thread +from time import time +from traceback import print_exc +from urllib import urlopen, urlencode +import sys + +from BaseLib.Core.BitTornado.BT1.MessageID import protocol_name, EXTEND +from BaseLib.Core.BitTornado.BT1.convert import toint, tobinary +from BaseLib.Core.BitTornado.RawServer import RawServer +from BaseLib.Core.BitTornado.SocketHandler import SocketHandler +from BaseLib.Core.BitTornado.bencode import bencode, bdecode +from BaseLib.Core.Utilities.Crypto import sha + +UT_EXTEND_HANDSHAKE = chr(0) +UT_PEX_ID = chr(1) +UT_METADATA_ID = chr(2) +METADATA_PIECE_SIZE = 16 * 1024 +MAX_CONNECTIONS = 30 +MAX_TIME_INACTIVE = 30 + +DEBUG = True + +# todo: extend testcases +# todo: add tracker support +# todo: stop the dht + +class Connection: + """ + A single BitTorrent connection. + """ + def __init__(self, swarm, raw_server, address): + self._swarm = swarm + self._closed = False + self._in_buffer = StringIO() + self._next_len = 1 + self._next_func = self.read_header_len + self._address = address + self._last_activity = time() + + self._her_ut_metadata_id = chr(0) + + # outstanding requests for pieces + self._metadata_requests = [] + + if DEBUG: print >> sys.stderr, self._address, "MiniBitTorrent: New connection" + self._socket = raw_server.start_connection(address, self) + self.write_handshake() + + def write_handshake(self): + # if DEBUG: print >> sys.stderr, "MiniBitTorrent.write_handshake()" + self._socket.write("".join((chr(len(protocol_name)), protocol_name, + "\x00\x00\x00\x00\x00\x30\x00\x00", + self._swarm.get_info_hash(), + self._swarm.get_peer_id()))) + + def write_extend_message(self, metadata_message_id, payload): + assert isinstance(payload, dict), "PAYLOAD has invalid type: %s" % type(payload) + assert isinstance(metadata_message_id, str), "METADATA_MESSAGE_ID has invalid type: %s" % type(metadata_message_id) + assert len(metadata_message_id) == 1, "METADATA_MESSAGE_ID has invalid length: %d" % len(metadata_message_id) + if DEBUG: print >> sys.stderr, self._address, "MiniBitTorrent.write_extend_message()" + payload = bencode(payload) + self._socket.write("".join((tobinary(2 + len(payload)), # msg len + EXTEND, # msg id + metadata_message_id, # extend msg id + payload))) # bencoded msg + + def read_header_len(self, s): + if ord(s) != len(protocol_name): + return None + return len(protocol_name), self.read_header + + def read_header(self, s): + if s != protocol_name: + return None + return 8, self.read_reserved + + def read_reserved(self, s): + if ord(s[5]) & 16: + # extend module is enabled + if DEBUG: print >> sys.stderr, self._address, "MiniBitTorrent.read_reserved() extend module is supported" + self.write_extend_message(UT_EXTEND_HANDSHAKE, {"m":{"ut_pex":ord(UT_PEX_ID), "ut_metadata":ord(UT_METADATA_ID), "metadata_size":self._swarm.get_metadata_size()}}) + return 20, self.read_download_id + else: + if DEBUG: print >> sys.stderr, self._address, "MiniBitTorrent.read_reserved() extend module not supported" + return None + + def read_download_id(self, s): + if s != self._swarm.get_info_hash(): + if DEBUG: print >> sys.stderr, self._address, "MiniBitTorrent.read_download_id() invalid info hash" + return None + return 20, self.read_peer_id + + def read_peer_id(self, s): + self._swarm.add_good_peer(self._address) + return 4, self.read_len + + def read_len(self, s): + l = toint(s) + # if l > self.Encoder.max_len: + # return None + # if DEBUG: print >> sys.stderr, "waiting for", l, "bytes" + return l, self.read_message + + def read_message(self, s): + if s != '': + if not self.got_message(s): + return None + return 4, self.read_len + + def got_message(self, data): + if data[0] == EXTEND and len(data) > 2: + + # we only care about EXTEND messages. So all other + # messages will NOT reset the _last_activity timestamp. + self._last_activity = time() + + return self.got_extend_message(data) + + # ignore all other messages, but stay connected + return True + + def _request_some_metadata_piece(self): + if not self._closed: + piece = self._swarm.reserve_metadata_piece() + if isinstance(piece, int): + if DEBUG: print >> sys.stderr, self._address, "MiniBitTorrent.got_extend_message() Requesting metadata piece", piece + self._metadata_requests.append(piece) + self.write_extend_message(self._her_ut_metadata_id, {"msg_type":0, "piece":piece}) + + else: + self._swarm._raw_server.add_task(self._request_some_metadata_piece, 1) + + def got_extend_message(self, data): + try: + message = bdecode(data[2:], sloppy=True) + if DEBUG: print >> sys.stderr, self._address, "MiniBitTorrent.got_extend_message()", len(message), "bytes as payload" + # if DEBUG: print >> sys.stderr, message + except: + if DEBUG: print >> sys.stderr, self._address, "MiniBitTorrent.got_extend_message() Received invalid UT_METADATA message" + return False + + if data[1] == UT_EXTEND_HANDSHAKE: # extend handshake + if "metadata_size" in message and isinstance(message["metadata_size"], int) and message["metadata_size"] > 0: + self._swarm.add_metadata_size_opinion(message["metadata_size"]) + + if "m" in message and isinstance(message["m"], dict) and "ut_metadata" in message["m"] and isinstance(message["m"]["ut_metadata"], int): + self._her_ut_metadata_id = chr(message["m"]["ut_metadata"]) + self._request_some_metadata_piece() + + else: + # other peer does not support ut_metadata. Try to get + # some PEX peers, otherwise close connection + if not ("m" in message and isinstance(message["m"], dict) and "ut_pex" in message["m"]): + return False + + elif data[1] == UT_PEX_ID: # our ut_pex id + if "added" in message and isinstance(message["added"], str) and len(message["added"]) % 6 == 0: + added = message["added"] + addresses = [] + for offset in xrange(0, len(added), 6): + address = ("%s.%s.%s.%s" % (ord(added[offset]), ord(added[offset+1]), ord(added[offset+2]), ord(added[offset+3])), ord(added[offset+4]) << 8 | ord(added[offset+5])) + addresses.append(address) + if DEBUG: print >> sys.stderr, self._address, "MiniBitTorrent.got_extend_message()", len(addresses), "peers from PEX" + self._swarm.add_potential_peers(addresses) + + # when this peer does not support ut_metadata we can + # close the connection after receiving a PEX message + if self._her_ut_metadata_id == chr(0): + return False + + elif data[1] == UT_METADATA_ID: # our ut_metadata id + if "msg_type" in message: + if message["msg_type"] == 0 and "piece" in message and isinstance(message["piece"], int): + # She send us a request. However, since + # MiniBitTorrent disconnects after obtaining the + # metadata, we can not provide any pieces + # whatsoever. + # So... send reject + if DEBUG: print >> sys.stderr, self._address, "MiniBitTorrent.got_extend_message() Rejecting request for piece", message["piece"] + self.write_extend_message(self._her_ut_metadata_id, {"msg_type":2, "piece":message["piece"]}) + + elif message["msg_type"] == 1 and \ + "piece" in message and (isinstance(message["piece"], int) or isinstance(message["piece"], long)) and message["piece"] in self._metadata_requests and \ + "total_size" in message and (isinstance(message["total_size"], int) or isinstance(message["total_size"], long)) and message["total_size"] <= METADATA_PIECE_SIZE: + # Received a metadata piece + if DEBUG: print >> sys.stderr, self._address, "MiniBitTorrent.got_extend_message() Received metadata piece", message["piece"] + self._metadata_requests.remove(message["piece"]) + self._swarm.add_metadata_piece(message["piece"], data[-message["total_size"]:]) + self._request_some_metadata_piece() + + elif message["msg_type"] == 2 and "piece" in message and isinstance(message["piece"], int) and message["piece"] in self._metadata_requests: + # Received a reject + if DEBUG: print >> sys.stderr, self._address, "MiniBitTorrent.got_extend_message() Our request for", message["piece"], "was rejected" + self._metadata_requests.remove(message["piece"]) + self._swarm.unreserve_metadata_piece(message["piece"]) + + # Register a task to run in 'some time' to start + # requesting again (reject is usually caused by + # flood protection) + self._swarm._raw_server.add_task(self._request_some_metadata_piece, 5) + + else: + if DEBUG: print >> sys.stderr, self._address, "MiniBitTorrent.got_extend_message() Received unknown message" + return False + + else: + if DEBUG: print >> sys.stderr, self._address, "MiniBitTorrent.got_extend_message() Received invalid extend message (no msg_type)" + return False + + else: + if DEBUG: print >> sys.stderr, self._address, "MiniBitTorrent.got_extend_message() Received unknown extend message" + return False + + return True + + def data_came_in(self, socket, data): + while not self._closed: + left = self._next_len - self._in_buffer.tell() + # if DEBUG: print >> sys.stderr, self._in_buffer.tell() + len(data), "/", self._next_len + if left > len(data): + self._in_buffer.write(data) + return + self._in_buffer.write(data[:left]) + data = data[left:] + message = self._in_buffer.getvalue() + self._in_buffer.reset() + self._in_buffer.truncate() + next_ = self._next_func(message) + if next_ is None: + self.close() + return + self._next_len, self._next_func = next_ + + def connection_lost(self, socket): + if DEBUG: print >> sys.stderr, self._address, "MiniBitTorrent.connection_lost()" + self._closed = True + self._swarm.connection_lost(self) + + def connection_flushed(self, socket): + pass + + def check_for_timeout(self, deadline): + """ + Close when no activity since DEADLINE + """ + if self._last_activity < deadline: + if DEBUG: print >> sys.stderr, self._address, "MiniBitTorrent.check_for_timeout() Timeout!" + self.close() + + def close(self): + if DEBUG: print >> sys.stderr, self._address, "MiniBitTorrent.close()" + self._closed = True + if self._socket.connected: + self._socket.close() + else: + self._swarm.connection_lost(self) + +class MiniSwarm: + """ + A MiniSwarm instance maintains an overview of what is going on in + a single BitTorrent swarm. + """ + def __init__(self, info_hash, raw_server, callback): + # _info_hash is the 20 byte binary info hash that identifies + # the swarm. + assert isinstance(info_hash, str), str + assert len(info_hash) == 20 + self._info_hash = info_hash + + # _raw_server provides threading support. All socket events + # will run in this thread. + self._raw_server = raw_server + + # _callback is called with the raw metadata string when it is + # retrieved + self._callback = callback + + # _peer_id contains 20 semi random bytes + self._peer_id = "-ST0100-" + "".join([chr(getrandbits(8)) for _ in range(12)]) + assert isinstance(self._peer_id, str) + assert len(self._peer_id) == 20, len(self._peer_id) + + # _lock protects several member variables that are accessed + # from our RawServer and other threads. + self._lock = Lock() + + # _connections contains all open socket connections. This + # variable is protected by _lock. + self._connections = [] + + # _metadata_blocks contains the blocks that form the metadata + # that we want to download. This variable is protected by + # _lock. + self._metadata_blocks = [] # [requested, piece, data] + + # _metadata_size contains the size in bytes of the metadata. + # This value is based on the opinions of other peers which is + # accumulated in _metadata_size_opinions. + self._metadata_size = 0 + self._metadata_size_opinions = {} # size:number-of-votes + + # _potential_peers contains a dictionary of address::timestamp + # pairs where potential BitTorrent peers can be found + self._potential_peers = {} + + # _good_peers contains a dictionary of address:timestamp pairs + # where valid BitTorrent peers can be found + self._good_peers = {} + + # _closed indicates that we no longer need this swarm instance + self._closed = False + + # scan for old connections + self._raw_server.add_task(self._timeout_connections, 5) + + def add_good_peer(self, address): + assert isinstance(address, tuple) + assert len(address) == 2 + assert isinstance(address[0], str) + assert isinstance(address[1], int) + self._good_peers[address] = time() + + def get_info_hash(self): + return self._info_hash + + def get_peer_id(self): + return self._peer_id + + def get_metadata_size(self): + return self._metadata_size + + def add_metadata_size_opinion(self, metadata_size): + """ + A peer told us the metadata size. Assume it is correct, + however, keep track of potential differences. + """ + if metadata_size in self._metadata_size_opinions: + self._metadata_size_opinions[metadata_size] += 1 + else: + self._metadata_size_opinions[metadata_size] = 1 + + # what do we believe the metadata size is + if len(self._metadata_size_opinions) == 1: + metadata_size = self._metadata_size_opinions.keys()[0] + if DEBUG: print >> sys.stderr, "MiniBitTorrent.add_metadata_size_opinion() Metadata size is:", metadata_size, "(%d unanimous vote)" % sum(self._metadata_size_opinions.values()) + + else: + options = [(weight, metadata_size) for metadata_size, weight in self._metadata_size_opinions.iteritems()] + options.sort(reverse=True) + if DEBUG: print >> sys.stderr, "MiniBitTorrent.add_metadata_size_opinion() Choosing metadata size from multiple options:", options + metadata_size = options[0][1] + + if self._metadata_size != metadata_size: + self._metadata_size = metadata_size + + pieces = metadata_size / METADATA_PIECE_SIZE + if metadata_size % METADATA_PIECE_SIZE != 0: + pieces += 1 + + # we were led to believe that there are more blocks than + # there actually are, remove some + if len(self._metadata_blocks) > pieces: + if DEBUG: print >> sys.stderr, "MiniBitTorrent.add_metadata_size_opinion() removing some blocks..." + self._metadata_blocks = [block_tuple for block_tuple in self._metadata_blocks if block_tuple[1] < pieces] + + # we were led to believe that there are fewer blocks than + # there actually are, add some + elif len(self._metadata_blocks) < pieces: + blocks = [[0, piece, None] for piece in xrange(len(self._metadata_blocks), pieces)] + if DEBUG: print >> sys.stderr, "MiniBitTorrent.add_metadata_size_opinion() adding", len(blocks), "blocks..." + self._metadata_blocks.extend(blocks) + + def reserve_metadata_piece(self): + """ + A metadata piece request can be made. Find the most usefull + piece to request. + """ + for block_tuple in self._metadata_blocks: + if block_tuple[2] is None: + block_tuple[0] += 1 + self._metadata_blocks.sort() + return block_tuple[1] + return None + + def unreserve_metadata_piece(self, piece): + """ + A metadata piece request is refused or cancelled. Update the + priorities. + """ + for index, block_tuple in zip(xrange(len(self._metadata_blocks)), self._metadata_blocks): + if block_tuple[1] == piece: + block_tuple[0] = max(0, block_tuple[0] - 1) + self._metadata_blocks.sort() + break + + def add_metadata_piece(self, piece, data): + """ + A metadata piece was received + """ + if not self._closed: + + for index, block_tuple in zip(xrange(len(self._metadata_blocks)), self._metadata_blocks): + if block_tuple[1] == piece: + block_tuple[0] = max(0, block_tuple[0] - 1) + block_tuple[2] = data + self._metadata_blocks.sort() + break + + # def p(s): + # if s is None: return 0 + # return len(s) + # if DEBUG: print >> sys.stderr, "Progress:", [p(t[2]) for t in self._metadata_blocks] + + # see if we are done + for requested, piece, data in self._metadata_blocks: + if data is None: + break + + else: + metadata = "".join([data for requested, piece, data in self._metadata_blocks]) + info_hash = sha(metadata).digest() + + if info_hash == self._info_hash: + if DEBUG: print >> sys.stderr, "MiniBitTorrent.add_metadata_piece() Done!" + + # get nice list with recent BitTorrent peers, sorted + # by most recently connected + peers = [(timestamp, address) for address, timestamp in self._good_peers.iteritems()] + peers.sort(reverse=True) + peers = [address for timestamp, address in peers] + + self._callback(bdecode(metadata), peers) + + else: + # todo: hash failed... now what? + # quick solution... remove everything and try again + self._metadata_blocks = [(requested, piece, None) for requested, piece, data in self._metadata_blocks] + + def add_potential_peers(self, addresses): + if not self._closed: + for address in addresses: + if not address in self._potential_peers: + self._potential_peers[address] = 0 + + if len(self._connections) < MAX_CONNECTIONS: + self._create_connections() + + def _create_connections(self): + # order by last connection attempt + addresses = [(timestamp, address) for address, timestamp in self._potential_peers.iteritems()] + addresses.sort() + + now = time() + + # print >> sys.stderr, len(self._connections), "/", len(self._potential_peers) + + for timestamp, address in addresses: + if len(self._connections) >= MAX_CONNECTIONS: + break + + if address in self._connections: + continue + + try: + self._potential_peers[address] = now + connection = Connection(self, self._raw_server, address) + + except: + if DEBUG: print >> sys.stderr, "MiniBitTorrent.add_potential_peers() ERROR" + print_exc() + + else: + self._lock.acquire() + try: + self._connections.append(connection) + finally: + self._lock.release() + + def _timeout_connections(self): + deadline = time() - MAX_TIME_INACTIVE + for connection in self._connections: + connection.check_for_timeout(deadline) + + if not self._closed: + self._raw_server.add_task(self._timeout_connections, 1) + + def connection_lost(self, connection): + try: + self._connections.remove(connection) + except: + # it is possible that a connection timout occurs followed + # by another connection close from the socket handler when + # the connection can not be established. + pass + if not self._closed: + self._create_connections() + + def close(self): + self._closed = True + for connection in self._connections: + connection.close() + +class MiniTracker(Thread): + """ + A MiniTracker instance makes a single connection to a tracker to + attempt to obtain peer addresses. + """ + def __init__(self, swarm, tracker): + Thread.__init__(self) + self._swarm = swarm + self._tracker = tracker + self.start() + + def run(self): + announce = self._tracker + "?" + urlencode({"info_hash":self._swarm.get_info_hash(), + "peer_id":self._swarm.get_peer_id(), + "port":"12345", + "compact":"1", + "uploaded":"0", + "downloaded":"0", + "left":"-1", + "event":"started"}) + handle = urlopen(announce) + if handle: + body = handle.read() + if body: + try: + body = bdecode(body) + + except: + pass + + else: + # using low-bandwidth binary format + peers = [] + peer_data = body["peers"] + for x in range(0, len(peer_data), 6): + key = peer_data[x:x+6] + ip = ".".join([str(ord(i)) for i in peer_data[x:x+4]]) + port = (ord(peer_data[x+4]) << 8) | ord(peer_data[x+5]) + peers.append((ip, port)) + + if DEBUG: print >> sys.stderr, "MiniTracker.run() received", len(peers), "peer addresses from tracker" + self._swarm.add_potential_peers(peers) + + diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/MagnetLink/__init__.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/MagnetLink/__init__.py new file mode 100644 index 0000000..0038dec --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/MagnetLink/__init__.py @@ -0,0 +1,6 @@ +# Written by Boudewijn Schoon +# see LICENSE.txt for license information + +# the extention id for the 'ut_metadata' message +EXTEND_MSG_METADATA = 'ut_metadata' +EXTEND_MSG_METADATA_ID = chr(224) diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/__init__.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/__init__.py new file mode 100644 index 0000000..8a8e66a --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/__init__.py @@ -0,0 +1,2 @@ +# written by Arno Bakker +# see LICENSE.txt for license information diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/MDHT_Spec.txt b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/MDHT_Spec.txt new file mode 100644 index 0000000..b2ce812 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/MDHT_Spec.txt @@ -0,0 +1,73 @@ +M24 release. BitTorrent and Mainline DHT protocol extensions. + +Mainline DHT is used in the Nextshare content delivery platform for the peer discovery process. +Currently, the performance of the protocol is very poor, with the median lookup time taking up to 1 minute. +We believe this poor performance is due to the bad management of the peer routing tables. + +Therefore, we propose a modification to the current MDHT protocol with a particular +focus on a better routing table management, while maintaining its backward compatibility. +Our MDHT protocol extensions are a follow up of our previous experiments on more then 3 million deployed nodes. + +The extensions in MDHT include (general): + +- A main routing table and a replacement routing table. The main routing table contains best known nodes; +such nodes are considered best nodes when they are reachable from us and when our requests to them +do no time out. +- The nodes in the replacement table are nodes with relatively good attributes; for example, they +demonstrate a significant number of responses to our requests and they have a relatively low number of +timeouts. Such nodes are used (moved to the main routing table) as main nodes, when the current nodes +in the main routing table fail to respond to our queries. + +- A node always starts in quarantine, no matter in which routing table the node is in. +A node is in quarantine for as long as there is no response from it, after having sent us a query for about +3 minutes ago. The quarantine ends when there is a 3 minutes window between a query from the node and +the next response. This quarantine period is designed to detect possible NATed nodes. If a node is in quarantine, +we are not sure whether the node is behind a NAT, but if the node is not in the quarantine - then we are +pretty confident that the node in not behind the NAT. A node that is not in quarantine never comes back to the +quarantine (unless it is completely kicked out from both tables and we loose all rnode information, and therefore starts over). + +- Nodes in the main table are refreshed every 3 minutes (if nodes are in quarantine), and every 10 minutes +(if nodes aren't in quarantine). Nodes in the replacement table are not refreshed (no matter whether they are in +quarantine or not). + +- The nodes that are in one of the routing tables (main or replacement) are called rnodes. They store node-specific +information such as: the number of queries to the node, the number of responses from the node, the number of timeouts and errors. + +- Nodes are added to the main routing table only after they have been checked for reachability; nodes are +not added to the main routing table if they don't respond to our reachability check queries. + +- When a node in the main table gets a timeout, it goes to the replacement table. In fact, this node gets replaced with a better +node from the replacement table. The following happens inside the replacement table in order to select the best node for the main table: + - All the nodes in the correct bucket of the replacement table are pinged - checked for reachability + - Pings to the NextShare (NS) nodes are delayed for 200ms (in order to give priority to NS nodes) + - The node that replies first (the fastest) to our query is chosen as the best node for the main table + + +More details on the routing table management: + +- When a query is received from a node which is not in any of the routing tables, then this node is checked for reachability. +If there is room in the main table, the node will be added only if it responded to our reachability check (ping query). +Otherwise, the worst node in the replacement table will be replaced with this new-coming node. +A node in the replacement table is considered the "worst node" when its accummulated number of timeouts exceeds 3. + +- When a response is received from a node that is already in the main routing table, then it is simply refreshed. +Otherwise, if the response comes from a node that is not an rnode and if there is room in the replacement table, then the +node is simply replaced with the worst node of the replacement table. + +- If there is a timeout from a node that is in the main routing table, then it is simply removed from the main table and +put into the replacement table. In fact, the node that did timeout is put inside the replacement table in place of the worst node. + +- Regarding the worst node selection inside the replacement table, we emphasize that the NS nodes are favored to remain inside +the table. When the replacement table is refreshed - in order to identify the worst node in the bucket - the first pings (reachability checks) +are sent to the NS nodes, and only after a delay, they are sent to the rest of the nodes. + + +Additional Mainline DHT extensions - nodes2 replies: + +- For IPv4 nodes, we use the standard 'compact node info' encoding, specified in the BitTorrent protocol. However, +the protocol specification does not have support for IPv6 nodes. The DHT messages - the 'nodes' replies - don't support IPv6, +because all the node contacts are encoded as 6-bytes but IPv6 nodes need 18-bytes. Therefore, in this protocol extension we +use libtorrent - which implements a few extensions to MDHT - in order to make use of the 'nodes2' for IPv6 contact encoding. + +- According to the libtorrent specification, replies with the 'nodes2' key are contacts that are encoded as 20-bytes node ID and +then a variable length encoded IP address (6 bytes in IPv4 case and 18 bytes in IPv6 case). diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/Makefile b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/Makefile new file mode 100644 index 0000000..ca4f667 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/Makefile @@ -0,0 +1,5 @@ +all: + rm .coverage; \ + python2.5 /usr/bin/nosetests --with-doctest \ + --with-coverage --cover-package=trunk #\ +# --pdb --pdb-failures diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/README b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/README new file mode 100644 index 0000000..37cd14c --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/README @@ -0,0 +1,33 @@ +KadTracker 0.1.0 + +INSTALLATION + +This package uses Python 2.5 standard library. No extra modules need to be +installed. + +A Makefile is provided to run the tests. The tests require the nose test +framework to work. + +API + +The API is located in kadtracker.py. This is the only module necessary to use the +package. + +Users must ONLY use the methods provided in kadtracker.py. + +Users can use the Id and Node containers as they wish. These classes are located +in identifier.py and node.py + +EXAMPLES + +Two examples are provided in the distribution: + +- server_dht.py + +Do the routing maintainance tasks plus a get_peers lookup to a random identifier +every 10 minutes. + +- interactive_dht.py + +Do the routing maintainance task plus get_peers lookups asked by the user during +the interactive session. \ No newline at end of file diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/__init__.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/bencode.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/bencode.py new file mode 100644 index 0000000..9e0eebb --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/bencode.py @@ -0,0 +1,210 @@ +# Copyright (C) 2009 Raul Jimenez +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +import cStringIO +import logging + +logger = logging.getLogger('dht') + +class LoggingException(Exception): + + def __init__(self, msg): + logger.info('%s: %s' % (self.__class__, msg)) + + +class EncodeError(LoggingException): + """Raised by encoder when invalid input.""" + +class DecodeError(LoggingException): + """Raised by decoder when invalid bencode input.""" + def __init__(self, msg, bencoded): + LoggingException.__init__(self, '\nBencoded: '.join((msg, bencoded))) + +class RecursionDepthError(DecodeError): + """Raised when the bencoded recursivity is too deep. + + This check prevents from using too much recursivity when an + accidentally/maliciously constructed bencoded string looks like + 'llllllllllllllllllllllllllllllllllll'. + + """ + + +def encode(data): + output = cStringIO.StringIO() + encode_f = _get_encode_f(data) + encode_f(data, output) + result = output.getvalue() + output.close() + return result + +def decode(bencoded, max_depth=4): + if not bencoded: + raise DecodeError('Empty bencoded string', bencoded) + try: + decode_f = _get_decode_f(bencoded, 0) + data, next_pos, = decode_f(bencoded, 0, max_depth) + except (KeyError, IndexError, ValueError): + raise DecodeError('UNEXPECTED>>>>>>>>>>>>', bencoded) + else: + if next_pos != len(bencoded): + raise DecodeError('Extra characters after valid bencode.', bencoded) + return data + + +def _encode_str(data, output): + """Encode a string object + + The result format is: + : + + """ + output.write('%d:%s' % (len(data), data)) + +def _encode_int(data, output): + """Encode an integer (or long) object + + The result format is: + ie + + """ + output.write('i%de' % data) + +def _encode_list(data, output): + """Encode a list object + + The result format is: + l...e + + """ + output.write('l') + for item in data: + encode_f = _get_encode_f(item) + encode_f(item, output) + output.write('e') + +def _encode_dict(data, output): + """Encode a dict object + + The result format is: + d...e + Keys must be string and will be encoded in lexicographical order + + """ + output.write('d') + keys = data.keys() + keys.sort() + for key in keys: + if type(key) != str: # key must be a string) + raise EncodeError, 'Found a non-string key. Data: %r' % data + value = data[key] + _encode_fs[str](key, output) + encode_f = _get_encode_f(value) + encode_f(value, output) + output.write('e') + + + + +def _decode_str(bencoded, pos, _): + """ + + + """ + str_len, str_begin = _get_int(bencoded, pos, ':') + str_end = str_begin + str_len + return (bencoded[str_begin:str_end], str_end) + +def _decode_int(bencoded, pos, _): + """ + + + """ + return _get_int(bencoded, pos + 1, 'e') # +1 to skip 'i' + +def _decode_list(bencoded, pos, max_depth): + """ + + + """ + if max_depth == 0: + raise RecursionDepthError('maximum recursion depth exceeded', bencoded) + + result = [] + next_pos = pos + 1 # skip 'l' + while bencoded[next_pos] != 'e': + decode_f = _get_decode_f(bencoded, next_pos) + item, next_pos = decode_f(bencoded, + next_pos, max_depth - 1) + result.append(item) + return result, next_pos + 1 # correct for 'e' + +def _decode_dict(bencoded, pos, max_depth): + """ + + """ + if max_depth == 0: + raise RecursionDepthError, 'maximum recursion depth exceeded' + + result = {} + next_pos = pos + 1 # skip 'd' + while bencoded[next_pos] != 'e': + # Decode key + decode_f = _get_decode_f(bencoded, next_pos) + if decode_f != _decode_str: + raise DecodeError('Keys must be string. Found: <%s>' % ( + bencoded[next_pos]), + bencoded) + key, next_pos = decode_f(bencoded, + next_pos, max_depth - 1) + # Decode value + decode_f = _get_decode_f(bencoded, next_pos) + value, next_pos = decode_f(bencoded, + next_pos, max_depth - 1) + result[key] = value + return result, next_pos + 1 # skip 'e' + + + +def _get_encode_f(value): + try: + return _encode_fs[type(value)] + except (KeyError), e: + raise EncodeError, 'Invalid type: <%r>' % e + +def _get_int(bencoded, pos, char): + try: + end = bencoded.index(char, pos) + except (ValueError): + raise DecodeError('Character %s not found.', bencoded) + try: + result = int(bencoded[pos:end]) + except (ValueError), e: + raise DecodeError('Not an integer: %r' %e, bencoded) + return (result, end + 1) # +1 to skip final character + +def _get_decode_f(bencoded, pos): + try: + return _decode_fs[bencoded[pos]] + except (KeyError), e: + raise DecodeError('Caracter in position %d raised %r' % (pos, e), + bencoded) + + +_encode_fs = {str : _encode_str, + int : _encode_int, + long : _encode_int, + tuple : _encode_list, + list : _encode_list, + dict : _encode_dict + } + +_decode_fs = {'i' : _decode_int, + 'l' : _decode_list, + 'd' : _decode_dict} +for i in xrange(10): + _decode_fs[str(i)] = _decode_str + + + diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/controller.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/controller.py new file mode 100644 index 0000000..e26f878 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/controller.py @@ -0,0 +1,77 @@ +# Copyright (C) 2009 Raul Jimenez +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +import time + +import logging, logging_conf + +import identifier +import message +import token_manager +import tracker +from routing_manager import RoutingManager +from minitwisted import ThreadedReactor +from rpc_manager import RPCManager +from querier import Querier +from responder import Responder +from message import QUERY, RESPONSE, ERROR, OutgoingGetPeersQuery +from lookup_manager import LookupManager +from node import Node + +logger = logging.getLogger('dht') + +class Controller: + + def __init__(self, dht_addr): + my_addr = dht_addr + my_id = identifier.RandomId() + my_node = Node(my_addr, my_id) + tracker_ = tracker.Tracker() + token_m = token_manager.TokenManager() + + self.reactor = ThreadedReactor() + rpc_m = RPCManager(self.reactor, my_addr[1]) + querier_ = Querier(rpc_m, my_id) + routing_m = RoutingManager(my_node, querier_, + bootstrap_nodes) + responder_ = Responder(my_id, routing_m, + tracker_, token_m) + + responder_.set_on_query_received_callback( + routing_m.on_query_received) + querier_.set_on_response_received_callback( + routing_m.on_response_received) + querier_.set_on_error_received_callback( + routing_m.on_error_received) + querier_.set_on_timeout_callback(routing_m.on_timeout) + querier_.set_on_nodes_found_callback(routing_m.on_nodes_found) + + routing_m.do_bootstrap() + + rpc_m.add_msg_callback(QUERY, + responder_.on_query_received) + + self.lookup_m = LookupManager(my_id, querier_, + routing_m) + self._routing_m = routing_m + + + def start(self): + self.reactor.start() + + def stop(self): + #TODO2: stop each manager + self.reactor.stop() + + def get_peers(self, info_hash, callback_f, bt_port=None): + return self.lookup_m.get_peers(info_hash, callback_f, bt_port) + + def print_routing_table_stats(self): + self._routing_m.print_stats() + +bootstrap_nodes = ( + + Node(('67.215.242.138', 6881)), #router.bittorrent.com + Node(('192.16.127.98', 7005)), #KTH node + ) diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/floodbarrier.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/floodbarrier.py new file mode 100644 index 0000000..0ac3167 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/floodbarrier.py @@ -0,0 +1,95 @@ +# Copyright (C) 2009 Raul Jimenez +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +""" +Floodbarrier is a protection mechanism which protects us from +host processing too many messages from a single host. + +""" + +import time +import collections +import logging, logging_conf + +logger = logging.getLogger('dht') + + +CHECKING_PERIOD = 2 # seconds +MAX_PACKETS_PER_PERIOD = 20 +BLOCKING_PERIOD = 100 # seconds + +class HalfPeriodRegister(object): + + """Helper class. Not meant to be used outside this module""" + + def __init__(self): + self.ip_dict = {} + + def get_num_packets(self, ip): + return self.ip_dict.get(ip, 0) + + def register_ip(self, ip): + self.ip_dict[ip] = self.ip_dict.get(ip, 0) + 1 + +class FloodBarrier(object): + + """ + Object which keeps track of packets received from different + hosts. Default values are coded but users can choose their own. + The main function is ip_blocked(). + + """ + + def __init__(self, checking_period=CHECKING_PERIOD, + max_packets_per_period=MAX_PACKETS_PER_PERIOD, + blocking_period=BLOCKING_PERIOD): + self.checking_period = checking_period + self.max_packets_per_period = max_packets_per_period + self.blocking_period = blocking_period + + self.last_half_period_time = time.time() + self.ip_registers = [HalfPeriodRegister(), HalfPeriodRegister()] + self.blocked_ips = {} + + def ip_blocked(self, ip): + """ + Register that a packet has been received from the given IP and return + whether the host is blocked and, hence, the packet should be dropped + + """ + current_time = time.time() + if current_time > self.last_half_period_time + self.checking_period / 2: + self.half_period_timeout = current_time + self.ip_registers = [self.ip_registers[1], HalfPeriodRegister()] + if current_time > self.last_half_period_time + self.checking_period: + self.ip_registers = [self.ip_registers[1], HalfPeriodRegister()] + self.ip_registers[1].register_ip(ip) + num_packets = self.ip_registers[0].get_num_packets(ip) + \ + self.ip_registers[1].get_num_packets(ip) + if num_packets > self.max_packets_per_period: + logger.debug('Got %d packets: blocking %r...' % ( + num_packets, ip)) + self.blocked_ips[ip] = current_time + self.blocking_period + return True + # At this point there are no enough packets to block ip (in current + # period). Now, we need to check whether the ip is currently blocked + if ip in self.blocked_ips: + logger.debug('Ip %r (%d) currently blocked' % (ip, + num_packets)) + if current_time > self.blocked_ips[ip]: + logger.debug( + 'Block for %r (%d) has expired: unblocking...' % + (ip, num_packets)) + # Blocking period already expired + del self.blocked_ips[ip] + return False + else: + # IP is currently blocked (block hasn't expired) + return True + else: + + + # IP is not blocked + return False + diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/identifier.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/identifier.py new file mode 100644 index 0000000..7842167 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/identifier.py @@ -0,0 +1,237 @@ +# Copyright (C) 2009 Raul Jimenez +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +""" +This module provides the Id object and necessary tools. + +""" + +import sys +import random + +import logging + +logger = logging.getLogger('dht') + + +BITS_PER_BYTE = 8 +ID_SIZE_BYTES = 20 +ID_SIZE_BITS = ID_SIZE_BYTES * BITS_PER_BYTE + + +def _bin_to_hex(bin_str): + """Convert a binary string to a hex string.""" + hex_list = ['%02x' % ord(c) for c in bin_str] + return ''.join(hex_list) + +def _hex_to_bin_byte(hex_byte): + #TODO2: Absolutely sure there is a library function for this + hex_down = '0123456789abcdef' + hex_up = '0123456789ABDCEF' + value = 0 + for i in xrange(2): + value *= 16 + try: + value += hex_down.index(hex_byte[i]) + except ValueError: + try: + value += hex_up.index(hex_byte[i]) + except ValueError: + raise IdError + return chr(value) + +def _hex_to_bin(hex_str): + return ''.join([_hex_to_bin_byte(hex_byte) for hex_byte in zip( + hex_str[::2], hex_str[1::2])]) + + +def _byte_xor(byte1, byte2): + """Xor two characters as if they were bytes.""" + return chr(ord(byte1) ^ ord(byte2)) + +def _first_different_byte(str1, str2): + """Return the position of the first different byte in the strings. + Raise IndexError when no difference was found (str1 == str2). + """ + for i in range(len(str1)): + if str1[i] != str2[i]: + return i + raise IndexError + +def _first_different_bit(byte1, byte2): + """Return the position of the first different bit in the bytes. + The bytes must not be equal. + + """ + assert byte1 != byte2 + byte = ord(byte1) ^ ord(byte2) + i = 0 + while byte >> (BITS_PER_BYTE - 1) == 0: + byte <<= 1 + i += 1 + return i + +class IdError(Exception): + pass + +class Id(object): + + """Convert a string to an Id object. + + The bin_id string's lenght must be ID_SIZE bytes (characters). + + You can use both binary and hexadecimal strings. Example + #>>> Id('\x00' * ID_SIZE_BYTES) == Id('0' * ID_SIZE_BYTES * 2) + #True + #>>> Id('\xff' * ID_SIZE_BYTES) == Id('f' * ID_SIZE_BYTES * 2) + #True + """ + + def __init__(self, hex_or_bin_id): + if not isinstance(hex_or_bin_id, str): + raise IdError + if len(hex_or_bin_id) == ID_SIZE_BYTES: + self._bin_id = hex_or_bin_id + elif len(hex_or_bin_id) == ID_SIZE_BYTES*2: + self._bin_id = _hex_to_bin(hex_or_bin_id) + else: + raise IdError + + def __hash__(self): + return self.bin_id.__hash__() + + @property + def bin_id(self): + """bin_id is read-only.""" + return self._bin_id + + def __eq__(self, other): + return self.bin_id == other.bin_id + + def __ne__(self, other): + return not self == other + + def __str__(self): + return self.bin_id + + def __repr__(self): + return '' % _bin_to_hex(self.bin_id) + + def distance(self, other): + """ + Do XOR distance between two Id objects and return it as Id + object. + + """ + byte_list = [_byte_xor(a, b) for a, b in zip(self.bin_id, + other.bin_id)] + return Id(''.join(byte_list)) + + def log_distance(self, other): + """Return log (base 2) of the XOR distance between two Id + objects. Return -1 when the XOR distance is 0. + + That is, this function returns 'n' when the distance between + the two objects is [2^n, 2^(n+1)). + When the two identifiers are equal, the distance is 0. Therefore + log_distance is -infinity. In this case, -1 is returned. + Example: + >>> z = Id(chr(0) * ID_SIZE_BYTES) + + >>> # distance = 0 [-inf, 1) -> log(0) = -infinity + >>> z.log_distance(z) + -1 + >>> # distance = 1 [1, 2) -> log(1) = 0 + >>> z.log_distance(Id(chr(0)*(ID_SIZE_BYTES-1)+chr(1))) + 0 + >>> # distance = 2 [2, 4) -> log(2) = 1 + >>> z.log_distance(Id(chr(0)*(ID_SIZE_BYTES-1)+chr(2))) + 1 + >>> # distance = 3 [2, 4) -> log(3) = + >>> z.log_distance(Id(chr(0)*(ID_SIZE_BYTES-1)+chr(3))) + 1 + >>> # distance = 4 [4, 8) -> log(2^2) = 2 + >>> z.log_distance(Id(chr(0)*(ID_SIZE_BYTES-1)+chr(4))) + 2 + >>> # distance = 5 [4, 8) -> log(5) = 2 + >>> z.log_distance(Id(chr(0)*(ID_SIZE_BYTES-1)+chr(5))) + 2 + >>> # distance = 6 [4, 8) -> log(6) = 2 + >>> z.log_distance(Id(chr(0)*(ID_SIZE_BYTES-1)+chr(6))) + 2 + >>> # distance = 7 [4, 8) -> log(7) = 2 + >>> z.log_distance(Id(chr(0)*(ID_SIZE_BYTES-1)+chr(7))) + 2 + >>> # distance = 128 = 2^(8*0+7) [128, 256) -> log(7^2) = 7 + >>> z.log_distance(Id(chr(0)*(ID_SIZE_BYTES-1)+chr(128))) + 7 + >>> # distance = 2^(8*18+8) = 2^148+8 -> log(1) = 152 + >>> z.log_distance(Id(chr(1)+chr(0)*(ID_SIZE_BYTES-1))) + 152 + >>> # distance = 2^(8*19+1) = 2^159 -> log(1) = 159 + >>> z.log_distance(Id(chr(128)+chr(0)*(ID_SIZE_BYTES-1))) + 159 + + """ + try: + byte_i = _first_different_byte(self.bin_id, other.bin_id) + except IndexError: + # _first_different_byte did't find differences, thus the + # distance is 0 and log_distance is -1 + return -1 + unmatching_bytes = ID_SIZE_BYTES - byte_i - 1 + byte1 = self.bin_id[byte_i] + byte2 = other.bin_id[byte_i] + bit_i = _first_different_bit(byte1, byte2) + # unmatching_bits (in byte: from least significant bit) + unmatching_bits = BITS_PER_BYTE - bit_i - 1 + return unmatching_bytes * BITS_PER_BYTE + unmatching_bits + + + def order_closest(self, id_list): + """Return a list with the Id objects in 'id_list' ordered + according to the distance to self. The closest id first. + + The original list is not modified. + + """ + id_list_copy = id_list[:] + max_distance = ID_SIZE_BITS + 1 + log_distance_list = [] + for element in id_list: + log_distance_list.append(self.log_distance(element)) + + result = [] + for _ in range(len(id_list)): + lowest_index = None + lowest_distance = max_distance + for j in range(len(id_list_copy)): + if log_distance_list[j] < lowest_distance: + lowest_index = j + lowest_distance = log_distance_list[j] + result.append(id_list_copy[lowest_index]) + del log_distance_list[lowest_index] + del id_list_copy[lowest_index] + return result + + def generate_close_id(self, log_distance): + if log_distance < 0: + return self + byte_num, bit_num = divmod(log_distance, BITS_PER_BYTE) + byte_index = len(self.bin_id) - byte_num - 1 # -1 correction + id_byte = self.bin_id[byte_index] + id_byte = chr(ord(id_byte) ^ (1 << bit_num)) + bin_id = self.bin_id[:byte_index] +\ + id_byte + self.bin_id[byte_index + 1:] + return Id(bin_id) + + +class RandomId(Id): + + """Create a random Id object.""" + def __init__(self): + random_str = ''.join([chr(random.randint(0, 255)) \ + for _ in xrange(ID_SIZE_BYTES)]) + Id.__init__(self, random_str) + diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/interactive_dht.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/interactive_dht.py new file mode 100644 index 0000000..684dcb3 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/interactive_dht.py @@ -0,0 +1,52 @@ +# Copyright (C) 2009 Raul Jimenez +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +import time +import sys + +import logging, logging_conf +logs_path = '.' +logs_level = logging.DEBUG # This generates HUGE (and useful) logs +#logs_level = logging.INFO # This generates some (useful) logs +#logs_level = logging.WARNING # This generates warning and error logs + +import identifier +import kadtracker + +def peers_found(peers): + print 'Peers found:' + for peer in peers: + print peer + print 'Type an info_hash (in hex digits): ', + +def lookup_done(): + print 'Lookup DONE' + print 'Type an info_hash (in hex digits): ', + +if len(sys.argv) == 4 and sys.argv[0] == 'interactive_dht.py': + logging_conf.setup(logs_path, logs_level) + RUN_DHT = True + my_addr = (sys.argv[1], int(sys.argv[2])) #('192.16.125.242', 7000) + logs_path = sys.argv[3] + dht = kadtracker.KadTracker(my_addr, logs_path) +else: + RUN_DHT = False + print 'usage: python interactive_dht.py dht_ip dht_port log_path' + +print 'Type "exit" to stop the DHT and exit' +while (RUN_DHT): + print 'Type an info_hash (in hex digits): ', + input = sys.stdin.readline()[:-1] + if input == 'exit': + dht.stop() + break + try: + info_hash = identifier.Id(input) + except (identifier.IdError): + print 'Invalid input (%s)' % input + continue + print 'Getting peers for info_hash %r' % info_hash + dht.get_peers(info_hash, peers_found) + + diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/kadtracker.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/kadtracker.py new file mode 100644 index 0000000..4af2916 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/kadtracker.py @@ -0,0 +1,60 @@ +# Copyright (C) 2009 Raul Jimenez +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +""" +This module is the API for the whole package. + +You can use the KadTracker class and its methods to interact with +the DHT. + +Find usage examples in server_dht.py and interactive_dht.py. + +If you want to generate logs. You will have to setup logging_conf +BEFORE importing this module. See the examples above for details. + +""" + +import controller +import logging, logging_conf + +class KadTracker: + """KadTracker is the interface for the whole package. + + Setting up the DHT is as simple as creating this object. + The parameters are: + - dht_addr: a tuple containing IP address and port number. + - logs_path: a string containing the path to the log files. + + """ + def __init__(self, dht_addr, logs_path): + logging_conf.setup(logs_path, logging.DEBUG) + self.controller = controller.Controller(dht_addr) + self.controller.start() + + def stop(self): + """Stop the DHT.""" + self.controller.stop() + + def get_peers(self, info_hash, callback_f, bt_port=None): + """ Start a get peers lookup. Return a Lookup object. + + The info_hash must be an identifier.Id object. + + The callback_f must expect one parameter. When peers are + discovered, the callback is called with a list of peers as parameter. + The list of peers is a list of addresses ( pairs). + + The bt_port parameter is optional. When provided, ANNOUNCE messages + will be send using the provided port number. + + """ + return self.controller.get_peers(info_hash, callback_f, bt_port) + + def print_routing_table_stats(self): + self.controller.print_routing_table_stats() + + + #TODO2: Future Work + #TODO2: def add_bootstrap_node(self, node_addr, node_id=None): + #TODO2: def lookup.back_off() diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/ktracker_example.py.no b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/ktracker_example.py.no new file mode 100644 index 0000000..8b3fc73 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/ktracker_example.py.no @@ -0,0 +1,47 @@ +''' +This example shows how ktracker can be used as a library +''' + +import time + +import ktracker_query +import ktracker + + +my_dht_port = 1111 + +bootstrap_node_addr = ('127.0.0.1', 2222) +bootstrap_node_id = '0' * 20 + +info_hash = 'z' * 20 +my_bt_addr = ('127.0.0.1', my_dht_port) + + +def on_query_event(query): + print '-' * 40 + print 'Query status:', query.status + print query.peers + print '-' * 40 + + + + +ktracker = ktracker.KTracker(my_dht_port) +ktracker.start() + +# join an existing DHT via bootstraping node (node_id is optional) +ktracker.add_bootstrap_node(bootstrap_node_addr) + +# create an announce_and_get_peers query +query = ktracker_query.AnnounceAndGetPeersQuery(info_hash, my_bt_addr) +ktracker.do_query(query, on_query_event) + +# let's give some time for the first query to finish +time.sleep(60) + +# create a get_peers query +query = ktracker_query.GetPeersQuery(info_hash) +ktracker.do_query(query, on_query_event) + +# we are done +ktracker.stop() diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/ktracker_query.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/ktracker_query.py new file mode 100644 index 0000000..2179738 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/ktracker_query.py @@ -0,0 +1,28 @@ + +class GetPeersLookup(object): + + def __init__(self, info_hash): + self._info_hash = info_hash + + @property + def info_hash(self): + return self._info_hash + + def get_status(self): + #lock + return self._status + def set_status(self, query_status): + #lock + self._status = query_status + status = property(get_status, set_status) + + + + def add_peers(self, peer_list): + ''' + Library users should not use this method. + ''' + #lock + self._peers.append(peer_list) + + diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/logging_conf.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/logging_conf.py new file mode 100644 index 0000000..2f09c13 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/logging_conf.py @@ -0,0 +1,42 @@ +# Copyright (C) 2009 Raul Jimenez +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +import sys +import os +import logging + +FORMAT = '%(asctime)s %(levelname)s %(filename)s:%(lineno)s - %(funcName)s()\n\ +%(message)s\n' + +devnullstream = open(os.devnull,"w") + +logging.basicConfig(level=logging.CRITICAL, + format='%(asctime)s %(levelname)-8s %(message)s', + datefmt='%a, %d %b %Y %H:%M:%S', + stream=devnullstream) + +def testing_setup(module_name): + logger = logging.getLogger('dht') + # Arno, 2010-06-11: Alt way of disabling logging from DHT instead of global + logger.setLevel(logging.CRITICAL+100) + filename = ''.join((str(module_name), '.log')) + logger_file = os.path.join('test_logs', filename) + + logger_conf = logging.FileHandler(logger_file, 'w') + logger_conf.setLevel(logging.DEBUG) + logger_conf.setFormatter(logging.Formatter(FORMAT)) + logger.addHandler(logger_conf) + +def setup(logs_path, logs_level): + logger = logging.getLogger('dht') + logger.setLevel(logs_level) + + logger_conf = logging.FileHandler( + os.path.join(logs_path, 'dht.log'), 'w') + #print "Logging to", os.path.join(logs_path, 'dht.log') + logger_conf.setLevel(logs_level) + logger_conf.setFormatter(logging.Formatter(FORMAT)) + logger.addHandler(logger_conf) + + diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/lookup_manager.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/lookup_manager.py new file mode 100644 index 0000000..7d29f95 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/lookup_manager.py @@ -0,0 +1,260 @@ +# Copyright (C) 2009 Raul Jimenez +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +import sys +import threading + +import logging + +import identifier as identifier +import message as message + + +logger = logging.getLogger('dht') + + +MAX_PARALLEL_QUERIES = 16 + +ANNOUNCE_REDUNDANCY = 3 + +class _QueuedNode(object): + + def __init__(self, node_, log_distance): + self.node = node_ + self.log_distance = log_distance + self.queried = False + +class _LookupQueue(object): + + def __init__(self, target_id, queue_size): + self.target_id = target_id + self.queue_size = queue_size + self.queue = [_QueuedNode(None, identifier.ID_SIZE_BITS+1)] + # Queued_ips is used to prevent that many Ids are + # claimed from a single IP address. + self.queued_ips = {} + + def add(self, nodes): + for node_ in nodes: + if node_.ip in self.queued_ips: + continue # Already queued + self.queued_ips[node_.ip] = None + + log_distance = self.target_id.log_distance(node_.id) + for i, qnode in enumerate(self.queue): + if log_distance < qnode.log_distance: + break + self.queue = self.queue[:i] \ + + [_QueuedNode(node_, log_distance)] \ + + self.queue[i:self.queue_size-1] + + def pop_closest_node(self): + """ Raise IndexError when empty queue""" + for qnode in self.queue: + if qnode.node and not qnode.queried: + qnode.queried = True + return qnode.node + raise IndexError('No more nodes in the queue.') + + +class GetPeersLookup(object): + """DO NOT use underscored variables, they are thread-unsafe. + Variables without leading underscore are thread-safe. + + All nodes in bootstrap_nodes MUST have ID. + """ + + def __init__(self, my_id, querier_, max_parallel_queries, + info_hash, callback_f, bootstrap_nodes, + bt_port=None): + logger.debug('New lookup (info_hash: %r)' % info_hash) + self._my_id = my_id + self._querier = querier_ + self._max_parallel_queries = max_parallel_queries + self._get_peers_msg = message.OutgoingGetPeersQuery( + my_id, info_hash) + self._callback_f = callback_f + self._lookup_queue = _LookupQueue(info_hash, + max_parallel_queries * 2) + self._lookup_queue.add(bootstrap_nodes) + self._num_parallel_queries = 0 + + self._info_hash = info_hash + self._bt_port = bt_port + self._lock = threading.RLock() + + self._announce_candidates = [] + self._num_responses_with_peers = 0 + self._is_done = False + + @property + def is_done(self): + #with self._lock: + self._lock.acquire() + try: + is_done = self._is_done + finally: + self._lock.release() + return is_done + + @property + def num_parallel_queries(self): + #with self._lock: + self._lock.acquire() + try: + num_parallel_queries = self._num_parallel_queries + finally: + self._lock.release() + return num_parallel_queries + + def start(self): + self._send_queries() + + + def _on_response(self, response_msg, node_): + logger.debug('response from %r\n%r' % (node_, + response_msg)) + #with self._lock: + self._lock.acquire() + try: + self._num_parallel_queries -= 1 + try: + peers = response_msg.peers + logger.debug('PEERS\n%r' % peers) + self._num_responses_with_peers += 1 + #TODO2: Halve queue size as well? + # We've got some peers, let's back off a little + self._max_parallel_queries = max( + self._max_parallel_queries / 2, 1) + self._callback_f(peers) + except (AttributeError): + pass + nodes = [] + try: + nodes.extend(response_msg.nodes) + except (AttributeError): + pass + try: + nodes.extend(response_msg.nodes2) + except (AttributeError): + pass + logger.info('NODES: %r' % (nodes)) + self._add_to_announce_candidates(node_, + response_msg.token) + self._lookup_queue.add(nodes) + self._send_queries() + finally: + self._lock.release() + + def _on_timeout(self, node_): + logger.debug('TIMEOUT node: %r' % node_) + #with self._lock: + self._lock.acquire() + try: + self._num_parallel_queries -= 1 + self._send_queries() + finally: + self._lock.release() + + def _on_error(self, error_msg, node_): + logger.debug('ERROR node: %r' % node_) + #with self._lock: + self._lock.acquire() + try: + self._num_parallel_queries -= 1 + self._send_queries() + finally: + self._lock.release() + + def _send_queries(self): + #with self._lock: + self._lock.acquire() + try: + while self._num_parallel_queries < self._max_parallel_queries: + try: + node_ = self._lookup_queue.pop_closest_node() + logger.debug('popped node %r' % node_) + except(IndexError): + logger.debug('no more candidate nodes!') + if not self._num_parallel_queries: + logger.debug('Lookup DONE') + self._announce() + return + if node_.id == self._my_id: + # Don't send to myself + continue + self._num_parallel_queries += 1 + logger.debug('sending to: %r, parallelism: %d/%d' % + (node_, + self._num_parallel_queries, + self._max_parallel_queries)) + self._querier.send_query(self._get_peers_msg, node_, + self._on_response, + self._on_timeout, + self._on_error) + finally: + self._lock.release() + + def _add_to_announce_candidates(self, node_, token): + node_log_distance = node_.id.log_distance(self._info_hash) + self._announce_candidates.append((node_log_distance, + node_, + token)) + for i in xrange(len(self._announce_candidates)-1, 0, -1): + if self._announce_candidates[i][1] \ + < self._announce_candidates[i-1][1]: + tmp1, tmp2 = self._announce_candidates[i-1:i+1] + self._announce_candidates[i-1:i+1] = tmp2, tmp1 + else: + break + self._announce_candidates = \ + self._announce_candidates[:ANNOUNCE_REDUNDANCY] + + def _do_nothing(self, *args, **kwargs): + #TODO2: generate logs + pass + + def _announce(self): + self._is_done = True + if not self._bt_port: + return + for (_, node_, token) in self._announce_candidates: + logger.debug('announcing to %r' % node_) + msg = message.OutgoingAnnouncePeerQuery( + self._my_id, self._info_hash, self._bt_port, token) + self._querier.send_query(msg, node_, + self._do_nothing, + self._do_nothing, + self._do_nothing) + + + def _get_announce_candidates(self): + return [e[1] for e in self._announce_candidates] + + +class LookupManager(object): + + def __init__(self, my_id, querier_, routing_m, + max_parallel_queries=MAX_PARALLEL_QUERIES): + self.my_id = my_id + self.querier = querier_ + self.routing_m = routing_m + self.max_parallel_queries = max_parallel_queries + + + def get_peers(self, info_hash, callback_f, bt_port=None): + lookup_q = GetPeersLookup( + self.my_id, self.querier, + self.max_parallel_queries, info_hash, callback_f, + self.routing_m.get_closest_rnodes(info_hash), + bt_port) + lookup_q.start() + return lookup_q + + def stop(self): + self.querier.stop() + + +#TODO2: During the lookup, routing_m gets nodes_found and sends find_node + # to them (in addition to the get_peers sent by lookup_m) diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/message.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/message.py new file mode 100644 index 0000000..85ee830 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/message.py @@ -0,0 +1,521 @@ +# Copyright (C) 2009 Raul Jimenez +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +""" +This module provides message classes. + +Outgoing messages are built from a few parameters. They are immutable and can be +reused (TID is not part of the message). + +Incoming messages are built from bencoded data. They are immutable and must be +sanitized before attempting to use message's attributes. + +""" + +import sys + +import logging + +import bencode +from identifier import Id, ID_SIZE_BYTES, IdError +from node import Node + + +logger = logging.getLogger('dht') + + +NEXTSHARE = 'NS\0\0\0' + +# High level keys +TYPE = 'y' # Message's type +ARGS = 'a' # Query's arguments in a dictionary +RESPONSE = 'r' # Reply dictionary +ERROR = 'e' # Error message string +TID = 't' # Transaction ID +QUERY = 'q' # Query command (only for queries) +VERSION = 'v' # Client's version + +# Valid values for key TYPE +QUERY = 'q' # Query +RESPONSE = 'r' # Response +ERROR = 'e' # Error + +# Valid values for key QUERY +PING = 'ping' +FIND_NODE = 'find_node' +GET_PEERS = 'get_peers' +ANNOUNCE_PEER = 'announce_peer' + +# Valid keys for ARGS +ID = 'id' # Node's nodeID (all queries) +TARGET = 'target' # Target's nodeID (find_node) +INFO_HASH = 'info_hash' # Torrent's info_hash (get_peers and announce) +PORT = 'port' # BitTorrent port (announce) +TOKEN = 'token' # Token (announce) + +# Valid keys for RESPONSE +ID = 'id' # Node's nodeID (all replies) +NODES = 'nodes' # String of nodes in compact format (find_nodes and get_peers) +NODES2 = 'nodes2' # Same as previous (with IPv6 support) +TOKEN = 'token' # Token (get_peers) +VALUES = 'values' # List of peers in compact format (get_peers) + +# Valid values for ERROR +GENERIC_E = [201, 'Generic Error'] +SERVER_E = [202, 'Server Error'] +PROTOCOL_E = [203, 'Protocol Error'] +UNKNOWN_E = [201, 'Method Unknown'] + +BLANK = 'BLANK' +BENCODED_BLANK = bencode.encode(BLANK) + +# Valid values for TID and VERSION +# binary string + + + +IP4_SIZE = 4 #bytes +IP6_SIZE = 16 #bytes +ADDR4_SIZE = IP4_SIZE + 2 # IPv4 address plus port +ADDR6_SIZE = IP6_SIZE + 2 # IPv6 address plus port +C_NODE_SIZE = ID_SIZE_BYTES + ADDR4_SIZE +C_NODE2_SIZE = ID_SIZE_BYTES + ADDR6_SIZE + +IP6_PADDING = '\0' * 10 + '\xff\xff' + + +class AddrError(Exception): + pass + +#class IP6Addr(AddrError): +# pass +# TODO2: deal with IPv6 address (we ignore them now) + +def bin_to_int(bin_str): + return ord(bin_str[0]) * 256 + ord(bin_str[1]) + +def int_to_bin(i): + return chr(i/256) + chr(i%256) + +def bin_to_ip(bin_str): + if len(bin_str) == IP4_SIZE: + return '.'.join([str(ord(b)) for b in bin_str]) + if len(bin_str) != IP6_SIZE: + raise MsgError, 'compact_ip: invalid size (%d)' % len(bin_str) + if not bin_str.startswith(IP6_PADDING): + raise AddrError, 'IPv4 and v6 should not be mixed!' + c_ip = bin_str[len(IP6_PADDING):] + return '.'.join([`ord(byte)` for byte in c_ip]) + +def ip_to_bin(ip_str): + return ''.join([chr(int(b)) for b in ip_str.split('.')]) + +def compact_addr(addr): + return ''.join((ip_to_bin(addr[0]), int_to_bin(addr[1]))) + +def uncompact_addr(c_addr): + if c_addr[-2:] == '\0\0': + logger.warning('c_addr: %r > port is ZERO' % c_addr) + raise AddrError + return (bin_to_ip(c_addr[:-2]), bin_to_int(c_addr[-2:])) + +def _compact_peers(peers): + return [compact_addr(peer) for peer in peers] + +def _uncompact_peers(c_peers): + peers = [] + for c_peer in c_peers: + try: + peers.append(uncompact_addr(c_peer)) + except (AddrError): + pass + return peers + +def _compact_nodes(nodes): + return ''.join([node.id.bin_id + compact_addr(node.addr) \ + for node in nodes]) + +def _uncompact_nodes(c_nodes): + if len(c_nodes) % C_NODE_SIZE != 0: + raise MsgError, 'invalid size (%d) %s' % (len(c_nodes), + c_nodes) + nodes = [] + for begin in xrange(0, len(c_nodes), C_NODE_SIZE): + node_id = Id(c_nodes[begin:begin + ID_SIZE_BYTES]) + try: + node_addr = uncompact_addr( + c_nodes[begin+ID_SIZE_BYTES:begin+C_NODE_SIZE]) + except AddrError: + pass + else: + node = Node(node_addr, node_id) + nodes.append(node) + return nodes + +def _compact_nodes2(nodes): + return [node.id.bin_id + IP6_PADDING + compact_addr(node.addr) \ + for node in nodes] + +def _uncompact_nodes2(c_nodes): + nodes = [] + for c_node in c_nodes: + node_id = Id(c_node[:ID_SIZE_BYTES]) + try: + node_addr = uncompact_addr(c_node[ID_SIZE_BYTES:]) + except (AddrError): + logger.warning('IPv6 addr in nodes2: %s' % c_node) + else: + node = Node(node_addr, node_id) + nodes.append(node) + return nodes + + +def matching_tid(query_tid, response_tid): + return query_tid[0] == response_tid[0] + + + +MSG_DICTS = {} + +MSG_DICTS['og_ping_q'] = {VERSION: NEXTSHARE, + TID: BLANK, + TYPE: QUERY, + QUERY: PING, + ARGS: {ID: BLANK} + } +MSG_DICTS['og_find_node_q'] = {VERSION: NEXTSHARE, + TID: BLANK, + TYPE: QUERY, + QUERY: FIND_NODE, + ARGS: {ID: BLANK, TARGET: BLANK} + } +MSG_DICTS['og_get_peers_q'] = {VERSION: NEXTSHARE, + TID: BLANK, + TYPE: QUERY, + QUERY: GET_PEERS, + ARGS: {ID: BLANK, INFO_HASH: BLANK} + } +MSG_DICTS['og_announce_peer_q'] = {VERSION: NEXTSHARE, + TID: BLANK, + TYPE: QUERY, + QUERY: ANNOUNCE_PEER, + ARGS: {ID: BLANK, INFO_HASH: BLANK, + PORT: BLANK, TOKEN: BLANK} + } + +MSG_DICTS['og_ping_r'] = {VERSION: NEXTSHARE, + TID: BLANK, + TYPE: RESPONSE, + RESPONSE: {ID: BLANK} + } +MSG_DICTS['og_find_node_r'] = {VERSION: NEXTSHARE, + TID: BLANK, + TYPE: RESPONSE, + RESPONSE: {ID: BLANK, NODES2: BLANK} + } +MSG_DICTS['og_get_peers_r_nodes'] = {VERSION: NEXTSHARE, + TID: BLANK, + TYPE: RESPONSE, + RESPONSE: {ID: BLANK, NODES2: BLANK, + TOKEN: BLANK} + } +MSG_DICTS['og_get_peers_r_values'] = {VERSION: NEXTSHARE, + TID: BLANK, + TYPE: RESPONSE, + RESPONSE: {ID: BLANK, VALUES: BLANK, + TOKEN: BLANK} + } +MSG_DICTS['og_announce_peer_r'] = {VERSION: NEXTSHARE, + TID: BLANK, + TYPE: RESPONSE, + RESPONSE: {ID: BLANK} + } +MSG_DICTS['og_error'] = {VERSION: NEXTSHARE, + TID: BLANK, + TYPE: ERROR, + ERROR: BLANK + } +BENCODED_MSG_TEMPLATES = {} +for msg_type, msg_dict in MSG_DICTS.iteritems(): + bencoded_msg = bencode.encode(msg_dict) + BENCODED_MSG_TEMPLATES[msg_type] = bencoded_msg.split(BENCODED_BLANK) + + +class MsgError(Exception): + """Raised anytime something goes wrong (specially when decoding/sanitizing). + + """ + + +class OutgoingMsgBase(object): + """Base class for outgoing messages. You shouldn't have instances of it. + + """ + + def __str__(self): + return str(self._bencoded_msg) + str(self._values) + + def __repr__(self): + return str(self.__class__) + str(self) + + def encode(self, tid): + self._values[-1] = tid + num_blank_slots = len(self._bencoded_msg) -1 + # Reserve space for prebencoded chunks and blank slots. + splitted_msg = [None] * (len(self._bencoded_msg) + num_blank_slots) + # Let's fill in every blank slot. + for i in range(num_blank_slots): + splitted_msg[2*i] = self._bencoded_msg[i] # prebencoded chunk + splitted_msg[2*i+1] = bencode.encode(self._values[i]) # value + splitted_msg[-1] = self._bencoded_msg[-1] # last prebencoded chunk + return ''.join(splitted_msg) # put all bencode in a single string + + +class OutgoingPingQuery(OutgoingMsgBase): + + def __init__(self, sender_id): + self._bencoded_msg = BENCODED_MSG_TEMPLATES['og_ping_q'] + self._values = [sender_id.bin_id, + ''] #TID + self.query = PING + + +class OutgoingFindNodeQuery(OutgoingMsgBase): + + def __init__(self, sender_id, target_id): + self._bencoded_msg = BENCODED_MSG_TEMPLATES['og_find_node_q'] + self._values = [sender_id.bin_id, + target_id.bin_id, + ''] #TID + self.query = FIND_NODE + + +class OutgoingGetPeersQuery(OutgoingMsgBase): + + def __init__(self, sender_id, info_hash): + self._bencoded_msg = BENCODED_MSG_TEMPLATES['og_get_peers_q'] + self._values = [sender_id.bin_id, + info_hash.bin_id, + ''] #TID + self.query = GET_PEERS + + +class OutgoingAnnouncePeerQuery(OutgoingMsgBase): + + def __init__(self, sender_id, info_hash, port, token): + self._bencoded_msg = BENCODED_MSG_TEMPLATES['og_announce_peer_q'] + self._values = [sender_id.bin_id, + info_hash.bin_id, + port, + token, + ''] #TID + self.query = ANNOUNCE_PEER + + +class OutgoingPingResponse(OutgoingMsgBase): + + def __init__(self, sender_id): + self._bencoded_msg = BENCODED_MSG_TEMPLATES['og_ping_r'] + self._values = [sender_id.bin_id, + ''] #TID + + +class OutgoingFindNodeResponse(OutgoingMsgBase): + + def __init__(self, sender_id, nodes2=None, nodes=None): + if nodes is not None: + raise MsgError, 'not implemented' + if nodes2 is not None: + self._bencoded_msg = BENCODED_MSG_TEMPLATES['og_find_node_r'] + self._values = [sender_id.bin_id, + _compact_nodes2(nodes2), + ''] #TID + else: + raise MsgError, 'must have nodes OR nodes2' + +class OutgoingGetPeersResponse(OutgoingMsgBase): + + def __init__(self, sender_id, token, + nodes2=None, peers=None): + if peers: + self._bencoded_msg = BENCODED_MSG_TEMPLATES['og_get_peers_r_values'] + self._values = [sender_id.bin_id, + token, + _compact_peers(peers), + ''] #TID + + elif nodes2: + self._bencoded_msg = BENCODED_MSG_TEMPLATES['og_get_peers_r_nodes'] + self._values = [sender_id.bin_id, + _compact_nodes2(nodes2), + token, + ''] #TID + else: + raise MsgError, 'must have nodes OR peers' + +class OutgoingAnnouncePeerResponse(OutgoingMsgBase): + + def __init__(self, sender_id): + self._bencoded_msg = BENCODED_MSG_TEMPLATES['og_announce_peer_r'] + self._values = [sender_id.bin_id, + ''] #TID + +class OutgoingErrorMsg(OutgoingMsgBase): + + def __init__(self, error): + self._bencoded_msg = BENCODED_MSG_TEMPLATES['og_error'] + self._values = [error, + ''] #TID + return + + +class IncomingMsg(object): + + def __init__(self, bencoded_msg): + try: + self._msg_dict = bencode.decode(bencoded_msg) + except (bencode.DecodeError): + logger.exception('invalid bencode') + raise MsgError, 'invalid bencode' + # Make sure the decoded data is a dict and has a TID key + try: + self.tid = self._msg_dict[TID] + except (TypeError): + raise MsgError, 'decoded data is not a dictionary' + except (KeyError): + raise MsgError, 'key TID not found' + # Sanitize TID + if not (isinstance(self.tid, str) and self.tid): + raise MsgError, 'TID must be a non-empty binary string' + + # Sanitize TYPE + try: + self.type = self._msg_dict[TYPE] + except (KeyError): + raise MsgError, 'key TYPE not found' + + if not self.type in (QUERY, RESPONSE, ERROR): + raise MsgError, 'Unknown TYPE value' + if self.type == QUERY: + self._sanitize_query() + elif self.type == ERROR: + self._sanitize_error() + return + + def __repr__(self): + return repr(self._msg_dict) + + + def _get_value(self, k, kk=None, optional=False): + try: + v = self._msg_dict[k] + if kk: + v = v[kk] + except (KeyError): + if optional: + return None + else: + raise MsgError, 'Non-optional key (%s:%s) not found' % (k, kk) + except (TypeError): + raise MsgError, 'Probably k (%r) is not a dictionary' % (k) + return v + + def _get_str(self, k, kk=None, optional=False): + v = self._get_value(k, kk, optional) + if v is None: + return None + if not isinstance(v, str): + raise MsgError, 'Value (%s:%s,%s) must be a string' % (k, kk, v) + return v + + def _get_id(self, k, kk=None): + try: + v = self._get_value(k, kk) + v = Id(v) + except (IdError): + raise MsgError, 'Value (%s:%s,%s) must be a valid Id' % (k, kk, v) + return v + + def _get_int(self, k, kk=None): + v = self._get_value(k, kk) + try: + v= int(v) + except (TypeError, ValueError): + raise MsgError, 'Value (%s:%s,%s) must be an int' % (k, kk, v) + return v + + def _sanitize_common(self): + # version (optional) + self.version = self._get_str(VERSION, optional=True) + self.ns_node = self.version \ + and self.version.startswith(NEXTSHARE[:2]) + + def _sanitize_query(self): + self._sanitize_common() + # sender_id + self.sender_id = self._get_id(ARGS, ID) + # query + self.query = self._get_str(QUERY) + if self.query in [GET_PEERS, ANNOUNCE_PEER]: + # info_hash + self.info_hash = self._get_id(ARGS, INFO_HASH) + if self.query == ANNOUNCE_PEER: + self.port = self._get_int(ARGS, PORT) + self.token = self._get_str(ARGS, TOKEN) + elif self.query == FIND_NODE: + # target + self.target = self._get_id(ARGS, TARGET) + return + + def sanitize_response(self, query): + self._sanitize_common() + # sender_id + self.sender_id = self._get_id(RESPONSE, ID) + if query in [FIND_NODE, GET_PEERS]: + # nodes + nodes_found = False + c_nodes = self._get_str(RESPONSE, NODES, optional=True) + if c_nodes: + self.nodes = _uncompact_nodes(c_nodes) + nodes_found = True + # nodes2 + try: + self.nodes2 = _uncompact_nodes2( + self._msg_dict[RESPONSE][NODES2]) + if nodes_found: + logger.info('Both nodes and nodes2 found') + nodes_found = True + except (KeyError): + pass + if query == FIND_NODE: + if not nodes_found: + logger.warning('No nodes in find_node response') + raise MsgError, 'No nodes in find_node response' + elif query == GET_PEERS: + # peers + try: + self.peers = _uncompact_peers( + self._msg_dict[RESPONSE][VALUES]) + if nodes_found: + logger.debug( + 'Nodes and peers found in get_peers response') + except (KeyError): + if not nodes_found: + logger.warning( + 'No nodes or peers found in get_peers response') + raise (MsgError, + 'No nodes or peers found in get_peers response') + # token + self.token = self._get_str(RESPONSE, TOKEN) + + def _sanitize_error(self): + self._sanitize_common() + try: + self.error = [int(self._msg_dict[ERROR][0]), + str(self._msg_dict[ERROR][1])] + except (KeyError, IndexError, ValueError, TypeError): + raise MsgError, 'Invalid error message' + if self.error not in [GENERIC_E, SERVER_E, PROTOCOL_E, UNKNOWN_E]: + logger.info('Unknown error: %s', self.error) + diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/minitwisted.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/minitwisted.py new file mode 100644 index 0000000..1ad5819 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/minitwisted.py @@ -0,0 +1,312 @@ +# Copyright (C) 2009 Raul Jimenez +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +''' +Minitwisted is inspired by the Twisted framework. Although, it is much +simpler. +- It can only handle one UDP connection per reactor. +- Reactor runs in a thread +- You can use call_later and call_now to run your code in thread-safe mode + +''' + +#from __future__ import with_statement + +import sys +import socket +import threading +import time + +import logging + +from floodbarrier import FloodBarrier + +logger = logging.getLogger('dht') + + +BUFFER_SIZE = 1024 + + +class Task(object): + + '''Simple container for a task ''' + + def __init__(self, delay, callback_fs, *args, **kwds): + ''' + Create a task instance. Here is when the call time is calculated. + + ''' + self.delay = delay + if callable(callback_fs): + # single callback + self.callback_fs = [callback_fs] + else: + self.callback_fs = callback_fs + self.args = args + self.kwds = kwds + self.call_time = time.time() + self.delay + self._cancelled = False + + @property + def cancelled(self): + return self._cancelled + + def fire_callbacks(self): + """Fire a callback (if it hasn't been cancelled).""" + if not self._cancelled: + for callback_f in self.callback_fs: + callback_f(*self.args, **self.kwds) + ''' + Tasks usually have arguments which reference to the objects which + created the task. That is, they create a memory cycle. In order + to break the memoery cycle, those arguments are deleted. + ''' + del self.callback_fs + del self.args + del self.kwds + + def cancel(self): + """Cancel a task (callback won't be called when fired)""" + self._cancelled = True + + +class TaskManager(object): + + """Manage tasks""" + + def __init__(self): + self.tasks = {} + self.next_task = None + + def add(self, task): + """Add task to the TaskManager""" + + ms_delay = int(task.delay * 1000) + # we need integers for the dictionary (floats are not hashable) + self.tasks.setdefault(ms_delay, []).append(task) + if self.next_task is None or task.call_time < self.next_task.call_time: + self.next_task = task + +# def __iter__(self): +# """Makes (along with next) this objcet iterable""" +# return self + + def _get_next_task(self): + """Return the task which should be fired next""" + + next_task = None + for _, task_list in self.tasks.items(): + task = task_list[0] + if next_task is None: + next_task = task + if task.call_time < next_task.call_time: + next_task = task + return next_task + + + def consume_task(self): + """ + Return the task which should be fire next and removes it from + TaskManager + + """ + current_time = time.time() + if self.next_task is None: + # no pending tasks + return None #raise StopIteration + if self.next_task.call_time > current_time: + # there are pending tasks but it's too soon to fire them + return None #raise StopIteration + # self.next_task is ready to be fired + task = self.next_task + # delete consummed task and get next one (if any) + ms_delay = int(self.next_task.delay * 1000) + del self.tasks[ms_delay][0] + if not self.tasks[ms_delay]: + # delete list when it's empty + del self.tasks[ms_delay] + self.next_task = self._get_next_task() + #TODO2: make it yield + return task + +class ThreadedReactor(threading.Thread): + + """ + Object inspired in Twisted's reactor. + Run in its own thread. + It is an instance, not a nasty global + + """ + def __init__(self, task_interval=0.1, floodbarrier_active=True): + threading.Thread.__init__(self) + self.setName("KADTracker"+self.getName()) + self.setDaemon(True) + + self.stop_flag = False + self._lock = threading.RLock() + self.task_interval = task_interval + self.floodbarrier_active = floodbarrier_active + self.tasks = TaskManager() + if self.floodbarrier_active: + self.floodbarrier = FloodBarrier() + + def run(self): + """Main loop activated by calling self.start()""" + + last_task_run = time.time() + stop_flag = self.stop_flag + while not stop_flag: + timeout_raised = False + try: + data, addr = self.s.recvfrom(BUFFER_SIZE) + except (AttributeError): + logger.warning('udp_listen has not been called') + time.sleep(self.task_interval) + #TODO2: try using Event and wait + timeout_raised = True + except (socket.timeout): + timeout_raised = True + except (socket.error), e: + logger.critical( + 'Got socket.error when receiving (more info follows)') + logger.exception('See critical log above') + else: + ip_is_blocked = self.floodbarrier_active and \ + self.floodbarrier.ip_blocked(addr[0]) + if ip_is_blocked: + logger.warning('%s blocked' % `addr`) + else: + self.datagram_received_f(data, addr) + + if timeout_raised or \ + time.time() - last_task_run > self.task_interval: + #with self._lock: + self._lock.acquire() + try: + while True: + task = self.tasks.consume_task() + if task is None: + break +# logger.critical('TASK COUNT 2 %d' % sys.getrefcount(task)) + task.fire_callbacks() + stop_flag = self.stop_flag + finally: + self._lock.release() + logger.debug('Reactor stopped') + + def stop(self): + """Stop the thread. It cannot be resumed afterwards????""" + #with self._lock: + self._lock.acquire() + try: + self.stop_flag = True + finally: + self._lock.release() + # wait a little for the thread to end + time.sleep(self.task_interval) + + +# def stop_and_wait(self): +# """Stop the thread and wait a little (task_interval).""" + +# self.stop() + # wait a little before ending the thread's life +# time.sleep(self.task_interval * 2) + + def listen_udp(self, port, datagram_received_f): + """Listen on given port and call the given callback when data is + received. + + """ + self.datagram_received_f = datagram_received_f + self.s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.s.settimeout(self.task_interval) + my_addr = ('', port) + self.s.bind(my_addr) + + + def call_later(self, delay, callback_fs, *args, **kwds): + """Call the given callback with given arguments in the future (delay + seconds). + + """ + #with self._lock: + self._lock.acquire() + try: + task = Task(delay, callback_fs, *args, **kwds) +# logger.critical('TASK COUNT CREATION 2 %d' % sys.getrefcount(task)) + self.tasks.add(task) +# logger.critical('TASK COUNT CREATION 3 %d' % sys.getrefcount(task)) + finally: + self._lock.release() + return task + + def call_now(self, callback_f, *args, **kwds): + """Same as call_later with delay 0 seconds.""" + return self.call_later(0, callback_f, *args, **kwds) + + + def sendto(self, data, addr): + """Send data to addr using the UDP port used by listen_udp.""" + #with self._lock: + self._lock.acquire() + try: + try: + bytes_sent = self.s.sendto(data, addr) + if bytes_sent != len(data): + logger.critical( + 'Just %d bytes sent out of %d (Data follows)' % ( + bytes_sent, + len(data))) + logger.critical('Data: %s' % data) + except (socket.error): + logger.critical( + 'Got socket.error when sending (more info follows)') + logger.critical('Sending data to %r\n%r' % (addr, + data)) + logger.exception('See critical log above') + finally: + self._lock.release() + + +class ThreadedReactorSocketError(ThreadedReactor): + + def listen_udp(self, delay, callback_f, *args, **kwds): + self.s = _SocketMock() + + +class ThreadedReactorMock(object): + + def __init__(self, task_interval=0.1): + pass + + def start(self): + pass + + stop = start +# stop_and_wait = stop + + def listen_udp(self, port, data_received_f): + self.s = _SocketMock() + + + def call_later(self, delay, callback_f, *args, **kwds): + return Task(delay, callback_f, *args, **kwds) + + def sendto(self, data, addr): + pass + + + + +class _SocketMock(object): + + def sendto(self, data, addr): + if len(data) > BUFFER_SIZE: + return BUFFER_SIZE + raise socket.error + + def recvfrom(self, buffer_size): + raise socket.error diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/node.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/node.py new file mode 100644 index 0000000..4b2e5c9 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/node.py @@ -0,0 +1,165 @@ +# Copyright (C) 2009 Raul Jimenez +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +import time + +import utils +import identifier + +class Node(object): + + def __init__(self, addr, node_id=None, ns_node=False): + self._addr = addr + self._id = node_id + self.is_ns = ns_node + self._compact_addr = utils.compact_addr(addr) + + def get_id(self): + return self._id + def set_id(self, node_id): + if self._id is None: + self._id = node_id + else: + raise AttributeError, "Node's id is read-only" + id = property(get_id, set_id) + + @property + def addr(self): + return self._addr + + @property + def compact_addr(self): + return self._compact_addr + + @property + def ip(self): + return self._addr[0] + + def __eq__(self, other): + try: + return self.addr == other.addr and self.id == other.id + except AttributeError: #self.id == None + return self.id is None and other.id is None \ + and self.addr == other.addr + + def __ne__(self, other): + return not self == other + + def __repr__(self): + return '' % (self.addr, self.id) + + def log_distance(self, other): + return self.id.log_distance(other.id) + + def compact(self): + """Return compact format""" + return self.id.bin_id + self.compact_addr + + def get_rnode(self): + return RoutingNode(self) + + + +QUERY = 'query' +REPLY = 'reply' +TIMEOUT = 'timeout' + +LAST_RTT_W = 0.2 # The weight of the last RTT to calculate average + +MAX_NUM_TIMEOUT_STRIKES = 2 +QUARANTINE_PERIOD = 3 * 60 # 3 minutes + + +class RoutingNode(Node): + + def __init__(self, node_): + Node.__init__(self, node_.addr, node_.id, node_.is_ns) + self._rtt_avg = None + self._num_queries = 0 + self._num_responses = 0 + self._num_timeouts = 0 + self._msgs_since_timeout = 0 + self._last_events = [] + self._max_last_events = 10 + self.refresh_task = None + self._rank = 0 + current_time = time.time() + self._creation_ts = current_time + self._last_action_ts = current_time + self.in_quarantine = True + + def __repr__(self): + return '' % (self.addr, self.id) + + def get_rnode(self): + return self + + def on_query_received(self): + """Register a query from node. + + You should call this method when receiving a query from this node. + + """ + self._last_action_ts = time.time() + self._msgs_since_timeout += 1 + self._num_queries += 1 + self._last_events.append((time.time(), QUERY)) + self._last_events[:self._max_last_events] + + def on_response_received(self, rtt=0): + """Register a reply from rnode. + + You should call this method when receiving a response from this rnode. + + """ + current_time = time.time() + #self._reset_refresh_task() + if self.in_quarantine: + self.in_quarantine = \ + self._last_action_ts < current_time - QUARANTINE_PERIOD + + self._last_action_ts = current_time + self._msgs_since_timeout += 1 + try: + self._rtt_avg = \ + self._rtt_avg * (1 - LAST_RTT_W) + rtt * LAST_RTT_W + except TypeError: # rtt_avg is None + self._rtt_avg = rtt + self._num_responses += 1 + self._last_events.append((time.time(), REPLY)) + self._last_events[:self._max_last_events] + + def on_timeout(self): + """Register a timeout for this rnode. + + You should call this method when getting a timeout for this node. + + """ + self._last_action_ts = time.time() + self._msgs_since_timeout = 0 + self._num_timeouts += 1 + self._last_events.append((time.time(), TIMEOUT)) + self._last_events[:self._max_last_events] + + def timeouts_in_a_row(self, consider_queries=True): + """Return number of timeouts in a row for this rnode.""" + result = 0 + for timestamp, event in reversed(self._last_events): + if event == TIMEOUT: + result += 1 + elif event == REPLY or \ + (consider_queries and event == QUERY): + return result + return result # all timeouts (and queries), or empty list + +# def rank(self): +# if self._num_responses == 0: +# # No responses received, the node might be unreachable +# return 0 +# if self.timeouts_in_a_row() > MAX_NUM_TIMEOUT_STRIKES: +# return 0 +# return self._num_queries + self._num_responses + \ +# -3 * self._num_timeouts + + diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/querier.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/querier.py new file mode 100644 index 0000000..5bf632d --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/querier.py @@ -0,0 +1,251 @@ +# Copyright (C) 2009 Raul Jimenez +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +import sys + +import logging + +import message +import identifier + +logger = logging.getLogger('dht') + + +TIMEOUT_DELAY = 3 + +class Query(object): + + def __init__(self, tid, query_type, node_, + on_response_f, on_error_f, on_timeout_f, + notify_routing_m_on_response_f, + notify_routing_m_on_error_f, + notify_routing_m_on_timeout_f, + notify_routing_m_on_nodes_found_f): + #assert on_response_f + #assert on_error_f + #assert on_timeout_f + #assert notify_routing_m_on_response_f + #assert notify_routing_m_on_error_f + #assert notify_routing_m_on_timeout_f + #assert notify_routing_m_on_nodes_found_f + + self.tid = tid + self.query = query_type + self.node = node_ + self.on_response_f = on_response_f + self.on_error_f = on_error_f + self.on_timeout_f = on_timeout_f + self.notify_routing_m_on_response_f = \ + notify_routing_m_on_response_f + self.notify_routing_m_on_error_f = \ + notify_routing_m_on_error_f + self.notify_routing_m_on_timeout_f = \ + notify_routing_m_on_timeout_f + self.notify_routing_m_on_nodes_found_f = \ + notify_routing_m_on_nodes_found_f + self.timeout_task = None + + def on_response_received(self, response_msg): + try: + response_msg.sanitize_response(self.query) + except (message.MsgError): + logger.exception( + "We don't like dirty reponses: %r|nresponse ignored" + % response_msg) + return # Response ignored + self.node.is_ns = response_msg.ns_node + if self.node.id: + if response_msg.sender_id != self.node.id: + return # Ignore response + else: + self.node.id = response_msg.sender_id + #TODO2: think whether late responses should be accepted + if self.timeout_task.cancelled: + logger.warning( + "Response recevived but it's too late!!\n%r, %r" % + (response_msg, + self.timeout_task)) + return # Ignore response + self.timeout_task.cancel() + nodes = [] + try: + nodes.extend(response_msg.nodes) + except (AttributeError): + pass + try: + nodes.extend(response_msg.nodes2) + except (AttributeError): + pass + # Notify routing manager (if nodes found). + # Do not notify when the query was a GET_PEERS because + # the lookup is in progress and the routing_m shouldn't + # generate extra traffic. + if self.query == message.FIND_NODE and \ + nodes and self.notify_routing_m_on_nodes_found_f: + self.notify_routing_m_on_nodes_found_f(nodes) + # Notify routing manager (response) + self.node.is_ns = response_msg.ns_node + if self.notify_routing_m_on_response_f: + self.notify_routing_m_on_response_f(self.node) + # Do callback to whomever did the query + if self.on_response_f: + self.on_response_f(response_msg, self.node) + return True # the response was fine + + def on_error_received(self, error_msg): + if self.on_error_f: + self.on_error_f(error_msg, self.node) + if self.notify_routing_m_on_error_f: + self.notify_routing_m_on_error_f(self.node) + + def on_timeout(self): + # Careful here. Node might not have ID. + if self.on_timeout_f: + self.on_timeout_f(self.node) + if self.notify_routing_m_on_timeout_f: + self.notify_routing_m_on_timeout_f(self.node) + + def matching_tid(self, response_tid): + return message.matching_tid(self.tid, response_tid) + +class Querier(object): + + def __init__(self, rpc_m, my_id, default_timeout_delay=TIMEOUT_DELAY): + self.rpc_m = rpc_m + self.my_id = my_id + self.default_timeout_delay = default_timeout_delay + self.rpc_m.add_msg_callback(message.RESPONSE, self.on_response_received) + self.rpc_m.add_msg_callback(message.ERROR, self.on_error_received) + self.pending = {} # collections.defaultdict(list) + self._tid = [0, 0] + self.notify_routing_m_on_response = None + self.notify_routing_m_on_error = None + self.notify_routing_m_on_timeout = None + self.notify_routing_m_on_nodes_found = None + + def _next_tid(self): + current_tid_str = ''.join([chr(c) for c in self._tid]) + self._tid[0] = (self._tid[0] + 1) % 256 + if self._tid[0] == 0: + self._tid[1] = (self._tid[1] + 1) % 256 + return current_tid_str # raul: yield created trouble + + def set_on_response_received_callback(self, callback_f): + self.notify_routing_m_on_response = callback_f + + def set_on_error_received_callback(self, callback_f): + self.notify_routing_m_on_error = callback_f + + def set_on_timeout_callback(self, callback_f): + self.notify_routing_m_on_timeout = callback_f + + def set_on_nodes_found_callback(self, callback_f): + self.notify_routing_m_on_nodes_found = callback_f + + def send_query(self, msg, node_, on_response_f, + on_timeout_f, on_error_f, + timeout_delay=None): + timeout_delay = timeout_delay or self.default_timeout_delay + tid = self._next_tid() + logger.debug('sending to node: %r\n%r' % (node_, msg)) + query = Query(tid, msg.query, node_, + on_response_f, on_error_f, + on_timeout_f, + self.notify_routing_m_on_response, + self.notify_routing_m_on_error, + self.notify_routing_m_on_timeout, + self.notify_routing_m_on_nodes_found) + # if node is not in the dictionary, it will create an empty list + self.pending.setdefault(node_.addr, []).append(query) + bencoded_msg = msg.encode(tid) + query.timeout_task = self.rpc_m.get_timeout_task(node_.addr, + timeout_delay, + self.on_timeout) + self.rpc_m.send_msg_to(bencoded_msg, node_.addr) + return query + + def send_query_later(self, delay, msg, node_, on_response_f, + on_timeout_f, on_error_f, + timeout_delay=None): + return self.rpc_m.call_later(delay, self.send_query, + msg, node_, + on_response_f, + on_timeout_f, + on_error_f, + timeout_delay) + + def on_response_received(self, response_msg, addr): + # TYPE and TID already sanitized by rpc_manager + logger.debug('response received: %s' % repr(response_msg)) + try: + addr_query_list = self.pending[addr] + except (KeyError): + logger.warning('No pending queries for %s', addr) + return # Ignore response + # There are pending queries from node (let's find the right one (TID) + query_found = False + for query_index, query in enumerate(addr_query_list): + logger.debug('response node: %s, query:\n(%s, %s)' % ( + `addr`, + `query.tid`, + `query.query`)) + if query.matching_tid(response_msg.tid): + query_found = True + break + if not query_found: + logger.warning('No query for this response\n%s\nsource: %s' % ( + response_msg, addr)) + return # ignore response + # This response matches query. Trigger query's callback + response_is_ok = query.on_response_received(response_msg) + if response_is_ok: + # Remove this query from pending + if len(addr_query_list) == 1: + # There is one item in the list. Remove the whole list. + del self.pending[addr] + else: + del addr_query_list[query_index] + else: + logger.warning('Bad response from %r\n%r' % (addr, + response_msg)) + + def on_error_received(self, error_msg, addr): + logger.warning('Error message received:\n%s\nSource: %s', + `error_msg`, + `addr`) + # TODO2: find query (with TID) + # and fire query.on_error_received(error_msg) + + def on_timeout(self, addr): + #try + addr_query_list = self.pending[addr] + #except (KeyError): + # logger.warning('No pending queries for %s', addr) + # return # Ignore response + # There are pending queries from node (oldest query) + query = addr_query_list.pop(0) + # Remove this query from pending + if not addr_query_list: + # The list is empty. Remove the whole list. + del self.pending[addr] + # Trigger query's on_timeout callback + query.on_timeout() + + + def stop(self): + self.rpc_m.stop() + + +class QuerierMock(Querier): + + def __init__(self, my_id): + import minitwisted + import rpc_manager + import test_const as tc + reactor = minitwisted.ThreadedReactorMock() + rpc_m = rpc_manager.RPCManager(reactor, 1) + Querier.__init__(self, rpc_m, my_id) + + + diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/responder.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/responder.py new file mode 100644 index 0000000..864171f --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/responder.py @@ -0,0 +1,67 @@ +# Copyright (C) 2009 Raul Jimenez +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +import message +import node +import sys + +import logging + +logger = logging.getLogger('dht') + + +class Responder(object ): + "docstring for Responder" + def __init__(self, my_id, routing_m, tracker, token_m): + self.my_id = my_id + self.routing_m = routing_m + self.tracker = tracker + self.token_m = token_m + self.query_handler = {message.PING: self._on_ping, + message.FIND_NODE: self._on_find_node, + message.GET_PEERS: self._on_get_peers, + message.ANNOUNCE_PEER: self._on_announce_peer, + } + self.notify_routing_m_on_query = None + + def set_on_query_received_callback(self, callback_f): + self.notify_routing_m_on_query = callback_f + + def on_query_received(self, query_msg, addr): + logger.debug('query received\n%s\nSource: %s' % (`query_msg`, + `addr`)) + try: + handler = self.query_handler[query_msg.query] + except (KeyError, ValueError): + logger.exception('Invalid QUERY') + return # ignore query #TODO2: send error back? + response_msg = handler(query_msg) + self.notify_routing_m_on_query(node.Node(addr, + query_msg.sender_id, + query_msg.ns_node)) + return response_msg + + def _on_ping(self, query_msg): + return message.OutgoingPingResponse(self.my_id) + + def _on_find_node(self, query_msg): + rnodes = self.routing_m.get_closest_rnodes(query_msg.target) + return message.OutgoingFindNodeResponse(self.my_id, + nodes2=rnodes) + + def _on_get_peers(self, query_msg): + #get peers from the tracker (if any) + token = self.token_m.get() + peers = self.tracker.get(query_msg.info_hash) + if peers: + return message.OutgoingGetPeersResponse(self.my_id, + token, + peers=peers) + rnodes = self.routing_m.get_closest_rnodes(query_msg.info_hash) + return message.OutgoingGetPeersResponse(self.my_id, + token, + nodes2=rnodes) + def _on_announce_peer(self, query_msg): + return message.OutgoingAnnouncePeerResponse(self.my_id) + diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/routing_manager.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/routing_manager.py new file mode 100644 index 0000000..1d8cc4f --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/routing_manager.py @@ -0,0 +1,307 @@ +# Copyright (C) 2009 Raul Jimenez +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +import random + +import logging + +import identifier as identifier +import message as message +from node import Node, RoutingNode +from routing_table import RoutingTable, RnodeNotFound, BucketFullError + + +logger = logging.getLogger('dht') + + +#TODO2: Stop expelling nodes from tables when there are many consecutive +# timeouts (and enter off-line mode) + +NUM_BUCKETS = identifier.ID_SIZE_BITS + 1 +""" +We need (+1) to cover all the cases. See the following table: +Index | Distance | Comment +0 | [2^0,2^1) | All bits equal but the least significant bit +1 | [2^1,2^2) | All bits equal till the second least significant bit +... +158 | [2^159,2^160) | The most significant bit is equal the second is not +159 | [2^159,2^160) | The most significant bit is different +-1 | 0 | The bit strings are equal +""" + +DEFAULT_NUM_NODES = 8 +NODES_PER_BUCKET = [] # 16, 32, 64, 128, 256] +NODES_PER_BUCKET[:0] = [DEFAULT_NUM_NODES] \ + * (NUM_BUCKETS - len(NODES_PER_BUCKET)) + +REFRESH_PERIOD = 10 * 60 # 10 minutes +QUARANTINE_PERIOD = 3 * 60 # 3 minutes + +MAX_NUM_TIMEOUTS = 3 +PING_DELAY_AFTER_TIMEOUT = 30 #seconds + + +MIN_RNODES_BOOTSTRAP = 50 +NUM_NODES_PER_BOOTSTRAP_STEP = 10 +BOOTSTRAP_DELAY = 1 + +BOOTSTRAP_MODE = 'bootstrap_node' +NORMAL_MODE = 'normal_mode' + +MAX_CONCURRENT_REFRESH_MSGS = 20 +NO_PRIORITY = 0 +PRIORITY = 10 + +REFRESH_DELAY_FOR_NON_NS = .200 #seconds + +class RoutingManager(object): + + def __init__(self, my_node, querier, bootstrap_nodes): + self.my_node = my_node + self.querier = querier + #Copy the bootstrap list + self.bootstrap_nodes = [n for n in bootstrap_nodes] + + self.main = RoutingTable(my_node, NODES_PER_BUCKET) + self.replacement = RoutingTable(my_node, NODES_PER_BUCKET) + self.ping_msg = message.OutgoingPingQuery(my_node.id) + self.find_node_msg = message.OutgoingFindNodeQuery( + my_node.id, + my_node.id) + self.mode = BOOTSTRAP_MODE + self.num_concurrent_refresh_msgs = 0 + #This must be called by an external party: self.do_bootstrap() + #After initializing callbacks + + # Add myself to the routing table + rnode = self.main.add(my_node) + self._reset_refresh_task(rnode) + + def do_bootstrap(self): + if self.main.num_rnodes > MIN_RNODES_BOOTSTRAP: + # Enough nodes. Stop bootstrap. + return + for _ in xrange(NUM_NODES_PER_BOOTSTRAP_STEP): + if not self.bootstrap_nodes: + self.mode = NORMAL_MODE + return + index = random.randint(0, + len(self.bootstrap_nodes) - 1) + self.querier.send_query(self.find_node_msg, + self.bootstrap_nodes[index], + None, + None, + None) + del self.bootstrap_nodes[index] + #TODO2: Don't use querier's rpc_m + self.querier.rpc_m.call_later(BOOTSTRAP_DELAY, + self.do_bootstrap) + + def on_query_received(self, node_): + try: + rnode = self.main.get_rnode(node_) + except RnodeNotFound: + pass # node is not in the main table + else: + # node in routing table: inform rnode + rnode.on_query_received() + self._reset_refresh_task(rnode) + return + # Node is not in routing table + # Check reachability (if the bucket is not full) + if self.main.there_is_room(node_): + # there is room in the bucket: ping node to check reachability + self._refresh_now(node_) + return + # No room in the main routing table + # Add to replacement table (if the bucket is not full) + bucket = self.replacement.get_bucket(node_) + worst_rnode = self._worst_rnode(bucket.rnodes) + if worst_rnode \ + and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS: + self.replacement.remove(worst_rnode) + self.replacement.add(node_) + + + def on_response_received(self, node_): #TODO2:, rtt=0): + try: + rnode = self.main.get_rnode(node_) + except (RnodeNotFound): + pass + else: + # node in routing table: refresh it + rnode.on_response_received() + self._reset_refresh_task(rnode) + return + # The node is not in main + try: + rnode = self.replacement.get_rnode(node_) + except (RnodeNotFound): + pass + else: + # node in replacement table + # let's see whether there is room in the main + rnode.on_response_received() + if self.main.there_is_room(node_): + rnode = self.main.add(rnode) + self._reset_refresh_task(rnode) + self.replacement.remove(rnode) + return + # The node is nowhere + # Add to replacement table (if the bucket is not full) + bucket = self.replacement.get_bucket(node_) + if self.main.there_is_room(node_): + if not bucket.rnodes: + # Replacement is empty + rnode = self.main.add(node_) + self._reset_refresh_task(rnode) + return + # The main bucket is full or the repl bucket is not empty + worst_rnode = self._worst_rnode(bucket.rnodes) + # Get the worst node in replacement bucket and see whether + # it's bad enough to be replaced by node_ + if worst_rnode \ + and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS: + # This node is better candidate than worst_rnode + self.replacement.remove(worst_rnode) + try: + self.replacement.add(node_) + except (BucketFullError): + pass + + + def on_error_received(self, node_): + pass + + def on_timeout(self, node_): + if node_ is self.my_node: + raise Exception, 'I got a timeout from myself!!!' + if not node_.id: + return # This is a bootstrap node (just addr, no id) + try: + rnode = self.main.get_rnode(node_) + except RnodeNotFound: + pass + else: + # node in routing table: check whether it should be removed + rnode.on_timeout() + replacement_bucket = self.replacement.get_bucket(node_) + self._refresh_replacement_bucket(replacement_bucket) + self.main.remove(rnode) + try: + self.replacement.add(rnode) + except (BucketFullError): + worst_rnode = self._worst_rnode(replacement_bucket.rnodes) + if worst_rnode: + # Replace worst node in replacement table + self.replacement.remove(worst_rnode) + self._refresh_replacement_bucket(replacement_bucket) + # We don't want to ping the node which just did timeout + self.replacement.add(rnode) + # Node is not in main table + try: + rnode = self.replacement.get_rnode(node_) + except RnodeNotFound: + pass # the node is not in any table. Nothing to do here. + else: + # Node in replacement table: just update rnode + rnode.on_timeout() + + def on_nodes_found(self, nodes): + #FIXME: this will send ping at exponential rate + #not good!!!! + logger.debug('nodes found: %r', nodes) + for node_ in nodes: + try: + rnode = self.main.get_rnode(node_) + except RnodeNotFound: + # Not in the main: ping it if there is room in main + if self.main.there_is_room(node_): + logger.debug('pinging node found: %r', node_) + self._refresh_now(node_, NO_PRIORITY) + #TODO2: prefer NS + + def get_closest_rnodes(self, target_id, num_nodes=DEFAULT_NUM_NODES): + return self.main.get_closest_rnodes(target_id, num_nodes) + + def get_all_rnodes(self): + return (self.main.get_all_rnodes(), + self.replacement.get_all_rnodes()) + + def print_stats(self): + print '=== MAIN ===' + self.main.print_stats() + print '=== REPLACEMENT ===' + self.replacement.print_stats() + print '=== ===' + + def _refresh_now(self, node_, priority=PRIORITY): + if priority == NO_PRIORITY and \ + self.num_concurrent_refresh_msgs > MAX_CONCURRENT_REFRESH_MSGS: + return + self.num_concurrent_refresh_msgs += 1 + return self.querier.send_query(self.find_node_msg, + node_, + self._refresh_now_callback, + self._refresh_now_callback, + self._refresh_now_callback) + + def _reset_refresh_task(self, rnode): + if rnode.refresh_task: + # Cancel the current refresh task + rnode.refresh_task.cancel() + if rnode.in_quarantine: + rnode.refresh_task = self._refresh_later(rnode, + QUARANTINE_PERIOD) + else: + rnode.refresh_task = self._refresh_later(rnode) + + + def _refresh_later(self, rnode, delay=REFRESH_PERIOD): + return self.querier.send_query_later(delay, + self.find_node_msg, + rnode, + None, + None, + None) + def _do_nothing(self, *args, **kwargs): + pass + + def _refresh_now_callback(self, *args, **kwargs): + self.num_concurrent_refresh_msgs -= 1 + + + def _refresh_replacement_bucket(self, bucket): + for rnode in bucket.rnodes: + if rnode.is_ns: + # We give advantage to NS nodes + self._refresh_now(rnode) + else: + self._refresh_later(rnode, REFRESH_DELAY_FOR_NON_NS) + + def _worst_rnode(self, rnodes): + max_num_timeouts = -1 + worst_rnode_so_far = None + for rnode in rnodes: + num_timeouots = rnode.timeouts_in_a_row() + if num_timeouots >= max_num_timeouts: + max_num_timeouts = num_timeouots + worst_rnode_so_far = rnode + return worst_rnode_so_far + + + + +class RoutingManagerMock(object): + + def get_closest_rnodes(self, target_id): + import test_const as tc + if target_id == tc.INFO_HASH_ZERO: + return (tc.NODES_LD_IH[155][4], + tc.NODES_LD_IH[157][3], + tc.NODES_LD_IH[158][1], + tc.NODES_LD_IH[159][0], + tc.NODES_LD_IH[159][2],) + else: + return tc.NODES diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/routing_table.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/routing_table.py new file mode 100644 index 0000000..70cae23 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/routing_table.py @@ -0,0 +1,125 @@ +# Copyright (C) 2009 Raul Jimenez +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +import logging + +logger = logging.getLogger('dht') + + +class BucketFullError(Exception): + pass +class RnodeNotFound(IndexError): + pass + +class Bucket(object): + + def __init__(self, max_nodes): + self.max_nodes = max_nodes + self.rnodes = [] + + def __getitem__(self, node_): + try: + return self.rnodes[self._index(node_)] + except (KeyError): + raise RnodeNotFound + + def add(self, rnode): + if len(self.rnodes) == self.max_nodes: + raise BucketFullError + self.rnodes.append(rnode) + + def remove(self, node_): + del self.rnodes[self._index(node_)] + + def __repr__(self): + return '\n'.join([repr(rnode) for rnode in self.rnodes]) + + def __len__(self): + return len(self.rnodes) + + def is_full(self): + return len(self.rnodes) == self.max_nodes + + def _index(self, node_): + for i, rnode in enumerate(self.rnodes): + if rnode == node_: + return i + raise KeyError # not found + +NUM_BUCKETS = 160 + 1 # log_distance returns a number in range [-1,159] +NUM_NODES = 8 +class RoutingTable(object): + ''' + ''' + + def __init__(self, my_node, nodes_per_bucket): + assert len(nodes_per_bucket) == NUM_BUCKETS + self.my_node = my_node + self.buckets = [Bucket(num_nodes) + for num_nodes in nodes_per_bucket] + self.num_rnodes = 0 + + def get_rnode(self, node_): + index = node_.log_distance(self.my_node) + return self.buckets[index][node_] + + def get_bucket(self, node_): + index = node_.log_distance(self.my_node) + return self.buckets[index] + + def there_is_room(self, node_): + return not self.get_bucket(node_).is_full() + + def add(self, node_): + rnode = node_.get_rnode() + index = node_.log_distance(self.my_node) + bucket = self.buckets[index].add(rnode) + self.num_rnodes += 1 + return rnode + + def remove(self, node_): + index = node_.log_distance(self.my_node) + bucket = self.buckets[index].remove(node_) + self.num_rnodes -= 1 + + def get_closest_rnodes(self, id_, num_nodes=NUM_NODES): + # Myself + if id_ == self.my_node.id: + return [self.my_node] + # id_ is not myself + result = [] + highest_index = id_.log_distance(self.my_node.id) + for i, bucket in enumerate(self.buckets[highest_index::-1]): + result.extend(bucket.rnodes[:num_nodes-len(result)]) + #TODO2: get all nodes in the bucket and order + if len(result) == num_nodes: + break + if len(result) < num_nodes: + result.extend(self.buckets[-1].rnodes) # myself + return result + + def get_all_rnodes(self): + rnodes = [] + for bucket in self.buckets: + rnodes.extend(bucket.rnodes) + return rnodes + + def print_stats(self): + num_nodes = 0 + for i, bucket in enumerate(self.buckets): + if len(bucket): + print i, len(bucket) + num_nodes += len(bucket) + print 'Total:', num_nodes + + def __repr__(self): + msg = ['==============RoutingTable============= BEGIN'] + for i, bucket in enumerate(self.buckets): + msg.append('%d %r' % (i, bucket)) + msg.append('==============RoutingTable============= END') + return '\n'.join(msg) + + + + diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/rpc_manager.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/rpc_manager.py new file mode 100644 index 0000000..f0fa951 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/rpc_manager.py @@ -0,0 +1,66 @@ +# Copyright (C) 2009 Raul Jimenez +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + + +import logging + +import message + +logger = logging.getLogger('dht') + + +class RPCManager(object): + + def __init__(self, reactor, port): + self.reactor = reactor + self.reactor.listen_udp(port, self._on_datagram_received) + self.msg_callbacks_d = {} + self.timeout_callbacks = [] + + def get_timeout_task(self, addr, timeout_delay, timeout_callback): + timeout_callbacks = self.timeout_callbacks + [timeout_callback] + return self.reactor.call_later(timeout_delay, + timeout_callbacks, addr) + def send_msg_to(self, bencoded_msg, addr): + """This must be called right after get_timeout_task + (when timeout is needed). + """ + self.reactor.sendto(bencoded_msg, addr) + + def call_later(self, delay, callback_fs, *args, **kwargs): + return self.reactor.call_later(delay, callback_fs, *args, **kwargs) + + def add_msg_callback(self, msg_type, callback_f): + self.msg_callbacks_d.setdefault(msg_type, []).append(callback_f) + + def add_timeout_callback(self, callback_f): + self.timeout_callbacks.append(callback_f) + + def stop(self): + self.reactor.stop() + + def _on_datagram_received(self, data, addr): + # Sanitize bencode + try: + msg = message.IncomingMsg(data) + except (message.MsgError): + logger.info('MsgError when decoding\n%s\nsouce: %s' % ( + data, addr)) + return # ignore message + try: + # callback according to message's type + callback_fs = self.msg_callbacks_d[msg.type] + except (KeyError): + logger.info('Key TYPE has an invalid value\n%s\nsouce: %s' % ( + data, addr)) + return #ignore message + # Call the proper callback (selected according msg's TYPE) + response_msg = None + for callback_f in callback_fs: + # if there is a response we should keep it + response_msg = callback_f(msg, addr) or response_msg + if response_msg: + bencoded_response = response_msg.encode(msg.tid) + self.send_msg_to(bencoded_response, addr) + diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/run_dht_node_forever.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/run_dht_node_forever.py new file mode 100644 index 0000000..55800d7 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/run_dht_node_forever.py @@ -0,0 +1,47 @@ +# Copyright (C) 2009 Raul Jimenez +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +import time +import sys + +import logging, logging_conf + +import identifier +import kadtracker + +def peers_found(peers): + for peer in peers: + print peer + print 'Type "EXIT" to stop the DHT and exit' + print 'Type an info_hash (in hex digits):' + +def lookup_done(): + print 'No peers found' + print 'Type "EXIT" to stop the DHT and exit' + print 'Type an info_hash (in hex digits):' + +if len(sys.argv) == 5 and sys.argv[1] == 'interactive_dht': + logging.critical('argv %r' % sys.argv) + assert 0 + RUN_DHT = True + my_addr = (sys.argv[1], sys.argv[2]) #('192.16.125.242', 7000) + logs_path = sys.argv[3] + dht = kadtracker.KadTracker(my_addr, logs_path) +else: + RUN_DHT = False + print 'usage: python interactive_dht ip port paht' + +while (RUN_DHT): + input = sys.stdin.readline()[-1] + if input == 'EXIT': + dht.stop() + break + try: + info_hash = identifier.Id(hex_id) + except (IdError): + print 'Invalid info_hash (%s)' % hex_id + continue + dht.get_peers(info_hash, do_nothing) + + diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/server_dht.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/server_dht.py new file mode 100644 index 0000000..ae50021 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/server_dht.py @@ -0,0 +1,87 @@ +# Copyright (C) 2009 Raul Jimenez +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +import time +import sys +import pdb +#import guppy + +import logging, logging_conf +logs_path = '.' +logs_level = logging.DEBUG # This generates HUGE (and useful) logs +#logs_level = logging.INFO # This generates some (useful) logs +#logs_level = logging.WARNING # This generates warning and error logs +#logs_level = logging.CRITICAL + +import identifier +import kadtracker + + +#hp = guppy.hpy() + +def peers_found(peers): + print 'Peers found:', time.time() + return + for peer in peers: + print peer + print '-'*20 + +def lookup_done(): + print 'Lookup DONE' + + +info_hashes = ( + identifier.RandomId(), + identifier.Id('28f2e5ea2bf87eae4bcd5e3fc9021844c01a4df9'), + identifier.RandomId(), + identifier.Id('dd5c25b4b8230e108fbf9d07f87a86c6b05c9b6d'), + identifier.RandomId(), + identifier.Id('bcbdb9c2e7b49c65c9057431b492cb7957c8a330'), + identifier.RandomId(), + identifier.Id('d93df7a507f3c9d2ebfbe49762a217ab318825bd'), + identifier.RandomId(), + identifier.Id('6807e5d151e2ac7ae92eabb76ddaf4237e4abb60'), + identifier.RandomId(), + identifier.Id('83c7b3b7d36da4df289670592be68f9dc7c7096e'), + identifier.RandomId(), + identifier.Id('9b16aecf952597f9bb051fecb7a0d8475d060fa0'), + identifier.RandomId(), + identifier.Id('24f2446365d3ef782ec16ad63aea1206df4b8d21'), + identifier.RandomId(), + identifier.Id('a91af3cde492e29530754591b862b1beecab10ff'), + identifier.RandomId(), + identifier.Id('3119baecadea3f31bed00de5e7e76db5cfea7ca1'), + ) + +if len(sys.argv) == 4 and sys.argv[0] == 'server_dht.py': + logging.critical('argv %r' % sys.argv) + RUN_DHT = True + my_addr = (sys.argv[1], int(sys.argv[2])) #('192.16.125.242', 7000) + logs_path = sys.argv[3] + print 'logs_path:', logs_path + logging_conf.setup(logs_path, logs_level) + dht = kadtracker.KadTracker(my_addr, logs_path) +else: + RUN_DHT = False + print 'usage: python server_dht.py dht_ip dht_port path' + +try: + print 'Type Control-C to exit.' + i = 0 + while (RUN_DHT): + for info_hash in info_hashes: + #splitted_heap_str = str(hp.heap()).split() + #print i, splitted_heap_str[10] + dht.print_routing_table_stats() + time.sleep(2 * 60) + print 'Getting peers:', time.time() + dht.get_peers(info_hash, peers_found) + #time.sleep(1.5) + #dht.stop() + #pdb.set_trace() + i = i + 1 +except (KeyboardInterrupt): + dht.stop() + + diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_bencode.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_bencode.py new file mode 100644 index 0000000..df548fd --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_bencode.py @@ -0,0 +1,140 @@ +# Copyright (C) 2009 Raul Jimenez +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +from nose.tools import assert_raises, raises + +import cStringIO +import logging, logging_conf + +from bencode import * + +logging_conf.testing_setup(__name__) +logger = logging.getLogger('dht') + +test_data = [ + # strings + ('a', '1:a'), + ('1', '1:1'), + ('0123456789abcdef', '16:0123456789abcdef'), + ('A' * 100, '100:' + 'A' * 100), + ('{', '1:{'), + ('[', '1:['), + (chr(2), '1:' + chr(2)), + # integers + (0, 'i0e'), + (000, 'i0e'), + (1234567890, 'i1234567890e'), + (-1, 'i-1e'), + # lists + ([], 'le'), + ([[[[]]]], 'lllleeee'), # maximum recursivity depht + ([1, 2, 3], 'li1ei2ei3ee'), + (['A', 'B', 'C'], 'l1:A1:B1:Ce'), + (['A', 2, 'C'], 'l1:Ai2e1:Ce'), + ([1, ['X'], 2, 'Z'], 'li1el1:Xei2e1:Ze'), + # dictionaries + ({}, 'de'), + ({'key': 'a'}, 'd3:key1:ae'), + ({'ZZZ': 12345}, 'd3:ZZZi12345ee'), + # ordered dictionaries + ({'a':{'A':1, 'C':2, 'B':3}, 'b':2, 'z':3, 'c':[]}, + 'd1:ad1:Ai1e1:Bi3e1:Ci2ee1:bi2e1:cle1:zi3ee'), + # mixed types + ({'A': [], 'B': {'B': [1], 'C': [], 'D':{}}, 'C': 9}, + 'd1:Ale1:Bd1:Bli1ee1:Cle1:Ddee1:Ci9ee'), + ] + +test_data_encode_error = [ + (False, EncodeError), + # Using no-string types in dict + ({1:1}, EncodeError), + ({None:1}, EncodeError), + ({(1,2):1}, EncodeError), + # There is no recursion limit when encoding + ] + +test_data_decode_error = [ + ('', DecodeError), # empty bencode + ('leEXTRA', DecodeError), # extra characters after bencode + ('xWHATEVER', DecodeError), # start with invalid character + ('dxe', DecodeError), # invalid special character + ('ixe', DecodeError), # invalid integer + ('li2e', DecodeError), # list end missing + ('li2eee', DecodeError), # extra end + ('d3:KEYe', DecodeError), # value missing + ('lllll', RecursionDepthError), + ('ddddd', DecodeError), # Notice that a dictionary is NOT a valid KEY. + ] + + +def debug_print(test_num, input_, expected, output): + logger.debug('''test_num: %d + input: %s + expected: %s + output: %s''' % (test_num, input_, expected, output)) + + +class TestEncode(): + + def setup(self): + pass + + def test_encode(self): + for i, (data, expected) in enumerate(test_data): + bencoded = None + try: + bencoded = encode(data) + except(Exception), e: + debug_print(i, data, expected, e) + raise + if bencoded != expected: + debug_print(i, data, expected, bencoded) + assert False + + def test_encode_error(self): + for i, (data, expected) in enumerate(test_data_encode_error): + logger.debug( + '>>>>>>>>>>>EXPECTED ERROR LOG: %r' % expected) + try: + encode(data) + except expected: + pass # Good. We got the expected exception. + except (Exception), e: + debug_print(i, data, expected, e) + raise # Fail. We got some other exception. + else: + debug_print(i, data, expected, 'NO EXCEPTION RAISED') + assert False # Fail. We got no exception at all. + + +class TestDecode: + + def setup(self): + pass + + def test_decode(self): + for i, (expected, bencoded) in enumerate(test_data): + data = None + try: + data = decode(bencoded) + except (Exception), e: + debug_print(i, bencoded, expected, e) + raise + else: + if data != expected: + debug_print(i, bencoded, expected, data) + assert False + + def test_decode_error(self): + for i, (bencoded, expected) in enumerate(test_data_decode_error): + try: + decode(bencoded) + except expected: + pass + except (Exception), e: + debug_print(i, bencoded, expected, e) + raise + else: + debug_print(i, bencoded, expected, 'NO EXCEPTION RAISED') + assert False diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_const.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_const.py new file mode 100644 index 0000000..78bebc8 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_const.py @@ -0,0 +1,79 @@ +# Copyright (C) 2009 Raul Jimenez +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +#from nose.tools import eq_, ok_ +''' +import logging, logging_conf +logs_path = 'test_logs' +logs_level = logging.DEBUG +logging_conf.setup(logs_path, logs_level) +''' + +import identifier +from identifier import Id, ID_SIZE_BITS, BITS_PER_BYTE +import node + + + +TASK_INTERVAL = .01 +TIMEOUT_DELAY = .4 + +CLIENT_ID = identifier.Id('\x41' * identifier.ID_SIZE_BYTES) +CLIENT_ADDR = ('127.0.0.1', 6000) +CLIENT_NODE = node.Node(CLIENT_ADDR, CLIENT_ID) +BT_PORT = 7777 + +SERVER_ID = identifier.Id('\x01' * identifier.ID_SIZE_BYTES) +SERVER_ADDR = ('127.0.0.1', 6001) +SERVER_NODE = node.Node(SERVER_ADDR, SERVER_ID) + +SERVER2_ID = identifier.Id('\x43' * identifier.ID_SIZE_BYTES) +SERVER2_ADDR = ('127.0.0.2', 6002) +SERVER2_NODE = node.Node(SERVER2_ADDR, SERVER2_ID) + +EXTERNAL_NODE_ADDR = ('127.0.0.1', 6881) +EXTERNAL_NODE = node.Node(EXTERNAL_NODE_ADDR) + +NO_ADDR = ('127.0.0.1', 1) +DEAD_NODE = node.Node(NO_ADDR) + +NODE_ID = identifier.Id('\x02' * identifier.ID_SIZE_BYTES) +TARGET_ID = NODE_ID +INFO_HASH = identifier.Id('\x60\xd5\xd8\x23\x28\xb4\x54\x75\x11\xfd\xea\xc9\xbf\x4d\x01\x12\xda\xa0\xce\x00') +INFO_HASH_ZERO = identifier.Id('\x00' * identifier.ID_SIZE_BYTES) +TID = 'a' +TID2 = 'b' +TOKEN = 'aa' + +NUM_NODES = 8 +NODE_IDS = [identifier.Id(chr(i) * identifier.ID_SIZE_BYTES) \ + for i in xrange(NUM_NODES)] +ADDRS = [('127.0.0.'+str(i), 7000 + i) for i in xrange(NUM_NODES)] +NODES = [node.Node(addr, node_id) \ + for addr, node_id in zip(ADDRS, NODE_IDS)] +PEERS = ADDRS + +NODE2_IDS = [identifier.Id('\x01'+chr(i) * (identifier.ID_SIZE_BYTES-1)) \ + for i in xrange(100, 100+NUM_NODES)] +ADDRS2 = [('127.0.0.'+str(i), 7000 + i) \ + for i in xrange(100, 100+NUM_NODES)] +NODES2 = [node.Node(addr, node_id) \ + for addr, node_id in zip(ADDRS2, NODE2_IDS)] +PEERS2 = ADDRS2 + +IPS = ['1.2.3.' + str(i) for i in xrange(NUM_NODES)] + +#TODO2: make this faster +num_nodes_per_ld = 20 +NODES_LD_IH = [[]] * BITS_PER_BYTE +for ld in xrange(BITS_PER_BYTE, ID_SIZE_BITS): + NODES_LD_IH.append([]) + common_id = INFO_HASH_ZERO.generate_close_id(ld) + #eq_(common_id.log_distance(INFO_HASH_ZERO), ld) + for i in xrange(num_nodes_per_ld): + this_id = Id(common_id.bin_id[:-1] + chr(i)) + #eq_(this_id.log_distance(INFO_HASH_ZERO), ld) + NODES_LD_IH[ld].append( + node.Node(('127.0.0.' + str(i), i), this_id)) + diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_controller.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_controller.py new file mode 100644 index 0000000..b2f8dc3 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_controller.py @@ -0,0 +1,18 @@ +# Copyright (C) 2009 Raul Jimenez +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +import test_const as tc + +import controller + + + +class TestController: + + def setup(self): + self.controller = controller.Controller(tc.CLIENT_ADDR) + + def test_start_stop(self): + self.controller.start() + self.controller.stop() diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_floodbarrier.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_floodbarrier.py new file mode 100644 index 0000000..76ff17b --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_floodbarrier.py @@ -0,0 +1,60 @@ +# Copyright (C) 2009 Raul Jimenez +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +import time +import logging, logging_conf + +from test_const import * + +from floodbarrier import FloodBarrier + +logging_conf.testing_setup(__name__) +logger = logging.getLogger('dht') + + +class TestFloodBarrier: + + def setup(self): + #logger.critical('************* BEGIN **************') + pass + + def test(self): + fb = FloodBarrier(checking_period=.4, + max_packets_per_period=4, + blocking_period=1) + for ip in IPS: + for _ in xrange(4): + assert not fb.ip_blocked(ip) + # Every ip is on the limit + assert fb.ip_blocked(IPS[0]) + assert fb.ip_blocked(IPS[1]) + # 0 and 3 blocked + time.sleep(.2) + # Half a period gone + assert fb.ip_blocked(IPS[0]) + # IP 0 refreshes the blocking (extra .2 seconds) + time.sleep(.2) + # The initial floods are forgotten + # IP 0,1,3 are blocked + assert fb.ip_blocked(IPS[0]) + # The blocking doesn't get refreshed now (.8 secs to expire) + assert fb.ip_blocked(IPS[1]) + # The blocking doesn't get refreshed (.6 secs to expire) + assert not fb.ip_blocked(IPS[2]) + time.sleep(.7) + # IP 0 is the only one still blocked (it got refreshed) + assert fb.ip_blocked(IPS[0]) + assert not fb.ip_blocked(IPS[1]) + assert not fb.ip_blocked(IPS[2]) + assert not fb.ip_blocked(IPS[3]) + time.sleep(.4) + for ip in IPS: + for _ in xrange(4): + assert not fb.ip_blocked(ip) + time.sleep(.4) + for ip in IPS: + for _ in xrange(4): + assert not fb.ip_blocked(ip) + + diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_identifier.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_identifier.py new file mode 100644 index 0000000..5191d7a --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_identifier.py @@ -0,0 +1,199 @@ +# Copyright (C) 2009 Raul Jimenez +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +import random + +import logging, logging_conf + +from nose.tools import eq_, ok_, assert_raises, raises +import test_const as tc + +import identifier +from identifier import Id, RandomId, IdError +from identifier import ID_SIZE_BYTES, ID_SIZE_BITS, BITS_PER_BYTE + +logging_conf.testing_setup(__name__) +logger = logging.getLogger('dht') + + +BIN_ID0 = '\x00' * ID_SIZE_BYTES +BIN_ID1 = '\x01' * ID_SIZE_BYTES +BIN_ID2 = '\x02' * ID_SIZE_BYTES +DIST0_1 = '\x01' * ID_SIZE_BYTES +DIST1_2 = '\x03' * ID_SIZE_BYTES + +HEX_ID1 = '01' * ID_SIZE_BYTES + + +class TestId(object): + + def test_create(self): + _ = Id(BIN_ID1) + _ = RandomId() + assert_raises(IdError, Id, 1) + assert_raises(IdError, Id, '1') + _ = Id('1' * 40) # Hexadecimal + assert_raises(IdError, Id, 'Z'*40) + eq_(Id('\x00'*20).bin_id, Id('0'*40).bin_id) + eq_(Id('\xff'*20), Id('f'*40)) + + def test_has_repr(self): + eq_(repr(Id(BIN_ID1)), '') + + def test_is_hashable(self): + d = {Id(BIN_ID1): 1} + + def test_util(self): + assert identifier._bin_to_hex(BIN_ID1) == HEX_ID1 + assert identifier._byte_xor('\0', '\1') == '\1' + + def test_first_different_byte(self): + str1 = '0' * ID_SIZE_BYTES + for i in range(ID_SIZE_BYTES): + str2 = '0' * i + '1' * (ID_SIZE_BYTES - i) + logger.debug('test_num: %d, _first_different_byte: %d' % ( + i, identifier._first_different_byte(str1, str2))) + assert identifier._first_different_byte(str1, str2) == i + assert_raises(IndexError, + identifier._first_different_byte, str1, str1) + + def test_first_different_bit(self): + assert identifier._first_different_bit('\0', '\x01') == 7 + assert identifier._first_different_bit('\0', '\x02') == 6 + assert identifier._first_different_bit('\0', '\x04') == 5 + assert identifier._first_different_bit('\0', '\x09') == 4 + assert identifier._first_different_bit('\0', '\x10') == 3 + assert identifier._first_different_bit('\0', '\x23') == 2 + assert identifier._first_different_bit('\0', '\x40') == 1 + assert identifier._first_different_bit('\0', '\xa5') == 0 + assert identifier._first_different_bit('\0', '\xff') == 0 + assert_raises(AssertionError, identifier._first_different_bit, + '\5', '\5') + + def test_bin_id(self): + assert Id(BIN_ID1).bin_id == BIN_ID1 + + def test_equal(self): + id1 = Id(BIN_ID0) + assert id1 == id1 # same instance + assert id1 == Id(BIN_ID0) #different instance, same value + assert id1 != Id(BIN_ID1) + + + @raises(AttributeError) + def test_bin_id_read_only(self): + id1 = Id(BIN_ID1) + id1.bin_id = BIN_ID2 + + def test_str(self): + id1 = Id(BIN_ID1) + assert BIN_ID1 == '%s' % id1 + + def test_distance(self): + id1 = Id(BIN_ID1) + id2 = Id(BIN_ID2) + dist1_2 = Id(DIST1_2) + assert id1.distance(id2).bin_id == dist1_2.bin_id + assert id2.distance(id1).bin_id == dist1_2.bin_id + #assert id1.distance(id1).bin_id == ZeroId().bin_id + + def test_log_distance(self): + id0 = Id(BIN_ID0) + id1 = Id(BIN_ID1) + id2 = Id(BIN_ID2) + eq_(id0.log_distance(id0), -1) + eq_(id0.log_distance(id1), ID_SIZE_BITS - 8) + eq_(id0.log_distance(id2), ID_SIZE_BITS - 7) + + id_log = ( + (Id('\x00' + '\xff' * (ID_SIZE_BYTES - 1)), + BITS_PER_BYTE * (ID_SIZE_BYTES - 1) - 1), + + (Id('\x53' * ID_SIZE_BYTES), + BITS_PER_BYTE * ID_SIZE_BYTES - 2), + + (Id(BIN_ID0[:7] + '\xff' * (ID_SIZE_BYTES - 7)), + (ID_SIZE_BYTES - 7) * BITS_PER_BYTE - 1), + + (Id(BIN_ID0[:9] + '\x01' * (ID_SIZE_BYTES - 9)), + (ID_SIZE_BYTES - 10) * BITS_PER_BYTE), + + (Id(BIN_ID0[:-1] + '\x06'), + 2), + ) + id2_log = ( + (Id('\x41' * ID_SIZE_BYTES), + Id('\x41' * ID_SIZE_BYTES), + -1), + + (Id('\x41' * ID_SIZE_BYTES), + Id('\x01' * ID_SIZE_BYTES), + 158), + + (Id('\x41' * ID_SIZE_BYTES), + Id('\x43' * ID_SIZE_BYTES), + 153), + ) + + for (id_, log_) in id_log: + logger.debug('log_distance: %d' % id0.log_distance(id_)) + logger.debug('expected: %d' % log_) + eq_(id0.log_distance(id_), log_) + for id1, id2, expected in id2_log: + eq_(id1.log_distance(id2), expected) + + z = Id('\0'*20) + eq_(z.log_distance(Id('\x00'*19+'\x00')), -1) + eq_(z.log_distance(Id('\x00'*19+'\x00')), -1) + eq_(z.log_distance(Id('\x00'*19+'\x00')), -1) + eq_(z.log_distance(Id('\x00'*19+'\x00')), -1) + eq_(z.log_distance(Id('\x00'*19+'\x00')), -1) + + + + def test_order_closest(self): + id0 = Id(BIN_ID0) + ordered_list = [ + Id('\x00' * ID_SIZE_BYTES), + Id(BIN_ID0[:-1] + '\x06'), + Id(BIN_ID0[:9] + '\x01' * (ID_SIZE_BYTES - 9)), + Id(BIN_ID0[:7] + '\xff' * (ID_SIZE_BYTES - 7)), + Id(BIN_ID0[:7] + '\xff' * (ID_SIZE_BYTES - 7)), + Id('\x00' + '\xff' * (ID_SIZE_BYTES - 1)), + Id('\x53' * ID_SIZE_BYTES), + Id('\xff' * ID_SIZE_BYTES), + ] + random_list = random.sample(ordered_list, len(ordered_list)) + + random_list_copy = random_list[:] + + logger.debug('ordered list') + for e in ordered_list: logger.debug('%s' % e) + logger.debug('random order') + for e in random_list: logger.debug('%s' % e) + + result_list = id0.order_closest(random_list) + logger.debug('order_closest result') + for e in result_list: logger.debug('%s' % e) + logger.debug('random order (it should not change)') + for e in random_list: logger.debug('%s' % e) + + # make sure order_closest does not modify random_list + assert random_list == random_list_copy + + for i, ordered_id in enumerate(ordered_list): + logger.debug('%d, %s, %s' % (i, ordered_id, result_list[i])) + assert ordered_id.bin_id == result_list[i].bin_id + # Notice that 'assert ordered_id is result_id' + # do not work when two Id instances have the same bin_id + + def test_generate_closest_id(self): + id_ = RandomId() + for i in [-1] + range(ID_SIZE_BITS): + eq_(id_.log_distance(id_.generate_close_id(i)), i) + + +class TestRandomId: + for i in xrange(123): + assert RandomId() != RandomId() diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_kadtracker.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_kadtracker.py new file mode 100644 index 0000000..1462ca9 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_kadtracker.py @@ -0,0 +1,20 @@ +# Copyright (C) 2009 Raul Jimenez +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +import test_const as tc + +import kadtracker + +class TestKadTracker: + + def _callback(self, *args, **kwargs): + return + + def setup(self): + self.dht = kadtracker.KadTracker(tc.CLIENT_ADDR, '.') + + def test_all(self): + #self.dht.start() + self.dht.get_peers(tc.INFO_HASH, self._callback, tc.BT_PORT) + self.dht.stop() diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_lookup_manager.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_lookup_manager.py new file mode 100644 index 0000000..d93fb02 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_lookup_manager.py @@ -0,0 +1,381 @@ +# Copyright (C) 2009 Flutra Osmani, Raul Jimenez +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +from nose.tools import eq_, ok_, assert_raises +import test_const as tc +import logging, logging_conf + +import time + +import querier +from routing_manager import RoutingManagerMock +import lookup_manager +import message +from identifier import Id, ID_SIZE_BYTES +from node import Node + +logging_conf.testing_setup(__name__) +logger = logging.getLogger('dht') + + +class TestLookupQueue: + + def setup(self): + self.lookup = lookup_manager._LookupQueue(tc.INFO_HASH_ZERO, 4) + + def test_add_pop1(self): + nodes = (tc.NODES_LD_IH[157][0], + tc.NODES_LD_IH[158][1], + tc.NODES_LD_IH[154][2], + tc.NODES_LD_IH[159][3], + tc.NODES_LD_IH[158][4], + tc.NODES_LD_IH[152][5],) + self.lookup.add(nodes) + # Just the 4 closest nodes are added + #This second add doesn't affect (duplicates are ignored) + self.lookup.add(nodes) + eq_(self.lookup.pop_closest_node(), tc.NODES_LD_IH[152][5]) + eq_(self.lookup.pop_closest_node(), tc.NODES_LD_IH[154][2]) + eq_(self.lookup.pop_closest_node(), tc.NODES_LD_IH[157][0]) + eq_(self.lookup.pop_closest_node(), tc.NODES_LD_IH[158][1]) + # Now the queue is empty + assert_raises(IndexError, self.lookup.pop_closest_node) + self.lookup.add(nodes) + # The nodes added are ingnored + assert_raises(IndexError, self.lookup.pop_closest_node) + + + def _test_add_pop2(self): + self.lookup.add(tc.NODES[3:6]) + eq_(self.lookup.pop_closest_node(), tc.NODES[3]) + eq_(self.lookup.pop_closest_node(), tc.NODES[4]) + self.lookup.add(tc.NODES[2:3]) + eq_(self.lookup.pop_closest_node(), tc.NODES[2]) + eq_(self.lookup.pop_closest_node(), tc.NODES[5]) + # Empty + assert_raises(IndexError, self.lookup.pop_closest_node) + # This add only affects 0,1,6,7 + self.lookup.add(tc.NODES) + eq_(self.lookup.pop_closest_node(), tc.NODES[0]) + eq_(self.lookup.pop_closest_node(), tc.NODES[1]) + eq_(self.lookup.pop_closest_node(), tc.NODES[6]) + eq_(self.lookup.pop_closest_node(), tc.NODES[7]) + + +class TestGetPeersLookup: + + def _callback(self, peers): + self.got_peers = peers + + def setup(self): + self.got_peers = None + querier_ = querier.QuerierMock(tc.CLIENT_ID) + bootstrap_nodes = RoutingManagerMock( + ).get_closest_rnodes(tc.INFO_HASH_ZERO) + self.lookup = lookup_manager.GetPeersLookup(tc.CLIENT_ID, + querier_, + 2, + tc.INFO_HASH_ZERO, + self._callback, + bootstrap_nodes) + + def test_n(self): + pass + + def _test_complete(self): + self.lookup.start() + """Start sends two parallel queries to the closest + bootstrap nodes (to the INFO_HASH) + + """ + # Ongoing queries to (sorted: oldest first): + # 155-4, 157-3, + # Queued nodes to query (sorted by log_distance to info_hash): + # 158-1, 159-0 + # Notice 159-2 is kicked out from the queue + logger.critical("") + eq_(self.lookup.num_parallel_queries, 2) + nodes = [tc.NODES_LD_IH[157][5], + tc.NODES_LD_IH[152][6], + tc.NODES_LD_IH[158][7]] + self.lookup._on_response(*_gen_nodes_args( + tc.NODES_LD_IH[157][3], + nodes)) + eq_(self.lookup._get_announce_candidates(), + [tc.NODES_LD_IH[157][3], + ]) + # This response triggers a new query (to 152-6) + eq_(self.lookup.num_parallel_queries, 2) + # Ongoing queries to (sorted: oldest first): + # 155-4, 152-6 + # Queued nodes to query (sorted by log_distance to info_hash): + # 157-5, 158-1, 158-7, 159-0 + self.lookup._on_timeout(tc.NODES_LD_IH[155][4]) + eq_(self.lookup.num_parallel_queries, 2) + # This timeout triggers a new query (to 157-5) + eq_(self.lookup.num_parallel_queries, 2) + # Ongoing queries to (sorted: oldest first): + # 155-4, 157-5 + # Queued nodes to query (sorted by log_distance to info_hash): + # 158-1, 158-7, 159-0 + self.lookup._on_timeout(tc.NODES_LD_IH[155][4]) + # This timeout triggers a new query (to 158-1) + eq_(self.lookup.num_parallel_queries, 2) + # Ongoing queries to (sorted: oldest first): + # 152-6, 158-1 + # Queued nodes to query (sorted by log_distance to info_hash): + # 158-7, 159-0 + nodes = [tc.NODES_LD_IH[151][8], + tc.NODES_LD_IH[150][9]] + self.lookup._on_response(*_gen_nodes_args( + tc.NODES_LD_IH[152][6], + nodes)) + eq_(self.lookup._get_announce_candidates(), + [tc.NODES_LD_IH[152][6], + tc.NODES_LD_IH[157][3], + ]) + # This response triggers a new query (to 150-9) + eq_(self.lookup.num_parallel_queries, 2) + # Ongoing queries to (sorted: oldest first): + # 157-5, 150-9 + # Queued nodes to query (sorted by log_distance to info_hash): + # 151-8, 158-7, 159-0 + nodes = [tc.NODES_LD_IH[151][10], + tc.NODES_LD_IH[151][11], + tc.NODES_LD_IH[156][12], + tc.NODES_LD_IH[156][13], + ] + self.lookup._on_response(*_gen_nodes_args( + tc.NODES_LD_IH[157][5], + nodes)) + eq_(self.lookup._get_announce_candidates(), + [tc.NODES_LD_IH[152][6], + tc.NODES_LD_IH[157][3], + tc.NODES_LD_IH[157][5], + ]) + # This response triggers a new query (to 151-8) + eq_(self.lookup.num_parallel_queries, 2) + # Ongoing queries to (sorted: oldest first): + # 150-9, 151-8 + # Queued nodes to query (sorted by log_distance to info_hash): + # 151-10, 151-11, 156-12, 156-13 + # Notice that the lookup queue size limit is 4, therefore + # 158-7 and 159-0 are removed from the queue + self.lookup._on_error(None, tc.NODES_LD_IH[151][8]) + # This error triggers a new query (to 151-8) + eq_(self.lookup.num_parallel_queries, 2) + # Ongoing queries to (sorted: oldest first): + # 150-9, 151-10 + # Queued nodes to query (sorted by log_distance to info_hash): + # 151-11, 156-12, 156-13 + self.lookup._on_timeout(tc.NODES_LD_IH[151][8]) + # This timeout triggers a new query (to 151-11) + eq_(self.lookup.num_parallel_queries, 2) + # Ongoing queries to (sorted: oldest first): + # 151-10, 151-11 + # Queued nodes to query (sorted by log_distance to info_hash): + # 156-12, 156-13 + nodes = [tc.NODES_LD_IH[144][14], + tc.NODES_LD_IH[145][15], + tc.NODES_LD_IH[145][16], + tc.NODES_LD_IH[145][17], + ] + self.lookup._on_response(*_gen_nodes_args( + tc.NODES_LD_IH[151][10], + nodes)) + eq_(self.lookup._get_announce_candidates(), [tc.NODES_LD_IH[151][10], + tc.NODES_LD_IH[152][6], + tc.NODES_LD_IH[157][3], + ]) + # This response triggers a new query (to 144-14) + eq_(self.lookup.num_parallel_queries, 2) + # Ongoing queries to (sorted: oldest first): + # 151-11, 144-14 + # Queued nodes to query (sorted by log_distance to info_hash): + # Notice 156-13 is removed + # 145-15, 145-16, 145-17, 156-12 + peers = [tc.NO_ADDR] + ok_(not self.got_peers) + self.lookup._on_response(*_gen_peers_args( + tc.NODES_LD_IH[144][14], + peers)) + eq_(self.lookup._get_announce_candidates(), [tc.NODES_LD_IH[144][14], + tc.NODES_LD_IH[151][10], + tc.NODES_LD_IH[152][6], + ]) + ok_(self.got_peers) + self.got_peers = False + # The response with peers halves parallelism to 1. + # No new query is triggered. + eq_(self.lookup.num_parallel_queries, 1) + # Ongoing queries to (sorted: oldest first): + # 151-11 + # Queued nodes to query (sorted by log_distance to info_hash): + # 145-15, 145-16, 156-12 + self.lookup._on_timeout(tc.NODES_LD_IH[151][11]) + # This timeout triggers a new query (to 145-15) + eq_(self.lookup.num_parallel_queries, 1) + # Ongoing queries to (sorted: oldest first): + # 145-15 + # Queued nodes to query (sorted by log_distance to info_hash): + # 145-16, 145-17, 156-12 + peers = [tc.NO_ADDR] + ok_(not self.got_peers) + self.lookup._on_response(*_gen_peers_args( + tc.NODES_LD_IH[145][15], + peers)) + # This response triggers a new query (to 145-16) + # The parallelism is not halved (remains 1). + eq_(self.lookup.num_parallel_queries, 1) + # Ongoing queries to (sorted: oldest first): + # 145-16 + # Queued nodes to query (sorted by log_distance to info_hash): + # 145-17, 156-12 + eq_(self.lookup._get_announce_candidates(), [tc.NODES_LD_IH[144][14], + tc.NODES_LD_IH[145][15], + tc.NODES_LD_IH[151][10], + ]) + ok_(self.got_peers) + self.got_peers = False + self.lookup._on_timeout(tc.NODES_LD_IH[145][16]) + # This timeout triggers a new query (to 145-17) + eq_(self.lookup.num_parallel_queries, 1) + # Ongoing queries to (sorted: oldest first): + # 145-17 + # Queued nodes to query (sorted by log_distance to info_hash): + # 156-12 + self.lookup._on_timeout(tc.NODES_LD_IH[145][17]) + # This timeout triggers a new query (to 156-12) + return + eq_(self.lookup.num_parallel_queries, 1) + # Ongoing queries to (sorted: oldest first): + # 156-12 + # Queued nodes to query (sorted by log_distance to info_hash): + # + nodes = [tc.NODES_LD_IH[144][18], + tc.NODES_LD_IH[145][19], + ] + self.lookup._on_response(*_gen_nodes_args( + tc.NODES_LD_IH[156][12], + nodes)) + eq_(self.lookup._get_announce_candidates(), [tc.NODES_LD_IH[144][14], + tc.NODES_LD_IH[145][15], + tc.NODES_LD_IH[151][10], + ]) + # This response triggers a new query (to 144-18) + eq_(self.lookup.num_parallel_queries, 1) + # Ongoing queries to (sorted: oldest first): + # 144-18 + # Queued nodes to query (sorted by log_distance to info_hash): + # 145-19 + peers = [tc.NO_ADDR] + ok_(not self.got_peers) + self.lookup._on_response(*_gen_peers_args( + tc.NODES_LD_IH[144][18], + peers)) + eq_(self.lookup._get_announce_candidates(), [tc.NODES_LD_IH[144][14], + tc.NODES_LD_IH[144][18], + tc.NODES_LD_IH[145][15], + ]) + ok_(self.got_peers) + self.got_peers = False + # This timeout triggers a new query (145-19) + eq_(self.lookup.num_parallel_queries, 0) + # Ongoing queries to (sorted: oldest first): + # 145-19 + # Queued nodes to query (sorted by log_distance to info_hash): + # + ok_(not self.lookup.is_done) + self.lookup._on_timeout(tc.NODES_LD_IH[145][19]) + # THE END + eq_(self.lookup.num_parallel_queries, 0) + ok_(self.lookup.is_done) + + def test_dont_query_myself(self): + logger.debug('test start') + self.lookup.start() + # Ongoing queries to (sorted: oldest first): + # 155-4, 157-3, + # Queued nodes to query (sorted by log_distance to info_hash): + # 158-1, 159-0 + # Notice 159-2 is kicked out from the queue + eq_(self.lookup.num_parallel_queries, 2) + nodes = [Node(tc.CLIENT_ADDR, self.lookup._my_id)] + self.lookup._on_response(*_gen_nodes_args( + tc.NODES_LD_IH[157][3], + nodes)) + eq_(self.lookup._get_announce_candidates(), + [tc.NODES_LD_IH[157][3], + ]) + # This response triggers a new query to 158-1 (ignoring myself) + eq_(self.lookup.num_parallel_queries, 2) + # Ongoing queries to (sorted: oldest first): + # 155-4, 158-1 + # Queued nodes to query (sorted by log_distance to info_hash): + # 159-0 + self.lookup._on_timeout(tc.NODES_LD_IH[155][4]) + # This timeout triggers a new query (to 159-0) + eq_(self.lookup.num_parallel_queries, 2) + self.lookup._on_timeout(tc.NODES_LD_IH[158][1]) + # No more nodes to send queries to + eq_(self.lookup.num_parallel_queries, 1) + ok_(not self.lookup.is_done) + self.lookup._on_timeout(tc.NODES_LD_IH[159][0]) + # No more nodes to send queries to + eq_(self.lookup.num_parallel_queries, 0) + ok_(self.lookup.is_done) + + +class TestLookupManager: + + def _on_got_peers(self, peers): + self.got_peers = peers + + + def setup(self): + self.got_peers = None + querier_ = querier.QuerierMock(tc.CLIENT_ID) + routing_m = RoutingManagerMock() + self.bootstrap_nodes = routing_m.get_closest_rnodes( + tc.INFO_HASH_ZERO) + self.lm = lookup_manager.LookupManager(tc.CLIENT_ID, + querier_, + routing_m, + 2) + self.lookup = self.lm.get_peers(tc.INFO_HASH, self._on_got_peers, + tc.BT_PORT) + + def test_all_nodes_timeout(self): + for node_ in self.bootstrap_nodes: + self.lookup._on_timeout(node_) + ok_(self.lookup.is_done) + + def test_peers(self): + self.lookup._on_response(*_gen_peers_args( + self.bootstrap_nodes[0], + [tc.NO_ADDR])) + for node_ in self.bootstrap_nodes[1:]: + self.lookup._on_timeout(node_) + ok_(self.lookup.is_done) + def teardown(self): + self.lm.stop() + +def _gen_nodes_args(node_, nodes): + out_msg = message.OutgoingGetPeersResponse( + node_.id, + tc.TOKEN, + nodes2=nodes).encode(tc.TID) + in_msg = message.IncomingMsg(out_msg) + in_msg.sanitize_response(message.GET_PEERS) + return in_msg, node_ + +def _gen_peers_args(node_, peers): + out_msg = message.OutgoingGetPeersResponse( + node_.id, + tc.TOKEN, + peers=peers).encode(tc.TID) + in_msg = message.IncomingMsg(out_msg) + in_msg.sanitize_response(message.GET_PEERS) + return in_msg, node_ + diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_message.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_message.py new file mode 100644 index 0000000..d30b5c1 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_message.py @@ -0,0 +1,446 @@ +# Copyright (C) 2009 Raul Jimenez +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +from nose.tools import * + +import node +import logging, logging_conf + +import test_const as tc +import message +from message import * + +logging_conf.testing_setup(__name__) +logger = logging.getLogger('dht') + + +class TestMsg: + + def setup(self): + pass + + def test_tools(self): + bin_strs = ['23', '\1\5', 'a\3'] + for bs in bin_strs: + i = bin_to_int(bs) + bs2 = int_to_bin(i) + logger.debug('bs: %s, bin_to_int(bs): %d, bs2: %s' % (bs, + i, bs2)) + assert bs == bs2 + + ips = ['127.0.0.1', '222.222.222.222', '1.2.3.4'] + ports = [12345, 99, 54321] + for addr in zip(ips, ports): + c_addr = compact_addr(addr) + addr2 = uncompact_addr(c_addr) + assert addr == addr2 + + c_peers = message._compact_peers(tc.PEERS) + peers = message._uncompact_peers(c_peers) + for p1, p2 in zip(tc.PEERS, peers): + assert p1[0] == p2[0] + assert p1[0] == p2[0] + + c_nodes = message._compact_nodes(tc.NODES) + nodes = message._uncompact_nodes(c_nodes) + for n1, n2 in zip(tc.NODES, nodes): + assert n1 == n2 + + bin_ipv6s = ['\x00' * 10 + '\xff\xff' + '\1\2\3\4', + '\x22' * 16, + ] + assert bin_to_ip(bin_ipv6s[0]) == '1.2.3.4' + assert_raises(AddrError, bin_to_ip, bin_ipv6s[1]) + + + PORT = 7777 + BIN_PORT = int_to_bin(PORT) + c_nodes2 = [tc.CLIENT_ID.bin_id + ip + BIN_PORT for ip in bin_ipv6s] + nodes2 = [node.Node(('1.2.3.4', PORT), tc.CLIENT_ID)] + logger.debug(message._uncompact_nodes2(c_nodes2)) + assert message._uncompact_nodes2(c_nodes2) == nodes2 + logger.warning( + "**IGNORE WARNING LOG** This exception was raised by a test") + + + def test_tools_error(self): + c_nodes = message._compact_nodes(tc.NODES) + # Compact nodes is one byte short + assert_raises(MsgError, message._uncompact_nodes, c_nodes[:-1]) + # IP size is weird + assert_raises(MsgError, bin_to_ip, '123') + # Port is 0 ( + eq_(message._uncompact_nodes(c_nodes), tc.NODES) + n = tc.NODES[0] + tc.NODES[0] = node.Node((n.addr[0], 0), n.id) + c_nodes = message._compact_nodes(tc.NODES) + eq_(message._uncompact_nodes(c_nodes), tc.NODES[1:]) + c_nodes2 = message._compact_nodes2(tc.NODES) + eq_(message._uncompact_nodes2(c_nodes2), tc.NODES[1:]) + tc.NODES[0] = n + + def test_matching_tid(self): + # It _only_ matches the first byte) + ok_(matching_tid('aaa', 'aaa')) + ok_(matching_tid('axa', 'a1a')) + ok_(matching_tid('aQWEREWTWETWTWETWETEWT', 'a')) + ok_(not matching_tid('a', 'b')) + ok_(not matching_tid('aZZ', 'bZZ')) + + def test_ping(self): + #client + outgoing_query = OutgoingPingQuery(tc.CLIENT_ID) + data = outgoing_query.encode(tc.TID) # query_manager would do it + #server + incoming_query = IncomingMsg(data) + assert incoming_query.type is QUERY + outgoing_response = OutgoingPingResponse(tc.SERVER_ID) + data = outgoing_response.encode(incoming_query.tid) + #client + incoming_response = IncomingMsg(data) + assert incoming_response.type is RESPONSE + incoming_response.sanitize_response(outgoing_query.query) + + def _test_ping_error(self): + outgoing_query = OutgoingPingQuery(tc.CLIENT_ID) + #outgoing_query.my_id = CLIENT_ID + #outgoing_query.tid = tc.TID + # TID and ARGS ID are None + assert_raises(MsgError, outgoing_query.encode) + logger.error( + "**IGNORE 2 ERROR LOGS** This exception was raised by a test") + + outgoing_query = OutgoingPingQuery() + outgoing_query.my_id = tc.CLIENT_ID + #outgoing_query.tid = tc.TID + assert_raises(MsgError, outgoing_query.encode) + logger.error( + "**IGNORE 2 ERROR LOGS** This exception was raised by a test") + + outgoing_query = OutgoingPingQuery() + #outgoing_query.my_id = tc.CLIENT_ID + outgoing_query.tid = tc.TID + assert_raises(MsgError, outgoing_query.encode) + logger.error( + "**IGNORE 2 ERROR LOGS** This exception was raised by a test") + + outgoing_query = OutgoingPingQuery() + assert_raises(MsgError, outgoing_query.__setattr__, 'my_id', '') + logger.error( + "**IGNORE 2 ERROR LOGS** This exception was raised by a test") + + outgoing_query = OutgoingPingQuery() + outgoing_query.my_id = tc.CLIENT_ID + outgoing_query.tid = 567 + data = outgoing_query.encode() + assert_raises(MsgError, decode, data) + logger.error( + "**IGNORE 2 ERROR LOGS** This exception was raised by a test") + + outgoing_query = OutgoingPingQuery() + outgoing_query.my_id = tc.CLIENT_ID + outgoing_query.tid = tc.TID + data = outgoing_query.encode() + data += 'this string ruins the bencoded msg' + assert_raises(MsgError, decode, data) + logger.error( + "**IGNORE 2 ERROR LOGS** This exception was raised by a test") + + + + + outgoing_response = OutgoingPingResponse(tc.TID, tc.SERVER_ID) + outgoing_response.tid = None + assert_raises(MsgError, outgoing_response.encode) + logger.error( + "**IGNORE ERROR LOGS** This exception was raised by a test") + + + def test_find_node(self): + #client + outgoing_query = OutgoingFindNodeQuery(tc.CLIENT_ID, tc.NODE_ID) + data = outgoing_query.encode(tc.TID) + #server + incoming_query = IncomingMsg(data) + assert incoming_query.type is QUERY + outgoing_response = OutgoingFindNodeResponse(tc.SERVER_ID, + tc.NODES) + data = outgoing_response.encode(incoming_query.tid) + #client + incoming_response = IncomingMsg(data) + eq_(incoming_response.type, RESPONSE) + incoming_response.sanitize_response(outgoing_query.query) + for n1, n2 in zip(tc.NODES, incoming_response.nodes2): + eq_(n1, n2) + + + def test_find_node_error(self): + assert_raises(MsgError, OutgoingFindNodeResponse, + tc.CLIENT_ID, nodes=tc.NODES) + assert_raises(MsgError, OutgoingFindNodeResponse, + tc.CLIENT_ID) + + + def test_get_peers_nodes(self): + #client + outgoing_query = OutgoingGetPeersQuery(tc.CLIENT_ID, tc.INFO_HASH) + data = outgoing_query.encode(tc.TID) + #server + incoming_query = IncomingMsg(data) + assert incoming_query.type is QUERY + outgoing_response = OutgoingGetPeersResponse(tc.SERVER_ID, + tc.TOKEN, + nodes2=tc.NODES) + data = outgoing_response.encode(incoming_query.tid) + #client + incoming_response = IncomingMsg(data) + assert incoming_response.type is RESPONSE + incoming_response.sanitize_response(outgoing_query.query) + for n1, n2 in zip(tc.NODES, incoming_response.nodes2): + assert n1 == n2 + + def test_get_peers_nodes_error(self): + assert_raises(MsgError, OutgoingGetPeersResponse, + tc.CLIENT_ID, tc.TOKEN) + + def test_get_peers_peers(self): + #client + outgoing_query = OutgoingGetPeersQuery(tc.CLIENT_ID, tc.INFO_HASH) + data = outgoing_query.encode(tc.TID) + #server + incoming_query = IncomingMsg(data) + assert incoming_query.type is QUERY + outgoing_response = OutgoingGetPeersResponse(tc.SERVER_ID, + tc.TOKEN, + peers=tc.PEERS) + data = outgoing_response.encode(incoming_query.tid) + #client + incoming_response = IncomingMsg(data) + assert incoming_response.type is RESPONSE + incoming_response.sanitize_response(outgoing_query.query) + for p1, p2 in zip(tc.PEERS, incoming_response.peers): + assert p1[0] == p2[0] + assert p1[1] == p2[1] + + def test_get_peers_peers_error(self): + assert 1 + + def test_announce_peer(self): + #client + outgoing_query = OutgoingAnnouncePeerQuery(tc.CLIENT_ID, + tc.INFO_HASH, + tc.BT_PORT, + tc.TOKEN) + outgoing_query.tid = tc.TID + data = outgoing_query.encode(tc.TID) + #server + incoming_query = IncomingMsg(data) + assert incoming_query.type is QUERY + outgoing_response = OutgoingAnnouncePeerResponse(tc.SERVER_ID) + data = outgoing_response.encode(incoming_query.tid) + #client + incoming_response = IncomingMsg(data) + assert incoming_response.type is RESPONSE + incoming_response.sanitize_response(outgoing_query.query) + + def test_announce_peer_error(self): + assert 1 + + def _test_error(self): + outgoing_error_msg = OutgoingErrorMsg(tc.TID, GENERIC_E) + data = outgoing_error_msg.encode() + tid, msg_type, msg_dict = decode(data) + incoming_error_msg = IncomingErrorMsg(msg_dict) + logger.debug(incoming_error_msg.error) + assert incoming_error_msg.error == GENERIC_E + + +def value_is_string(msg_d, k, valid_values=None): + v = msg_d[k] + ok_(isinstance(v, str)) + + + +class TestIncomingMsg: + + def setup(self): + b_ping = OutgoingPingQuery(tc.CLIENT_ID).encode(tc.TID) + self.msg_d = IncomingMsg(b_ping)._msg_dict + + def test_bad_bencode(self): + assert_raises(MsgError, IncomingMsg, 'z') + assert_raises(MsgError, IncomingMsg, '1:aa') + assert_raises(MsgError, IncomingMsg, 'd') + + def test_not_a_dict(self): + msgs = ([], 'a', 1) + for msg in msgs: + assert_raises(MsgError, IncomingMsg, bencode.encode(msg)) + + def test_tid_error(self): + # no TID + del self.msg_d[TID] + assert_raises(MsgError, IncomingMsg, bencode.encode(self.msg_d)) + # invalid TID + self.msg_d[TID] = 1 + assert_raises(MsgError, IncomingMsg, bencode.encode(self.msg_d)) + self.msg_d[TID] = [] + assert_raises(MsgError, IncomingMsg, bencode.encode(self.msg_d)) + self.msg_d[TID] = {} + assert_raises(MsgError, IncomingMsg, bencode.encode(self.msg_d)) + + def test_type_error(self): + # no TYPE + del self.msg_d[TYPE] + assert_raises(MsgError, IncomingMsg, bencode.encode(self.msg_d)) + # invalid TYPE + self.msg_d[TYPE] = 1 + assert_raises(MsgError, IncomingMsg, bencode.encode(self.msg_d)) + self.msg_d[TYPE] = [] + assert_raises(MsgError, IncomingMsg, bencode.encode(self.msg_d)) + self.msg_d[TYPE] = {} + assert_raises(MsgError, IncomingMsg, bencode.encode(self.msg_d)) + # unknown TYPE + self.msg_d[TYPE] = 'z' + assert_raises(MsgError, IncomingMsg, bencode.encode(self.msg_d)) + + def test_version_not_present(self): + del self.msg_d[VERSION] + IncomingMsg(bencode.encode(self.msg_d)) + + def test_unknown_error(self): + error_code = (999, "some weird error string") + b_err = OutgoingErrorMsg(error_code).encode(tc.TID) + + logger.info( + "TEST LOGGING ** IGNORE EXPECTED INFO ** Unknown error: %r", + error_code) + _ = IncomingMsg(b_err) + + + +b_ping_q = OutgoingPingQuery(tc.CLIENT_ID).encode(tc.TID) +b_fn_q = OutgoingFindNodeQuery(tc.CLIENT_ID, tc.NODE_ID).encode(tc.TID) +b_gp_q = OutgoingGetPeersQuery(tc.CLIENT_ID, tc.INFO_HASH).encode(tc.TID) +b_ap_q = OutgoingAnnouncePeerQuery(tc.CLIENT_ID, tc.INFO_HASH, + tc.BT_PORT,tc.TOKEN).encode(tc.TID) + +class TestSanitizeQueryError: + + def setup(self): + self.ping_d = IncomingMsg(b_ping_q)._msg_dict + self.fn_d = IncomingMsg(b_fn_q)._msg_dict + self.gp_d = IncomingMsg(b_gp_q)._msg_dict + self.ap_d = IncomingMsg(b_ap_q)._msg_dict + + def test_weird_msg(self): + self.ping_d[ARGS] = [] + assert_raises(MsgError, IncomingMsg, bencode.encode(self.ping_d)) + self.ping_d[ARGS] = 1 + assert_raises(MsgError, IncomingMsg, bencode.encode(self.ping_d)) + self.ping_d[ARGS] = 'ZZZZ' + assert_raises(MsgError, IncomingMsg, bencode.encode(self.ping_d)) + + + + def test_sender_id(self): + # no sender_id + del self.ping_d[ARGS][ID] + assert_raises(MsgError, IncomingMsg, bencode.encode(self.ping_d)) + # bad ID + self.ping_d[ARGS][ID] = 'a' + assert_raises(MsgError, IncomingMsg, bencode.encode(self.ping_d)) + self.ping_d[ARGS][ID] = 1 + assert_raises(MsgError, IncomingMsg, bencode.encode(self.ping_d)) + self.ping_d[ARGS][ID] = [] + assert_raises(MsgError, IncomingMsg, bencode.encode(self.ping_d)) + self.ping_d[ARGS][ID] = {} + assert_raises(MsgError, IncomingMsg, bencode.encode(self.ping_d)) + + def test_query(self): + # no QUERY + del self.ping_d[QUERY] + assert_raises(MsgError, IncomingMsg, bencode.encode(self.ping_d)) + # bad QUERY + self.ping_d[QUERY] = 1 + assert_raises(MsgError, IncomingMsg, bencode.encode(self.ping_d)) + self.ping_d[QUERY] = [] + assert_raises(MsgError, IncomingMsg, bencode.encode(self.ping_d)) + self.ping_d[QUERY] = {} + assert_raises(MsgError, IncomingMsg, bencode.encode(self.ping_d)) + # unknown QUERY is not an error at this point + # responder will process it and send an errror msg if necesary + self.ping_d[QUERY] = 'a' + IncomingMsg(bencode.encode(self.ping_d)) + + def test_announce(self): + # Port must be integer + self.ap_d[ARGS][PORT] = 'a' + assert_raises(MsgError, IncomingMsg, bencode.encode(self.ap_d)) + + +b_ping_r = OutgoingPingResponse(tc.CLIENT_ID).encode(tc.TID) +b_fn2_r = OutgoingFindNodeResponse(tc.CLIENT_ID, nodes2=tc.NODES).encode(tc.TID) +b_gp_r = OutgoingGetPeersResponse(tc.CLIENT_ID, token=tc.TOKEN, + peers=tc.PEERS).encode(tc.TID) +b_ap_r = OutgoingAnnouncePeerResponse(tc.CLIENT_ID).encode(tc.TID) + +class TestSanitizeResponseError: + + def setup(self): + self.ping_r = IncomingMsg(b_ping_r) + self.fn2_r = IncomingMsg(b_fn2_r) + self.gp_r = IncomingMsg(b_gp_r) + self.ap_r = IncomingMsg(b_ap_r) + + def test_nodes_not_implemented(self): + assert_raises(MsgError, OutgoingFindNodeResponse, tc.CLIENT_ID, + nodes=tc.NODES) + def test_sanitize(self): + self.ping_r.sanitize_response(PING) + + del self.fn2_r._msg_dict[RESPONSE][NODES2] + # No NODES and no NODES2 + assert_raises(MsgError, self.fn2_r.sanitize_response, FIND_NODE) + self.fn2_r._msg_dict[RESPONSE][NODES] = \ + message._compact_nodes(tc.NODES) + # Just NODES + self.fn2_r.sanitize_response(FIND_NODE) + self.fn2_r._msg_dict[RESPONSE][NODES2] = \ + message._compact_nodes2(tc.NODES) + # Both NODES and NODES2 + self.fn2_r.sanitize_response(FIND_NODE) + + # Both NODES and PEERS in response + self.gp_r._msg_dict[RESPONSE][NODES] = \ + message._compact_nodes(tc.NODES) + self.gp_r.sanitize_response(GET_PEERS) + # No NODES and no PEERS + del self.gp_r._msg_dict[RESPONSE][NODES] + del self.gp_r._msg_dict[RESPONSE][VALUES] + assert_raises(MsgError, self.gp_r.sanitize_response, GET_PEERS) + + +class TestSanitizeErrorError: + + def test(self): + msg_out = OutgoingErrorMsg(1).encode(tc.TID) + assert_raises(MsgError, IncomingMsg, msg_out) + # Unknown error doesn't raise MsgError + msg_out = OutgoingErrorMsg((1,1)).encode(tc.TID) + _ = IncomingMsg(msg_out) + + + + +class TestPrinting: + + def test_printing(self): + out_msg = OutgoingPingQuery(tc.CLIENT_ID) + in_msg = IncomingMsg(out_msg.encode(tc.TID)) + str(out_msg) + repr(out_msg) + repr(in_msg) + + diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_minitwisted.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_minitwisted.py new file mode 100644 index 0000000..92cc8a2 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_minitwisted.py @@ -0,0 +1,320 @@ +# Copyright (C) 2009 Raul Jimenez +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +from __future__ import with_statement +import threading +import time + +import logging, logging_conf + +from nose.tools import eq_, ok_, assert_raises +import test_const as tc + +logging_conf.testing_setup(__name__) +logger = logging.getLogger('dht') + + +from minitwisted import Task, TaskManager, \ + ThreadedReactor, ThreadedReactorMock, \ + ThreadedReactorSocketError + + +ADDRS= (tc.CLIENT_ADDR, tc.SERVER_ADDR) +DATA = 'testing...' + + +class TestTaskManager: + + def callback_f(self, callback_id): + self.callback_order.append(callback_id) + + def setup(self): + # Order in which callbacks have been fired + self.callback_order = [] + self.task_m = TaskManager() + + def test_simple(self): + for i in xrange(5): + self.task_m.add(Task(.01, self.callback_f, i)) + while True: + task = self.task_m.consume_task() + if task is None: + break + task.fire_callback() + logger.debug('%s' % self.callback_order) + assert self.callback_order == [] + time.sleep(.01) + while True: + task = self.task_m.consume_task() + if task is None: + break + task.fire_callbacks() + assert self.callback_order == range(5) + + def test_cancel(self): + for i in xrange(5): + self.task_m.add(Task(.1, self.callback_f, i)) + c_task = Task(.1, self.callback_f, 5) + self.task_m.add(c_task) + for i in xrange(6,10): + self.task_m.add(Task(.1, self.callback_f, i)) + while True: + task = self.task_m.consume_task() + if task is None: + break + task.fire_callback() + logger.debug('%s' % self.callback_order) + assert self.callback_order == [] + ok_(not c_task.cancelled) + c_task.cancel() + ok_(c_task.cancelled) + + time.sleep(.1) + while True: + task = self.task_m.consume_task() + if task is None: + break + task.fire_callbacks() + logger.debug('%s' % self.callback_order) + assert self.callback_order == [0,1,2,3,4, 6,7,8,9] + # task 5 was cancelled + + def test_different_delay(self): +# NOTICE: this test might fail if your configuration +# (interpreter/processor) is too slow + + task_delays = (1, 1, 1, .5, 1, 1, 2, 1, 1, 1, + 1, 1.5, 1, 1, 1, 1, .3) + + expected_list = ([], + ['a', 16, 3, 'b'], #9 is cancelled + ['a', 0, 1, 2, 4, 5, 7, 8, 10, 12, 13, 15, 'c', 'b'], + ['a', 11, 'c', 'b'], + ['a', 6, 'c', 'b'], + ) + tasks = [Task(delay, self.callback_f, i) \ + for i, delay in enumerate(task_delays)] + for task in tasks: + self.task_m.add(task) + + for i, expected in enumerate(expected_list): + while True: + task = self.task_m.consume_task() + if task is None: + break + task.fire_callbacks() + logger.debug('#: %d, result: %s, expected: %s' % (i, + self.callback_order, expected)) + assert self.callback_order == expected + self.callback_order = [] + self.task_m.add(Task(0, self.callback_f, 'a')) + self.task_m.add(Task(.5, self.callback_f, 'b')) + self.task_m.add(Task(1, self.callback_f, 'c')) + time.sleep(.5) + tasks[9].cancel() # too late (already fired) + tasks[14].cancel() # should be cancelled + + def _callback1(self, arg1, arg2): + if arg1 == 1 and arg2 == 2: + self.callback_order.append(1) + def _callback2(self, arg1, arg2): + if arg1 == 1 and arg2 == 2: + self.callback_order.append(2) + + def test_callback_list(self): + self.task_m.add(Task(tc.TASK_INTERVAL/2, + [self._callback1, self._callback2], + 1, 2)) + ok_(self.task_m.consume_task() is None) + eq_(self.callback_order, []) + time.sleep(tc.TASK_INTERVAL) + self.task_m.consume_task().fire_callbacks() + eq_(self.callback_order, [1,2]) + +class TestMinitwisted: + + def on_datagram_received(self, data, addr): + with self.lock: + self.datagrams_received.append((data, addr)) + + def callback_f(self, callback_id): + with self.lock: + self.callback_order.append(callback_id) + + def setup(self): + self.lock = threading.Lock() + self.datagrams_received = [] + self.callback_order = [] + self.client_r = ThreadedReactor(task_interval=tc.TASK_INTERVAL) + self.server_r = ThreadedReactor(task_interval=tc.TASK_INTERVAL) + self.client_r.listen_udp(tc.CLIENT_ADDR[1], self.on_datagram_received) + self.server_r.listen_udp(tc.SERVER_ADDR[1], self.on_datagram_received) + self.client_r.start() + self.server_r.start() + + def test_listen_upd(self): + r = ThreadedReactor() + r.start() + logger.warning(''.join( + ('TESTING LOGS ** IGNORE EXPECTED WARNING ** ', + '(udp_listen has not been called)'))) + self.client_r.sendto(DATA, tc.SERVER_ADDR) + while 1: #waiting for data + with self.lock: + if self.datagrams_received: + break + time.sleep(tc.TASK_INTERVAL) + with self.lock: + first_datagram = self.datagrams_received.pop(0) + logger.debug('first_datagram: %s, %s' % ( + first_datagram, + (DATA, tc.CLIENT_ADDR))) + assert first_datagram, (DATA, tc.CLIENT_ADDR) + r.stop() + + def test_network_callback(self): + self.client_r.sendto(DATA, tc.SERVER_ADDR) + time.sleep(tc.TASK_INTERVAL) + with self.lock: + first_datagram = self.datagrams_received.pop(0) + logger.debug('first_datagram: %s, %s' % ( + first_datagram, + (DATA, tc.CLIENT_ADDR))) + assert first_datagram, (DATA, tc.CLIENT_ADDR) + + def test_block_flood(self): + from floodbarrier import MAX_PACKETS_PER_PERIOD as FLOOD_LIMIT + for _ in xrange(FLOOD_LIMIT): + self.client_r.sendto(DATA, tc.SERVER_ADDR) + for _ in xrange(10): + self.client_r.sendto(DATA, tc.SERVER_ADDR) + logger.warning( + "TESTING LOGS ** IGNORE EXPECTED WARNING **") + time.sleep(tc.TASK_INTERVAL) + with self.lock: + logger.debug('datagram processed: %d/%d' % ( + len(self.datagrams_received), + FLOOD_LIMIT)) + assert len(self.datagrams_received) <= FLOOD_LIMIT + + def test_call_later(self): + self.client_r.call_later(.13, self.callback_f, 1) + self.client_r.call_later(.11, self.callback_f, 2) + self.client_r.call_later(.01, self.callback_f, 3) + task4 = self.client_r.call_later(.01, self.callback_f, 4) + task4.cancel() + time.sleep(.03) + with self.lock: + logger.debug('callback_order: %s' % self.callback_order) + assert self.callback_order == [3] + self.callback_order = [] + self.client_r.call_now(self.callback_f, 5) + time.sleep(.03) + with self.lock: + logger.debug('callback_order: %s' % self.callback_order) + assert self.callback_order == [5] + self.callback_order = [] + task6 = self.client_r.call_later(.03, self.callback_f, 6) + task6.cancel() + time.sleep(.1) + with self.lock: + logger.debug('callback_order: %s' % self.callback_order) + assert self.callback_order == [2, 1] + + def test_network_and_delayed(self): + self.client_r.call_later(.2, self.callback_f, 0) + self.client_r.call_now(self.callback_f, 1) + task2 = self.client_r.call_later(.2, self.callback_f, 2) + with self.lock: + eq_(self.callback_order, []) + time.sleep(.1) + + with self.lock: + logger.debug('callback_order: %s' % self.callback_order) + assert self.callback_order == [1] + self.callback_order = [] + assert not self.datagrams_received + self.server_r.sendto(DATA, tc.CLIENT_ADDR) + time.sleep(.02) # wait for network interruption + with self.lock: + logger.debug('callback_order: %s' % self.callback_order) + assert self.callback_order == [] + logger.debug('callback_order: %s' % self.callback_order) + assert self.datagrams_received.pop(0) == (DATA, tc.SERVER_ADDR) + task2.cancel() #inside critical region?? + time.sleep(.1) # wait for task 0 (task 2 should be cancelled) + with self.lock: + assert self.callback_order == [0] + assert not self.datagrams_received + + def test_sendto_socket_error(self): + logger.critical('TESTING: IGNORE CRITICAL MESSAGE') + self.client_r.sendto('z', (tc.NO_ADDR[0], 0)) + + def teardown(self): + self.client_r.stop() + self.server_r.stop() + +class TestSocketErrors: + + def _callback(self, *args, **kwargs): + self.callback_fired = True + + def setup(self): + self.callback_fired = False + self.r = ThreadedReactorSocketError() + self.r.listen_udp(tc.CLIENT_ADDR[1], lambda x,y:None) + + def test_sendto(self): + logger.critical('TESTING: IGNORE CRITICAL MESSAGE') + self.r.sendto('z', tc.NO_ADDR) + + def test_recvfrom(self): + self.r.start() + r2 = ThreadedReactor() + r2.listen_udp(tc.SERVER_ADDR[1], lambda x,y:None) + logger.critical('TESTING: IGNORE CRITICAL MESSAGE') + r2.sendto('z', tc.CLIENT_ADDR) + # self.r will call recvfrom (which raises socket.error) + time.sleep(tc.TASK_INTERVAL) + ok_(not self.callback_fired) + self.r.stop() + + def test_sendto_too_large_data_string(self): + logger.critical('TESTING: IGNORE CRITICAL MESSAGE') + self.r.sendto('z'*12345, tc.NO_ADDR) + + + + +class TestMockThreadedReactor: + + def setup(self): + pass + + def _callback(self, *args): + pass + + def test_mock_threaded_reactor(self): + ''' + Just making sure that the interface is the same + + ''' + r = ThreadedReactor(task_interval=.1) + rm = ThreadedReactorMock(task_interval=.1) + + r.listen_udp(tc.CLIENT_ADDR[1], lambda x,y:None) + rm.listen_udp(tc.CLIENT_ADDR[1], lambda x,y:None) + + r.start() + rm.start() + + r.sendto(DATA, tc.CLIENT_ADDR) + rm.sendto(DATA, tc.CLIENT_ADDR) + + r.call_later(.1, self._callback) + rm.call_later(.1, self._callback) +# time.sleep(.002) + r.stop() + rm.stop() diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_node.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_node.py new file mode 100644 index 0000000..525088a --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_node.py @@ -0,0 +1,130 @@ +# Copyright (C) 2009 Raul Jimenez +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +from nose.tools import ok_, eq_, raises, assert_raises +import test_const as tc + +import logging, logging_conf + +import utils +from identifier import Id, ID_SIZE_BYTES +from node import Node, RoutingNode +from node import LAST_RTT_W + +logging_conf.testing_setup(__name__) +logger = logging.getLogger('dht') + + + +bin_id1 = '1' * ID_SIZE_BYTES +bin_id2 = '2' * ID_SIZE_BYTES +id1 = Id(bin_id1) +id2 = Id(bin_id2) +addr1 = ('127.0.0.1', 1111) +addr2 = ('127.0.0.1', 2222) + + +class TestNode: + + def setup(self): + pass + + def test_node(self): + node1 = Node(addr1, id1) + node2 = Node(addr2, id2) + node1b = Node(addr1, None) + node1ip = Node(('127.0.0.2', 1111), id1) + node1port = Node(addr2, id1) + node1id = Node(addr1, id2) + + assert str(node1) == '' % (addr1, id1) + # + + assert node1.id == id1 + assert node1.id != id2 + assert node1.addr == addr1 + assert node1.addr != addr2 + assert node1 == node1 + + assert node1 != node1b + node1b.id = id1 + assert node1 == node1b + + assert node1 != node2 + assert node1 != node1ip + assert node1 != node1port + assert node1 != node1id + + def test_compact_addr(self): + eq_(tc.CLIENT_NODE.compact_addr, + utils.compact_addr(tc.CLIENT_ADDR)) + + def test_log_distance(self): + eq_(tc.CLIENT_NODE.log_distance(tc.SERVER_NODE), + tc.CLIENT_ID.log_distance(tc.SERVER_ID)) + + def test_compact(self): + eq_(tc.CLIENT_NODE.compact(), + tc.CLIENT_ID.bin_id + utils.compact_addr(tc.CLIENT_ADDR)) + + def test_get_rnode(self): + eq_(tc.CLIENT_NODE.get_rnode(), + RoutingNode(tc.CLIENT_NODE)) + + @raises(AttributeError) + def test_node_exceptions(self): + Node(addr1, id1).id = id2 + + + +class TestRoutingNode: + + def setup(self): + self.rnode1 = RoutingNode(Node(addr1, id1)) + self.rnode2 = RoutingNode(Node(addr2, id2)) + + def test_rnode(self): + RTT1 = 1 + RTT2 = 2 + assert self.rnode1.timeouts_in_a_row() == 0 + self.rnode1.on_timeout() + self.rnode1.on_timeout() + self.rnode1.on_timeout() + assert self.rnode1.timeouts_in_a_row() == 3 + assert self.rnode1.timeouts_in_a_row(False) == 3 + self.rnode1.on_query_received() + assert self.rnode1.timeouts_in_a_row() == 0 + eq_(self.rnode1.timeouts_in_a_row(False), 3) + self.rnode1.on_response_received(1) + assert self.rnode1.timeouts_in_a_row() == 0 + assert self.rnode1.timeouts_in_a_row(False) == 0 + assert self.rnode1._num_queries == 1 + assert self.rnode1._num_responses == 1 + assert self.rnode1._num_timeouts == 3 + self.rnode1.on_response_received(RTT1) + self.rnode1.on_response_received(RTT1) + self.rnode1.on_response_received(RTT1) + self.rnode1.on_response_received(RTT1) + self.rnode1.on_response_received(RTT1) + self.rnode1.on_response_received(RTT1) + self.rnode1.on_response_received(RTT1) + self.rnode1.on_response_received(RTT2) + eq_(self.rnode1._rtt_avg, + RTT1 * (1 - LAST_RTT_W) + RTT2 * LAST_RTT_W) + self.rnode1.on_timeout() + self.rnode1.on_timeout() + + def _test_rank(self): + eq_(self.rnode1.rank(), 0) + self.rnode1.on_query_received() + eq_(self.rnode1.rank(), 0) + self.rnode1.on_response_received() + eq_(self.rnode1.rank(), 1) + + def test_repr(self): + _ = repr(RoutingNode(tc.CLIENT_NODE)) + + def test_get_rnode(self): + eq_(self.rnode1.get_rnode(), + self.rnode1) diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_querier.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_querier.py new file mode 100644 index 0000000..f5e83d5 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_querier.py @@ -0,0 +1,415 @@ +# Copyright (C) 2009 Raul Jimenez +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +from nose.tools import ok_, eq_ + +import sys +import time +import logging, logging_conf + +import node +import identifier +import message +import minitwisted +import rpc_manager +import test_const as tc + +import querier + +logging_conf.testing_setup(__name__) +logger = logging.getLogger('dht') + + +RUN_CPU_INTENSIVE_TESTS = False +RUN_NETWORK_TESTS = False # Requires a running external DHT node + +class TestQuery: + + def setup(self): + self.ping_msg = message.OutgoingPingQuery(tc.CLIENT_ID) + ping_r_out = message.OutgoingPingResponse(tc.SERVER_ID) + self.ping_r_in = message.IncomingMsg(ping_r_out.encode(tc.TID)) + fn_r_out = message.OutgoingFindNodeResponse(tc.SERVER_ID, + nodes2=tc.NODES) + self.fn_r_in = message.IncomingMsg(fn_r_out.encode(tc.TID)) + + self.got_response = False + self.got_error = False + self.got_timeout = False + + self.got_routing_response = False + self.got_routing_error = False + self.got_routing_timeout = False + self.got_routing_nodes_found = False + + self.query = querier.Query(tc.TID, self.ping_msg.query, tc.SERVER_NODE, + self.on_response, + self.on_error, + self.on_timeout, + self.on_routing_response, + self.on_routing_error, + self.on_routing_timeout, + self.on_routing_nodes_found) + self.query.timeout_task = minitwisted.Task(1, self.on_timeout, + tc.SERVER_NODE) + + def on_response(self, response_msg, addr): + self.got_response = True + + def on_error(self, error_msg, addr): + self.got_error = True + + def on_timeout(self, addr): + self.got_timeout = True + + def on_routing_response(self, node_): + self.got_routing_response = True + + def on_routing_error(self, node_): + self.got_routing_error = True + + def on_routing_timeout(self, node_): + self.got_routing_timeout = True + + def on_routing_nodes_found(self, node_): + self.got_routing_nodes_found = True + + + def test_fire_callback_on_response(self): + # the server creates the response + pong_msg = message.OutgoingPingResponse(tc.SERVER_ID) + pong_data = pong_msg.encode(tc.TID) + # rpc_m decodes the response received + pong_msg = message.IncomingMsg(pong_data) + # querier notifies of the message (callback) + self.query.on_response_received(pong_msg) + ok_(self.got_response) + ok_(not self.got_error) + ok_(not self.got_timeout) + + def test_fire_callback_on_error(self): + # the server creates the response + error_msg = message.OutgoingErrorMsg(message.GENERIC_E) + error_data = error_msg.encode(tc.TID) + # rpc_m decodes the response received + error_msg = message.IncomingMsg(error_data) + # querier notifies of the message (callback) + self.query.on_error_received(error_msg) + assert not self.got_response and self.got_error + + def test_on_timeout(self): + ok_(not self.got_timeout) + ok_(not self.got_routing_timeout) + self.query.on_timeout() + ok_(self.got_timeout) + ok_(self.got_routing_timeout) + + def test_fire_callback_on_timeout(self): + self.query.timeout_task.fire_callbacks() + self.query.timeout_task.cancel() + assert not self.got_response and not self.got_error \ + and self.got_timeout + + def test_fire_callback_on_late_response(self): + self.query.timeout_task.fire_callbacks() + self.query.timeout_task.cancel() + # the server creates the response + pong_msg = message.OutgoingPingResponse(tc.SERVER_ID) + pong_data = pong_msg.encode(tc.TID) + # rpc_m decodes the response received + pong_msg = message.IncomingMsg(pong_data) + # querier notifies of the message (but it's too late) + self.query.on_response_received(pong_msg) + logger.warning( + "**IGNORE WARNING LOG**") + assert not self.got_response and not self.got_error \ + and self.got_timeout + + def test_invalid_response_received(self): + # Response received is invalid + self.ping_r_in._msg_dict[message.RESPONSE] = 'zz' + ok_(not self.got_response) + logger.warning( + "**IGNORE WARNING LOG**") + self.query.on_response_received(self.ping_r_in) + ok_(not self.got_response) + + def test_response_contains_nodes(self): + # Trick query to accept find node response + self.query.query = message.FIND_NODE + ok_(not self.got_response) + ok_(not self.got_routing_response) + ok_(not self.got_routing_nodes_found) + self.query.on_response_received(self.fn_r_in) + ok_(self.got_response) + ok_(self.got_routing_response) + ok_(self.got_routing_nodes_found) + + def test_different_node_id(self): + # We are expecting response from SERVER_NODE + # Here we test if the response contains an ID + # different to SERVER_ID + self.query.node = node.Node(tc.SERVER_ADDR, + tc.CLIENT_ID) + ok_(not self.got_response) + ok_(not self.got_routing_response) + ok_(not self.got_routing_nodes_found) + self.query.on_response_received(self.fn_r_in) + ok_(not self.got_response) + ok_(not self.got_routing_response) + ok_(not self.got_routing_nodes_found) + + def teardown(self): + pass + +class TestQuerier: + + def setup(self): + if RUN_NETWORK_TESTS: + time.sleep(1) # Reduce test interdependence + self.got_response = False + self.got_timeout = False + self.got_error = False + self.found_nodes = False + + self.got_routing_response = False + self.got_routing_error = False + self.got_routing_timeout = False + self.got_routing_nodes_found = False + + self.querier_mock = querier.QuerierMock(tc.CLIENT_ID) + + self.r = minitwisted.ThreadedReactor(task_interval=.01) + self.rpc_m = rpc_manager.RPCManager(self.r, + tc.CLIENT_ADDR[1]) + self.querier = querier.Querier(self.rpc_m, + tc.CLIENT_NODE) + self.querier_routing = querier.Querier(self.rpc_m, + tc.CLIENT_NODE) + self.querier_routing.set_on_response_received_callback( + self.on_routing_response) + self.querier_routing.set_on_error_received_callback( + self.on_routing_error) + self.querier_routing.set_on_timeout_callback( + self.on_routing_timeout) + self.querier_routing.set_on_nodes_found_callback( + self.on_routing_nodes_found) + + self.r.start() + + + + def on_response(self, response_msg, node_): + self.got_response = True + + def on_timeout(self, node_): + self.got_timeout = True + + def on_error(self, error_msg, node_): + self.got_error = True + + def on_routing_response(self, node_): + self.got_routing_response = True + + def on_routing_error(self, node_): + self.got_routing_error = True + + def on_routing_timeout(self, node_): + self.got_routing_timeout = True + + def on_routing_nodes_found(self, node_): + self.got_routing_nodes_found = True + + + def test_generate_tids(self): + num_tids = 1000 + if RUN_CPU_INTENSIVE_TESTS: + num_tids = pow(2, 16) + 2 #CPU intensive + for i in xrange(num_tids): + eq_(self.querier._next_tid(), + chr(i%256)+chr((i/256)%256)) + + + + def send_query_and_get_response(self, querier_, later_delay=0): + ping_msg = message.OutgoingPingQuery(tc.CLIENT_ID) + msg = message.OutgoingFindNodeQuery(tc.CLIENT_ID, + tc.CLIENT_ID) + if later_delay: + task = querier_.send_query_later(later_delay, + msg, + tc.EXTERNAL_NODE, + self.on_response, + self.on_timeout, + self.on_error, + tc.TIMEOUT_DELAY) + # This second query is just to have two elements + # in the querier_.pending[tc.EXTERNAL_ADDR] list + task = querier_.send_query_later(later_delay, + msg, + tc.EXTERNAL_NODE, + self.on_response, + self.on_timeout, + self.on_error, + tc.TIMEOUT_DELAY) + else: + node_ = (querier_ == self.querier_mock) and tc.SERVER_NODE + query = querier_.send_query(ping_msg, node_ or tc.EXTERNAL_NODE, + self.on_response, + self.on_timeout, self.on_error, + timeout_delay=tc.TIMEOUT_DELAY) + # querier creates TID + msg_tid = '\0\0' + if querier_ is self.querier_mock: + # server gets query + # the server creates the response + pong_msg = message.OutgoingPingResponse(tc.SERVER_ID) + pong_msg_data = pong_msg.encode(msg_tid) + # the client gets the response + # rpc_m decodes msg and calls callback + pong_msg = message.IncomingMsg(pong_msg_data) + querier_.on_response_received(pong_msg, tc.SERVER_ADDR) + if later_delay: + ok_(not self.got_response) + ok_(not self.got_timeout) + time.sleep(later_delay*2) + time.sleep(tc.TIMEOUT_DELAY+.1) + ### It crashed (timeout_task.cancelled??????) + + + #TODO2: move the 'real' tests to integration + + ############################################### + ### A DHT node must be running on peer_addr ### + ############################################### + ok_(self.got_response) + ok_(not self.got_timeout) + ############################################### + ############################################### + + def send_query_and_get_error(self, querier): + + + ping_msg = message.OutgoingPingQuery() + query = querier.send_query(ping_msg, tc.EXTERNAL_NODE, + self.on_response, + self.on_timeout, self.on_error, + timeout_delay=tc.TIMEOUT_DELAY) + if querier is self.querier_mock: + # the server creates the response + error_msg = message.OutgoingErrorMsg(ping_msg.tid, + message.GENERIC_E) + error_data = error_msg.encode() + # rpc_m decodes the response received + _, _, error_msg_dict = message.decode(error_data) + # rpc_m notifies of the message (callback) + querier.on_error_received(error_msg_dict, tc.EXTERNAL_NODE) + time.sleep(tc.TIMEOUT_DELAY + .1) + + ### It crashed (timeout_task.cancelled??????) + + + #TODO2: move the 'real' tests to integration + + ############################################### + ### A DHT node must be running on peer_addr ### + ############################################### + ########## assert self.got_response and not self.got_timeout + ############################################### + ############################################### + + + + def send_query_and_get_timeout(self, querier): + ping_msg = message.OutgoingPingQuery(tc.CLIENT_ID) + query = querier.send_query(ping_msg, tc.DEAD_NODE, + self.on_response, + self.on_timeout, self.on_error, + timeout_delay=tc.TIMEOUT_DELAY) + if querier is self.querier_mock: + query.timeout_task.fire_callbacks() + time.sleep(tc.TIMEOUT_DELAY + .1) + assert not self.got_response and self.got_timeout + + def test_send_query_mock(self): + self.send_query_and_get_response(self.querier_mock) + ok_(not self.got_routing_response) + ok_(not self.got_routing_nodes_found) + ok_(not self.got_routing_timeout) + + def test_send_query(self): + if RUN_NETWORK_TESTS: + self.send_query_and_get_response(self.querier) + ok_(not self.got_routing_response) + ok_(not self.got_routing_nodes_found) + ok_(not self.got_routing_timeout) + + def test_send_query_routing(self): + if RUN_NETWORK_TESTS: + self.send_query_and_get_response(self.querier_routing) + ok_(self.got_routing_response) + ok_(not self.got_routing_nodes_found) + ok_(not self.got_routing_timeout) + + def test_send_query_timeout_mock(self): + self.send_query_and_get_timeout(self.querier_mock) + ok_(not self.got_routing_response) + ok_(not self.got_routing_nodes_found) + ok_(not self.got_routing_timeout) + + def test_send_query_timeout(self): + self.send_query_and_get_timeout(self.querier) + ok_(not self.got_routing_response) + ok_(not self.got_routing_nodes_found) + ok_(not self.got_routing_timeout) + + def test_send_query_timeout_routing(self): + self.send_query_and_get_timeout(self.querier_routing) + ok_(not self.got_routing_response) + ok_(not self.got_routing_nodes_found) + ok_(self.got_routing_timeout) + + def test_send_query_later(self): + if RUN_NETWORK_TESTS: + self.send_query_and_get_response(self.querier_routing, .001) + ok_(self.got_routing_response) + ok_(self.got_routing_nodes_found) + ok_(not self.got_routing_timeout) + + def test_unsolicited_response(self): + # We have a pending response from NO_ADDR (TID \0\0) + # but we get a response with different TID + + # client + ping = message.OutgoingPingQuery(tc.CLIENT_ID) + self.querier.send_query(ping, + tc.SERVER_NODE, + self.on_response, + self.on_error, + self.on_timeout, + tc.TIMEOUT_DELAY) + ok_(not self.got_response) + ok_(not self.got_routing_response) + ok_(not self.got_routing_nodes_found) + # server + pong = message.OutgoingPingResponse(tc.SERVER_ID) + pong_in = message.IncomingMsg(pong.encode(tc.TID)) + # client + self.querier.on_response_received(pong_in, + tc.SERVER_ADDR) + ok_(not self.got_response) + ok_(not self.got_routing_response) + ok_(not self.got_routing_nodes_found) + + def test_error(self): + msg = message.OutgoingErrorMsg(message.GENERIC_E) + self.querier.on_error_received(msg, tc.SERVER_ADDR) + + + def teardown(self): + self.querier_mock.stop() + self.querier.stop() + self.querier_routing.stop() + + diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_responder.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_responder.py new file mode 100644 index 0000000..fa7df0e --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_responder.py @@ -0,0 +1,153 @@ +# Copyright (C) 2009 Raul Jimenez +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +from nose.tools import * + +import message + +import logging, logging_conf +import test_const as tc + +import routing_manager +import token_manager +import tracker +import responder + +logging_conf.testing_setup(__name__) +logger = logging.getLogger('dht') + + +class TestResponder: + + def _notify_routing_m(self, node): + self.notification_callback_done = True + + def setup(self): + routing_m = routing_manager.RoutingManagerMock() + self.tracker = tracker.Tracker() + self.token_m = token_manager.TokenManager() + self.responder = responder.Responder(tc.SERVER_ID, routing_m, + self.tracker, self.token_m) + self.notification_callback_done = False + self.responder.set_on_query_received_callback(self._notify_routing_m) + + def test_return_response_for_ping(self): + # client side + query_msg = message.OutgoingPingQuery(tc.CLIENT_ID) + # rpc_manager.sendto() encodes + query_data = query_msg.encode(tc.TID) + # server side + # rpc_manager.datagram_received() decodes + query_msg = message.IncomingMsg(query_data) + assert not self.notification_callback_done + response_msg = self.responder.on_query_received(query_msg, + tc.CLIENT_ADDR) + response_data = response_msg.encode(query_msg.tid) + + assert self.notification_callback_done + expected_msg = message.OutgoingPingResponse(tc.SERVER_ID) + expected_data = expected_msg.encode(tc.TID) + eq_(response_data, expected_data) + + def test_return_response_for_find_node(self): + # client side + query_msg = message.OutgoingFindNodeQuery(tc.CLIENT_ID, + tc.TARGET_ID) + # querier encodes + query_data = query_msg.encode(tc.TID) + # server side + # rpc_manager.datagram_received() decodes + query_msg = message.IncomingMsg(query_data) + # rpc calls responder + assert not self.notification_callback_done + response_msg = self.responder.on_query_received(query_msg, + tc.CLIENT_ADDR) + response_data = response_msg.encode(query_msg.tid) + assert self.notification_callback_done + expected_msg = message.OutgoingFindNodeResponse(tc.SERVER_ID, + tc.NODES) + expected_data = expected_msg.encode(tc.TID) + eq_(response_data, expected_data) + + def test_return_response_for_get_peers_when_peers(self): + # server's tracker has peers + for peer in tc.PEERS: + self.tracker.put(tc.INFO_HASH, peer) + + # client side + query_msg = message.OutgoingGetPeersQuery(tc.CLIENT_ID, + tc.INFO_HASH) + # querier encodes + query_data = query_msg.encode(tc.TID) + # server side + # rpc_manager.datagram_received() decodes + query_msg = message.IncomingMsg(query_data) + # rpc calls responder + assert not self.notification_callback_done + response_msg = self.responder.on_query_received(query_msg, + tc.CLIENT_ADDR) + response_data = response_msg.encode(query_msg.tid) + assert self.notification_callback_done + expected_msg = message.OutgoingGetPeersResponse(tc.SERVER_ID, + self.token_m.get(), + peers=tc.PEERS) + expected_data = expected_msg.encode(tc.TID) + eq_(response_data, expected_data) + + def test_return_response_for_get_peers_when_no_peers(self): + # client side + query_msg = message.OutgoingGetPeersQuery(tc.CLIENT_ID, tc.NODE_ID) + # rpc_manager.sendto() encodes + query_data = query_msg.encode(tc.TID) + # server side + # rpc_manager.datagram_received() decodes + query_msg = message.IncomingMsg(query_data) + assert not self.notification_callback_done + response_msg = self.responder.on_query_received(query_msg, + tc.CLIENT_ADDR) + response_data = response_msg.encode(query_msg.tid) + assert self.notification_callback_done + expected_msg = message.OutgoingGetPeersResponse(tc.SERVER_ID, + self.token_m.get(), + nodes2=tc.NODES) + expected_data = expected_msg.encode(query_msg.tid) + eq_(response_data, expected_data) + + def test_return_response_for_announce_peer_with_valid_tocken(self): + # client side + query_msg = message.OutgoingAnnouncePeerQuery(tc.CLIENT_ID, + tc.INFO_HASH, + tc.CLIENT_ADDR[1], + self.token_m.get()) + # querier.send_query() encodes + query_data = query_msg.encode(tc.TID) + # server side + # rpc_manager.datagram_received() decodes and calls responder (callback) + query_msg = message.IncomingMsg(query_data) + assert not self.notification_callback_done + response_msg = self.responder.on_query_received(query_msg, + tc.CLIENT_ADDR) + response_data = response_msg.encode(query_msg.tid) + assert self.notification_callback_done + # responder returns to querier + expected_msg = message.OutgoingAnnouncePeerResponse(tc.SERVER_ID) + expected_data = expected_msg.encode(tc.TID) + assert response_data == expected_data + + def test_errors(self): + # client side + query_msg = message.OutgoingPingQuery(tc.CLIENT_ID) + # querier.send_query() encodes + query_data = query_msg.encode(tc.TID) + # server side + # rpc_manager.datagram_received() decodes and calls responder (callback) + query_msg = message.IncomingMsg(query_data) + ## 'xxxxxx' is not a valid QUERY + query_msg.query = 'zzzzzzzz' + assert not self.notification_callback_done + ok_(self.responder.on_query_received(query_msg, + tc.CLIENT_ADDR) is None) + + + diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_routing_manager.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_routing_manager.py new file mode 100644 index 0000000..cb14477 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_routing_manager.py @@ -0,0 +1,212 @@ +# Copyright (C) 2009 Raul Jimenez +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +from nose.tools import ok_, eq_, assert_raises + +import test_const as tc + +import minitwisted +import rpc_manager +import querier +import message + +from routing_manager import RoutingManager, RoutingManagerMock + +import logging, logging_conf + +logging_conf.testing_setup(__name__) +logger = logging.getLogger('dht') + + +class TestRoutingManager: + + def setup(self): + for n in tc.NODES + [tc.SERVER_NODE, tc.SERVER2_NODE]: + n.is_ns = False + for n in tc.NODES2 + [tc.CLIENT_NODE]: + n.is_ns = True + + self.querier = querier.QuerierMock(tc.CLIENT_ID) + self.routing_m = RoutingManager(tc.CLIENT_NODE, self.querier, + tc.NODES) + + def exercise_mock(self, mode): + # If this happens, we want to know + assert_raises(Exception, self.routing_m.on_timeout, tc.CLIENT_NODE) + + # node is nowhere (timeout is ignored) + self.routing_m.on_timeout(tc.SERVER_NODE) + + # main: CLIENT_NODE, replacement: empty + eq_(self.routing_m.get_closest_rnodes(tc.SERVER_ID), + [tc.CLIENT_NODE]) + + self.routing_m.on_response_received(tc.SERVER_NODE) + # main: client_node, server_node, replacement: empty + + # this should reset refresh task + self.routing_m.on_response_received(tc.SERVER_NODE) + + eq_(self.routing_m.get_closest_rnodes(tc.SERVER_ID), + [tc.SERVER_NODE, tc.CLIENT_NODE]) + + self.routing_m.on_timeout(tc.SERVER_NODE) + # main: client_node, replacement: server_node + eq_(self.routing_m.get_closest_rnodes(tc.SERVER_ID), + [tc.CLIENT_NODE]) + + self.routing_m.on_response_received(tc.SERVER2_NODE) + # main: client_node, server_node, replacement: server2_node(q) + eq_(self.routing_m.get_closest_rnodes(tc.SERVER_ID), + [tc.SERVER2_NODE, tc.CLIENT_NODE]) + + self.routing_m.on_response_received(tc.SERVER_NODE) + # main: client_node, server_node, replacement: server2_node(q) + eq_(self.routing_m.get_closest_rnodes(tc.SERVER_ID), + [tc.SERVER_NODE, tc.SERVER2_NODE, tc.CLIENT_NODE]) + eq_(self.routing_m.get_closest_rnodes(tc.SERVER2_ID), + [tc.SERVER2_NODE, tc.CLIENT_NODE]) + eq_(self.routing_m.get_closest_rnodes(tc.CLIENT_ID), + [tc.CLIENT_NODE]) + for n in tc.NODES: + self.routing_m.on_response_received(n) + """ + Main Routing Table + # -1 + client + # 154 + server2 + # 159 + server nodes[0:7] + """ + eq_(self.routing_m.get_closest_rnodes(tc.CLIENT_ID), + [tc.CLIENT_NODE]) + for n in tc.NODES: + eq_(self.routing_m.get_closest_rnodes(n.id), + [tc.SERVER_NODE] + tc.NODES[:7]) + # bucket full (NODES[7] in replacement + + self.routing_m.on_query_received(tc.NODES[7]) + eq_(self.routing_m.get_closest_rnodes(n.id), + [tc.SERVER_NODE] + tc.NODES[:7]) + + # nodes[0] is kicked out from main + # all nodes in replacement are refreshed (pinged) + self.routing_m.on_timeout(tc.NODES[0]) + eq_(self.routing_m.get_closest_rnodes(tc.NODES[0].id), + [tc.SERVER_NODE] + tc.NODES[1:7] + [tc.SERVER2_NODE]) + + # nodes[7] is refreshed + self.routing_m.on_query_received(tc.NODES[7]) + # nodes[7] still in replacement (queries don't cause movements) + eq_(self.routing_m.get_closest_rnodes(tc.NODES[0].id), + [tc.SERVER_NODE] + tc.NODES[1:7] + [tc.SERVER2_NODE]) + + # nodes[7] is moved to the main table (response to refresh ping) + self.routing_m.on_response_received(tc.NODES[7]) + eq_(self.routing_m.get_closest_rnodes(tc.NODES[0].id), + [tc.SERVER_NODE] + tc.NODES[1:8]) + + # nodes[7] is refreshed (no change to the tables) + self.routing_m.on_query_received(tc.NODES[7]) + eq_(self.routing_m.get_closest_rnodes(tc.NODES[0].id), + [tc.SERVER_NODE] + tc.NODES[1:8]) + + # nodes[7] is in main and get response + self.routing_m.on_response_received(tc.NODES[7]) + + + # nodes[0] gets strike 2, 3 and 4 (timeouts) + self.routing_m.on_timeout(tc.NODES[0]) + self.routing_m.on_timeout(tc.NODES[0]) + self.routing_m.on_timeout(tc.NODES[0]) + # and can be expelled from the replacement table + # nodes2[:] send responses + + #TODO2: rnode(nodes[0] report 5 timeouts + eq_(self.routing_m.replacement.get_rnode( + tc.NODES[0]).timeouts_in_a_row(), 5) + + if mode is message.QUERY: + for n in tc.NODES2: + self.routing_m.on_query_received(n) + elif mode is message.RESPONSE: + for n in tc.NODES2: + self.routing_m.on_response_received(n) + # nodes[0] comes back but the repl bucket is full + self.routing_m.on_response_received(tc.NODES[0]) + # nodes[0] sends error (routing manager ignores it) + self.routing_m.on_error_received(tc.NODES[0]) + + # timeout from node without id (ignored) + # nodes[0] comes back but the repl bucket is full + self.routing_m.on_timeout(tc.EXTERNAL_NODE) + + # nodes found (but no room in main + self.routing_m.on_nodes_found(tc.NODES) + + # nodes[1] (in main) timeout and repl bucket is full + # find worst node in repl (nodes[7]) and replace it + # all nodes in repl bucket get refreshed (not nodes[1] + self.routing_m.on_timeout(tc.NODES[1]) + eq_(self.routing_m.get_closest_rnodes(tc.NODES[0].id), + [tc.SERVER_NODE] + tc.NODES[2:8] +[tc.SERVER2_NODE]) + + # nodes found (there is room now) + # nodes2[0:1] get refreshed (pinged + self.routing_m.on_nodes_found(tc.NODES2) + # nodes2[0] replies (and is added to main) + self.routing_m.on_response_received(tc.NODES2[0]) + eq_(self.routing_m.get_closest_rnodes(tc.NODES2[0].id), + [tc.SERVER_NODE] + tc.NODES[2:8] +tc.NODES2[0:1]) + + + if mode == message.QUERY: + expected_main = [tc.SERVER2_NODE] + \ + [tc.SERVER_NODE] + tc.NODES[2:8] + tc.NODES2[0:1] + \ + [tc.CLIENT_NODE] + + expected_replacement = tc.NODES[0:2] + + elif mode == message.RESPONSE: + expected_main = [tc.SERVER2_NODE] + \ + [tc.SERVER_NODE] + tc.NODES[2:8] + tc.NODES2[0:1] + \ + [tc.CLIENT_NODE] + + expected_replacement = tc.NODES2[1:7] + tc.NODES[1:2] + + all_main, all_replacement = self.routing_m.get_all_rnodes() + + for n, expected in zip(all_main, expected_main): + eq_(n, expected) + for n, expected in zip(all_replacement, expected_replacement): + eq_(n, expected) + eq_(len(all_main), len(expected_main)) + eq_(len(all_replacement), len(expected_replacement)) + + + def test_query(self): + self.exercise_mock(message.QUERY) + def test_response(self): + self.exercise_mock(message.RESPONSE) + + + + + def test_bootstrap(self): + self.routing_m.do_bootstrap() + fn_r = message.OutgoingFindNodeResponse(tc.NODES[0].id, + tc.NODES2[0:1]) + fn_r = message.IncomingMsg(fn_r.encode('\0\0')) + self.querier.on_response_received(fn_r, tc.NODES[0].addr) + + def test_routing_m_mock(self): + # Just testing interface + rm = RoutingManagerMock() + eq_(rm.get_closest_rnodes(tc.TARGET_ID), tc.NODES) + + + def test_complete_coverage(self): + self.routing_m._do_nothing() + self.routing_m._refresh_now_callback() diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_routing_table.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_routing_table.py new file mode 100644 index 0000000..443065a --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_routing_table.py @@ -0,0 +1,77 @@ +# Copyright (C) 2009 Raul Jimenez +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +import logging, logging_conf + +from nose.tools import eq_, ok_, assert_raises + +import test_const as tc + +import node + +from routing_table import * + +logging_conf.testing_setup(__name__) +logger = logging.getLogger('dht') + + +class TestRoutingTable: + + def setup(self): + nodes_per_bucket = [2] * 161 + self.rt = RoutingTable(tc.CLIENT_NODE, + nodes_per_bucket) + + def test_basics(self): + eq_(self.rt.get_bucket(tc.SERVER_NODE).rnodes, []) + ok_(self.rt.there_is_room(tc.SERVER_NODE)) + assert_raises(RnodeNotFound, self.rt.get_rnode, tc.SERVER_NODE) + ok_(not self.rt.get_bucket(tc.SERVER_NODE).is_full()) + eq_(self.rt.num_rnodes, 0) + eq_(self.rt.get_all_rnodes(), []) + + self.rt.add(tc.SERVER_NODE) + ok_(self.rt.there_is_room(tc.SERVER_NODE)) + eq_(self.rt.get_bucket(tc.SERVER_NODE).rnodes, [tc.SERVER_NODE]) + eq_(self.rt.get_rnode(tc.SERVER_NODE), tc.SERVER_NODE) + ok_(not self.rt.get_bucket(tc.SERVER_NODE).is_full()) + eq_(self.rt.num_rnodes, 1) + eq_(self.rt.get_all_rnodes(), [tc.SERVER_NODE]) + + # Let's add a node to the same bucket + new_node = node.Node(tc.SERVER_NODE.addr, + tc.SERVER_NODE.id.generate_close_id(1)) + self.rt.add(new_node) + # full bucket + ok_(not self.rt.there_is_room(tc.SERVER_NODE)) + eq_(self.rt.get_bucket(new_node).rnodes, [tc.SERVER_NODE, new_node]) + eq_(self.rt.get_rnode(new_node), new_node) + ok_(self.rt.get_bucket(tc.SERVER_NODE).is_full()) + eq_(self.rt.num_rnodes, 2) + eq_(self.rt.get_all_rnodes(), [tc.SERVER_NODE, new_node]) + + + eq_(self.rt.get_closest_rnodes(tc.SERVER_ID, 1), + [tc.SERVER_NODE]) + eq_(self.rt.get_closest_rnodes(tc.SERVER_ID), + [tc.SERVER_NODE, new_node]) + + assert_raises(BucketFullError, self.rt.add, new_node) + + self.rt.remove(new_node) + # there is one slot in the bucket + ok_(self.rt.there_is_room(tc.SERVER_NODE)) + assert_raises(RnodeNotFound, self.rt.get_rnode, new_node) + eq_(self.rt.get_bucket(tc.SERVER_NODE).rnodes, [tc.SERVER_NODE]) + eq_(self.rt.get_rnode(tc.SERVER_NODE), tc.SERVER_NODE) + ok_(not self.rt.get_bucket(tc.SERVER_NODE).is_full()) + eq_(self.rt.num_rnodes, 1) + eq_(self.rt.get_all_rnodes(), [tc.SERVER_NODE]) + + + eq_(self.rt.get_closest_rnodes(tc.SERVER_ID), [tc.SERVER_NODE]) + + def test_complete_coverage(self): + str(self.rt.get_bucket(tc.SERVER_NODE)) + repr(self.rt) diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_rpc_manager.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_rpc_manager.py new file mode 100644 index 0000000..75dc733 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_rpc_manager.py @@ -0,0 +1,131 @@ +# Copyright (C) 2009 Raul Jimenez +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +from nose.tools import ok_, eq_, assert_raises + +import logging, logging_conf + +import minitwisted +import message +import test_const as tc + +import rpc_manager + + +logging_conf.testing_setup(__name__) +logger = logging.getLogger('dht') +#FIXME: more tests!!!! + +class TestRPCManager: + + def on_query_received(self, msg, addr): + self.got_query = True + return message.OutgoingPingResponse(tc.SERVER_ID) + + def on_response_received(self, msg, addr): + self.got_response = True + + def on_routing_response_received(self, msg, addr): + self.got_routing_response = True + + def on_error_received(self, msg, addr): + self.got_error = True + + def on_timeout(self, addr): + self.got_timeout = True + + def on_routing_timeout(self, addr): + self.got_routing_timeout = True + + def setup(self): + self.reactor = minitwisted.ThreadedReactor() + self.c = rpc_manager.RPCManager(self.reactor, + tc.CLIENT_ADDR[1]) + self.s = rpc_manager.RPCManager(self.reactor, + tc.SERVER_ADDR[1]) + + self.got_query = False + self.got_response = False + self.got_routing_response = False + self.got_error = False + self.got_timeout = False + self.got_routing_timeout = False + + def test_querier_responder(self): + # client + # setup + self.c.add_msg_callback(message.RESPONSE, + self.on_response_received) + self.c.add_msg_callback(message.RESPONSE, + self.on_routing_response_received) + self.c.add_msg_callback(message.ERROR, + self.on_error_received) + self.c.add_timeout_callback(self.on_routing_timeout) + + # server + # setup + self.s.add_msg_callback(message.QUERY, + self.on_query_received) + + # client creates and sends query + t_task = self.c.get_timeout_task(tc.SERVER_ADDR, + tc.TIMEOUT_DELAY, + self.on_timeout) + msg = message.OutgoingPingQuery(tc.CLIENT_ID) + msg_data = msg.encode(tc.TID) + self.c.send_msg_to(msg_data, tc.SERVER_ADDR) + # client sets up timeout + + # server receives query, creates response and sends it back + self.s._on_datagram_received(msg_data, tc.CLIENT_ADDR) + # rpc_manager would send the message back automatically + ok_(self.got_query); self.got_query = False + msg = message.OutgoingPingResponse(tc.SERVER_ID) + msg_data = msg.encode(tc.TID) + self.s.send_msg_to(msg_data, tc.CLIENT_ADDR) + + # client gets response + self.c._on_datagram_received(msg_data, tc.SERVER_ADDR) + ok_(self.got_response); self.got_response = False + ok_(self.got_routing_response) + self.got_routing_response = False + + # client gets error + msg_data = message.OutgoingErrorMsg(message.GENERIC_E + ).encode(tc.TID) + self.c._on_datagram_received(msg_data, tc.SERVER_ADDR) + ok_(self.got_error); self.got_error = False + + # client gets timeout + t_task.fire_callbacks() + ok_(self.got_timeout); self.got_timeout = False + ok_(self.got_routing_timeout) + self.got_routing_timeout = False + + # server gets invalid message + self.s._on_datagram_received('zzz', tc.CLIENT_ADDR) + ok_(not self.got_query) + ok_(not self.got_response) + ok_(not self.got_routing_response) + + + def test_call_later(self): + t = self.c.call_later(tc.TIMEOUT_DELAY, + self.on_timeout, + 1234) + t.fire_callbacks() + ok_(self.got_timeout) + + def test_no_callback_for_type(self): + msg = message.OutgoingPingQuery(tc.CLIENT_ID) + msg_data = msg.encode(tc.TID) + self.s._on_datagram_received(msg_data, + tc.CLIENT_ADDR) + ok_(not self.got_query) + ok_(not self.got_response) + ok_(not self.got_routing_response) + + def teardown(self): + self.c.stop() + self.s.stop() diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_token_manager.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_token_manager.py new file mode 100644 index 0000000..f597e26 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_token_manager.py @@ -0,0 +1,24 @@ +# Copyright (C) 2009 Raul Jimenez, Flutra Osmani +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +from nose.tools import * + +import token_manager + + + +token_str = '123' +invalid_token_str = '' + +class TestTokenManager: + + def setup(self): + self.token_m = token_manager.TokenManager() + + def test_get_token(self): + eq_(self.token_m.get(), token_str) + + def test_check_token(self): + ok_(self.token_m.check(token_str)) + ok_(not self.token_m.check(invalid_token_str)) diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_tracker.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_tracker.py new file mode 100644 index 0000000..59038cb --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_tracker.py @@ -0,0 +1,107 @@ +# Copyright (C) 2009 Raul Jimenez, Flutra Osmani +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +from nose.tools import * + +import time + +import tracker +import minitwisted + +import logging, logging_conf + +logging_conf.testing_setup(__name__) +logger = logging.getLogger('dht') + + +keys = ('1','2') +peers = (('1.2.3.4', 1234), ('2.3.4.5', 2222)) + +class TestTracker(object): + + def setup(self): + self.t = tracker.Tracker(.01, 5) + + def test_put(self): + self.t.put(keys[0], peers[0]) + + def test_get_empty_key(self): + eq_(self.t.get(keys[0]), []) + + def test_get_nonempty_key(self): + self.t.put(keys[0], peers[0]) + eq_(self.t.get(keys[0]), [peers[0]]) + + def test_get_expired_value(self): + self.t.put(keys[0], peers[0]) + time.sleep(.015) + eq_(self.t.get(keys[0]), []) + + def test_many_puts_and_gets(self): + #0 + self.t.put(keys[0], peers[0]) + time.sleep(.02) + #.02 + self.t.put(keys[0], peers[0]) + time.sleep(.02) + #.04 + self.t.put(keys[0], peers[1]) + eq_(self.t.get(keys[0]), [peers[0], peers[1]]) + time.sleep(.07) + #.11 + self.t.put(keys[0], peers[0]) + eq_(self.t.get(keys[0]), [peers[1], peers[0]]) + time.sleep(.02) + #.13 + eq_(self.t.get(keys[0]), [peers[0]]) + + def test_hundred_puts(self): + # test > 5 puts + eq_(len(self.t.debug_view()), 0) + time.sleep(0) + eq_(len(self.t.debug_view()), 0) + self.t.put(1,1) + eq_(len(self.t.debug_view()), 1) + time.sleep(.006) + eq_(len(self.t.debug_view()), 1) + self.t.put(2,2) + eq_(len(self.t.debug_view()), 2) + time.sleep(.004) + eq_(len(self.t.debug_view()), 2) + self.t.put(3,3) + eq_(len(self.t.debug_view()), 3) + time.sleep(.0) + eq_(len(self.t.debug_view()), 3) + self.t.put(4,4) + eq_(len(self.t.debug_view()), 4) + time.sleep(.0) + eq_(len(self.t.debug_view()), 4) + self.t.put(5,5) + # cleaning... 1 out + eq_(len(self.t.debug_view()), 4) + time.sleep(.0) + eq_(len(self.t.debug_view()), 4) + self.t.put(6,6) + eq_(len(self.t.debug_view()), 5) + time.sleep(.00) + eq_(len(self.t.debug_view()), 5) + self.t.put(7,7) + eq_(len(self.t.debug_view()), 6) + time.sleep(.01) + eq_(len(self.t.debug_view()), 6) + self.t.put(8,8) + eq_(len(self.t.debug_view()), 7) + time.sleep(.00) + eq_(len(self.t.debug_view()), 7) + self.t.put(9,9) + eq_(len(self.t.debug_view()), 8) + time.sleep(.00) + eq_(len(self.t.debug_view()), 8) + self.t.put(0,0) + # cleaning ... 2,3,4,5,6,7 out + eq_(len(self.t.debug_view()), 3) + + + def teardown(self): + pass diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_utils.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_utils.py new file mode 100644 index 0000000..9551c68 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/test_utils.py @@ -0,0 +1,19 @@ +# Copyright (C) 2009 Raul Jimenez +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +from nose.tools import * + +import utils + + +class TestUtils: + + def test_compact_addr(self): + cases = ((('1.2.3.4', 255), (1,2,3,4,0,255)), + (('199.2.3.4', 256), (199,2,3,4,1,0)), + ) + for case in cases: + expected = ''.join([chr(i) for i in case[1]]) + eq_(utils.compact_addr(case[0]), expected) + diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/token_manager.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/token_manager.py new file mode 100644 index 0000000..d5f9649 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/token_manager.py @@ -0,0 +1,15 @@ +# Copyright (C) 2009 Raul Jimenez, Flutra Osmani +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + + + +class TokenManager(object): + def __init__(self): + self.current_token = '123' + + def get(self): + return self.current_token + + def check(self, token): + return token == self.current_token diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/tracker.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/tracker.py new file mode 100644 index 0000000..7693d91 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/tracker.py @@ -0,0 +1,64 @@ +# Copyright (C) 2009 Raul Jimenez, Flutra Osmani +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +import time + +VALIDITY_PERIOD = 30 * 60 #30 minutes +CLEANUP_COUNTER = 100 + +class Tracker(object): + + def __init__(self, validity_period=VALIDITY_PERIOD, + cleanup_counter=CLEANUP_COUNTER): + self.tracker_dict = {} + self.validity_period = validity_period + self.cleanup_counter = cleanup_counter + self.put_counter = 0 + + + def _cleanup_list(self, ts_peers): + ''' + Clean up the list as side effect. + ''' + oldest_valid_ts = time.time() - self.validity_period + for i in range(len(ts_peers)): + if ts_peers[i][0] < oldest_valid_ts: + del ts_peers[i] + break + + + def put(self, k, peer): + #Clean up every n puts + self.put_counter += 1 + if self.put_counter == self.cleanup_counter: + self.put_counter = 0 + for k_ in self.tracker_dict.keys(): + ts_peers = self.tracker_dict[k_] + self._cleanup_list(ts_peers) + if not ts_peers: #empty list. Delete key + del self.tracker_dict[k_] + + ts_peers = self.tracker_dict.setdefault(k,[]) + if ts_peers: + # let's see whether the peer is already there + for i in range(len(ts_peers)): + if ts_peers[i] == peer: + del ts_peers[i] + break + ts_peers.append((time.time(), peer)) + + def get(self, k): + ts_peers = self.tracker_dict.get(k, []) + self._cleanup_list(ts_peers) + return [ts_peer[1] for ts_peer in ts_peers] + + def debug_view(self): + return self.tracker_dict + + +class TrackerMock(object): + + def get(self, k): + import test_const + return test_const.PEERS diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/utils.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/utils.py new file mode 100644 index 0000000..f4551e4 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/utils.py @@ -0,0 +1,35 @@ +# Copyright (C) 2009 Raul Jimenez +# Released under GNU LGPL 2.1 +# See LICENSE.txt for more information + +import socket + + +class AddrError(Exception): + pass + +class IP6Addr(AddrError): + pass +#TODO2: IPv6 support + + +#TODO2: move binary functions from identifier + +def compact_port(port): + return ''.join( + [chr(port_byte_int) for port_byte_int in divmod(port, 256)]) + +def uncompact_port(c_port_net): + return ord(bin_str[0]) * 256 + ord(bin_str[1]) + +def compact_addr(addr): + return socket.inet_aton(addr[0]) + compact_port(addr[1]) + +def uncompact_addr(c_addr): + try: + return (socket.inet_ntoa(c_addr[:-2], + uncompact_port(c_addr[-2:]))) + except (socket.error): + raise AddrError + +compact_peer = compact_addr diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/version.txt b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/version.txt new file mode 100644 index 0000000..eb56657 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/kadtracker/version.txt @@ -0,0 +1,11 @@ +Path: . +URL: https://ttuki.vtt.fi/svn/p2p-next/KTH/kadtracker/branches/20100208-from_trunk_r882-kadtracker-1.0.1_integration +Repository Root: https://ttuki.vtt.fi/svn/p2p-next +Repository UUID: e16421f0-f15b-0410-abcd-98678b794739 +Revision: 899 +Node Kind: directory +Schedule: normal +Last Changed Author: raul +Last Changed Rev: 898 +Last Changed Date: 2010-02-10 12:11:31 +0100 (Wed, 10 Feb 2010) + diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/mainlineDHT.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/mainlineDHT.py new file mode 100644 index 0000000..81e94fd --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/mainlineDHT.py @@ -0,0 +1,41 @@ +# written by Fabian van der Werf, Arno Bakker +# Modified by Raul Jimenez to integrate KTH DHT +# see LICENSE.txt for license information + +import sys +from traceback import print_exc + +dht_imported = False + +if sys.version.split()[0] >= '2.5': + try: + from BaseLib.Core.DecentralizedTracking.kadtracker.kadtracker import KadTracker + dht_imported = True + except (ImportError), e: + print_exc() + +DEBUG = False + +dht = None + +def init(*args, **kws): + global dht + global dht_imported + if DEBUG: + print >>sys.stderr,'dht: DHT initialization', dht_imported + if dht_imported and dht is None: + dht = KadTracker(*args, **kws) + if DEBUG: + print >>sys.stderr,'dht: DHT running' + +def control(): + import pdb + pdb.set_trace() + +def deinit(): + global dht + if dht is not None: + try: + dht.stop() + except: + pass diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/mainlineDHTChecker.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/mainlineDHTChecker.py new file mode 100644 index 0000000..18b8998 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/mainlineDHTChecker.py @@ -0,0 +1,66 @@ +# written by Arno Bakker, Yuan Yuan +# Modified by Raul Jimenez to integrate KTH DHT +# see LICENSE.txt for license information + +import sys +from threading import currentThread +from BaseLib.Core.CacheDB.CacheDBHandler import TorrentDBHandler + +DEBUG = False + +class mainlineDHTChecker: + __single = None + + def __init__(self): + + if DEBUG: + print >>sys.stderr,'mainlineDHTChecker: initialization' + if mainlineDHTChecker.__single: + raise RuntimeError, "mainlineDHTChecker is Singleton" + mainlineDHTChecker.__single = self + + self.dht = None + self.torrent_db = TorrentDBHandler.getInstance() + + def getInstance(*args, **kw): + if mainlineDHTChecker.__single is None: + mainlineDHTChecker(*args, **kw) + return mainlineDHTChecker.__single + getInstance = staticmethod(getInstance) + + def register(self,dht): + self.dht = dht + + def lookup(self,infohash): + if DEBUG: + print >>sys.stderr,"mainlineDHTChecker: Lookup",`infohash` + + if self.dht is not None: + from BaseLib.Core.DecentralizedTracking.kadtracker.identifier import Id, IdError + try: + infohash_id = Id(infohash) + func = lambda p:self.got_peers_callback(infohash,p) + self.dht.get_peers(infohash_id,func) + except (IdError): + print >>sys.stderr,"Rerequester: _dht_rerequest: self.info_hash is not a valid identifier" + return + elif DEBUG: + print >>sys.stderr,"mainlineDHTChecker: No lookup, no DHT support loaded" + + + def got_peers_callback(self,infohash,peers): + """ Called by network thread """ + if DEBUG: + print >>sys.stderr,"mainlineDHTChecker: Got",len(peers),"peers for torrent",`infohash`,currentThread().getName() + + alive = len(peers) > 0 + if alive: + # Arno, 2010-02-19: this can be called frequently with the new DHT, + # so first check before doing commit. + # + torrent = self.torrent_db.getTorrent(infohash) # ,keys=('torrent_id','status_id') don't work, st*pid code + if torrent['status'] != "good": + status = "good" + kw = {'status': status} + self.torrent_db.updateTorrent(infohash, commit=True, **kw) + diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/repex.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/repex.py new file mode 100644 index 0000000..9dbdca3 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/repex.py @@ -0,0 +1,1116 @@ +# Written by Raynor Vliegendhart +# see LICENSE.txt for license information +import sys +import os +from time import time as ts_now +from random import shuffle +from traceback import print_exc,print_stack +from threading import RLock,Condition,Event,Thread,currentThread +from binascii import b2a_hex + +from BaseLib.Core.simpledefs import * +from BaseLib.Core.exceptions import * +from BaseLib.Core.osutils import * +from BaseLib.Core.DecentralizedTracking.ut_pex import check_ut_pex_peerlist + +DEBUG = False +REPEX_DISABLE_BOOTSTRAP = False + +# TODO: Move constants to simpledefs or make it configurable? +REPEX_SWARMCACHE_SIZE = 4 # Number of peers per SwarmCache table +REPEX_STORED_PEX_SIZE = 5 # Number of PEX addresses per peer per SwarmCache +REPEX_PEX_MINSIZE = 1 # minimum number of peers in PEX message before considered a good peer + # TODO: Currently set at 1, but what if a swarm consists of 1 user? +REPEX_INTERVAL = 20*60 # Refresh SwarmCache every 20 minutes. +REPEX_MIN_INTERVAL = 5*60 # Minimum time between attempts to prevent starvation in cases like continuous failures. +REPEX_PEX_MSG_MAX_PEERS = 200 # only consider the first 200 peers (Opera10 sends its *whole* neighborhood set) +REPEX_LISTEN_TIME = 50 # listen max. 50 seconds for PEX message +REPEX_INITIAL_SOCKETS = 4 # number of sockets used initially +REPEX_MAX_SOCKETS = 8 # max number of sockets when all initial peers are checked or after the first failure has occured +REPEX_SCAN_INTERVAL = 1*60 # Scan for stopped Downloads every minute. + + +# Testing values +# REPEX_INTERVAL = 10 +# REPEX_SCAN_INTERVAL = 30 +# REPEX_MIN_INTERVAL = 60 +# REPEX_DISABLE_BOOTSTRAP = True + +class RePEXerInterface: + """ + Describes the RePEXer interface required by the SingleDownload and + the download engine classes. + """ + + def repex_ready(self, infohash, connecter, encoder, rerequester): + """ + Called by network thread. SingleDownload calls this method when + everything is set up. + @param infohash Infohash of download. + @param connecter Connecter (Connecter.py from the download engine). + @param encoder Encoder (Encrypter.py from the download engine). + @param rerequester Rerequester (Rerequester.py from the download engine) + """ + def repex_aborted(self, infohash, dlstatus=None): + """ + Called by network thread. SingleDownload calls this method when + the download is stopped or restarted, interrupting the RePEX mode. + @param infohash Infohash of download. + @param dlstatus Status of the download when the RePEX mode was + interrupted, or None if unknown. + """ + def rerequester_peers(self, peers): + """ + Called by network thread. Rerequester (accessible via Encoder) + schedules this method call when peers have arrived. + @param peers [(dns,id)] or None in case of error. + """ + def connection_timeout(self, connection): + """ + Called by network thread. Encoder calls this when a connection + was not established within Encrypter's autoclose timeout. + @param connection Encrypter.Connection + """ + def connection_closed(self, connection): + """ + Called by network thread. Encoder or Connecter calls this when + a connection was closed, either locally or remotely. It is also + called right after a timeout. The RePEXer should maintain state + on connections it has started. + @param connection Encrypter.Connection or Connecter.Connection + """ + + def connection_made(self, connection, ext_support): + """ + Called by network thread. Connecter calls this when a connection + was established. + @param connection Connecter.Connection + @param ext_support Flag indicating whether the connection supports + the extension protocol. + """ + + def got_extend_handshake(self, connection, version=None): + """ + Called by network thread. Connecter calls this when a extended + handshake is received. Use connection's supports_extend_msg(msg_name) + method to figure out whether a message is supported. + @param connection Connecter.Connection + @param version Version string or None if not available. + """ + + def got_ut_pex(self, connection, d): + """ + Called by network thread. Connecter calls this when a PEX message is + received. + @param connection Connecter.Connection + @param d The PEX dictionary containing 'added' and 'added.f' + """ + +def c2infohash_dns(connection): + """ + Utility function to retrieve the infohash and dns of a Encrypter or + Connecter Connection. + + Note, if the returned dns is None, it is an incoming connection. + """ + # luckily the same interface + infohash = connection.connecter.infohash + #dns = (connection.get_ip(True), connection.get_port(False)) # buggy, get_port might return -1 + if hasattr(connection, 'got_ut_pex'): + encr_connection = connection.connection + else: + encr_connection = connection + dns = encr_connection.dns + return infohash, dns + +def swarmcache_ts(swarmcache): + """ + Computes the timestamp of a SwarmCache or None if SwarmCache is empty. + """ + ts = None + if swarmcache: + ts = max(v['last_seen'] for v in swarmcache.values()) + # Currently the greatest timestamp is chosen as *the* + # timestamp of a SwarmCache. TODO: is this ok? + return ts + +class RePEXer(RePEXerInterface): + """ + A RePEXer is associated with a single SingleDownload. While the interface + is set up in a way that allows a RePEXer to be associated with multiple + SingleDownloads, it is easier to maintain state when one RePEXer is created + per Download instance. + """ + # (Actually, the interface does not quite work that way... when the + # rerequester delivers peers, the RePEXer cannot tell for which + # Download they are meant) + + _observers = [] + lock = RLock() # needed to atomically update observers list + + @classmethod + def attach_observer(cls, observer): + """ + Attaches an observer to observe all RePEXer instances. + + @param observer RePEXerStatusCallback. + """ + cls.lock.acquire() + try: + cls._observers.append(observer) + finally: + cls.lock.release() + + @classmethod + def detach_observer(cls, observer): + """ + Detaches a previously attached observer. + + @param observer RePEXerStatusCallback. + """ + cls.lock.acquire() + try: + cls._observers.remove(observer) + finally: + cls.lock.release() + + def __init__(self, infohash, swarmcache): + """ + Constructs a RePEXer object, associated with a download's infohash. + + @param infohash Infohash of download. + @param swarmcache Previous SwarmCache to check, which is a dict + mapping dns to a dict with at least 'last_seen' and 'pex' keys. + """ + # Note: internally in this class we'll use the name 'peertable', + # but the outside world calls it more appropiately the SwarmCache. + self.infohash = infohash + self.connecter = None + self.encoder = None + self.rerequest = None + + self.starting_peertable = swarmcache + self.final_peertable = None + self.to_pex = [] + self.active_sockets = 0 + self.max_sockets = REPEX_INITIAL_SOCKETS + self.attempted = set() + self.live_peers = {} # The pex-capable and useful peers. + + # The following two sets are usable in a debugging/logging context + self.bt_connectable = set() # sent BT handshake + self.bt_ext = set() # supported ext + self.bt_pex = set() # supported ut_pex + + self.dns2version = {} # additional data + + self.onlinecount = 0 # number of initial peers found online + self.shufflecount = 0 # number of peers in peertable unconnectable or useless + # sum of these two must become len(peertable) since we prefer the initial peertable + + self.datacost_bandwidth_keys = ['no_pex_support', 'no_pex_msg', 'pex', 'other'] + self.datacost_counter_keys = ['connection_attempts','connections_made','bootstrap_peers','pex_connections'] + self.datacost = {} + self.datacost['no_pex_support'] = (0,0) # down,up + self.datacost['no_pex_msg'] = (0,0) # down,up + self.datacost['pex'] = (0,0) # down,up + self.datacost['other'] = (0,0) # down,up + self.datacost['connection_attempts'] = 0 # number of times connect() successfully created a connection + self.datacost['connections_made'] = 0 # number of times connection_made() was called + self.datacost['bootstrap_peers'] = 0 # total number of peers given to rerequester_peers() + self.datacost['pex_connections'] = 0 # total number of connections that sent a PEX reply + + self.requesting_tracker = False # needed to interact with Rerequester in case of failure + self.bootstrap_counter = 0 # how often did we call bootstrap()? + + self.is_closing = False # flag so that we only call close_all once + self.done = False # flag so we know when we're done or are aborted + self.aborted = False # flag so we know the exact done-reason + self.ready = False # flag so we know whether repex_ready has been called + self.ready_ts = -1 # for logging purposes, store the time repex_ready event was triggered + self.end_ts = -1 # for logging purposes, store the time done or aborted was sent + + # Added robustness, check whether received SwarmCache is not None + if self.starting_peertable is None: + print >>sys.stderr, 'RePEXer: __init__: swarmcache was None, defaulting to {}' + self.starting_peertable = {} + + + # + # RePEXerInterface + # + def repex_ready(self, infohash, connecter, encoder, rerequester): + if infohash != self.infohash: + print >>sys.stderr, "RePEXer: repex_ready: wrong infohash:", b2a_hex(infohash) + return + if self.done: + print >>sys.stderr, "RePEXer: repex_ready: already done" + return + if DEBUG: + print >>sys.stderr, "RePEXer: repex_ready:", b2a_hex(infohash) + self.ready = True + self.ready_ts = ts_now() + self.connecter = connecter + self.encoder = encoder + self.rerequest = rerequester + + # Fill connect queue + self.to_pex = self.starting_peertable.keys() + self.max_sockets = REPEX_INITIAL_SOCKETS + + # We'll also extend the queue with all peers from the pex messages + # TODO: investigate whether a more sophisticated queueing scheme is more appropiate + # For example, only fill the queue when countering a failure + for dns in self.starting_peertable: + self.to_pex.extend([pexdns for pexdns,flags in self.starting_peertable[dns].get('pex',[])]) + self.connect_queue() + + + def repex_aborted(self, infohash, dlstatus): + if self.done: + return + if infohash != self.infohash: + print >>sys.stderr, "RePEXer: repex_aborted: wrong infohash:", b2a_hex(infohash) + return + if DEBUG: + if dlstatus is None: + status_string = str(None) + else: + status_string = dlstatus_strings[dlstatus] + print >>sys.stderr, "RePEXer: repex_aborted:", b2a_hex(infohash),status_string + self.done = True + self.aborted = True + self.end_ts = ts_now() + for observer in self._observers: + observer.repex_aborted(self, dlstatus) + # Note that we do not need to close active connections + # 1) If repex_aborted is called because the download was stopped, + # the connections are closed automatically. + # 2) If repex_aborted is called because the download was restarted, + # open connections are actually useful. + + def rerequester_peers(self, peers): + self.requesting_tracker = False + if peers is not None: + numpeers = len(peers) + else: + numpeers = -1 + if DEBUG: + print >>sys.stderr, "RePEXer: rerequester_peers: received %s peers" % numpeers + if numpeers > 0: + self.to_pex.extend([dns for dns,id in peers]) + self.datacost['bootstrap_peers'] += numpeers + self.connect_queue() + + + def connection_timeout(self, connection): + infohash, dns = c2infohash_dns(connection) + if infohash != self.infohash or dns is None: + return + if DEBUG: + print >>sys.stderr, "RePEXer: connection_timeout: %s:%s" % dns + + def connection_closed(self, connection): + self.active_sockets -= 1 + if self.active_sockets < 0: + self.active_sockets = 0 + infohash, dns = c2infohash_dns(connection) + c = None # Connecter.Connection + if hasattr(connection, 'got_ut_pex'): + c = connection + connection = c.connection # Encrypter.Connection + if infohash != self.infohash or dns is None: + return + if DEBUG: + print >>sys.stderr, "RePEXer: connection_closed: %s:%s" % dns + + singlesocket = connection.connection + + # Update costs and determine success + success = False + costtype = 'other' + if c is not None: + if c.pex_received > 0: + costtype = 'pex' + success = True + elif not c.supports_extend_msg('ut_pex'): + costtype = 'no_pex_support' + elif c.pex_received == 0: + costtype = 'no_pex_msg' + + if costtype: + d, u = self.datacost[costtype] + d += singlesocket.data_received + u += singlesocket.data_sent + self.datacost[costtype] = (d,u) + + # If the peer was in our starting peertable, update online/shuffle count + if dns in self.starting_peertable: + if success: + self.onlinecount += 1 + self.live_peers[dns]['prev'] = True + else: + self.shufflecount += 1 + #self.to_pex.extend([pexdns for pexdns,flags in self.starting_peertable[dns]['pex']]) + # TODO: see repex_ready for now + + # Boost on failure of initial peer or when all initial peers are checked + if (dns in self.starting_peertable and not success) or self.initial_peers_checked(): + self.max_sockets = REPEX_MAX_SOCKETS + + # always try to connect + self.connect_queue() + + def connection_made(self, connection, ext_support): + infohash, dns = c2infohash_dns(connection) + if infohash != self.infohash or dns is None: + return + if DEBUG: + print >>sys.stderr, "RePEXer: connection_made: %s:%s ext_support = %s" % (dns + (ext_support,)) + self.datacost['connections_made'] += 1 + self.bt_connectable.add(dns) + if ext_support: + self.bt_ext.add(dns) + # Start timer on Encryption.Connection + def auto_close(connection = connection.connection, dns=dns): + if not connection.closed: + if DEBUG: + print >>sys.stderr, "RePEXer: auto_close: %s:%s" % dns + try: + # only in rare circumstances + # (like playing around in the REPL which is running in a diff. thread) + # an Assertion is thrown. + connection.close() + except AssertionError, e: + if DEBUG: + print >>sys.stderr, "RePEXer: auto_close:", `e` + self.connection_closed(connection) + self.connecter.sched(auto_close, REPEX_LISTEN_TIME) + else: + connection.close() + + def got_extend_handshake(self, connection, version=None): + infohash, dns = c2infohash_dns(connection) + ut_pex_support = connection.supports_extend_msg('ut_pex') + if infohash != self.infohash or dns is None: + return + if DEBUG: + print >>sys.stderr, "RePEXer: got_extend_handshake: %s:%s version = %s ut_pex_support = %s" % (dns + (`version`,ut_pex_support )) + if ut_pex_support: + self.bt_pex.add(dns) + else: + connection.close() + self.dns2version[dns] = version + + def got_ut_pex(self, connection, d): + infohash, dns = c2infohash_dns(connection) + is_tribler_peer = connection.is_tribler_peer() + added = check_ut_pex_peerlist(d,'added')[:REPEX_PEX_MSG_MAX_PEERS] + addedf = map(ord, d.get('addedf',[]))[:REPEX_PEX_MSG_MAX_PEERS] + addedf.extend( [0]*(len(added)-len(addedf)) ) + IS_SEED = 2 + IS_SAME = 4 + if infohash != self.infohash or dns is None: + return + if DEBUG: + print >>sys.stderr, "RePEXer: got_ut_pex: %s:%s pex_size = %s" % (dns + (len(added),)) + + # Remove bad IPs like 0.x.x.x (often received from Transmission peers) + for i in range(len(added)-1,-1,-1): + if added[i][0].startswith('0.'): + added.pop(i) + addedf.pop(i) + + # only store peer when sufficiently connected + if len(added) >= REPEX_PEX_MINSIZE: + # Clear flag IS_SAME if it was not a Tribler peer + if not is_tribler_peer: + addedf = [flag & ~IS_SAME for flag in addedf] + + # sample PEX message and + picks = range(len(added)) + shuffle(picks) + pex_peers = [(added[i],addedf[i]) for i in picks[:REPEX_STORED_PEX_SIZE]] + self.live_peers[dns] = {'last_seen' : ts_now(), + 'pex' : pex_peers, + 'version' : self.dns2version[dns]} + # Should we do the following? Might lower the load on the tracker even more? + # self.to_pex.extend(zip(*pex_peers)[0]) + # Possible danger: too much crawling, wasting resources? + + # TODO: Might be more sophisticated to sampling of PEX msg at the end? + # (allows us to get more diversity and perhaps also security?) + + self.datacost['pex_connections'] += 1 + + # Closing time + connection.close() + + # + # Status methods + # + def initial_peers_checked(self): + return len(self.starting_peertable) == (self.onlinecount + self.shufflecount) + + # + # Connect and bootstrap methods + # + def connect(self, dns, id=0): + if dns in self.attempted: + return + if DEBUG: + print >>sys.stderr, "RePEXer: connecting: %s:%s" % dns + self.active_sockets += 1 + self.datacost['connection_attempts'] += 1 + self.attempted.add(dns) + if not self.encoder.start_connection(dns, id, forcenew = True): + print >>sys.stderr, "RePEXer: connecting failed: %s:%s" % dns + self.active_sockets -= 1 + self.datacost['connection_attempts'] -= 1 + if dns in self.starting_peertable: + self.shufflecount += 1 + + def next_peer_from_queue(self): + # Only return a peer if we can connect + if self.can_connect() and self.to_pex: + return self.to_pex.pop(0) + else: + return None + + def can_connect(self): + return self.active_sockets < self.max_sockets + + def connect_queue(self): + if DEBUG: + print >>sys.stderr, "RePEXer: connect_queue: active_sockets: %s" % self.active_sockets + + # We get here from repex_ready, connection_closed or from rerequester_peers. + # First we check whether we can connect, whether we're done, or whether we are closing. + if self.done or self.is_closing or not self.can_connect(): + return + # when we have found sufficient live peers and at least the initial peers are checked, + # we are done and close the remaining connections: + if self.initial_peers_checked() and len(self.live_peers) >= REPEX_SWARMCACHE_SIZE: + # close_all() will result in generate several connection_closed events. + # To prevent reentry of this function, we'll set a flag we check at function entry. + self.is_closing = True + self.encoder.close_all() + assert self.active_sockets == 0 + if self.active_sockets == 0: + self.send_done() + return + + # Connect to peers in the queue + peer = self.next_peer_from_queue() + while peer is not None: + self.connect(peer) + peer = self.next_peer_from_queue() + + # if we didn't connect at all and we have checked all initial peers, we are forced to bootstrap + if self.active_sockets == 0 and self.initial_peers_checked(): + if self.bootstrap_counter == 0: + self.bootstrap() + elif not self.requesting_tracker: + # we have contacted the tracker before and that + # didn't give us any new live peers, so we are + # forced to give up + self.send_done() + + if DEBUG: + print >>sys.stderr, "RePEXer: connect_queue: active_sockets: %s" % self.active_sockets + + def bootstrap(self): + if DEBUG: + print >>sys.stderr, "RePEXer: bootstrap" + self.bootstrap_counter += 1 + if REPEX_DISABLE_BOOTSTRAP or self.rerequest is None: + self.rerequester_peers(None) + return + + # In the future, bootstrap needs to try 2-Hop TorrentSmell first... + # Or, Rerequester needs to modified to incorporate 2-Hop TorrentSmell. + if self.rerequest.trackerlist in [ [], [[]] ]: + # no trackers? + self.rerequester_peers(None) + return + + self.requesting_tracker = True + def tracker_callback(self=self): + if self.requesting_tracker: + # in case of failure, call rerequester_peers with None + self.requesting_tracker = False + self.rerequester_peers(None) + self.rerequest.announce(callback=tracker_callback) + + # + # Get SwarmCache + # + def get_swarmcache(self): + """ + Returns the updated SwarmCache and its timestamp when done (self.done), + otherwise the old SwarmCache and its timestamp. The timestamp is + None when the SwarmCache is empty. + + @return A dict mapping dns to a dict with at least 'last_seen' + and 'pex' keys. If it contains a 'prev'=True key-value pair, the peer + was known to be in the SwarmCache's predecessor. + """ + if self.done: + swarmcache = self.final_peertable + else: + swarmcache = self.starting_peertable + ts = swarmcache_ts(swarmcache) + return swarmcache, ts + + # + # When done (or partially in case of peer shortage) + # + def send_done(self): + self.done = True + self.end_ts = ts_now() + + # Construct the new SwarmCache by removing excess peers + swarmcache = dict(self.live_peers) + to_delete = max(len(swarmcache) - REPEX_SWARMCACHE_SIZE, 0) + deleted = 0 + for dns in swarmcache.keys(): + if deleted == to_delete: + break + if dns not in self.starting_peertable: + del swarmcache[dns] + deleted += 1 + + # TODO: Should we change the shuffle algorithm such that we + # prefer to replace an offline peer with one of the peers + # in its PEX message? + + # create shufflepeers dict, allowing us to deduce why a peer was shuffled out + shufflepeers = {} + for dns in self.starting_peertable: + if dns not in swarmcache: + shufflepeers[dns] = (dns in self.bt_connectable, dns in self.bt_pex, self.starting_peertable[dns].get('last_seen',0)) + + self.final_peertable = swarmcache + for observer in self._observers: + if DEBUG: + print >>sys.stderr, "RePEXer: send_done: calling repex_done on", `observer` + try: + observer.repex_done(self, + swarmcache, + self.shufflecount, + shufflepeers, + self.bootstrap_counter, + self.datacost) + except: + print_exc() + + # + # Informal string representation of a RePEXer + # + def __str__(self): + if self.done and self.aborted: + status = 'ABORTED' + elif self.done: + status = 'DONE' + elif self.ready: + status = 'REPEXING' + else: + status = 'WAITING' + infohash = '[%s]' % b2a_hex(self.infohash) + summary = '' + table = '' + datacost = '' + if self.done and not self.aborted: + infohash = '\n ' + infohash + swarmcache = self.final_peertable + summary = '\n table size/shuffle/bootstrap %s/%s/%s' % (len(swarmcache), self.shufflecount, self.bootstrap_counter) + prev_peers = set(self.starting_peertable.keys()) + cur_peers = set(swarmcache.keys()) + + for dns in sorted(set.symmetric_difference(prev_peers,cur_peers)): + if dns in cur_peers: + table += '\n A: %s:%s' % dns + else: + table += '\n D: %s:%s - BT/PEX %s/%s' % (dns + (dns in self.bt_connectable, dns in self.bt_pex)) + table += '\n' + datacost = ' datacost:\n %s(%s)/%s BT(PEX) connections made, received %s bootstrap peers\n' + datacost %= (self.datacost['connections_made'],self.datacost['pex_connections'], + self.datacost['connection_attempts'],self.datacost['bootstrap_peers']) + for k in self.datacost_bandwidth_keys: + v = self.datacost[k] + datacost += ' %s: %s bytes down / %s bytes up\n' % (k.ljust(16), str(v[0]).rjust(6), str(v[1]).rjust(6)) + + return '' % (status,infohash,summary,table,datacost) + +class RePEXerStatusCallback: + """ + Describes the interface required by RePEXer for status callbacks. + """ + def repex_aborted(self, repexer, dlstatus=None): + """ + Called by network thread. RePEXer calls this method when the + repex task is aborted. It is the propagation of the similarly + named method in RePEXerInterface. + @param repexer RePEXer + @param dlstatus Status of the download when the RePEX mode was + interrupted, or None when unknown. + """ + + def repex_done(self, repexer, swarmcache, shufflecount, shufflepeers, bootstrapcount, datacost): + """ + Called by network thread. RePEXer calls this method when it is done + repexing. + @param repexer RePEXer + @param swarmcache A dict mapping dns to a dict with 'last_seen' and + 'pex' keys. The 'pex' key contains a list of (dns,flags) tuples. + @param shufflecount The number of peers in the old SwarmCache that + were not responding with a PEX message. + @param shufflepeers A dict mapping a shuffle peer's dns to a triple, + indicating (a) whether it sent a BT handshake, (b) whether it supported + ut_pex, and (c) the last time the peer was seen. + @param bootstrapcount The number of times bootstrapping was needed. + @param datacost A dict with keys 'no_pex_support', 'no_pex_msg', + 'pex' and 'other', containing (download,upload) byte tuples, and + keys 'connection_attempts', 'connections_made', 'bootstrap_peers', + containing simple counters. + """ + +# TODO: move this class to a module in Policies +class RePEXScheduler(RePEXerStatusCallback): + """ + The RePEXScheduler periodically requests a list of DownloadStates from + the Session and repexes the stopped downloads in a round robin fashion. + """ + __single = None # used for multithreaded singletons pattern + lock = RLock() + + @classmethod + def getInstance(cls, *args, **kw): + # Singleton pattern with double-checking to ensure that it can only create one object + if cls.__single is None: + cls.lock.acquire() + try: + if cls.__single is None: + cls.__single = cls(*args, **kw) + finally: + cls.lock.release() + return cls.__single + + def __init__(self): + # always use getInstance() to create this object + # ARNOCOMMENT: why isn't the lock used on this read?! + if self.__single != None: + raise RuntimeError, "RePEXScheduler is singleton" + from BaseLib.Core.Session import Session # Circular import fix + self.session = Session.get_instance() + self.lock = RLock() + self.active = False + self.current_repex = None # infohash + self.downloads = {} # infohash -> Download; in order to stop Downloads that are done repexing + self.last_attempts = {} # infohash -> ts; in order to prevent starvation when a certain download + # keeps producing empty SwarmCaches + + + def start(self): + """ Starts the RePEX scheduler. """ + if DEBUG: + print >>sys.stderr, "RePEXScheduler: start" + self.lock.acquire() + try: + if self.active: + return + self.active = True + self.session.set_download_states_callback(self.network_scan) + RePEXer.attach_observer(self) + finally: + self.lock.release() + + def stop(self): + """ Stops the RePEX scheduler. """ + if DEBUG: + print >>sys.stderr, "RePEXScheduler: stop" + self.lock.acquire() + try: + if not self.active: + return + RePEXer.detach_observer(self) + self.active = False + self.session.set_download_states_callback(self.network_stop_repex) + finally: + self.lock.release() + + def network_scan(self, dslist): + """ + Called by network thread. Scans for stopped downloads and stores + them in a queue. + @param dslist List of DownloadStates""" + # TODO: only repex last X Downloads instead of all. + if DEBUG: + print >>sys.stderr, "RePEXScheduler: network_scan: %s DownloadStates" % len(dslist) + self.lock.acquire() + exception = None + try: + try: + if not self.active or self.current_repex is not None: + return -1, False + + now = ts_now() + found_infohash = None + found_download = None + found_age = -1 + for ds in dslist: + download = ds.get_download() + infohash = download.tdef.get_infohash() + debug_msg = None + if DEBUG: + print >>sys.stderr, "RePEXScheduler: network_scan: checking", `download.tdef.get_name_as_unicode()` + if ds.get_status() == DLSTATUS_STOPPED and ds.get_progress()==1.0: + # TODO: only repex finished downloads or also prematurely stopped ones? + age = now - (swarmcache_ts(ds.get_swarmcache()) or 0) + last_attempt_ago = now - self.last_attempts.get(infohash, 0) + + if last_attempt_ago < REPEX_MIN_INTERVAL: + debug_msg = "...too soon to try again, last attempt was %ss ago" % last_attempt_ago + elif age < REPEX_INTERVAL: + debug_msg = "...SwarmCache too fresh: %s seconds" % age + else: + if age >= REPEX_INTERVAL: + debug_msg = "...suitable for RePEX!" + if age > found_age: + found_download = download + found_infohash = infohash + found_age = age + else: + debug_msg = "...not repexable: %s %s%%" % (dlstatus_strings[ds.get_status()], ds.get_progress()*100) + if DEBUG: + print >>sys.stderr, "RePEXScheduler: network_scan:", debug_msg + + if found_download is None: + if DEBUG: + print >>sys.stderr, "RePEXScheduler: network_scan: nothing found yet" + return REPEX_SCAN_INTERVAL, False + else: + if DEBUG: + print >>sys.stderr, "RePEXScheduler: network_scan: found %s, starting RePEX phase." % `found_download.tdef.get_name_as_unicode()` + self.current_repex = found_infohash + self.downloads[found_infohash] = found_download + found_download.set_mode(DLMODE_NORMAL) + found_download.restart(initialdlstatus=DLSTATUS_REPEXING) + return -1, False + except Exception, e: + exception = e + finally: + self.lock.release() + if exception is not None: + # [E0702, RePEXScheduler.network_scan] Raising NoneType + # while only classes, instances or string are allowed + # pylint: disable-msg=E0702 + raise exception + + def network_stop_repex(self, dslist): + """Called by network thread. + @param dslist List of DownloadStates""" + if DEBUG: + print >>sys.stderr, "RePEXScheduler: network_stop_repex:" + for d in [ds.get_download() for ds in dslist if ds.get_status() == DLSTATUS_REPEXING]: + if DEBUG: + print >>sys.stderr, "\t...",`d.tdef.get_name_as_unicode()` + d.stop() + return -1, False + + # + # RePEXerStatusCallback interface (called by network thread) + # + def repex_aborted(self, repexer, dlstatus=None): + if DEBUG: + if dlstatus is None: + status_string = str(None) + else: + status_string = dlstatus_strings[dlstatus] + print >>sys.stderr, "RePEXScheduler: repex_aborted:", b2a_hex(repexer.infohash), status_string + self.current_repex = None + self.last_attempts[repexer.infohash] = ts_now() + self.session.set_download_states_callback(self.network_scan) + + def repex_done(self, repexer, swarmcache, shufflecount, shufflepeers, bootstrapcount, datacost): + if DEBUG: + print >>sys.stderr, 'RePEXScheduler: repex_done: %s\n\ttable size/shuffle/bootstrap %s/%s/%s' % ( + b2a_hex(repexer.infohash), len(swarmcache), shufflecount, bootstrapcount) + self.current_repex = None + self.last_attempts[repexer.infohash] = ts_now() + self.downloads[repexer.infohash].stop() + self.session.set_download_states_callback(self.network_scan) + +# +# Classes for logging/measurement purposes +# + +class RePEXLogger(RePEXerStatusCallback): + """ + For measurement: This class' sole purpose is to log all repex done + messages. + """ + __single = None # used for multithreaded singletons pattern + lock = RLock() + + @classmethod + def getInstance(cls, *args, **kw): + # Singleton pattern with double-checking to ensure that it can only create one object + if cls.__single is None: + cls.lock.acquire() + try: + if cls.__single is None: + cls.__single = cls(*args, **kw) + finally: + cls.lock.release() + return cls.__single + + def __init__(self): + # always use getInstance() to create this object + # ARNOCOMMENT: why isn't the lock used on this read?! + if self.__single != None: + raise RuntimeError, "RePEXLogger is singleton" + self.repexlog = RePEXLogDB.getInstance() + self.active = False + + def start(self): + """ Starts the RePEX logger. """ + if DEBUG: + print >>sys.stderr, "RePEXLogger: start" + self.lock.acquire() + try: + if self.active: + return + self.active = True + RePEXer.attach_observer(self) + finally: + self.lock.release() + + def stop(self): + """ Stops the RePEX logger. """ + if DEBUG: + print >>sys.stderr, "RePEXLogger: stop" + self.lock.acquire() + try: + if not self.active: + return + RePEXer.detach_observer(self) + self.active = False + finally: + self.lock.release() + + # + # RePEXerStatusCallback interface + # + def repex_aborted(self, repexer, dlstatus=None): + if dlstatus is None: + status_string = str(None) + else: + status_string = dlstatus_strings[dlstatus] + if DEBUG: + print >>sys.stderr, "RePEXLogger: repex_aborted:", b2a_hex(repexer.infohash), status_string + + def repex_done(self, repexer, swarmcache, shufflecount, shufflepeers, bootstrapcount, datacost): + if DEBUG: + print >>sys.stderr, 'RePEXLogger: repex_done: %s' % repexer + self.repexlog.storeSwarmCache(repexer.infohash, swarmcache, + (shufflecount,shufflepeers,bootstrapcount,datacost), + timestamp=repexer.ready_ts, endtimestamp=repexer.end_ts, + commit=True) + +class RePEXLogDB: + """ + For measurements, stores the intermediate RePEX results. + """ + __single = None # used for multithreaded singletons pattern + lock = RLock() + PEERDB_FILE = 'repexlog.pickle' + PEERDB_VERSION = '0.7' + MAX_HISTORY = 20480 # let's say 1K per SwarmCache, 20480 would be max 20 MB... + + @classmethod + def getInstance(cls, *args, **kw): + # Singleton pattern with double-checking to ensure that it can only create one object + if cls.__single is None: + cls.lock.acquire() + try: + if cls.__single is None: + cls.__single = cls(*args, **kw) + finally: + cls.lock.release() + return cls.__single + + def __init__(self,session): + # always use getInstance() to create this object + # ARNOCOMMENT: why isn't the lock used on this read?! + if self.__single != None: + raise RuntimeError, "RePEXLogDB is singleton" + #SQLiteCacheDBBase.__init__(self, *args, **kargs) + + state_dir = session.get_state_dir() + self.db = os.path.join(state_dir, self.PEERDB_FILE) + if not os.path.exists(self.db): + self.version = self.PEERDB_VERSION + self.history = [] + else: + import cPickle as pickle + f = open(self.db,'rb') + tuple = pickle.load(f) + self.version, self.history = tuple + f.close() + + def commit(self): + """ + Commits the last changes to file. + """ + self.lock.acquire() + try: + import cPickle as pickle + f = open(self.db,'wb') + pickle.dump((self.version, self.history), f) + f.close() + finally: + self.lock.release() + + def storeSwarmCache(self, infohash, swarmcache, stats = None, timestamp=-1, endtimestamp=-1, commit=False): + """ + Stores the SwarmCache for a given infohash. Does not automatically + commit the changes to file. + @param infohash SHA1 hash of the swarm. + @param swarmcache A dict mapping dns to a dict with at least + 'last_seen' and 'pex' keys. + @param stats (shufflecount, shufflepeers, bootstrapcount, datacost) + quadruple or None. + @param timestamp Optional timestamp, by default -1. Empty SwarmCaches + don't contain any time information at all, so it's useful to explicitly + specify the time when the SwarmCache was created. + @param commit Flag to commit automatically. + """ + if DEBUG: + print >>sys.stderr, 'RePEXLogDB: storeSwarmCache: DEBUG:\n\t%s\n\t%s\n\t%s' % ( + #b2a_hex(infohash), swarmcache, stats) # verbose + b2a_hex(infohash), '', '') # less cluttered + self.lock.acquire() + try: + self.history.append((infohash,swarmcache,stats,timestamp,endtimestamp)) + if len(self.history) > self.MAX_HISTORY: + del self.history[:-self.MAX_HISTORY] + if commit: + self.commit() + finally: + self.lock.release() + + def getHistoryAndCleanup(self): + """ + For measurement purposes, gets the history of all stored SwarmCaches + (infohash, swarmcache, stats). This method clears the history and + commits the empty history to file. + """ + self.lock.acquire() + try: + res = self.history + self.history = [] + self.commit() + return res + finally: + self.lock.release() + + +# +# Manual testing class +# + +class RePEXerTester(RePEXerStatusCallback): + """ + Manual testing class for in the Python REPL. + + Usage: + + >>> from BaseLib.Core.TorrentDef import TorrentDef + >>> from BaseLib.Core.DownloadConfig import * + >>> from BaseLib.Core.DecentralizedTracking.repex import * + >>> tdef = TorrentDef.load('foo.torrent') + >>> dscfg = DownloadStartupConfig() + >>> dscfg.set_dest_dir('/tmp') + >>> r = RePEXerTester() + >>> d = r.stopped_download(tdef,dscfg) + >>> sys.stdout=sys.stderr # optionally + >>> r.test_repex(d) + ... + >>> r.test_repex(d) + ... + >>> r.test_repex(d, swarmcache={('xxx.xxx.xxx.xxx',zzz) : {'last_seen':0, 'pex': []}}) + ... + >>> r.test_repex(d, use_peerdb=True) + ... + + r.repexers[Download] and r.swarmcaches[Download] contain a list of created + repexers and the SwarmCaches they have returned. + """ + def __init__(self): + from BaseLib.Core.Session import Session # Circular import fix + self.session = Session.get_instance() + self.peerdb = RePEXLogDB.getInstance() + self.downloads = {} # infohash -> Download + self.swarmcaches = {} # Download -> [SwarmCache] + self.repexers = {} # Download -> [repexer] + # register as global observer + RePEXer.attach_observer(self) + + def stopped_download(self, tdef, dcfg): + """ + For testing purposes, creates a stopped download given a TorrentDef + and config. + @param tdef A finalized TorrentDef. + @param dcfg DownloadStartupConfig or None, in which case + a new DownloadStartupConfig() is created with its default settings + and the result becomes the runtime config of this Download. + @return Download + """ + d = self.session.start_download(tdef,dcfg) + d.stop() + self.downloads[d.tdef.get_infohash()] = d + return d + + def test_repex(self, download, swarmcache=None): + """ + Performs a RePEX on a stopped Download. + @param download A stopped Download + @param swarmcache Initial SwarmCache to use. If None, the latest + SwarmCache in the Download's pstate will be used. + """ + download.stop() + self.downloads[download.tdef.get_infohash()] = download + if swarmcache is not None: + # Hacking into pstate must happen after network_stop! + def hack_into_pstate(d=download,swarmcache=swarmcache): + d.pstate_for_restart.setdefault('dlstate',{})['swarmcache'] = swarmcache + self.session.lm.rawserver.add_task(hack_into_pstate,0.0) + + download.set_mode(DLMODE_NORMAL) + download.restart(initialdlstatus=DLSTATUS_REPEXING) + + # + # RePEXerStatusCallback interface + # + def repex_aborted(self, repexer, dlstatus=None): + if dlstatus is None: + status_string = str(None) + else: + status_string = dlstatus_strings[dlstatus] + print >>sys.stderr, "RePEXerTester: repex_aborted:", `repexer`,status_string + download = self.downloads[repexer.infohash] + self.repexers.setdefault(download,[]).append(repexer) + self.swarmcaches.setdefault(download,[]).append(None) + + def repex_done(self, repexer, swarmcache, shufflecount, shufflepeers, bootstrapcount, datacost): + download = self.downloads[repexer.infohash] + print >>sys.stderr, 'RePEXerTester: repex_done: %s' % repexer + self.repexers.setdefault(download,[]).append(repexer) + self.swarmcaches.setdefault(download,[]).append(swarmcache) + + # Always log to RePEXLogDB + self.peerdb.storeSwarmCache(repexer.infohash, swarmcache, + (shufflecount,shufflepeers,bootstrapcount,datacost), + timestamp=repexer.ready_ts, endtimestamp=repexer.end_ts, + commit=True) diff --git a/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/ut_pex.py b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/ut_pex.py new file mode 100644 index 0000000..23f9814 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DecentralizedTracking/ut_pex.py @@ -0,0 +1,169 @@ +# Written by Arno Bakker, Bram Cohen +# see LICENSE.txt for license information + +__fool_epydoc = 481 +""" +uTorrent Peer Exchange (PEX) Support: +------------------------------------- +As documented in + https://trac.transmissionbt.com/browser/trunk/extras/extended-messaging.txt + BitTorrent-5.0.8/BitTorrent/Connector.py + (link no longer available) http://transmission.m0k.org/trac/browser/trunk/misc/utorrent.txt + +The PEX message payload is a bencoded dict with three keys: + 'added': the set of peers met since the last PEX + 'added.f': a flag for every peer, apparently with the following values: + \x00: unknown, assuming default + \x01: Prefers encryption (as suggested by LH-ABC-3.2.0/BitTorrent/BT1/Connector.py) + \x02: Is seeder (as suggested by BitTorrent-5.0.8/BitTorrent/Connector.py) + OR-ing them together is allowed as I've seen \x03 values. + 'dropped': the set of peers dropped since last PEX + +03/09/09 Boudewijn: Added a 'is same kind of peer as me' bit to the +'added.f' value. When a Tribler peer send this bit as True this means +'is also a Tribler peer'. + \x04: is same kind of peer + +The mechanism is insecure because there is no way to know if the peer addresses +are really of some peers that are running BitTorrent, or just DoS victims. +For peer addresses that come from trackers we at least know that the peer host +ran BitTorrent and was downloading this swarm (assuming the tracker is trustworthy). + +""" +import sys +from types import DictType,StringType +from BaseLib.Core.BitTornado.BT1.track import compact_peer_info +from BaseLib.Core.BitTornado.bencode import bencode + +EXTEND_MSG_UTORRENT_PEX_ID = chr(1) # Can be any value, the name 'ut_pex' is standardized +EXTEND_MSG_UTORRENT_PEX = 'ut_pex' # note case sensitive + +DEBUG = False + +def create_ut_pex(addedconns,droppedconns,thisconn): + #print >>sys.stderr,"ut_pex: create_ut_pex:",addedconns,droppedconns,thisconn + d = {} + compactedpeerstr = compact_connections(addedconns,thisconn) + d['added'] = compactedpeerstr + flags = '' + for i in range(len(addedconns)): + conn = addedconns[i] + if conn == thisconn: + continue + flag = 0 + if conn.get_extend_encryption(): + flag |= 1 + if conn.download is not None and conn.download.peer_is_complete(): + flag |= 2 + if conn.is_tribler_peer(): + flag |= 4 + + #print >>sys.stderr,"ut_pex: create_ut_pex: add flag",`flag` + flags += chr(flag) + d['added.f'] = flags + compactedpeerstr = compact_connections(droppedconns) + d['dropped'] = compactedpeerstr + return bencode(d) + +def check_ut_pex(d): + if type(d) != DictType: + raise ValueError('ut_pex: not a dict') + + # 'same' peers are peers that indicate (with a bit) that the peer + # in apeers is the same client type as itself. So if the sender of + # the pex message is a Tribler peer the same_apeers will also be + # tribler peers + same_apeers = [] + + apeers = check_ut_pex_peerlist(d,'added') + dpeers = check_ut_pex_peerlist(d,'dropped') + if 'added.f' in d: + addedf = d['added.f'] + if type(addedf) != StringType: + raise ValueError('ut_pex: added.f: not string') + if len(addedf) != len(apeers) and not len(addedf) == 0: + # KTorrent sends an empty added.f, be nice + raise ValueError('ut_pex: added.f: more flags than peers') + + # we need all flags to be integers + addedf = map(ord, addedf) + + # filter out all 'same' peers. the loop runs in reverse order + # so the indexes don't change as we pop them from the apeers + # list + for i in range(min(len(apeers),len(addedf))-1,-1,-1): + if addedf[i] & 4: + same_apeers.append(apeers.pop(i)) + + # for completeness we should also pop the item from + # addedf even though we don't use it anymore + addedf.pop(i) + + # Arno, 2008-09-12: Be liberal in what we receive + ##else: + ##raise ValueError('ut_pex: added.f: missing') + + if DEBUG: + print >>sys.stderr,"ut_pex: Got",apeers + + return (same_apeers,apeers,dpeers) + +def check_ut_pex_peerlist(d,name): + if name not in d: + # Arno, 2008-09-12: Be liberal in what we receive, some clients + # leave out 'dropped' key + ##raise ValueError('ut_pex:'+name+': missing') + return [] + peerlist = d[name] + if type(peerlist) != StringType: + raise ValueError('ut_pex:'+name+': not string') + if len(peerlist) % 6 != 0: + raise ValueError('ut_pex:'+name+': not multiple of 6 bytes') + peers = decompact_connections(peerlist) + for ip,port in peers: + if ip == '127.0.0.1': + raise ValueError('ut_pex:'+name+': address is localhost') + return peers + +def ut_pex_get_conns_diff(currconns,prevconns): + addedconns = [] + droppedconns = [] + for conn in currconns: + if not (conn in prevconns): + # new conn + addedconns.append(conn) + for conn in prevconns: + if not (conn in currconns): + # old conn, was dropped + droppedconns.append(conn) + return (addedconns,droppedconns) + + +def compact_connections(conns,thisconn=None): + """ See BitTornado/BT1/track.py """ + compactpeers = [] + for conn in conns: + if conn == thisconn: + continue + ip = conn.get_ip() + port = conn.get_extend_listenport() + if port is None: + raise ValueError("ut_pex: compact: listen port unknown?!") + else: + compactpeer = compact_peer_info(ip,port) + compactpeers.append(compactpeer) + + # Create compact representation of peers + compactpeerstr = ''.join(compactpeers) + return compactpeerstr + + +def decompact_connections(p): + """ See BitTornado/BT1/Rerequester.py """ + peers = [] + for x in xrange(0, len(p), 6): + ip = '.'.join([str(ord(i)) for i in p[x:x+4]]) + port = (ord(p[x+4]) << 8) | ord(p[x+5]) + peers.append((ip, port)) + return peers + diff --git a/instrumentation/next-share/BaseLib/Core/Download.py b/instrumentation/next-share/BaseLib/Core/Download.py new file mode 100644 index 0000000..19cdcdd --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/Download.py @@ -0,0 +1,174 @@ +# Written by Arno Bakker +# see LICENSE.txt for license information +""" The representation of a running BT download/upload. """ + +import sys +from traceback import print_exc,print_stack + +from BaseLib.Core.simpledefs import * +from BaseLib.Core.defaults import * +from BaseLib.Core.exceptions import * +from BaseLib.Core.Base import * +from BaseLib.Core.APIImplementation.DownloadRuntimeConfig import DownloadRuntimeConfig +from BaseLib.Core.APIImplementation.DownloadImpl import DownloadImpl +from BaseLib.Core.APIImplementation.miscutils import * +from BaseLib.Core.osutils import * + + +class Download(DownloadRuntimeConfig,DownloadImpl): + """ + Representation of a running BT download/upload. + + A Download implements the DownloadConfigInterface which can be used to + change download parameters are runtime (for selected parameters). + + cf. libtorrent torrent_handle + """ + + # + # Internal methods + # + def __init__(self,session,tdef): + """ Internal constructor + @param session Session + @param tdef TorrentDef + """ + DownloadImpl.__init__(self,session,tdef) + # + # Public methods + # + def get_def(self): + """ + Return the read-only torrent definition (TorrentDef) for this Download. + @return A TorrentDef object. + """ + return DownloadImpl.get_def(self) + + + def set_state_callback(self,usercallback,getpeerlist=False): + """ + Set a callback for retrieving the state of the download. This callback + will be called immediately with a DownloadState object as first parameter. + The callback method must return a tuple (when,getpeerlist) where "when" + indicates whether the callback should be called again and represents a + number of seconds from now. If "when" <= 0.0 the callback will not be + called again. "getpeerlist" is a boolean that indicates whether the + DownloadState passed to the callback on the next invocation should + contain info about the set of current peers. + + The callback will be called by a popup thread which can be used + indefinitely (within reason) by the higher level code. + + @param usercallback Function that accepts DownloadState as parameter and + returns a (float,boolean) tuple. + """ + DownloadImpl.set_state_callback(self,usercallback,getpeerlist=getpeerlist) + + + def stop(self): + """ Stops the Download, i.e. closes all connections to other peers. """ + # Called by any thread + DownloadImpl.stop(self) + + def restart(self,initialdlstatus=None): + """ + Restarts the stopped Download. + + @param initialdlstatus An optional parameter to restart the Download in + a specific state. + """ + # Called by any thread + DownloadImpl.restart(self, initialdlstatus) + + # + # Config parameters that only exists at runtime + # + def set_max_desired_speed(self,direct,speed): + """ Sets the maximum desired upload/download speed for this Download. + @param direct The direction (UPLOAD/DOWNLOAD) + @param speed The speed in KB/s. + """ + DownloadImpl.set_max_desired_speed(self,direct,speed) + + def get_max_desired_speed(self,direct): + """ Returns the maximum desired upload/download speed for this Download. + @return The previously set speed in KB/s + """ + return DownloadImpl.get_max_desired_speed(self,direct) + + def get_dest_files(self, exts = None): + """ Returns the filenames on disk to which this Download saves + @return A list of (filename-in-torrent, disk filename) tuples. + """ + return DownloadImpl.get_dest_files(self, exts) + + # + # Cooperative download + # + def ask_coopdl_helpers(self,permidlist): + """ Ask the specified list of peers to help speed up this download """ + # called by any thread + self.dllock.acquire() + try: + # ARNOCOMMENT: WE NEED PERMID+IP FOR COOP DL. How to access DB? Can't + # do it on main thread, can't do it on network thread. + + peerreclist = self.session.lm.peer_db.getPeers(permidlist, ['permid','ip','port']) + + if self.sd is not None: + ask_coopdl_helpers_lambda = lambda:self.sd is not None and self.sd.ask_coopdl_helpers(peerreclist) + self.session.lm.rawserver.add_task(ask_coopdl_helpers_lambda,0) + else: + raise OperationNotPossibleWhenStoppedException() + finally: + self.dllock.release() + + # To retrieve the list of current helpers, see DownloadState + + def stop_coopdl_helpers(self,permidlist): + """ Ask the specified list of peers to stop helping speed up this + download """ + # called by any thread + self.dllock.acquire() + try: + # ARNOCOMMENT: WE NEED PERMID+IP FOR COOP DL. How to access DB? Can't + # do it on main thread, can't do it on network thread. + peerreclist = self.session.lm.peer_db.getPeers(permidlist, ['permid','ip','port']) + + if self.sd is not None: + stop_coopdl_helpers_lambda = lambda:self.sd is not None and self.sd.stop_coopdl_helpers(peerreclist) + self.session.lm.rawserver.add_task(stop_coopdl_helpers_lambda,0) + else: + raise OperationNotPossibleWhenStoppedException() + finally: + self.dllock.release() + +# SelectiveSeeding_ + def set_seeding_policy(self,smanager): + """ Assign the seeding policy to use for this Download. + @param smanager An instance of Tribler.Policies.SeedingManager + """ + self.dllock.acquire() + try: + if self.sd is not None: + set_seeding_smanager_lambda = lambda:self.sd is not None and self.sd.get_bt1download().choker.set_seeding_manager(smanager) + self.session.lm.rawserver.add_task(set_seeding_smanager_lambda,0) + else: + raise OperationNotPossibleWhenStoppedException() + finally: + self.dllock.release() +# _SelectiveSeeding + + def get_peer_id(self): + """ Return the BitTorrent peer ID used by this Download, or None, when + the download is STOPPED. + @return 20-byte peer ID. + """ + self.dllock.acquire() + try: + if self.sd is not None: + return self.sd.peerid + else: + return None + finally: + self.dllock.release() diff --git a/instrumentation/next-share/BaseLib/Core/DownloadConfig.py b/instrumentation/next-share/BaseLib/Core/DownloadConfig.py new file mode 100644 index 0000000..8151991 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DownloadConfig.py @@ -0,0 +1,885 @@ +# Written by Arno Bakker, George Milescu +# see LICENSE.txt for license information +""" Controls how a TorrentDef is downloaded (rate, where on disk, etc.) """ + +# +# WARNING: When extending this class: +# +# 1. Add a JavaDoc description for each method you add. +# 2. Also add the methods to APIImplementation/DownloadRuntimeConfig.py +# 3. Document your changes in API.py +# +# + +import sys +import os +#import time +import copy +import pickle +from types import StringType + +from BaseLib.Core.simpledefs import * +from BaseLib.Core.defaults import * +from BaseLib.Core.exceptions import * +from BaseLib.Core.Base import * +from BaseLib.Core.APIImplementation.miscutils import * + +from BaseLib.Core.osutils import getfreespace, get_desktop_dir + + +class DownloadConfigInterface: + """ + (key,value) pair config of per-torrent runtime parameters, + e.g. destdir, file-allocation policy, etc. Also options to advocate + torrent, e.g. register in DHT, advertise via Buddycast. + + Use DownloadStartupConfig to manipulate download configs before download + startup time. This is just a parent class. + + cf. libtorrent torrent_handle + """ + def __init__(self,dlconfig=None): + + if dlconfig is not None: # copy constructor + self.dlconfig = dlconfig + return + + self.dlconfig = {} + + # Define the built-in default here + self.dlconfig.update(dldefaults) + + self.dlconfig['saveas'] = get_default_dest_dir() + + + def set_dest_dir(self,path): + """ Sets the directory where to save this Download. + @param path A path of a directory. + """ + self.dlconfig['saveas'] = path + + def get_dest_dir(self): + """ Gets the directory where to save this Download. + """ + return self.dlconfig['saveas'] + + def set_video_event_callback(self,usercallback,dlmode=DLMODE_VOD): + """ Download the torrent in Video-On-Demand mode or as live stream. + When a playback event occurs, the usercallback function will be + called, with the following list of arguments: + 
+            Download,event,params
+
+ In which event is a string, and params a dictionary. The following + events are supported: + 
+        VODEVENT_START:
+            The params dictionary will contain the fields
+        
+                mimetype,stream,filename,length,bitrate
+        
+            If the filename is set, the video can be read from there. If not,
+            the video can be read from the stream, which is a file-like object 
+            supporting the read(),seek(), and close() operations. The MIME type
+            of the video is given by "mimetype", the length of the stream in
+            bytes by "length" which may be None if the length is unknown (e.g.
+            when live streaming). bitrate is either the bitrate as specified
+            in the TorrentDef, or if that was lacking an dynamic estimate 
+            calculated using the videoanalyser (e.g. ffmpeg), see
+            SessionConfig.set_video_analyser_path()
+        
+            To fetch a specific file from a multi-file torrent, use the 
+            set_selected_files() method. This method sets the mode to DLMODE_VOD 
+
+        VODEVENT_PAUSE:
+            The download engine would like video playback to be paused as the
+            data is not coming in fast enough / the data due is not available
+            yet.
+            
+            The params dictionary contains the fields
+            
+                autoresume
+                
+            "autoresume" indicates whether or not the Core will generate
+            a VODEVENT_RESUME when it is ready again, or that this is left
+            to the core user.
+                    
+        VODEVENT_RESUME:
+            The download engine would like video playback to resume.
+
+ The usercallback should ignore events it does not support. + + The usercallback will be called by a popup thread which can be used + indefinitely (within reason) by the higher level code. + + @param usercallback A function with the above signature. + @param dlmode The download mode to start in (_VOD or _SVC) + """ + self.dlconfig['mode'] = dlmode + self.dlconfig['vod_usercallback'] = usercallback + + + def set_video_events(self,events=[]): + """ Sets which events will be supported with the usercallback set + by set_video_event_callback. Supporting the VODEVENT_START event is + mandatory, and can therefore be omitted from the list. + + @param events A list of supported VODEVENT_* events. + """ + + # create a copy to avoid loosing the info + self.dlconfig['vod_userevents'] = events[:] + + def set_video_source(self,videosource,authconfig=None,restartstatefilename=None): + """ Provides the live video source for this torrent from an external + source. + + @param videosource A file-like object providing the live video stream + (i.e., supports read() and close()) + @param authconfig The key information for source authentication of + packets. See LiveSourceAuthConfig and TorrentDef.create_live_torrent() + @param restartstatefilename A filename to read/write state needed for a + graceful restart of the source. + """ + self.dlconfig['video_source'] = videosource + if authconfig is None: + from BaseLib.Core.LiveSourceAuthConfig import LiveSourceAuthConfig + + authconfig = LiveSourceAuthConfig(LIVE_AUTHMETHOD_NONE) + self.dlconfig['video_source_authconfig'] = authconfig + self.dlconfig['video_source_restartstatefilename'] = restartstatefilename + + def set_video_ratelimit(self,ratelimit): + """ Sets a limit on the speed at which the video stream is to be read. + Useful when creating a live stream from file or any other faster-than-live + data stream. + + @param ratelimit The maximum speed at which to read from the stream (bps) + """ + self.dlconfig['video_ratelimit'] = ratelimit + + def set_mode(self,mode): + """ Sets the mode of this download. + @param mode DLMODE_NORMAL/DLMODE_VOD """ + self.dlconfig['mode'] = mode + + def set_live_aux_seeders(self,seeders): + """ Sets a number of live seeders, auxiliary servers that + get high priority at the source server to distribute its content + to others. + @param seeders A list of [IP address,port] lists. + """ + self.dlconfig['live_aux_seeders'] = seeders + + def get_mode(self): + """ Returns the mode of this download. + @return DLMODE_NORMAL/DLMODE_VOD """ + return self.dlconfig['mode'] + + def get_video_event_callback(self): + """ Returns the function that was passed to set_video_event_callback(). + @return A function. + """ + return self.dlconfig['vod_usercallback'] + + def get_video_events(self): + """ Returns the function that was passed to set_video_events(). + @return A list of events. + """ + return self.dlconfig['vod_userevents'] + + def get_video_source(self): + """ Returns the object that was passed to set_video_source(). + @return A file-like object. + """ + return self.dlconfig['video_source'] + + def get_video_ratelimit(self): + """ Returns the speed at which the video stream is read (bps). + @return An integer. + """ + return self.dlconfig['video_ratelimit'] + + def get_live_aux_seeders(self): + """ Returns the aux. live seeders set. + @return A list of [IP address,port] lists. """ + return self.dlconfig['live_aux_seeders'] + + + def set_selected_files(self,files): + """ Select which files in the torrent to download. The filenames must + be the names as they appear in the torrent def. Trivially, when the + torrent contains a file 'sjaak.avi' the files parameter must + be 'sjaak.avi'. When the torrent contains multiple files and is named + 'filecollection', the files parameter must be + os.path.join('filecollection','sjaak.avi') + + @param files Can be a single filename or a list of filenames (e.g. + ['harry.avi','sjaak.avi']). + """ + # TODO: can't check if files exists, don't have tdef here.... bugger + if type(files) == StringType: # convenience + files = [files] + + if self.dlconfig['mode'] == DLMODE_VOD and len(files) > 1: + raise ValueError("In Video-On-Demand mode only 1 file can be selected for download") + + # Ric: added svc case + elif self.dlconfig['mode'] == DLMODE_SVC and len(files) < 2: + raise ValueError("In SVC Video-On-Demand mode at least 2 files have to be selected for download") + + + self.dlconfig['selected_files'] = files + + + def get_selected_files(self): + """ Returns the list of files selected for download. + @return A list of strings. """ + return self.dlconfig['selected_files'] + + + + # + # Common download performance parameters + # + def set_max_speed(self,direct,speed): + """ Sets the maximum upload or download speed for this Download. + @param direct The direction (UPLOAD/DOWNLOAD) + @param speed The speed in KB/s. + """ + if direct == UPLOAD: + self.dlconfig['max_upload_rate'] = speed + else: + self.dlconfig['max_download_rate'] = speed + + def get_max_speed(self,direct): + """ Returns the configured maximum speed. + Returns the speed in KB/s. """ + if direct == UPLOAD: + return self.dlconfig['max_upload_rate'] + else: + return self.dlconfig['max_download_rate'] + + def set_max_conns_to_initiate(self,nconns): + """ Sets the maximum number of connections to initiate for this + Download. + @param nconns A number of connections. + """ + self.dlconfig['max_initiate'] = nconns + + def get_max_conns_to_initiate(self): + """ Returns the configured maximum number of connections to initiate. + @return A number of connections. + """ + return self.dlconfig['max_initiate'] + + def set_max_conns(self,nconns): + """ Sets the maximum number of connections to connections for this + Download. + @param nconns A number of connections. + """ + self.dlconfig['max_connections'] = nconns + + def get_max_conns(self): + """ Returns the configured maximum number of connections. + @return A number of connections. + """ + return self.dlconfig['max_connections'] + + # + # ProxyService_ parameters + # + def get_coopdl_role(self): + """ Returns the role which the download plays in a cooperative download, + 
+        - COOPDL_ROLE_COORDINATOR: other peers help this download
+        - COOPDL_ROLE_HELPER: this download helps another peer download faster.
+
+ The default is coordinator, and it is set to helper by the + set_coopdl_coordinator_permid() method. + """ + return self.dlconfig['coopdl_role'] + + def set_coopdl_coordinator_permid(self,permid): + """ Calling this method makes this download a helper in a cooperative + download, helping the peer identified by the specified permid. This peer + acts as coordinator, telling this download which parts of the content + to download. + @param permid A PermID. + """ + self.dlconfig['coopdl_role'] = COOPDL_ROLE_HELPER + self.dlconfig['coopdl_coordinator_permid'] = permid + + def get_coopdl_coordinator_permid(self): + """ Returns the configured coordinator permid. + @return A PermID + """ + return self.dlconfig['coopdl_coordinator_permid'] + + # See DownloadRuntime config for adding, removing and getting list of + # helping peers. + + def set_proxy_mode(self,value): + """ Set the proxymode for current download + . + @param value: the proxyservice mode: PROXY_MODE_OFF, PROXY_MODE_PRIVATE or PROXY_MODE_SPEED + """ + if value == PROXY_MODE_OFF or value == PROXY_MODE_PRIVATE or value == PROXY_MODE_SPEED: + self.dlconfig['proxy_mode'] = value + else: + # If the method is called with an incorrect value, turn off the ProxyMode for this download + self.dlconfig['proxy_mode'] = PROXY_MODE_OFF + + def get_proxy_mode(self): + """ Returns the proxymode of the client. + @return: one of the possible three values: PROXY_MODE_OFF, PROXY_MODE_PRIVATE, PROXY_MODE_SPEED + """ + return self.dlconfig['proxy_mode'] + + def set_no_helpers(self,value): + """ Set the maximum number of helpers used for a download. + @param value: a positive integer number + """ + if value >= 0: + self.dlconfig['max_helpers'] = value + else: + self.dlconfig['max_helpers'] = 0 + + def get_no_helpers(self): + """ Returns the maximum number of helpers used for a download. + @return: a positive integer number + """ + return self.dlconfig['max_helpers'] + # + # _ProxyService + # + + + # + # Advanced download parameters + # + def set_max_uploads(self,value): + """ Set the maximum number of uploads to allow at once. + @param value A number. + """ + self.dlconfig['max_uploads'] = value + + def get_max_uploads(self): + """ Returns the maximum number of uploads. + @return A number. """ + return self.dlconfig['max_uploads'] + + def set_keepalive_interval(self,value): + """ Set the number of seconds to pause between sending keepalives. + @param value An interval """ + self.dlconfig['keepalive_interval'] = value + + def get_keepalive_interval(self): + """ Returns the keepalive interval. + @return A number of seconds. """ + return self.dlconfig['keepalive_interval'] + + def set_download_slice_size(self,value): + """ Set how many bytes to query for per request. + @param value A number of bytes. + """ + self.dlconfig['download_slice_size'] = value + + def get_download_slice_size(self): + """ Returns the number of bytes to query per request. + @return A number of bytes. """ + return self.dlconfig['download_slice_size'] + + def set_upload_unit_size(self,value): + """ When limiting upload rate, how many bytes to send at a time. + @param value A number of bytes. """ + self.dlconfig['upload_unit_size'] = value + + def get_upload_unit_size(self): + """ Returns the set upload unit size. + @returns A number of bytes. + """ + return self.dlconfig['upload_unit_size'] + + def set_request_backlog(self,value): + """ Maximum number of requests to keep in a single pipe at once. + @param value A number of requests. + """ + self.dlconfig['request_backlog'] = value + + def get_request_backlog(self): + """ Returns the request backlog. + @return A number of requests. + """ + return self.dlconfig['request_backlog'] + + def set_max_message_length(self,value): + """ Maximum message-length prefix to accept over the wire - larger + values get the connection dropped. + @param value A number of bytes. + """ + self.dlconfig['max_message_length'] = value + + def get_max_message_length(self): + """ Returns the maximum message length that is accepted. + @return A number of bytes. + """ + return self.dlconfig['max_message_length'] + + def set_max_slice_length(self,value): + """ Maximum length slice to send to peers, larger requests are ignored. + @param value A number of bytes. + """ + self.dlconfig['max_slice_length'] = value + + def get_max_slice_length(self): + """ Returns the maximum slice length that is accepted. + @return A number of bytes. + """ + return self.dlconfig['max_slice_length'] + + def set_max_rate_period(self,value): + """ Maximum amount of time to guess the current rate estimate. + @param value A number of seconds. """ + self.dlconfig['max_rate_period'] = value + + def get_max_rate_period(self): + """ Returns the maximum rate period. + @return A number of seconds. + """ + return self.dlconfig['max_rate_period'] + + def set_upload_rate_fudge(self,value): + """ Time equivalent of writing to kernel-level TCP buffer, for rate + adjustment. + @param value A number of seconds. + """ + self.dlconfig['upload_rate_fudge'] = value + + def get_upload_rate_fudge(self): + """ Returns the upload rate fudge. + @return A number of seconds. + """ + return self.dlconfig['upload_rate_fudge'] + + def set_tcp_ack_fudge(self,value): + """ How much TCP ACK download overhead to add to upload rate + calculations. I.e. when a message is received we add X percent + of this message to our upload rate to account for TCP ACKs that + were sent during the reception process. (0 = disabled) + @param value A percentage + """ + self.dlconfig['tcp_ack_fudge'] = value + + def get_tcp_ack_fudge(self): + """ Returns the TCP ACK fudge. + @return A percentage. + """ + return self.dlconfig['tcp_ack_fudge'] + + def set_rerequest_interval(self,value): + """ Time to wait between requesting more peers from tracker. + @param value An interval in seconds. + """ + self.dlconfig['rerequest_interval'] = value + + def get_rerequest_interval(self): + """ Returns the tracker re-request interval. + @return A number of seconds. + """ + return self.dlconfig['rerequest_interval'] + + def set_min_peers(self,value): + """ Minimum number of peers to not do rerequesting. + @param value A number of peers. + """ + self.dlconfig['min_peers'] = value + + def get_min_peers(self): + """ Returns the minimum number of peers. + @return A number of peers. + """ + return self.dlconfig['min_peers'] + + def set_http_timeout(self,value): + """ Number of seconds to wait before assuming that a HTTP connection + has timed out. + @param value A number of seconds. + """ + self.dlconfig['http_timeout'] = value + + def get_http_timeout(self): + """ Returns the HTTP timeout. + @return A number of seconds. + """ + return self.dlconfig['http_timeout'] + + def set_check_hashes(self,value): + """ Whether to check the integrity of the data on disk using the + hashes from the torrent definition. + @param value Boolean + """ + self.dlconfig['check_hashes'] = value + + def get_check_hashes(self): + """ Returns whether to check hashes. + @return Boolean. """ + return self.dlconfig['check_hashes'] + + def set_alloc_type(self,value): + """ Set disk-allocation type: + 
+        * DISKALLOC_NORMAL:  Allocates space as data is received
+        * DISKALLOC_BACKGROUND: Also adds space in the background
+        * DISKALLOC_PREALLOCATE: Reserves space up front (slow)
+        * DISKALLOC_SPARSE: Is only for filesystems that support it by default 
+          (UNIX)
+
+ @param value A DISKALLOC_* policy. + """ + self.dlconfig['alloc_type'] = value + + def get_alloc_type(self): + """ Returns the disk-allocation policy. + @return DISKALLOC_* + """ + return self.dlconfig['alloc_type'] + + def set_alloc_rate(self,value): + """ Set the rate to allocate space at using background + allocation (DISKALLOC_BACKGROUND). + + @param value A rate in MB/s. + """ + self.dlconfig['alloc_rate'] = value + + def get_alloc_rate(self): + """ Returns the background disk-allocation rate. + @return A number of megabytes per second. + """ + return self.dlconfig['alloc_rate'] + + def set_buffer_reads(self,value): + """ Whether to buffer disk reads. + @param value Boolean + """ + self.dlconfig['buffer_reads'] = value + + def get_buffer_reads(self): + """ Returns whether to buffer reads. + @return Boolean. """ + return self.dlconfig['buffer_reads'] + + def set_write_buffer_size(self,value): + """ The maximum amount of space to use for buffering disk writes + (0 = disabled). + @param value A buffer size in megabytes. + """ + self.dlconfig['write_buffer_size'] = value + + def get_write_buffer_size(self): + """ Returns the write buffer size. + @return A number of megabytes. + """ + return self.dlconfig['write_buffer_size'] + + def set_breakup_seed_bitfield(self,value): + """ Whether to send an incomplete BITFIELD and then fills with HAVE + messages, in order to get around intellectually-challenged Internet + Service Provider manipulation. + @param value Boolean + """ + self.dlconfig['breakup_seed_bitfield'] = value + + def get_breakup_seed_bitfield(self): + """ Returns whether to send an incomplete BITFIELD message. + @return Boolean. """ + return self.dlconfig['breakup_seed_bitfield'] + + def set_snub_time(self,value): + """ Seconds to wait for data to come in over a connection before + assuming it's semi-permanently choked. + @param value A number of seconds. + """ + self.dlconfig['snub_time'] = value + + def get_snub_time(self): + """ Returns the snub time. + @return A number of seconds. + """ + return self.dlconfig['snub_time'] + + def set_rarest_first_cutoff(self,value): + """ Number of downloads at which to switch from random to rarest first. + @param value A number of downloads. + """ + self.dlconfig['rarest_first_cutoff'] = value + + def get_rarest_first_cutoff(self): + """ Returns the rarest first cutoff. + @return A number of downloads. + """ + return self.dlconfig['rarest_first_cutoff'] + + def set_rarest_first_priority_cutoff(self,value): + """ The number of peers which need to have a piece before other + partials take priority over rarest first policy. + @param value A number of peers. + """ + self.dlconfig['rarest_first_priority_cutoff'] = value + + def get_rarest_first_priority_cutoff(self): + """ Returns the rarest-first priority cutoff. + @return A number of peers. """ + return self.dlconfig['rarest_first_priority_cutoff'] + + def set_min_uploads(self,value): + """ The number of uploads to fill out to with extra optimistic unchokes. + @param value A number of uploads. + """ + self.dlconfig['min_uploads'] = value + + def get_min_uploads(self): + """ Returns the minimum number of uploads. + @return A number of uploads. """ + return self.dlconfig['min_uploads'] + + def set_max_files_open(self,value): + """ The maximum number of files to keep open at a time, 0 means no + limit. + @param value A number of files. + """ + self.dlconfig['max_files_open'] = value + + def get_max_files_open(self): + """ Returns the maximum number of open files. + @return A number of files. """ + return self.dlconfig['max_files_open'] + + def set_round_robin_period(self,value): + """ The number of seconds between the client's switching upload targets. + @param value A number of seconds. + """ + self.dlconfig['round_robin_period'] = value + + def get_round_robin_period(self): + """ Returns the round-robin period. + @return A number of seconds. """ + return self.dlconfig['round_robin_period'] + + def set_super_seeder(self,value): + """ whether to use special upload-efficiency-maximizing routines (only + for dedicated seeds). + @param value Boolean + """ + self.dlconfig['super_seeder'] = value + + def get_super_seeder(self): + """ Returns hether super seeding is enabled. + @return Boolean. """ + return self.dlconfig['super_seeder'] + + def set_security(self,value): + """ Whether to enable extra security features intended to prevent abuse, + such as checking for multiple connections from the same IP address. + @param value Boolean + """ + self.dlconfig['security'] = value + + def get_security(self): + """ Returns the security setting. + @return Boolean. """ + return self.dlconfig['security'] + + def set_auto_kick(self,value): + """ Whether to automatically kick/ban peers that send bad data. + @param value Boolean + """ + self.dlconfig['auto_kick'] = value + + def get_auto_kick(self): + """ Returns whether autokick is enabled. + @return Boolean. """ + return self.dlconfig['auto_kick'] + + def set_double_check_writes(self,value): + """ Whether to double-check data being written to the disk for errors + (may increase CPU load). + @param value Boolean + """ + self.dlconfig['double_check'] = value + + def get_double_check_writes(self): + """ Returns whether double-checking on writes is enabled. """ + return self.dlconfig['double_check'] + + def set_triple_check_writes(self,value): + """ Whether to thoroughly check data being written to the disk (may + slow disk access). + @param value Boolean """ + self.dlconfig['triple_check'] = value + + def get_triple_check_writes(self): + """ Returns whether triple-checking on writes is enabled. """ + return self.dlconfig['triple_check'] + + def set_lock_files(self,value): + """ Whether to lock files the Download is working with. + @param value Boolean """ + self.dlconfig['lock_files'] = value + + def get_lock_files(self): + """ Returns whether locking of files is enabled. """ + return self.dlconfig['lock_files'] + + def set_lock_while_reading(self,value): + """ Whether to lock access to files being read. + @param value Boolean + """ + self.dlconfig['lock_while_reading'] = value + + def get_lock_while_reading(self): + """ Returns whether locking of files for reading is enabled. + @return Boolean. """ + return self.dlconfig['lock_while_reading'] + + def set_auto_flush(self,value): + """ Minutes between automatic flushes to disk (0 = disabled). + @param value A number of minutes. + """ + self.dlconfig['auto_flush'] = value + + def get_auto_flush(self): + """ Returns the auto flush interval. + @return A number of minutes. """ + return self.dlconfig['auto_flush'] + + def set_exclude_ips(self,value): + """ Set a list of IP addresses to be excluded. + @param value A list of IP addresses in dotted notation. + """ + self.dlconfig['exclude_ips'] = value + + def get_exclude_ips(self): + """ Returns the list of excluded IP addresses. + @return A list of strings. """ + return self.dlconfig['exclude_ips'] + + def set_ut_pex_max_addrs_from_peer(self,value): + """ Maximum number of addresses to accept from peer via the uTorrent + Peer Exchange extension (0 = disable PEX) + @param value A number of IP addresses. + """ + self.dlconfig['ut_pex_max_addrs_from_peer'] = value + + def get_ut_pex_max_addrs_from_peer(self): + """ Returns the maximum number of IP addresses to accept from a peer + via ut_pex. + @return A number of addresses. + """ + return self.dlconfig['ut_pex_max_addrs_from_peer'] + + def set_poa(self, poa): + if poa: + from base64 import encodestring + self.dlconfig['poa'] = encodestring(poa.serialize()).replace("\n","") + import sys + print >> sys.stderr,"POA is set:",self.dlconfig['poa'] + + def get_poa(self): + if 'poa' in self.dlconfig: + if not self.dlconfig['poa']: + raise Exception("No POA specified") + from BaseLib.Core.ClosedSwarm import ClosedSwarm + from base64 import decodestring + print >> sys.stderr,"get_poa:",self.dlconfig['poa'] + poa = ClosedSwarm.POA.deserialize(decodestring(self.dlconfig['poa'])) + return poa + return None + + + def set_same_nat_try_internal(self,value): + """ Whether to try to detect if a peer is behind the same NAT as + this Session and then establish a connection over the internal + network + @param value Boolean + """ + self.dlconfig['same_nat_try_internal'] = value + + def get_same_nat_try_internal(self): + """ Returns whether same NAT detection is enabled. + @return Boolean """ + return self.dlconfig['same_nat_try_internal'] + + def set_unchoke_bias_for_internal(self,value): + """ Amount to add to unchoke score for peers on the internal network. + @param value A number + """ + self.dlconfig['unchoke_bias_for_internal'] = value + + def get_unchoke_bias_for_internal(self): + """ Returns the bias for peers on the internal network. + @return A number + """ + return self.dlconfig['unchoke_bias_for_internal'] + + +class DownloadStartupConfig(DownloadConfigInterface,Serializable,Copyable): + """ + (key,value) pair config of per-torrent runtime parameters, + e.g. destdir, file-allocation policy, etc. Also options to advocate + torrent, e.g. register in DHT, advertise via Buddycast. + + cf. libtorrent torrent_handle + """ + def __init__(self,dlconfig=None): + """ Normal constructor for DownloadStartupConfig (copy constructor + used internally) """ + DownloadConfigInterface.__init__(self,dlconfig) + # + # Class method + # + def load(filename): + """ + Load a saved DownloadStartupConfig from disk. + + @param filename An absolute Unicode filename + @return DownloadStartupConfig object + """ + # Class method, no locking required + f = open(filename,"rb") + dlconfig = pickle.load(f) + dscfg = DownloadStartupConfig(dlconfig) + f.close() + return dscfg + load = staticmethod(load) + + def save(self,filename): + """ Save the DownloadStartupConfig to disk. + @param filename An absolute Unicode filename + """ + # Called by any thread + f = open(filename,"wb") + pickle.dump(self.dlconfig,f) + f.close() + + # + # Copyable interface + # + def copy(self): + config = copy.copy(self.dlconfig) + return DownloadStartupConfig(config) + + +def get_default_dest_dir(): + """ Returns the default dir to save content to. + 
 
+    * For Win32/MacOS: Desktop\TriblerDownloads
+    * For UNIX: 
+        If Desktop exists: Desktop\TriblerDownloads
+        else: Home\TriblerDownloads
+
+ """ + uhome = get_desktop_dir() + return os.path.join(uhome,u'TriblerDownloads') + diff --git a/instrumentation/next-share/BaseLib/Core/DownloadState.py b/instrumentation/next-share/BaseLib/Core/DownloadState.py new file mode 100644 index 0000000..990ff10 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/DownloadState.py @@ -0,0 +1,386 @@ +# Written by Arno Bakker +# see LICENSE.txt for license information +""" Contains a snapshot of the state of the Download at a specific point in time. """ +import time + +import sys +from traceback import print_exc,print_stack + +from BaseLib.Core.simpledefs import * +from BaseLib.Core.defaults import * +from BaseLib.Core.exceptions import * +from BaseLib.Core.Base import * +from BaseLib.Core.DecentralizedTracking.repex import REPEX_SWARMCACHE_SIZE + +DEBUG = False + +class DownloadState(Serializable): + """ + Contains a snapshot of the state of the Download at a specific + point in time. Using a snapshot instead of providing live data and + protecting access via locking should be faster. + + cf. libtorrent torrent_status + """ + def __init__(self,download,status,error,progress,stats=None,filepieceranges=None,logmsgs=None,coopdl_helpers=[],coopdl_coordinator=None,peerid=None,videoinfo=None,swarmcache=None): + """ Internal constructor. + @param download The Download this state belongs too. + @param status The status of the Download (DLSTATUS_*) + @param progress The general progress of the Download. + @param stats The BT engine statistics for the Download. + @param filepieceranges The range of pieces that we are interested in. + The get_pieces_complete() returns only completeness information about + this range. This is used for playing a video in a multi-torrent file. + @param logmsgs A list of messages from the BT engine which may be of + """ + # Raynor Vliegendhart, TODO: documentation of DownloadState seems incomplete? + # RePEX: @param swarmcache The latest SwarmCache known by Download. This + # cache will be used when the download is not running. + # RePEX TODO: instead of being passed the latest SwarmCache, DownloadState could + # also query it from Download? Perhaps add get_swarmcache to Download(Impl)? + + self.download = download + self.filepieceranges = filepieceranges # NEED CONC CONTROL IF selected_files RUNTIME SETABLE + self.logmsgs = logmsgs + self.coopdl_helpers = coopdl_helpers + self.coopdl_coordinator = coopdl_coordinator + + # RePEX: stored swarmcache from Download and store current time + if swarmcache is not None: + self.swarmcache = dict(swarmcache) + else: + self.swarmcache = None + self.time = time.time() + + if stats is None: + # No info available yet from download engine + self.error = error # readonly access + self.progress = progress + if self.error is not None: + self.status = DLSTATUS_STOPPED_ON_ERROR + else: + self.status = status + self.stats = None + elif error is not None: + self.error = error # readonly access + self.progress = 0.0 # really want old progress + self.status = DLSTATUS_STOPPED_ON_ERROR + self.stats = None + elif status is not None and status != DLSTATUS_REPEXING: + # For HASHCHECKING and WAITING4HASHCHECK + self.error = error + self.status = status + if self.status == DLSTATUS_WAITING4HASHCHECK: + self.progress = 0.0 + else: + self.progress = stats['frac'] + self.stats = None + else: + # Copy info from stats + self.error = None + self.progress = stats['frac'] + if stats['frac'] == 1.0: + self.status = DLSTATUS_SEEDING + else: + self.status = DLSTATUS_DOWNLOADING + #print >>sys.stderr,"STATS IS",stats + + # Safe to store the stats dict. The stats dict is created per + # invocation of the BT1Download returned statsfunc and contains no + # pointers. + # + self.stats = stats + + # for pieces complete + statsobj = self.stats['stats'] + if self.filepieceranges is None: + self.haveslice = statsobj.have # is copy of network engine list + else: + # Show only pieces complete for the selected ranges of files + totalpieces =0 + for t,tl,f in self.filepieceranges: + diff = tl-t + totalpieces += diff + + #print >>sys.stderr,"DownloadState: get_pieces_complete",totalpieces + + haveslice = [False] * totalpieces + haveall = True + index = 0 + for t,tl,f in self.filepieceranges: + for piece in range(t,tl): + haveslice[index] = statsobj.have[piece] + if haveall and haveslice[index] == False: + haveall = False + index += 1 + self.haveslice = haveslice + if haveall and len(self.filepieceranges) > 0: + # we have all pieces of the selected files + self.status = DLSTATUS_SEEDING + self.progress = 1.0 + + # RePEX: REPEXING status overrides SEEDING/DOWNLOADING status. + if status is not None and status == DLSTATUS_REPEXING: + self.status = DLSTATUS_REPEXING + + + def get_download(self): + """ Returns the Download object of which this is the state """ + return self.download + + def get_progress(self): + """ The general progress of the Download as a percentage. When status is + * DLSTATUS_HASHCHECKING it is the percentage of already downloaded + content checked for integrity. + * DLSTATUS_DOWNLOADING/SEEDING it is the percentage downloaded. + @return Progress as a float (0..1). + """ + return self.progress + + def get_status(self): + """ Returns the status of the torrent. + @return DLSTATUS_* """ + return self.status + + def get_error(self): + """ Returns the Exception that caused the download to be moved to + DLSTATUS_STOPPED_ON_ERROR status. + @return Exception + """ + return self.error + + # + # Details + # + def get_current_speed(self,direct): + """ + Returns the current up or download speed. + @return The speed in KB/s, as float. + """ + if self.stats is None: + return 0.0 + if direct == UPLOAD: + return self.stats['up']/1024.0 + else: + return self.stats['down']/1024.0 + + def get_total_transferred(self,direct): + """ + Returns the total amount of up or downloaded bytes. + @return The amount in bytes. + """ + if self.stats is None: + return 0L + # self.stats: BitTornado.BT1.DownloaderFeedback.py (return from gather method) + # self.stats["stats"]: BitTornado.BT1.Statistics.py (Statistics_Response instance) + if direct == UPLOAD: + return self.stats['stats'].upTotal + else: + return self.stats['stats'].downTotal + + def get_eta(self): + """ + Returns the estimated time to finish of download. + @return The time in ?, as ?. + """ + if self.stats is None: + return 0.0 + else: + return self.stats['time'] + + def get_num_peers(self): + """ + Returns the download's number of active connections. This is used + to see if there is any progress when non-fatal errors have occured + (e.g. tracker timeout). + @return An integer. + """ + if self.stats is None: + return 0 + + # Determine if we need statsobj to be requested, same as for spew + statsobj = self.stats['stats'] + return statsobj.numSeeds+statsobj.numPeers + + def get_num_seeds_peers(self): + """ + Returns the sum of the number of seeds and peers. This function + works only if the Download.set_state_callback() / + Session.set_download_states_callback() was called with the getpeerlist + parameter set to True, otherwise returns (None,None) + @return A tuple (num seeds, num peers) + """ + if self.stats is None or self.stats['spew'] is None: + return (None,None) + + total = len(self.stats['spew']) + seeds = len([i for i in self.stats['spew'] if i['completed'] == 1.0]) + return seeds, total-seeds + + def get_pieces_complete(self): + """ Returns a list of booleans indicating whether we have completely + received that piece of the content. The list of pieces for which + we provide this info depends on which files were selected for download + using DownloadStartupConfig.set_selected_files(). + @return A list of booleans + """ + if self.stats is None: + return [] + else: + return self.haveslice + + def get_vod_prebuffering_progress(self): + """ Returns the percentage of prebuffering for Video-On-Demand already + completed. + @return A float (0..1) """ + if self.stats is None: + if self.status == DLSTATUS_STOPPED and self.progress == 1.0: + return 1.0 + else: + return 0.0 + else: + return self.stats['vod_prebuf_frac'] + + def is_vod(self): + """ Returns if this download is currently in vod mode + + @return A Boolean""" + if self.stats is None: + return False + else: + return self.stats['vod'] + + def get_vod_playable(self): + """ Returns whether or not the Download started in Video-On-Demand + mode has sufficient prebuffer and download speed to be played out + to the user. + @return Boolean. + """ + if self.stats is None: + return False + else: + return self.stats['vod_playable'] + + def get_vod_playable_after(self): + """ Returns the estimated time until the Download started in Video-On-Demand + mode can be started to play out to the user. + @return A number of seconds. + """ + if self.stats is None: + return float(2 ** 31) + else: + return self.stats['vod_playable_after'] + + def get_vod_stats(self): + """ Returns a dictionary of collected VOD statistics. The keys contained are: + 
+        'played' = number of pieces played. With seeking this may be more than npieces
+        'late' = number of pieces arrived after they were due
+        'dropped' = number of pieces lost
+        'stall' = estimation of time the player stalled, waiting for pieces (seconds)
+        'pos' = playback position, as an absolute piece number
+        'prebuf' = amount of prebuffering time that was needed (seconds,
+                   set when playback starts)
+        'firstpiece' = starting absolute piece number of selected file
+        'npieces' = number of pieces in selected file
+
, or no keys if no VOD is in progress. + @return Dict. + """ + if self.stats is None: + return {} + else: + return self.stats['vod_stats'] + + + + def get_log_messages(self): + """ Returns the last 10 logged non-fatal error messages. + @return A list of (time,msg) tuples. Time is Python time() format. """ + if self.logmsgs is None: + return [] + else: + return self.logmsgs + + def get_peerlist(self): + """ Returns a list of dictionaries, one for each connected peer + containing the statistics for that peer. In particular, the + dictionary contains the keys: + 
+        'id' = PeerID or 'http seed'
+        'ip' = IP address as string or URL of httpseed
+        'optimistic' = True/False
+        'direction' = 'L'/'R' (outgoing/incoming)
+        'uprate' = Upload rate in KB/s
+        'uinterested' = Upload Interested: True/False
+        'uchoked' = Upload Choked: True/False
+        'downrate' = Download rate in KB/s
+        'dinterested' = Download interested: True/Flase
+        'dchoked' = Download choked: True/False
+        'snubbed' = Download snubbed: True/False
+        'utotal' = Total uploaded from peer in KB
+        'dtotal' = Total downloaded from peer in KB
+        'completed' = Fraction of download completed by peer (0-1.0) 
+        'speed' = The peer's current total download speed (estimated)
+
+ """ + if self.stats is None or 'spew' not in self.stats: + return [] + else: + return self.stats['spew'] + + + def get_coopdl_helpers(self): + """ Returns the peers currently helping. + @return A list of PermIDs. + """ + if self.coopdl_helpers is None: + return [] + else: + return self.coopdl_helpers + + def get_coopdl_coordinator(self): + """ Returns the permid of the coordinator when helping that peer + in a cooperative download + @return A PermID. + """ + return self.coopdl_coordinator + + # + # RePEX: get swarmcache + # + def get_swarmcache(self): + """ + Gets the SwarmCache of the Download. If the Download was RePEXing, + the latest SwarmCache is returned. If the Download was running + normally, a sample of the peerlist is merged with the last + known SwarmCache. If the Download was stopped, the last known + SwarmCache is returned. + + @return The latest SwarmCache for this Download, which is a dict + mapping dns to a dict with at least 'last_seen' and 'pex' keys. + """ + swarmcache = {} + if self.status == DLSTATUS_REPEXING and self.swarmcache is not None: + # the swarmcache given at construction comes from RePEXer + swarmcache = self.swarmcache + elif self.status in [DLSTATUS_DOWNLOADING, DLSTATUS_SEEDING]: + # get local PEX peers from peerlist and fill swarmcache + peerlist = [p for p in self.get_peerlist() if p['direction']=='L' and p.get('pex_received',0)][:REPEX_SWARMCACHE_SIZE] + swarmcache = {} + for peer in peerlist: + dns = (peer['ip'], peer['port']) + swarmcache[dns] = {'last_seen':self.time,'pex':[]} + # fill remainder with peers from old swarmcache + if self.swarmcache is not None: + for dns in self.swarmcache.keys()[:REPEX_SWARMCACHE_SIZE-len(swarmcache)]: + swarmcache[dns] = self.swarmcache[dns] + + # TODO: move peerlist sampling to a different module? + # TODO: perform swarmcache computation only once? + elif self.swarmcache is not None: + # In all other cases, use the old swarmcache + swarmcache = self.swarmcache + # TODO: rearrange if statement to merge 1st and 3rd case? + + return swarmcache + diff --git a/instrumentation/next-share/BaseLib/Core/LiveSourceAuthConfig.py b/instrumentation/next-share/BaseLib/Core/LiveSourceAuthConfig.py new file mode 100644 index 0000000..161cd17 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/LiveSourceAuthConfig.py @@ -0,0 +1,125 @@ +# Written by Arno Bakker +# see LICENSE.txt for license information + +from BaseLib.Core.simpledefs import * +import BaseLib.Core.Overlay.permid as permidmod +from BaseLib.Core.Utilities.Crypto import RSA_keypair_to_pub_key_in_der +from M2Crypto import RSA + + +class LiveSourceAuthConfig: + """ Base class for configuring authentication methods for data from the + source in live streaming. + """ + def __init__(self,authmethod): + self.authmethod = authmethod + + def get_method(self): + return self.authmethod + + +class ECDSALiveSourceAuthConfig(LiveSourceAuthConfig): + """ Class for configuring the ECDSA authentication method for data from the + source in live streaming. The ECDSA method adds a ECDSA signature to each + piece that is generated. + """ + def __init__(self,keypair=None): + """ Constructor for LIVE_AUTHMETHOD_ECDSA authentication of the + live source. If no keypair is specified, one is generated. + + @param keypair (Optional) An M2Crypto.EC keypair. + """ + LiveSourceAuthConfig.__init__(self,LIVE_AUTHMETHOD_ECDSA) + if keypair is None: + self.keypair = permidmod.generate_keypair() + else: + self.keypair = keypair + + def get_pubkey(self): + return str(self.keypair.pub().get_der()) + + def get_keypair(self): + return self.keypair + + # + # Class method + # + def load(filename): + """ + Load a saved ECDSALiveSourceAuthConfig from disk. + + @param filename An absolute Unicode filename + @return ECDSALiveSourceAuthConfig object + """ + keypair = permidmod.read_keypair(filename) + return ECDSALiveSourceAuthConfig(keypair) + load = staticmethod(load) + + def save(self,filename): + """ Save the ECDSALiveSourceAuthConfig to disk. + @param filename An absolute Unicode filename + """ + permidmod.save_keypair(self.keypair,filename) + + +class RSALiveSourceAuthConfig(LiveSourceAuthConfig): + """ Class for configuring the RSA authentication method for data from the + source in live streaming. The RSA method adds a RSA signature to each + piece that is generated. + """ + def __init__(self,keypair=None): + """ Constructor for LIVE_AUTHMETHOD_RSA authentication of the + live source. If no keypair is specified, one is generated. + + @param keypair (Optional) An M2Crypto.RSA keypair. + """ + LiveSourceAuthConfig.__init__(self,LIVE_AUTHMETHOD_RSA) + if keypair is None: + self.keypair = rsa_generate_keypair() + else: + self.keypair = keypair + + def get_pubkey(self): + return RSA_keypair_to_pub_key_in_der(self.keypair) + + def get_keypair(self): + return self.keypair + + # + # Class method + # + def load(filename): + """ + Load a saved RSALiveSourceAuthConfig from disk. + + @param filename An absolute Unicode filename + @return RSALiveSourceAuthConfig object + """ + keypair = rsa_read_keypair(filename) + return RSALiveSourceAuthConfig(keypair) + load = staticmethod(load) + + def save(self,filename): + """ Save the RSALiveSourceAuthConfig to disk. + @param filename An absolute Unicode filename + """ + rsa_write_keypair(self.keypair,filename) + + + +def rsa_generate_keypair(): + """ Create keypair using default params, use __init__(keypair) parameter + if you want to use custom params. + """ + # Choose fast exponent e. See Handbook of applied cryptography $8.2.2(ii) + # And small keysize, attackers have duration of broadcast to reverse + # engineer key. + e = 3 + keysize = 768 + return RSA.gen_key(keysize,e) + +def rsa_read_keypair(filename): + return RSA.load_key(filename) + +def rsa_write_keypair(keypair,filename): + keypair.save_key(filename,cipher=None) diff --git a/instrumentation/next-share/BaseLib/Core/Merkle/__init__.py b/instrumentation/next-share/BaseLib/Core/Merkle/__init__.py new file mode 100644 index 0000000..395f8fb --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/Merkle/__init__.py @@ -0,0 +1,2 @@ +# Written by Arno Bakker +# see LICENSE.txt for license information diff --git a/instrumentation/next-share/BaseLib/Core/Merkle/merkle.py b/instrumentation/next-share/BaseLib/Core/Merkle/merkle.py new file mode 100644 index 0000000..167d6bc --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/Merkle/merkle.py @@ -0,0 +1,272 @@ +# Written by Arno Bakker +# see LICENSE.txt for license information +""" +Reference Implementation of Merkle hash torrent extension, as now +standardized in http://www.bittorrent.org/beps/bep_0030.html (yay!) +""" + +from math import log,pow,floor +from BaseLib.Core.Utilities.Crypto import sha +import sys + +DEBUG = False + +# External classes + +class MerkleTree: + + def __init__(self,piece_size,total_length,root_hash=None,hashes=None): + """ + Create a Merkle hash tree + + When creating a .torrent: + root_hash is None and hashes is not None + When creating an initial seeder: + root_hash is None and hashes is not None + (root_hash is None to allow comparison with the calculated + root hash and the one in the .torrent) + When creating a downloader: + root_hash is not None and hashes is None + """ + self.npieces = len2npieces(piece_size,total_length) + self.treeheight = get_tree_height(self.npieces) + self.tree = create_tree(self.treeheight) + if hashes is None: + self.root_hash = root_hash + else: + fill_tree(self.tree,self.treeheight,self.npieces,hashes) + # root_hash is None during .torrent generation + if root_hash is None: + self.root_hash = self.tree[0] + else: + raise AssertionError, "merkle: if hashes not None, root_hash must be" + + def get_root_hash(self): + return self.root_hash + + def compare_root_hashes(self,other): + return self.root_hash == other + + def get_hashes_for_piece(self,index): + return get_hashes_for_piece(self.tree,self.treeheight,index) + + def check_hashes(self,hashlist): + return check_tree_path(self.root_hash,self.treeheight,hashlist) + + def update_hash_admin(self,hashlist,piece_hashes): + update_hash_admin(hashlist,self.tree,self.treeheight,piece_hashes) + + def get_piece_hashes(self): + """ + Get the pieces' hashes from the bottom of the hash tree. Used during + a graceful restart of a client that already downloaded stuff. + """ + return get_piece_hashes(self.tree,self.treeheight,self.npieces) + +def create_fake_hashes(info): + total_length = calc_total_length(info) + npieces = len2npieces(info['piece length'],total_length) + return ['\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' ] * npieces + + +# Internal functions +# Design choice: all algoritmics have been returned into stateless functions, +# i.e. they operate on the input parameters only. This to keep them extremely +# clear. + +def len2npieces(piece_size,total_length): + npieces = total_length / piece_size + if piece_size*npieces < total_length: + npieces += 1 + return npieces + + +def calc_total_length(info): + # Merkle: Calculate total length from .torrent info + if info.has_key('length'): + return info['length'] + # multi-file torrent + files = info['files'] + total_length = 0 + for i in range(0,len(files)): + total_length += files[i]['length'] + return total_length + + +def get_tree_height(npieces): + if DEBUG: + print >> sys.stderr,"merkle: number of pieces is",npieces + height = log(npieces,2) + if height - floor(height) > 0.0: + height = int(height)+1 + else: + height = int(height) + if DEBUG: + print >> sys.stderr,"merkle: tree height is",height + return height + +def create_tree(height): + # Create tree that has enough leaves to hold all hashes + treesize = int(pow(2,height+1)-1) # subtract unused tail + if DEBUG: + print >> sys.stderr,"merkle: treesize",treesize + tree = ['\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' ] * treesize + return tree + +def fill_tree(tree,height,npieces,hashes): + # 1. Fill bottom of tree with hashes + startoffset = int(pow(2,height)-1) + if DEBUG: + print >> sys.stderr,"merkle: bottom of tree starts at",startoffset + for offset in range(startoffset,startoffset+npieces): + #print >> sys.stderr,"merkle: copying",offset + #print >> sys.stderr,"merkle: hashes[",offset-startoffset,"]=",str(hashes[offset-startoffset]) + tree[offset] = hashes[offset-startoffset] + # 2. Note that unused leaves are NOT filled. It may be a good idea to fill + # them as hashing 0 values may create a security problem. However, the + # filler values would have to be known to any initial seeder, otherwise it + # will not be able build the same hash tree as the other initial seeders. + # Assume anyone should be able to autonomously become a seeder, the filler + # must be public info. I don't know whether having public info as filler + # instead of 0s is any safer, cryptographically speaking. Hence, we stick + # with 0 for the moment + + # 3. Calculate higher level hashes from leaves + for level in range(height,0,-1): + if DEBUG: + print >> sys.stderr,"merkle: calculating level",level + for offset in range(int(pow(2,level)-1),int(pow(2,level+1)-2),2): + #print >> sys.stderr,"merkle: data offset",offset + [ parentstartoffset, parentoffset ] = get_parent_offset(offset,level) + #print >> sys.stderr,"merkle: parent offset",parentoffset + data = tree[offset]+tree[offset+1] + digester = sha() + digester.update(data) + digest = digester.digest() + tree[parentoffset] = digest + #for offset in range(0,treesize-1): + # print offset,"HASH",str(tree[offset]) + return tree + + +def get_hashes_for_piece(tree,height,index): + startoffset = int(pow(2,height)-1) + myoffset = startoffset+index + if DEBUG: + print >> sys.stderr,"merkle: myoffset",myoffset + # 1. Add piece's own hash + hashlist = [ [myoffset,tree[myoffset]] ] + # 2. Add hash of piece's sibling, left or right + if myoffset % 2 == 0: + siblingoffset = myoffset-1 + else: + siblingoffset = myoffset+1 + if DEBUG: + print >> sys.stderr,"merkle: siblingoffset",siblingoffset + if siblingoffset != -1: + hashlist.append([siblingoffset,tree[siblingoffset]]) + # 3. Add hashes of uncles + uncleoffset = myoffset + for level in range(height,0,-1): + uncleoffset = get_uncle_offset(uncleoffset,level) + if DEBUG: + print >> sys.stderr,"merkle: uncleoffset",uncleoffset + hashlist.append( [uncleoffset,tree[uncleoffset]] ) + return hashlist + + +def check_tree_path(root_hash,height,hashlist): + """ + The hashes should be in the right order in the hashlist, otherwise + the peer will be kicked. The hashlist parameter is assumed to be + of the right type, and contain values of the right type as well. + The exact values should be checked for validity here. + """ + maxoffset = int(pow(2,height+1)-2) + mystartoffset = int(pow(2,height)-1) + i=0 + a = hashlist[i] + if a[0] < 0 or a[0] > maxoffset: + return False + i += 1 + b = hashlist[i] + if b[0] < 0 or b[0] > maxoffset: + return False + i += 1 + myindex = a[0]-mystartoffset + sibindex = b[0]-mystartoffset + for level in range(height,0,-1): + if DEBUG: + print >> sys.stderr,"merkle: checking level",level + a = check_fork(a,b,level) + b = hashlist[i] + if b[0] < 0 or b[0] > maxoffset: + return False + i += 1 + if DEBUG: + print >> sys.stderr,"merkle: ROOT HASH",`str(root_hash)`,"==",`str(a[1])` + if a[1] == root_hash: + return True + else: + return False + +def update_hash_admin(hashlist,tree,height,hashes): + mystartoffset = int(pow(2,height)-1) + for i in range(0,len(hashlist)): + if i < 2: + # me and sibling real hashes of piece data, save them + index = hashlist[i][0]-mystartoffset + # ignore siblings that are just tree filler + if index < len(hashes): + if DEBUG: + print >> sys.stderr,"merkle: update_hash_admin: saving hash of",index + hashes[index] = hashlist[i][1] + # put all hashes in tree, such that we incrementally learn it + # and can pass them on to others + tree[hashlist[i][0]] = hashlist[i][1] + + +def check_fork(a,b,level): + myoffset = a[0] + siblingoffset = b[0] + if myoffset > siblingoffset: + data = b[1]+a[1] + if DEBUG: + print >> sys.stderr,"merkle: combining",siblingoffset,myoffset + else: + data = a[1]+b[1] + if DEBUG: + print >> sys.stderr,"merkle: combining",myoffset,siblingoffset + digester = sha() + digester.update(data) + digest = digester.digest() + [parentstartoffset, parentoffset ] = get_parent_offset(myoffset,level-1) + return [parentoffset,digest] + +def get_parent_offset(myoffset,level): + parentstartoffset = int(pow(2,level)-1) + mystartoffset = int(pow(2,level+1)-1) + parentoffset = parentstartoffset + (myoffset-mystartoffset)/2 + return [parentstartoffset, parentoffset] + + +def get_uncle_offset(myoffset,level): + if level == 1: + return 0 + [parentstartoffset,parentoffset ] = get_parent_offset(myoffset,level-1) + if DEBUG: + print >> sys.stderr,"merkle: parent offset",parentoffset + parentindex = parentoffset-parentstartoffset + if parentoffset % 2 == 0: + uncleoffset = parentoffset-1 + else: + uncleoffset = parentoffset+1 + return uncleoffset + +def get_piece_hashes(tree,height,npieces): + startoffset = int(pow(2,height)-1) + hashes = ['\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' ] * npieces + for offset in range(startoffset,startoffset+npieces): + hashes[offset-startoffset] = tree[offset] + return hashes + diff --git a/instrumentation/next-share/BaseLib/Core/Multicast/Multicast.py b/instrumentation/next-share/BaseLib/Core/Multicast/Multicast.py new file mode 100644 index 0000000..d585ac4 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/Multicast/Multicast.py @@ -0,0 +1,692 @@ +# Written by Njaal Borch +# see LICENSE.txt for license information + +import socket +import threading +import struct +import select +import string +import sys +import time +import random # for ping +from traceback import print_exc + +import base64 # Must encode permid + + +from BaseLib.Core.BuddyCast.buddycast import BuddyCastFactory + + +DEBUG = False + +class MyLogger: + + """ + Dummy logger due to code re-use and no use of logger in Tribler + + """ + enabled = DEBUG + + def debug(self, message): + if self.enabled: + print >> sys.stderr, "pdisc: DEBUG:", message + + def info(self, message): + if self.enabled: + print >> sys.stderr, "pdisc: INFO:", message + + def warning(self, message): + if self.enabled: + print >> sys.stderr, "pdisc: WARNING:", message + + def error(self, message): + if self.enabled: + print >> sys.stderr, "pdisc: ERROR:", message + + def fatal(self, message): + if self.enabled: + print >> sys.stderr, "pdisc: FATAL:", message + + def exception(self, message): + if self.enabled: + print >> sys.stderr, "pdisc: EXCEPTION:", message + import traceback + traceback.print_exc() + +class Multicast: + + """ + This class allows nodes to communicate on a local network + using IP multicast + + """ + + def __init__(self, config, overlay_bridge, myport, myselversion, peerdb, + logger=None, capabilities=None): + """ + Initialize the multicast channel. Parameters: + - multicast_ipv4_enabled + - multicast_ipv6_enabled + - multicast_port + - multicast_announce - True if the node should announce itself + - permid - The ID of the node + - multicast_ipv4_address + - multicast_ipv6_address + + If both ipv4_enabled and ipv6_enabled is false, the channel + will not do anything. + + Other parameters: + logger - Send logs (debug/info/warning/error/exceptions) to a logger + capabilities - Announce a set of capabilities for this node. Should + be a list + + """ + self.myport = myport + self.myselversion = myselversion + self.overlay_bridge = overlay_bridge + self.peer_db = peerdb + + if logger: + self.log = logger + else: + self.log = MyLogger() + + self.config = config + self.capabilities = capabilities + + self.enabled = False + self.announceHandlers = [] + self.on_node_announce = None + self.incoming_pongs = {} + + self.interfaces = [] + + self.address_family = socket.AF_INET + if self.config['multicast_ipv6_enabled']: + if not socket.has_ipv6: + self.log.warning("Missing IPv6 support") + else: + self.address_family = socket.AF_INET6 + + self.sock = socket.socket(self.address_family, + socket.SOCK_DGRAM) + + self.sock.setsockopt(socket.SOL_SOCKET, + socket.SO_REUSEADDR, 1) + + for res in socket.getaddrinfo(None, + self.config['multicast_port'], + self.address_family, + socket.SOCK_DGRAM, 0, + socket.AI_PASSIVE): + + af, socktype, proto, canonname, sa = res + + try: + self.sock.bind(sa) + except: + self.log.exception("Error binding") + + try: + if self.config['multicast_ipv6_enabled']: + self.interfaces = self._joinMulticast(self.config['multicast_ipv6_address'], + self.config['multicast_port'], + self.sock) + self.enabled = True + except: + self.log.exception("Exception during IPv6 multicast join") + + try: + if self.config['multicast_ipv4_enabled']: + self._joinMulticast(self.config['multicast_ipv4_address'], + self.config['multicast_port'], + self.sock) + self.enabled = True + except: + self.log.exception("Exception during IPv4 multicast join") + + + def _getCapabilities(self, elements): + """ + Return a list of capabilities from a list of elements - internal function + """ + capabilities = [] + for elem in elements: + if elem.startswith("c:"): + capabilities.append(elem[2:]) + return capabilities + + def getSocket(self): + return self.sock + + def _joinMulticast(self, addr, port, sock): + """ + Join a multicast channel - internal function + """ + import struct + + for res in socket.getaddrinfo(addr, + port, + socket.AF_UNSPEC, + socket.SOCK_DGRAM): + + af, socktype, proto, canonname, sa = res + + break + + if af == socket.AF_INET6: + # Smurf, must manually reconstruct "::"??? + # Count the number of colons in the address + num_colons = addr.count(":") + + new_colons = ":" + + # Replace double colon with the appropriate number (7) + for i in range(num_colons, 8): + new_colons = "%s0:" % new_colons + + addr = addr.replace("::", new_colons) + + addr_pack = '' + + for l in addr.split(":"): + word = int(l,16) + addr_pack = addr_pack + struct.pack('!H', word) + + # Now we try to join the first 32 interfaces + # Not too nice, but it is absolutely portable :-) + interfaces = [] + for i in range (1, 32): + try: + mreq = addr_pack + struct.pack('l', i) + + # We're ready, at last + sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_JOIN_GROUP, + mreq) + ok = True + self.log.debug("Joined IPv6 multicast on interface %d"%i) + + # We return the interface indexes that worked + interfaces.append(i) + except Exception,e: + pass + + if len(interfaces) == 0: + self.log.fatal("Could not join on any interface") + raise Exception("Could not join multicast on any interface") + + return interfaces + + if af == socket.AF_INET: + + addr_pack = '' + grpaddr = 0 + bytes = map(int, string.split(addr, ".")) + for byte in bytes: + grpaddr = (grpaddr << 8) | byte + + # Construct struct mreq from grpaddr and ifaddr + ifaddr = socket.INADDR_ANY + mreq = struct.pack('ll', + socket.htonl(grpaddr), + socket.htonl(ifaddr)) + + # Add group membership + try: + self.sock.setsockopt(socket.IPPROTO_IP, + socket.IP_ADD_MEMBERSHIP, + mreq) + except Exception,e: + self.log.exception("Exception joining IPv4 multicast") + + return [] + + + def data_came_in(self, addr, data): + """ + Callback function for arriving data. This is non-blocking + and will return immediately after queuing the operation for + later processing. Called by NetworkThread + """ + # Must queue this for actual processing, we're not allowed + # to block here + process_data_func = lambda:self._data_came_in_callback(addr, data) + self.overlay_bridge.add_task(process_data_func, 0) + + + def _data_came_in_callback(self, addr, data): + """ + Handler function for when data arrives + """ + + self.log.debug("Got a message from %s"%str(addr)) + # Look at message + try: + elements = data.split("\n") + + if elements[0] == "NODE_DISCOVER": + if len(elements) < 3: + raise Exception("Too few elements") + + # Only reply if I'm announcing + if not self.config["multicast_announce"]: + self.log.debug("Not announcing myself") + return + + remotePermID = elements[2] + self.log.debug("Got node discovery from %s"%remotePermID) + # TODO: Do we reply to any node? + + # Reply with information about me + permid_64 = base64.b64encode(self.config['permid']).replace("\n","") + msg = "NODE_ANNOUNCE\n%s"%permid_64 + + # Add capabilities + if self.capabilities: + for capability in self.capabilities: + msg += "\nc:%s"%capability + try: + self.sock.sendto(msg, addr) + except Exception,e: + self.log.error("Could not send announce message to %s: %s"%(str(addr), e)) + return + + elif elements[0] == "ANNOUNCE": + self.handleAnnounce(addr, elements) + elif elements[0] == "NODE_ANNOUNCE": + # Some node announced itself - handle callbacks if + # the app wants it + if self.on_node_announce: + try: + self.on_node_announce(elements[1], addr, + self._getCapabilities(elements)) + except Exception,e: + self.log.exception("Exception handling node announce") + elif elements[0] == "PING": + permid = base64.b64decode(elements[1]) + if permid == self.config["permid"]: + # I should reply + msg = "PONG\n%s\n%s"%(elements[1], elements[2]) + self._sendMulticast(msg) + elif elements[0] == "PONG": + nonce = int(elements[2]) + if self.outstanding_pings.has_key(nonce): + self.incoming_pongs[nonce] = time.time() + else: + self.log.warning("Got bad discovery message from %s"%str(addr)) + except Exception,e: + self.log.exception("Illegal message '%s' from '%s'"%(data, addr[0])) + + + def _send(self, addr, msg): + """ + Send a message - internal function + """ + + for res in socket.getaddrinfo(addr, self.config['multicast_port'], + socket.AF_UNSPEC, + socket.SOCK_DGRAM): + + af, socktype, proto, canonname, sa = res + try: + sock = socket.socket(af, socktype) + sock.sendto(msg, sa) + except Exception,e: + self.log.warning("Error sending '%s...' to %s: %s"%(msg[:8], str(sa), e)) + + return sock + + def discoverNodes(self, timeout=3.0, requiredCapabilities=None): + """ + Try to find nodes on the local network and return them in a list + of touples on the form + (permid, addr, capabilities) + + Capabilities can be an empty list + + if requiredCapabilities is specified, only nodes matching one + or more of these will be returned + + """ + + # Create NODE_DISCOVER message + msg = "NODE_DISCOVER\nTr_OVERLAYSWARM node\npermid:%s"%\ + base64.b64encode(self.config['permid']).replace("\n","") + + # First send the discovery message + addrList = [] + sockList = [] + if self.config['multicast_ipv4_enabled']: + sockList.append(self._send(self.config['multicast_ipv4_address'], msg)) + + if self.config['multicast_ipv6_enabled']: + for iface in self.interfaces: + sockList.append(self._send("%s%%%s"%(self.config['multicast_ipv6_address'], iface), msg)) + + nodeList = [] + endAt = time.time() + timeout + while time.time() < endAt: + + # Wait for answers (these are unicast) + SelectList = sockList[:] + + (InList, OutList, ErrList) = select.select(SelectList, [], [], 1.0) + + if len(ErrList) < 0: + self.log.warning("Select gives error...") + + while len(InList) > 0: + + sock2 = InList.pop(0) + + try: + (data, addr) = sock2.recvfrom(1450) + except socket.error, e: + self.log.warning("Exception receiving: %s"%e) + continue + except Exception,e: + print_exc() + self.log.warning("Unknown exception receiving") + continue + + try: + elements = data.split("\n") + if len(elements) < 2: + self.log.warning("Bad message from %s: %s"%(addr, data)) + continue + + if elements[0] != "NODE_ANNOUNCE": + self.log.warning("Unknown message from %s: %s"%(addr, data)) + continue + + permid = base64.b64decode(elements[1]) + self.log.info("Discovered node %s at (%s)"%(permid, str(addr))) + capabilities = self._getCapabilities(elements) + if requiredCapabilities: + ok = False + for rc in requiredCapabilities: + if rc in capabilities: + ok = True + break + if not ok: + continue + nodeList.append((permid, addr, capabilities)) + except Exception,e: + self.log.warning("Could not understand message: %s"%e) + + return nodeList + + def sendNodeAnnounce(self): + + """ + Send a node announcement message on multicast + + """ + + msg = "NODE_ANNOUNCE\n%s"%\ + base64.b64encode(self.config['permid']).replace("\n","") + + if self.capabilities: + for capability in self.capabilities: + msg += "\nc:%s"%capability + try: + self._sendMulticast(msg) + except: + self.log.error("Could not send announce message") + + + def setNodeAnnounceHandler(self, handler): + + """ + Add a handler function for multicast node announce messages + + Will get a parameters (permid, address, capabilities) + + """ + self.on_node_announce = handler + + def addAnnounceHandler(self, handler): + + """ + Add an announcement handler for announcement messages (not + + node discovery) + + The callback function will get parameters: + (permid, remote_address, parameter_list) + + """ + self.announceHandlers.append(handler) + + def removeAnnouncehandler(self, handler): + + """ + Remove an announce handler (if present) + + """ + try: + self.announceHandlers.remove(handler) + except: + #handler not in list, ignore + pass + + def handleAnnounce(self, addr, elements): + + """ + Process an announcement and call any callback handlers + + """ + + if elements[0] != "ANNOUNCE": + raise Exception("Announce handler called on non-announce: %s"%\ + elements[0]) + + # Announce should be in the form: + # ANNOUNCE + # base64 encoded permid + # numElements + # element1 + # element2 + # ... + if len(elements) < 3: + raise Exception("Bad announce, too few elements in message") + + try: + permid = base64.b64decode(elements[1]) + numElements = int(elements[2]) + except: + raise Exception("Bad announce message") + + if len(elements) < 3 + numElements: + raise Exception("Incomplete announce message") + + _list = elements[3:3+numElements] + + # Loop over list to longs if numbers + list = [] + for elem in _list: + if elem.isdigit(): + list.append(long(elem)) + else: + list.append(elem) + + if len(self.announceHandlers) == 0: + self.log.warning("Got node-announce, but I'm missing announce handlers") + + # Handle the message + for handler in self.announceHandlers: + try: + self.log.debug("Calling callback handler") + handler(permid, addr, list) + except: + self.log.exception("Could not activate announce handler callback '%s'"%handler) + + + def handleOVERLAYSWARMAnnounce(self, permid, addr, params): + """ Callback function to handle multicast node announcements + + This one will trigger an overlay connection and then initiate a buddycast + exchange + """ + # todo: when the port or selversion change this will NOT be + # updated in the database. Solution: change the whole + # flag_peer_as_local_to_db into check_and_update_peer_in_db + # and let it check for the existance and current value of + # is_local, port, and selversion. (at no additional queries I + # might add) + + self.log.debug("Got Tr_OVERLAYSWARM announce!") + port, selversion = params + + if permid == self.config["permid"]: + self.log.debug("Discovered myself") + # Discovered myself, which is not interesting + return + + if self.flag_peer_as_local_to_db(permid, True): + self.log.debug("node flagged as local") + # Updated ok + return + + # We could not update - this is a new node! + try: + try: + self.log.debug("Adding peer at %s to database"%addr[0]) + self.add_peer_to_db(permid, (addr[0], port), selversion) + except Exception,e: + print >> sys.stderr, "pdisc: Could not add node:",e + + try: + self.flag_peer_as_local_to_db(permid, True) + self.log.debug("node flagged as local") + except Exception,e: + print >> sys.stderr, "pdisc: Could not flag node as local:",e + + # Now trigger a buddycast exchange + bc_core = BuddyCastFactory.getInstance().buddycast_core + if bc_core: + self.log.debug("Triggering buddycast") + bc_core.startBuddyCast(permid) + finally: + # Also announce myself so that the remote node can see me! + params = [self.myport, self.myselversion] + self.log.debug("Sending announce myself") + try: + self.sendAnnounce(params) + except: + self.log.exception("Sending announcement") + + def sendAnnounce(self, list): + + """ + Send an announce on local multicast, if enabled + + """ + + if not self.enabled: + return + + # Create ANNOUNCE message + msg = "ANNOUNCE\n%s\n%d\n"%\ + (base64.b64encode(self.config['permid']).replace("\n",""), len(list)) + + for elem in list: + msg += "%s\n"%elem + + self._sendMulticast(msg) + + def _sendMulticast(self, msg): + + """ + Send a message buffer on the multicast channels + + """ + + if self.config['multicast_ipv4_enabled']: + self._send(self.config['multicast_ipv4_address'], msg) + if self.config['multicast_ipv6_enabled']: + for iface in self.interfaces: + self._send("%s%%%s"%(self.config['multicast_ipv6_address'], iface), msg) + + + + def ping(self, permid, numPings=3): + """ + Ping a node and return (avg time, min, max) or (None, None, None) if no answer + Only one node can be pinged at the time - else this function will not work! + """ + + self.outstanding_pings = {} + self.incoming_pongs = {} + + # Send a PING via multicast and wait for a multicast response. + # Using multicast for both just in case it is different from + # unicast + + for i in range(0, numPings): + nonce = random.randint(0, 2147483647) + msg = "PING\n%s\n%s"%(base64.b64encode(permid).replace("\n",""), nonce) + self.outstanding_pings[nonce] = time.time() + self._sendMulticast(msg) + time.sleep(0.250) + + # Now we gather the results + time.sleep(0.5) + + if len(self.incoming_pongs) == 0: + return (None, None, None) + + max = 0 + min = 2147483647 + total = 0 + num = 0 + for nonce in self.outstanding_pings.keys(): + if self.incoming_pongs.has_key(nonce): + diff = self.incoming_pongs[nonce] - self.outstanding_pings[nonce] + if diff > max: + max = diff + if diff < min: + min = diff + total += diff + num += 1 + + avg = total/num + + self.outstanding_pings = {} + self.incoming_pongs = {} + return (avg, min, max) + + def add_peer_to_db(self,permid,dns,selversion): + # todo: should is_local be set to True? + now = int(time.time()) + peer_data = {'permid':permid, 'ip':dns[0], 'port':dns[1], 'oversion':selversion, 'last_seen':now, 'last_connected':now} + self.peer_db.addPeer(permid, peer_data, update_dns=True, update_connected=True, commit=True) + + def flag_peer_as_local_to_db(self, permid, is_local): + """ + Sets the is_local flag for PERMID to IS_LOCAL if and only if + PERMID exists in the database, in this case it returns + True. Otherwise it returns False. + """ + peer = self.peer_db.getPeer(permid, ('is_local',)) + + print >>sys.stderr,"pdisc: flag_peer_as_local returns",peer + + if not peer is None: + # Arno, 2010-02-09: Somehow return value is not std. + if isinstance(peer,list): + flag = peer[0] + else: + flag = peer + if not flag == is_local: + self.peer_db.setPeerLocalFlag(permid, is_local) + return True + return False + + # if is_local: + # pass + ##print >>sys.stderr,"pdisc: Flagging a peer as local" + # return self.peer_db.setPeerLocalFlag(permid, is_local) + diff --git a/instrumentation/next-share/BaseLib/Core/Multicast/__init__.py b/instrumentation/next-share/BaseLib/Core/Multicast/__init__.py new file mode 100644 index 0000000..9c0a205 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/Multicast/__init__.py @@ -0,0 +1,35 @@ +# Written by Njaal Borch +# see LICENSE.txt for license information + +""" +A local multicast discovery and communication + +Simple usage example: + +For config, please view the Multicast documentation. + +channel = multicast.Multicast(config) +for (id, address, capabilities) in channel.discoverNodes(): + print "Found node",id,"at",address,"with capabilities:",capabilities + +# Sending and handling announcements: +def on_announce(id, addr, list): + print 'Got an announcement from node",id,"at",addr,":",list + +channel = multicast.Multicast(config) +channel.addAnnounceHandler(on_announce) +channel.sendAnnounce(['element1', 'element2', 'element3']) + +# Handle multicast node announcements directly, with capabilities too +def on_node_announce(addr, id, capabilities): + print "Got a node announcement from",id,"at",addr,"with capabilities:",capabilities + +myCapabilities = ["CAN_PRINT", "CAN_FAIL"] +channel = multicast.Multicast(config, capabilities=myCapabilities) +channel.setNodeAnnounceHandler(on_node_announce) + +For more examples, take a look at the unit tests (MulticastTest) + +""" + +from Multicast import * diff --git a/instrumentation/next-share/BaseLib/Core/NATFirewall/ConnectionCheck.py b/instrumentation/next-share/BaseLib/Core/NATFirewall/ConnectionCheck.py new file mode 100644 index 0000000..5c78a33 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/NATFirewall/ConnectionCheck.py @@ -0,0 +1,153 @@ +import sys +from time import sleep +import thread +import random +from BaseLib.Core.NATFirewall.NatCheck import GetNATType +from BaseLib.Core.NATFirewall.TimeoutCheck import GetTimeout + +DEBUG = False + +class ConnectionCheck: + + __single = None + + def __init__(self, session): + if ConnectionCheck.__single: + raise RuntimeError, "ConnectionCheck is singleton" + ConnectionCheck.__single = self + self._lock = thread.allocate_lock() + self._running = False + self.session = session + self.permid = self.session.get_permid() + self.nat_type = None + self.nat_timeout = 0 + self._nat_callbacks = [] # list with callback functions that want to know the nat_type + self.natcheck_reply_callbacks = [] # list with callback functions that want to send a natcheck_reply message + + @staticmethod + def getInstance(*args, **kw): + if ConnectionCheck.__single is None: + ConnectionCheck(*args, **kw) + return ConnectionCheck.__single + + def try_start(self, reply_callback = None): + + if reply_callback: self.natcheck_reply_callbacks.append(reply_callback) + + if DEBUG: + if self._running: + print >>sys.stderr, "natcheckmsghandler: the thread is already running" + else: + print >>sys.stderr, "natcheckmsghandler: starting the thread" + + if not self._running: + thread.start_new_thread(self.run, ()) + + while True: + sleep(0) + if self._running: + break + + def run(self): + self._lock.acquire() + self._running = True + self._lock.release() + + try: + self.nat_discovery() + + finally: + self._lock.acquire() + self._running = False + self._lock.release() + + def timeout_check(self, pingback): + """ + Find out NAT timeout + """ + return GetTimeout(pingback) + + def natcheck(self, in_port, server1, server2): + """ + Find out NAT type and public address and port + """ + nat_type, ex_ip, ex_port, in_ip = GetNATType(in_port, server1, server2) + if DEBUG: print >> sys.stderr, "NATCheck:", "NAT Type: " + nat_type[1] + if DEBUG: print >> sys.stderr, "NATCheck:", "Public Address: " + ex_ip + ":" + str(ex_port) + if DEBUG: print >> sys.stderr, "NATCheck:", "Private Address: " + in_ip + ":" + str(in_port) + return nat_type, ex_ip, ex_port, in_ip + + def get_nat_type(self, callback=None): + """ + When a callback parameter is supplied it will always be + called. When the NAT-type is already known the callback will + be made instantly. Otherwise, the callback will be made when + the NAT discovery has finished. + """ + if self.nat_type: + if callback: + callback(self.nat_type) + return self.nat_type + else: + if callback: + self._nat_callbacks.append(callback) + self.try_start() + return "Unknown NAT/Firewall" + + def _perform_nat_type_notification(self): + nat_type = self.get_nat_type() + callbacks = self._nat_callbacks + self._nat_callbacks = [] + + for callback in callbacks: + try: + callback(nat_type) + except: + pass + + def nat_discovery(self): + """ + Main method of the class: launches nat discovery algorithm + """ + in_port = self.session.get_puncturing_internal_port() + stun_servers = self.session.get_stun_servers() + random.seed() + random.shuffle(stun_servers) + stun1 = stun_servers[1] + stun2 = stun_servers[0] + pingback_servers = self.session.get_pingback_servers() + random.shuffle(pingback_servers) + + if DEBUG: print >> sys.stderr, "NATCheck:", 'Starting ConnectionCheck on %s %s %s' % (in_port, stun1, stun2) + + performed_nat_type_notification = False + + # Check what kind of NAT the peer is behind + nat_type, ex_ip, ex_port, in_ip = self.natcheck(in_port, stun1, stun2) + self.nat_type = nat_type[1] + + # notify any callbacks interested in the nat_type only + self._perform_nat_type_notification() + performed_nat_type_notification = True + + + # If there is any callback interested, check the UDP timeout of the NAT the peer is behind + if len(self.natcheck_reply_callbacks): + + if nat_type[0] > 0: + for pingback in pingback_servers: + if DEBUG: print >> sys.stderr, "NatCheck: pingback is:", pingback + self.nat_timeout = self.timeout_check(pingback) + if self.nat_timeout <= 0: break + if DEBUG: print >> sys.stderr, "NATCheck: Nat UDP timeout is: ", str(self.nat_timeout) + + self.nat_params = [nat_type[1], nat_type[0], self.nat_timeout, ex_ip, int(ex_port), in_ip, in_port] + if DEBUG: print >> sys.stderr, "NATCheck:", str(self.nat_params) + + # notify any callbacks interested in sending a natcheck_reply message + for reply_callback in self.natcheck_reply_callbacks: + reply_callback(self.nat_params) + self.natcheck_reply_callbacks = [] + + if not performed_nat_type_notification: + self._perform_nat_type_notification() diff --git a/instrumentation/next-share/BaseLib/Core/NATFirewall/DialbackMsgHandler.py b/instrumentation/next-share/BaseLib/Core/NATFirewall/DialbackMsgHandler.py new file mode 100644 index 0000000..69b677d --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/NATFirewall/DialbackMsgHandler.py @@ -0,0 +1,467 @@ +# Written by Arno Bakker +# see LICENSE.txt for license information +# +# The dialback-message extension serves to (1)~see if we're externally reachable +# and (2)~to tell us what our external IP adress is. When an overlay connection +# is made when we're in dialback mode, we will send a DIALBACK_REQUEST message +# over the overlay connection. The peer is then support to initiate a new +# BT connection with infohash 0x00 0x00 ... 0x01 and send a DIALBACK_REPLY over +# that connection. Those connections are referred to as ReturnConnections +# +# TODO: security problem: if malicious peer connects 7 times to us and tells +# 7 times the same bad external iP, we believe him. Sol: only use locally +# initiated conns + IP address check (BC2 message could be used to attack +# still) +# +# TODO: Arno,2007-09-18: Bittorrent mainline tracker e.g. +# http://tracker.publish.bittorrent.com:6969/announce +# now also returns your IP address in the reply, i.e. there is a +# {'external ip': '\x82%\xc1@'} +# in the dict. We should use this info. +# + +import sys +from time import time +from random import shuffle +from traceback import print_exc,print_stack +from threading import currentThread + +from BaseLib.Core.BitTornado.BT1.MessageID import * +from BaseLib.Core.BitTornado.bencode import bencode,bdecode + +from BaseLib.Core.NATFirewall.ReturnConnHandler import ReturnConnHandler +from BaseLib.Core.Overlay.SecureOverlay import OLPROTO_VER_THIRD +from BaseLib.Core.Utilities.utilities import * +from BaseLib.Core.simpledefs import * + +DEBUG = False + +# +# Constants +# + +REPLY_WAIT = 60 # seconds +REPLY_VALIDITY = 2*24*3600.0 # seconds + +# Normally, one would allow just one majority to possibly exists. However, +# as current Buddycast has a lot of stale peer addresses, let's make +# PEERS_TO_ASK not 5 but 7. +# +PEERS_TO_AGREE = 4 # peers have to say X is my IP before I believe them +YOURIP_PEERS_TO_AGREE = 16 # peers have to say X is my IP via 'yourip' in EXTEND hs before I believe them +PEERS_TO_ASK = 7 # maximum number of outstanding requests +MAX_TRIES = 35 # 5 times 7 peers + +class DialbackMsgHandler: + + __single = None + + def __init__(self): + if DialbackMsgHandler.__single: + raise RuntimeError, "DialbackMsgHandler is singleton" + DialbackMsgHandler.__single = self + + self.peers_asked = {} + self.myips = [] + self.consensusip = None # IP address according to peers + self.fromsuperpeer = False + self.dbreach = False # Did I get any DIALBACK_REPLY? + self.btenginereach = False # Did BT engine get incoming connections? + self.ntries = 0 + self.active = False # Need defaults for test code + self.rawserver = None + self.launchmany = None + self.peer_db = None + self.superpeer_db = None + self.trust_superpeers = None + self.old_ext_ip = None + self.myips_according_to_yourip = [] + self.returnconnhand = ReturnConnHandler.getInstance() + + + def getInstance(*args, **kw): + if DialbackMsgHandler.__single is None: + DialbackMsgHandler(*args, **kw) + return DialbackMsgHandler.__single + getInstance = staticmethod(getInstance) + + def register(self,overlay_bridge,launchmany,rawserver,config): + """ Called by MainThread """ + self.overlay_bridge = overlay_bridge + self.rawserver = rawserver + self.launchmany = launchmany + self.peer_db = launchmany.peer_db + self.superpeer_db = launchmany.superpeer_db + self.active = config['dialback_active'], + self.trust_superpeers = config['dialback_trust_superpeers'] + self.returnconnhand.register(self.rawserver,launchmany.multihandler,launchmany.listen_port,config['overlay_max_message_length']) + self.returnconnhand.register_conns_callback(self.network_handleReturnConnConnection) + self.returnconnhand.register_recv_callback(self.network_handleReturnConnMessage) + self.returnconnhand.start_listening() + + self.old_ext_ip = launchmany.get_ext_ip() + + + def register_yourip(self,launchmany): + """ Called by MainThread """ + self.launchmany = launchmany + + + def olthread_handleSecOverlayConnection(self,exc,permid,selversion,locally_initiated): + """ + Called from OverlayApps to signal there is an overlay-connection, + see if we should ask it to dialback + """ + # Called by overlay thread + if DEBUG: + print >> sys.stderr,"dialback: handleConnection",exc,"v",selversion,"local",locally_initiated + if selversion < OLPROTO_VER_THIRD: + return True + + if exc is not None: + try: + del self.peers_asked[permid] + except: + if DEBUG: + print >> sys.stderr,"dialback: handleConnection: Got error on connection that we didn't ask for dialback" + pass + return + + if self.consensusip is None: + self.ntries += 1 + if self.ntries >= MAX_TRIES: + if DEBUG: + print >> sys.stderr,"dialback: tried too many times, giving up" + return True + + if self.dbreach or self.btenginereach: + self.launchmany.set_activity(NTFY_ACT_GET_EXT_IP_FROM_PEERS) + else: + self.launchmany.set_activity(NTFY_ACT_REACHABLE) + + # Also do this when the connection is not locally initiated. + # That tells us that we're connectable, but it doesn't tell us + # our external IP address. + if self.active: + self.olthread_attempt_request_dialback(permid) + return True + + def olthread_attempt_request_dialback(self,permid): + # Called by overlay thread + if DEBUG: + print >> sys.stderr,"dialback: attempt dialback request",show_permid_short(permid) + + dns = self.olthread_get_dns_from_peerdb(permid) + ipinuse = False + + # 1. Remove peers we asked but didn't succeed in connecting back + threshold = time()-REPLY_WAIT + newdict = {} + for permid2,peerrec in self.peers_asked.iteritems(): + if peerrec['reqtime'] >= threshold: + newdict[permid2] = peerrec + if peerrec['dns'][0] == dns[0]: + ipinuse = True + self.peers_asked = newdict + + # 2. Already asked? + if permid in self.peers_asked or ipinuse or len(self.peers_asked) >= PEERS_TO_ASK: + # ipinuse protects a little against attacker that want us to believe + # we have a certain IP address. + if DEBUG: + pipa = permid in self.peers_asked + lpa = len(self.peers_asked) + print >> sys.stderr,"dialback: No request made to",show_permid_short(permid),"already asked",pipa,"IP in use",ipinuse,"nasked",lpa + + return + dns = self.olthread_get_dns_from_peerdb(permid) + + # 3. Ask him to dialback + peerrec = {'dns':dns,'reqtime':time()} + self.peers_asked[permid] = peerrec + self.overlay_bridge.connect(permid,self.olthread_request_connect_callback) + + def olthread_request_connect_callback(self,exc,dns,permid,selversion): + # Called by overlay thread + if exc is None: + if selversion >= OLPROTO_VER_THIRD: + self.overlay_bridge.send(permid, DIALBACK_REQUEST+'',self.olthread_request_send_callback) + elif DEBUG: + print >> sys.stderr,"dialback: DIALBACK_REQUEST: peer speaks old protocol, weird",show_permid_short(permid) + elif DEBUG: + print >> sys.stderr,"dialback: DIALBACK_REQUEST: error connecting to",show_permid_short(permid),exc + + + def olthread_request_send_callback(self,exc,permid): + # Called by overlay thread + if exc is not None: + if DEBUG: + print >> sys.stderr,"dialback: DIALBACK_REQUEST error sending to",show_permid_short(permid),exc + pass + + def olthread_handleSecOverlayMessage(self,permid,selversion,message): + """ + Handle incoming DIALBACK_REQUEST messages + """ + # Called by overlay thread + t = message[0] + + if t == DIALBACK_REQUEST: + if DEBUG: + print >> sys.stderr,"dialback: Got DIALBACK_REQUEST",len(message),show_permid_short(permid) + return self.olthread_process_dialback_request(permid, message, selversion) + else: + if DEBUG: + print >> sys.stderr,"dialback: UNKNOWN OVERLAY MESSAGE", ord(t) + return False + + + def olthread_process_dialback_request(self,permid,message,selversion): + # Called by overlay thread + # 1. Check + if len(message) != 1: + if DEBUG: + print >> sys.stderr,"dialback: DIALBACK_REQUEST: message too big" + return False + + # 2. Retrieve peer's IP address + dns = self.olthread_get_dns_from_peerdb(permid) + + # 3. Send back reply + # returnconnhand uses the network thread to do stuff, so the callback + # will be made by the network thread + self.returnconnhand.connect_dns(dns,self.network_returnconn_reply_connect_callback) + + # 4. Message processed OK, don't know about sending of reply though + return True + + + def network_returnconn_reply_connect_callback(self,exc,dns): + # Called by network thread + + if not currentThread().getName().startswith("NetworkThread"): + print >>sys.stderr,"dialback: network_returnconn_reply_connect_callback: called by",currentThread().getName()," not NetworkThread" + print_stack() + + if exc is None: + hisip = str(dns[0]) # Arno, 2010-01-28: protection against DB returning Unicode IPs + try: + reply = bencode(hisip) + if DEBUG: + print >> sys.stderr,"dialback: DIALBACK_REPLY: sending to",dns + self.returnconnhand.send(dns, DIALBACK_REPLY+reply, self.network_returnconn_reply_send_callback) + except: + print_exc() + return False + elif DEBUG: + print >> sys.stderr,"dialback: DIALBACK_REPLY: error connecting to",dns,exc + + def network_returnconn_reply_send_callback(self,exc,dns): + # Called by network thread + if DEBUG: + print >> sys.stderr,"dialback: DIALBACK_REPLY: send callback:",dns,exc + + + if exc is not None: + if DEBUG: + print >> sys.stderr,"dialback: DIALBACK_REPLY: error sending to",dns,exc + pass + + # + # Receipt of connection that would carry DIALBACK_REPLY + # + def network_handleReturnConnConnection(self,exc,dns,locally_initiated): + # Called by network thread + if DEBUG: + print >> sys.stderr,"dialback: DIALBACK_REPLY: Got connection from",dns,exc + pass + + def network_handleReturnConnMessage(self,dns,message): + # Called by network thread + t = message[0] + + if t == DIALBACK_REPLY: + if DEBUG: + print >> sys.stderr,"dialback: Got DIALBACK_REPLY",len(message),dns + + # Hand over processing to overlay thread + olthread_process_dialback_reply_lambda = lambda:self.olthread_process_dialback_reply(dns, message) + self.overlay_bridge.add_task(olthread_process_dialback_reply_lambda,0) + + # We're done and no longer need the return connection, so + # call close explicitly + self.returnconnhand.close(dns) + return True + else: + if DEBUG: + print >> sys.stderr,"dialback: UNKNOWN RETURNCONN MESSAGE", ord(t) + return False + + + def olthread_process_dialback_reply(self,dns,message): + # Called by overlay thread + + # 1. Yes, we're reachable, now just matter of determining ext IP + self.dbreach = True + + # 2. Authentication: did I ask this peer? + permid = self.olthread_permid_of_asked_peer(dns) + if permid is None: + if DEBUG: + print >> sys.stderr,"dialback: DIALBACK_REPLY: Got reply from peer I didn't ask",dns + return False + + del self.peers_asked[permid] + + # 3. See what he sent us + try: + myip = bdecode(message[1:]) + except: + print_exc() + if DEBUG: + print >> sys.stderr,"dialback: DIALBACK_REPLY: error becoding" + return False + if not isValidIP(myip): + if DEBUG: + print >> sys.stderr,"dialback: DIALBACK_REPLY: invalid IP" + return False + + + # 4. See if superpeer, then we're done, trusted source + if self.trust_superpeers: + superpeers = self.superpeer_db.getSuperPeers() + if permid in superpeers: + if DEBUG: + print >> sys.stderr,"dialback: DIALBACK_REPLY: superpeer said my IP address is",myip,"setting it to that" + self.consensusip = myip + self.fromsuperpeer = True + else: + # 5, 6. 7, 8. Record this peers opinion and see if we get a + # majority vote. + # + self.myips,consensusip = tally_opinion(myip,self.myips,PEERS_TO_AGREE) + if self.consensusip is None: + self.consensusip = consensusip + + # 8. Change IP address if different + if self.consensusip is not None: + + self.launchmany.dialback_got_ext_ip_callback(self.consensusip) + if DEBUG: + print >> sys.stderr,"dialback: DIALBACK_REPLY: I think my IP address is",self.old_ext_ip,"others say",self.consensusip,", setting it to latter" + + # 9. Notify GUI that we are connectable + self.launchmany.dialback_reachable_callback() + + return True + + + # + # Information from other modules + # + def network_btengine_reachable_callback(self): + """ Called by network thread """ + if self.launchmany is not None: + self.launchmany.dialback_reachable_callback() + + # network thread updating our state. Ignoring concurrency, as this is a + # one time op. + self.btenginereach = True + + def isConnectable(self): + """ Called by overlay (BuddyCast) and network (Rerequester) thread + and now also any thread via Session.get_externally_reachable() """ + + # network thread updating our state. Ignoring concurrency, as these + # variables go from False to True once and stay there, or remain False + return self.dbreach or self.btenginereach + + + def network_btengine_extend_yourip(self,myip): + """ Called by Connecter when we receive an EXTEND handshake that + contains an yourip line. + + TODO: weigh opinion based on whether we locally initiated the connection + from a trusted tracker response, or that the address came from ut_pex. + """ + self.myips_according_to_yourip, yourip_consensusip = tally_opinion(myip,self.myips_according_to_yourip,YOURIP_PEERS_TO_AGREE) + if DEBUG: + print >> sys.stderr,"dialback: yourip: someone said my IP is",myip + if yourip_consensusip is not None: + self.launchmany.yourip_got_ext_ip_callback(yourip_consensusip) + if DEBUG: + print >> sys.stderr,"dialback: yourip: I think my IP address is",self.old_ext_ip,"others via EXTEND hs say",yourip_consensusip,"recording latter as option" + + # + # Internal methods + # + def olthread_get_dns_from_peerdb(self,permid): + dns = None + peer = self.peer_db.getPeer(permid) + #print >>sys.stderr,"dialback: get_dns_from_peerdb: Got peer",peer + if peer: + ip = self.to_real_ip(peer['ip']) + dns = (ip, int(peer['port'])) + return dns + + def to_real_ip(self,hostname_or_ip): + """ If it's a hostname convert it to IP address first """ + ip = None + try: + """ Speed up: don't go to DNS resolver unnecessarily """ + socket.inet_aton(hostname_or_ip) + ip = hostname_or_ip + except: + try: + ip = socket.gethostbyname(hostname_or_ip) + except: + print_exc() + return ip + + + def olthread_permid_of_asked_peer(self,dns): + for permid,peerrec in self.peers_asked.iteritems(): + if peerrec['dns'] == dns: + # Yes, we asked this peer + return permid + return None + + +def tally_opinion(myip,oplist,requiredquorum): + + consensusip = None + + # 5. Ordinary peer, just add his opinion + oplist.append([myip,time()]) + if DEBUG: + print >> sys.stderr,"dialback: DIALBACK_REPLY: peer said I have IP address",myip + + # 6. Remove stale opinions + newlist = [] + threshold = time()-REPLY_VALIDITY + for pair in oplist: + if pair[1] >= threshold: + newlist.append(pair) + oplist = newlist + + # 7. See if we have X peers that agree + opinions = {} + for pair in oplist: + ip = pair[0] + if not (ip in opinions): + opinions[ip] = 1 + else: + opinions[ip] += 1 + + for o in opinions: + if opinions[o] >= requiredquorum: + # We have a quorum + if consensusip is None: + consensusip = o + if DEBUG: + print >> sys.stderr,"dialback: DIALBACK_REPLY: Got consensus on my IP address being",consensusip + else: + # Hmmmm... more than one consensus + pass + + return oplist,consensusip diff --git a/instrumentation/next-share/BaseLib/Core/NATFirewall/NatCheck.py b/instrumentation/next-share/BaseLib/Core/NATFirewall/NatCheck.py new file mode 100644 index 0000000..e42d36f --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/NATFirewall/NatCheck.py @@ -0,0 +1,211 @@ +# Written by Lucia D'Acunto +# see LICENSE.txt for license information + +import socket +import sys + +DEBUG = False + +def Test1(udpsock, serveraddr): + """ + The client sends a request to a server asking it to send the + response back to the address and port the request came from + """ + + retVal = {"resp":False, "ex_ip":None, "ex_port":None} + BUFSIZ = 1024 + reply = "" + request = "ping1" + + udpsock.sendto(request, serveraddr) + + try: + reply, rcvaddr = udpsock.recvfrom(BUFSIZ) + except socket.timeout: + if DEBUG: print >> sys.stderr, "NATCheck:", "Connection attempt to %s timed out" % (serveraddr,) + return retVal + + except ValueError, (strerror): + if DEBUG: print >> sys.stderr, "NATCheck:", "Could not receive data: %s" % (strerror) + return retVal + except socket.error, (errno, strerror): + if DEBUG: print >> sys.stderr, "NATCheck:", "Could not receive data: %s" % (strerror) + return retVal + + ex_ip, ex_port = reply.split(":") + + retVal["resp"] = True + retVal["ex_ip"] = ex_ip + retVal["ex_port"] = ex_port + + return retVal + +def Test2(udpsock, serveraddr): + """ + The client sends a request asking to receive an echo from a + different address and a different port on the address and port the + request came from + """ + + retVal = {"resp":False} + BUFSIZ = 1024 + request = "ping2" + + udpsock.sendto(request, serveraddr) + + try: + reply, rcvaddr = udpsock.recvfrom(BUFSIZ) + except socket.timeout: + #if DEBUG: print >> sys.stderr, "NATCheck:", "Connection attempt to %s timed out" % (serveraddr,) + return retVal + except ValueError, (strerror): + if DEBUG: print >> sys.stderr, "NATCheck:", "Could not receive data: %s" % (strerror) + return retVal + except socket.error, (errno, strerror): + if DEBUG: print >> sys.stderr, "NATCheck:", "Could not receive data: %s" % (strerror) + return retVal + + retVal["resp"] = True + + return retVal + +def Test3(udpsock, serveraddr): + """ + The client sends a request asking to receive an echo from the same + address but from a different port on the address and port the + request came from + """ + + retVal = {"resp":False, "ex_ip":None, "ex_port":None} + BUFSIZ = 1024 + reply = "" + request = "ping3" + + udpsock.sendto(request, serveraddr) + + try: + reply, rcvaddr = udpsock.recvfrom(BUFSIZ) + except socket.timeout: + #if DEBUG: print >> sys.stderr, "NATCheck:", "Connection attempt to %s timed out" % (serveraddr,) + return retVal + except ValueError, (strerror): + if DEBUG: print >> sys.stderr, "NATCheck:", "Could not receive data: %s" % (strerror) + return retVal + except socket.error, (errno, strerror): + if DEBUG: print >> sys.stderr, "NATCheck:", "Could not receive data: %s" % (strerror) + return retVal + + ex_ip, ex_port = reply.split(":") + + retVal["resp"] = True + retVal["ex_ip"] = ex_ip + retVal["ex_port"] = ex_port + + return retVal + +# Returns information about the NAT the client is behind +def GetNATType(in_port, serveraddr1, serveraddr2): + """ + Returns the NAT type according to the STUN algorithm, as well as the external + address (ip, port) and the internal address of the host + """ + + serveraddr1 = ('stun1.tribler.org',6701) + serveraddr2 = ('stun2.tribler.org',6702) + + nat_type, ex_ip, ex_port, in_ip = [-1, "Unknown"], "0.0.0.0", "0", "0.0.0.0" + + # Set up the socket + udpsock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + udpsock.settimeout(5) + try: + udpsock.bind(('',in_port)) + except socket.error, err: + print >> sys.stderr, "Couldn't bind a udp socket on port %d : %s" % (in_port, err) + return (nat_type, ex_ip, ex_port, in_ip) + try: + # Get the internal IP address + connectaddr = ('tribler.org',80) + s = socket.socket() + s.connect(connectaddr) + in_ip = s.getsockname()[0] + del s + if DEBUG: print >> sys.stderr, "NATCheck: getting the internal ip address by connecting to tribler.org:80", in_ip + except socket.error, err: + print >> sys.stderr, "Couldn't connect to %s:%i" % (connectaddr[0], connectaddr[1]) + return (nat_type, ex_ip, ex_port, in_ip) + + """ + EXECUTE THE STUN ALGORITHM + """ + + # Do Test I + ret = Test1(udpsock, serveraddr1) + + if DEBUG: print >> sys.stderr, "NATCheck:", "Test I reported: " + str(ret) + + if ret["resp"] == False: + nat_type[1] = "Blocked" + + else: + ex_ip = ret["ex_ip"] + ex_port = ret["ex_port"] + + if ret["ex_ip"] == in_ip: # No NAT: check for firewall + + if DEBUG: print >> sys.stderr, "NATCheck:", "No NAT" + + # Do Test II + ret = Test2(udpsock, serveraddr1) + if DEBUG: print >> sys.stderr, "NATCheck:", "Test II reported: " + str(ret) + + if ret["resp"] == True: + nat_type[0] = 0 + nat_type[1] = "Open Internet" + else: + if DEBUG: print >> sys.stderr, "NATCheck:", "There is a Firewall" + + # Do Test III + ret = Test3(udpsock, serveraddr1) + if DEBUG: print >> sys.stderr, "NATCheck:", "Test III reported: " + str(ret) + + if ret["resp"] == True: + nat_type[0] = 2 + nat_type[1] = "Restricted Cone Firewall" + else: + nat_type[0] = 3 + nat_type[1] = "Port Restricted Cone Firewall" + + else: # There is a NAT + if DEBUG: print >> sys.stderr, "NATCheck:", "There is a NAT" + + # Do Test II + ret = Test2(udpsock, serveraddr1) + if DEBUG: print >> sys.stderr, "NATCheck:", "Test II reported: " + str(ret) + if ret["resp"] == True: + nat_type[0] = 1 + nat_type[1] = "Full Cone NAT" + else: + #Do Test I using a different echo server + ret = Test1(udpsock, serveraddr2) + if DEBUG: print >> sys.stderr, "NATCheck:", "Test I reported: " + str(ret) + + if ex_ip == ret["ex_ip"] and ex_port == ret["ex_port"]: # Public address is constant: consistent translation + + # Do Test III + ret = Test3(udpsock, serveraddr1) + if DEBUG: print >> sys.stderr, "NATCheck:", "Test III reported: " + str(ret) + + if ret["resp"] == True: + nat_type[0] = 2 + nat_type[1] = "Restricted Cone NAT" + else: + nat_type[0] = 3 + nat_type[1] = "Port Restricted Cone NAT" + + else: + nat_type[0] = -1 + nat_type[1] = "Symmetric NAT" + + udpsock.close() + return (nat_type, ex_ip, ex_port, in_ip) diff --git a/instrumentation/next-share/BaseLib/Core/NATFirewall/NatCheckMsgHandler.py b/instrumentation/next-share/BaseLib/Core/NATFirewall/NatCheckMsgHandler.py new file mode 100644 index 0000000..1eae139 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/NATFirewall/NatCheckMsgHandler.py @@ -0,0 +1,427 @@ +# Written by Lucia D'Acunto +# see LICENSE.txt for license information + +from time import strftime +from traceback import print_exc +import datetime +import random +import socket +import sys +import thread + +from BaseLib.Core.BitTornado.BT1.MessageID import CRAWLER_NATCHECK, CRAWLER_NATTRAVERSAL +from BaseLib.Core.BitTornado.bencode import bencode, bdecode +from BaseLib.Core.NATFirewall.ConnectionCheck import ConnectionCheck +from BaseLib.Core.NATFirewall.NatTraversal import tryConnect, coordinateHolePunching +from BaseLib.Core.Overlay.SecureOverlay import OLPROTO_VER_EIGHTH, OLPROTO_VER_NINETH, SecureOverlay +from BaseLib.Core.Statistics.Crawler import Crawler +from BaseLib.Core.Utilities.utilities import show_permid, show_permid_short +from types import IntType, StringType, ListType, TupleType +from BaseLib.Core.simpledefs import * + +DEBUG = False + +PEERLIST_LEN = 100 + +class NatCheckMsgHandler: + + __single = None + + def __init__(self): + if NatCheckMsgHandler.__single: + raise RuntimeError, "NatCheckMsgHandler is singleton" + NatCheckMsgHandler.__single = self + self.crawler_reply_callbacks = [] + self._secure_overlay = SecureOverlay.getInstance() + + self.crawler = Crawler.get_instance() + if self.crawler.am_crawler(): + self._file = open("natcheckcrawler.txt", "a") + self._file.write("\n".join(("# " + "*" * 80, strftime("%Y/%m/%d %H:%M:%S"), "# Crawler started\n"))) + self._file.flush() + self._file2 = open("nattraversalcrawler.txt", "a") + self._file2.write("\n".join(("# " + "*" * 80, strftime("%Y/%m/%d %H:%M:%S"), "# Crawler started\n"))) + self._file2.flush() + self.peerlist = [] + self.holePunchingIP = socket.gethostbyname(socket.gethostname()) + self.trav = {} + + else: + self._file = None + + @staticmethod + def getInstance(*args, **kw): + if NatCheckMsgHandler.__single is None: + NatCheckMsgHandler(*args, **kw) + return NatCheckMsgHandler.__single + + def register(self, launchmany): + if DEBUG: + print >> sys.stderr, "NatCheckMsgHandler: register" + + self.session = launchmany.session + self.doNatCheckSender = None + self.registered = True + + def doNatCheck(self, target_permid, selversion, request_callback): + """ + The nat-check initiator_callback + """ + + # for Tribler versions < 4.5.0 : do nothing + # TODO: change OLPROTO_VER_EIGHTH to OLPROTO_VER_SEVENTH + if selversion < OLPROTO_VER_NINETH: + if DEBUG: + print >> sys.stderr, "NatCheckMsgHandler: Tribler version too old for NATCHECK: do nothing" + return False + + if DEBUG: + print >> sys.stderr, "NatCheckMsgHandler: do NATCHECK" + + # send the message + request_callback(CRAWLER_NATCHECK, "", callback=self.doNatCheckCallback) + + return True + + def doNatCheckCallback(self, exc, permid): + + if exc is not None: + return False + if DEBUG: + print >> sys.stderr, "NATCHECK_REQUEST was sent to", show_permid_short(permid), exc + + # Register peerinfo on file + self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), + "REQUEST", + show_permid(permid), + str(self._secure_overlay.get_dns_from_peerdb(permid)), + "\n"))) + self._file.flush() + return True + + def gotDoNatCheckMessage(self, sender_permid, selversion, channel_id, payload, reply_callback): + """ + The handle-request callback + """ + + self.doNatCheckSender = sender_permid + self.crawler_reply_callbacks.append(reply_callback) + + try: + if DEBUG: + print >>sys.stderr,"NatCheckMsgHandler: start_nat_type_detect()" + conn_check = ConnectionCheck.getInstance(self.session) + conn_check.try_start(self.natthreadcb_natCheckReplyCallback) + except: + print_exc() + return False + + return True + + def natthreadcb_natCheckReplyCallback(self, ncr_data): + if DEBUG: + print >> sys.stderr, "NAT type: ", ncr_data + + # send the message to the peer who has made the NATCHECK request, if any + if self.doNatCheckSender is not None: + try: + ncr_msg = bencode(ncr_data) + except: + print_exc() + if DEBUG: print >> sys.stderr, "error ncr_data:", ncr_data + return False + if DEBUG: + print >> sys.stderr, "NatCheckMsgHandler:", ncr_data + + # todo: make sure that natthreadcb_natCheckReplyCallback is always called for a request + # send replies to all the requests that have been received so far + for reply_callback in self.crawler_reply_callbacks: + reply_callback(ncr_msg, callback=self.natCheckReplySendCallback) + self.crawler_reply_callbacks = [] + + + def natCheckReplySendCallback(self, exc, permid): + if DEBUG: + print >> sys.stderr, "NATCHECK_REPLY was sent to", show_permid_short(permid), exc + if exc is not None: + return False + return True + + def gotNatCheckReplyMessage(self, permid, selversion, channel_id, channel_data, error, payload, request_callback): + """ + The handle-reply callback + """ + if error: + if DEBUG: + print >> sys.stderr, "NatCheckMsgHandler: gotNatCheckReplyMessage" + print >> sys.stderr, "NatCheckMsgHandler: error", error + + # generic error: another crawler already obtained these results + self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), + " REPLY", + show_permid(permid), + str(self._secure_overlay.get_dns_from_peerdb(permid)), + "ERROR(%d)" % error, + payload, + "\n"))) + self._file.flush() + + else: + try: + recv_data = bdecode(payload) + except: + print_exc() + print >> sys.stderr, "bad encoded data:", payload + return False + + try: # check natCheckReply message + self.validNatCheckReplyMsg(recv_data) + except RuntimeError, e: + print >> sys.stderr, e + return False + + if DEBUG: + print >> sys.stderr, "NatCheckMsgHandler: received NAT_CHECK_REPLY message: ", recv_data + + # Register peerinfo on file + self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), + " REPLY", + show_permid(permid), + str(self._secure_overlay.get_dns_from_peerdb(permid)), + ":".join([str(x) for x in recv_data]), + "\n"))) + self._file.flush() + + # for Tribler versions < 5.0 : do nothing + if selversion < OLPROTO_VER_NINETH: + if DEBUG: + print >> sys.stderr, "NatCheckMsgHandler: Tribler version too old for NATTRAVERSAL: do nothing" + return True + + if DEBUG: + print >> sys.stderr, "NatCheckMsgHandler: do NATTRAVERSAL" + + # Save peer in peerlist + if len(self.peerlist) == PEERLIST_LEN: + del self.peerlist[0] + self.peerlist.append([permid,recv_data[1],recv_data[2]]) + if DEBUG: + print >> sys.stderr, "NatCheckMsgHandler: peerlist length is: ", len(self.peerlist) + + # Try to perform hole punching + if len(self.peerlist) >= 2: + self.tryHolePunching() + + return True + + def validNatCheckReplyMsg(self, ncr_data): + + if not type(ncr_data) == ListType: + raise RuntimeError, "NatCheckMsgHandler: received data is not valid. It must be a list of parameters." + return False + + if not type(ncr_data[0]) == StringType: + raise RuntimeError, "NatCheckMsgHandler: received data is not valid. The first element in the list must be a string." + return False + + if not type(ncr_data[1]) == IntType: + raise RuntimeError, "NatCheckMsgHandler: received data is not valid. The second element in the list must be an integer." + return False + + if not type(ncr_data[2]) == IntType: + raise RuntimeError, "NatCheckMsgHandler: received data is not valid. The third element in the list must be an integer." + return False + + if not type(ncr_data[3]) == StringType: + raise RuntimeError, "NatCheckMsgHandler: received data is not valid. The forth element in the list must be a string." + return False + + if not type(ncr_data[4]) == IntType: + raise RuntimeError, "NatCheckMsgHandler: received data is not valid. The fifth element in the list must be an integer." + return False + + if not type(ncr_data[5]) == StringType: + raise RuntimeError, "NatCheckMsgHandler: received data is not valid. The sixth element in the list must be a string." + return False + + if not type(ncr_data[6]) == IntType: + raise RuntimeError, "NatCheckMsgHandler: received data is not valid. The seventh element in the list must be an integer." + return False + + def tryHolePunching(self): + if DEBUG: + print >> sys.stderr, "NatCheckMsgHandler: first element in peerlist", self.peerlist[len(self.peerlist)-1] + print >> sys.stderr, "NatCheckMsgHandler: second element in peerlist", self.peerlist[len(self.peerlist)-2] + + holePunchingPort = random.randrange(3200, 4200, 1) + holePunchingAddr = (self.holePunchingIP, holePunchingPort) + + peer1 = self.peerlist[len(self.peerlist)-1] + peer2 = self.peerlist[len(self.peerlist)-2] + + request_id = str(show_permid_short(peer1[0]) + show_permid_short(peer2[0]) + str(random.randrange(0, 1000, 1))) + + self.udpConnect(peer1[0], request_id, holePunchingAddr) + self.udpConnect(peer2[0], request_id, holePunchingAddr) + + # Register peerinfo on file + self._file2.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), + "REQUEST", + request_id, + show_permid(peer1[0]), + str(peer1[1]), + str(peer1[2]), + str(self._secure_overlay.get_dns_from_peerdb(peer1[0])), + show_permid(peer2[0]), + str(peer2[1]), + str(peer2[2]), + str(self._secure_overlay.get_dns_from_peerdb(peer2[0])), + "\n"))) + self._file2.flush() + + self.trav[request_id] = (None, None) + thread.start_new_thread(coordinateHolePunching, (peer1, peer2, holePunchingAddr)) + + def udpConnect(self, permid, request_id, holePunchingAddr): + + if DEBUG: + print >> sys.stderr, "NatCheckMsgHandler: request UDP connection" + + mh_data = request_id + ":" + holePunchingAddr[0] + ":" + str(holePunchingAddr[1]) + + if DEBUG: + print >> sys.stderr, "NatCheckMsgHandler: udpConnect message is", mh_data + + try: + mh_msg = bencode(mh_data) + except: + print_exc() + if DEBUG: print >> sys.stderr, "NatCheckMsgHandler: error mh_data:", mh_data + return False + + # send the message + self.crawler.send_request(permid, CRAWLER_NATTRAVERSAL, mh_msg, frequency=0, callback=self.udpConnectCallback) + + if DEBUG: + print >> sys.stderr, "NatCheckMsgHandler: request for", show_permid_short(permid), "sent to crawler" + + def udpConnectCallback(self, exc, permid): + + if exc is not None: + if DEBUG: + print >> sys.stderr, "NATTRAVERSAL_REQUEST failed to", show_permid_short(permid), exc + + # Register peerinfo on file + self._file2.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), + "REQUEST FAILED", + show_permid(permid), + str(self._secure_overlay.get_dns_from_peerdb(permid)), + "\n"))) + return False + + if DEBUG: + print >> sys.stderr, "NATTRAVERSAL_REQUEST was sent to", show_permid_short(permid), exc + return True + + def gotUdpConnectRequest(self, permid, selversion, channel_id, mh_msg, reply_callback): + + if DEBUG: + print >> sys.stderr, "NatCheckMsgHandler: gotUdpConnectRequest from", show_permid_short(permid) + + try: + mh_data = bdecode(mh_msg) + except: + print_exc() + print >> sys.stderr, "NatCheckMsgHandler: bad encoded data:", mh_msg + return False + + if DEBUG: + print >> sys.stderr, "NatCheckMsgHandler: gotUdpConnectRequest is", mh_data + + + try: + request_id, host, port = mh_data.split(":") + except: + print_exc() + print >> sys.stderr, "NatCheckMsgHandler: error in received data:", mh_data + return False + + coordinator = (host, int(port)) + + if DEBUG: + print >> sys.stderr, "NatCheckMsgHandler: coordinator address is", coordinator + + mhr_data = request_id + ":" + tryConnect(coordinator) + + # Report back to coordinator + try: + mhr_msg = bencode(mhr_data) + except: + print_exc() + print >> sys.stderr, "NatCheckMsgHandler: error in encoding data:", mhr_data + return False + + reply_callback(mhr_msg, callback=self.udpConnectReplySendCallback) + + def udpConnectReplySendCallback(self, exc, permid): + + if DEBUG: + print >> sys.stderr, "NATTRAVERSAL_REPLY was sent to", show_permid_short(permid), exc + if exc is not None: + return False + return True + + + def gotUdpConnectReply(self, permid, selversion, channel_id, channel_data, error, mhr_msg, request_callback): + + if DEBUG: + print >> sys.stderr, "NatCheckMsgHandler: gotMakeHoleReplyMessage" + + try: + mhr_data = bdecode(mhr_msg) + except: + print_exc() + print >> sys.stderr, "NatCheckMsgHandler: bad encoded data:", mhr_msg + return False + + if DEBUG: + print >> sys.stderr, "NatCheckMsgHandler: message is", mhr_data + + try: + request_id, reply = mhr_data.split(":") + except: + print_exc() + print >> sys.stderr, "NatCheckMsgHandler: error in received data:", mhr_data + return False + + if DEBUG: + print >> sys.stderr, "NatCheckMsgHandler: request_id is", request_id + + if request_id in self.trav: + if DEBUG: + print >> sys.stderr, "NatCheckMsgHandler: request_id is in the list" + peer, value = self.trav[request_id] + if peer == None: # first peer reply + if DEBUG: + print >> sys.stderr, "NatCheckMsgHandler: first peer reply" + self.trav[request_id] = ( (permid, self._secure_overlay.get_dns_from_peerdb(permid)), reply ) + elif type(peer) == TupleType: # second peer reply + if DEBUG: + print >> sys.stderr, "NatCheckMsgHandler: second peer reply" + + # Register peerinfo on file + self._file2.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), + " REPLY", + request_id, + show_permid(peer[0]), + str(peer[1]), + value, + show_permid(permid), + str(self._secure_overlay.get_dns_from_peerdb(permid)), + reply, + "\n"))) + + del self.trav[request_id] + + self._file2.flush() + diff --git a/instrumentation/next-share/BaseLib/Core/NATFirewall/NatTraversal.py b/instrumentation/next-share/BaseLib/Core/NATFirewall/NatTraversal.py new file mode 100644 index 0000000..e301ed8 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/NATFirewall/NatTraversal.py @@ -0,0 +1,178 @@ +from time import strftime +from traceback import print_exc +import socket +import sys + +DEBUG = False + +def coordinateHolePunching(peer1, peer2, holePunchingAddr): + + if DEBUG: + print >> sys.stderr, "NatTraversal: coordinateHolePunching at", holePunchingAddr + + # Set up the sockets + try : + udpsock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + udpsock.bind(holePunchingAddr) + udpsock.settimeout(60) + + except socket.error, (errno, strerror) : + + if udpsock : + udpsock.close() + + if DEBUG: + print >> sys.stderr, "NatTraversal: Could not open socket: %s" % (strerror) + + return + + if DEBUG: + print >> sys.stderr, "NatTraversal: waiting for connection..." + + # Receive messages + peeraddr2 = None + while True: + + try: + data, peeraddr1 = udpsock.recvfrom(1024) + if not data: + continue + else: + if DEBUG: + print >> sys.stderr, "NatTraversal:", strftime("%Y/%m/%d %H:%M:%S"), "...connected from: ", peeraddr1 + if peeraddr2 == None: + peeraddr2 = peeraddr1 + elif peeraddr2 != peeraddr1: + udpsock.sendto(peeraddr1[0] + ":" + str(peeraddr1[1]), peeraddr2) + udpsock.sendto(peeraddr1[0] + ":" + str(peeraddr1[1]), peeraddr2) + udpsock.sendto(peeraddr1[0] + ":" + str(peeraddr1[1]), peeraddr2) + udpsock.sendto(peeraddr2[0] + ":" + str(peeraddr2[1]), peeraddr1) + udpsock.sendto(peeraddr2[0] + ":" + str(peeraddr2[1]), peeraddr1) + udpsock.sendto(peeraddr2[0] + ":" + str(peeraddr2[1]), peeraddr1) + break + + except socket.timeout, error: + if DEBUG: + print >> sys.stderr, "NatTraversal: timeout with peers", error + udpsock.close() + break + + # Close socket + udpsock.close() + +def tryConnect(coordinator): + + # Set up the socket + udpsock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + udpsock.settimeout(5) + + # Send messages + udpsock.sendto("ping",coordinator) + udpsock.sendto("ping",coordinator) + udpsock.sendto("ping",coordinator) + if DEBUG: + print >> sys.stderr, "NatTraversal: sending ping to ", coordinator + + # Wait for response from the coordinator + + while True: + data = None + addr = None + try: + data, addr = udpsock.recvfrom(1024) + except socket.timeout, (strerror): + if DEBUG: + print >> sys.stderr, "NatTraversal: timeout with coordinator" + return "ERR" + + if addr == coordinator: + if DEBUG: + print >> sys.stderr, "NatTraversal: received", data, "from coordinator" + break + + if DEBUG: + print >> sys.stderr, "NatTraversal: received", data, "from", addr + + #success = False + #try: + # host, port = data.split(":") + #except: + # print_exc() + # print >> sys.stderr, "NatCheckMsgHandler: error in received data:", data + # return success + # peer = (host, int(port)) + # for i in range(3): + # udpsock.sendto("hello",peer) + # udpsock.sendto("hello",peer) + # udpsock.sendto("hello",peer) + + # try: + # data, addr = udpsock.recvfrom(1024) + + # except socket.timeout, (strerror): + # if DEBUG: + # print >> sys.stderr, "NatTraversal: first timeout", strerror + # print >> sys.stderr, "NatTraversal: resend" + + # else: + # success = True + # break + + try: + host, port = data.split(":") + except: + print_exc() + print >> sys.stderr, "NatCheckMsgHandler: error in received data:", data + return "ERR" + + peer = (host, int(port)) + udpsock.sendto("hello",peer) + udpsock.sendto("hello",peer) + udpsock.sendto("hello",peer) + + # Wait for response + data = None + addr = None + + while True: + try: + data, addr = udpsock.recvfrom(1024) + except socket.timeout, (strerror): + if DEBUG: + print >> sys.stderr, "NatTraversal: first timeout", strerror + print >> sys.stderr, "NatTraversal: resend" + + udpsock.sendto("hello", peer) + udpsock.sendto("hello", peer) + udpsock.sendto("hello", peer) + + try: + data, addr = udpsock.recvfrom(1024) + except socket.timeout, (strerror): + if DEBUG: + print >> sys.stderr, "NatTraversal: second timeout", strerror + + return "NO" + + # data received, check address + if addr == peer: # peer is not symmetric NAT + break + + if addr[0] == peer[0]: # peer has a symmetric NAT + peer = addr + break + + + udpsock.sendto("hello",peer) + udpsock.sendto("hello",peer) + udpsock.sendto("hello",peer) + + # Close socket + udpsock.close() + + if DEBUG: + print >> sys.stderr, "NatTraversal: message from", addr, "is", data + + return "YES" + + diff --git a/instrumentation/next-share/BaseLib/Core/NATFirewall/ReturnConnHandler.py b/instrumentation/next-share/BaseLib/Core/NATFirewall/ReturnConnHandler.py new file mode 100644 index 0000000..904a0f4 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/NATFirewall/ReturnConnHandler.py @@ -0,0 +1,603 @@ +# Written by Arno Bakker, Bram Cohen, Jie Yang +# see LICENSE.txt for license information +# +# This class receives all connections and messages destined for +# infohash = 0x00 0x00 ... 0x01 +# The peer sends a DIALBACK_REPLY message, we send no reply. +# + +import sys +from struct import pack,unpack +from time import time +from sets import Set +from cStringIO import StringIO +from threading import currentThread +from socket import gethostbyname +from traceback import print_exc,print_stack + +from BaseLib.Core.BitTornado.__init__ import createPeerID +from BaseLib.Core.BitTornado.BT1.MessageID import protocol_name,option_pattern,getMessageName +from BaseLib.Core.BitTornado.BT1.convert import tobinary,toint + + +DEBUG = False + +# +# Public definitions +# +dialback_infohash = '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01' + +# +# Private definitions +# + +# States for overlay connection +STATE_INITIAL = 0 +STATE_HS_FULL_WAIT = 1 +STATE_HS_PEERID_WAIT = 2 +STATE_DATA_WAIT = 4 +STATE_CLOSED = 5 + +# Misc +EXPIRE_THRESHOLD = 30 # seconds:: keep consistent with sockethandler +EXPIRE_CHECK_INTERVAL = 60 # seconds + + +class ReturnConnHandler: + __single = None + + def __init__(self): + if ReturnConnHandler.__single: + raise RuntimeError, "ReturnConnHandler is Singleton" + ReturnConnHandler.__single = self + + # + # Interface for upper layer + # + def getInstance(*args, **kw): + if ReturnConnHandler.__single is None: + ReturnConnHandler(*args, **kw) + return ReturnConnHandler.__single + getInstance = staticmethod(getInstance) + + def register(self,rawserver,multihandler,mylistenport,max_len): + """ Called by MainThread """ + self.rawserver = rawserver # real rawserver, not overlay_bridge + self.sock_hand = self.rawserver.sockethandler + self.multihandler = multihandler + self.dialback_rawserver = multihandler.newRawServer(dialback_infohash, + self.rawserver.doneflag, + protocol_name) + self.myid = create_my_peer_id(mylistenport) + self.max_len = max_len + self.iplport2oc = {} # (IP,listen port) -> ReturnConnection + self.usermsghandler = None + self.userconnhandler = None + + def resetSingleton(self): + """ For testing purposes """ + ReturnConnHandler.__single = None + + def start_listening(self): + """ Called by MainThread """ + self.dialback_rawserver.start_listening(self) + + def connect_dns(self,dns,callback): + """ Connects to the indicated endpoint. Non-blocking. + + Pre: "dns" must be an IP address, not a hostname. + + Network thread calls "callback(exc,dns)" when the connection + is established or when an error occurs during connection + establishment. In the former case, exc is None, otherwise + it contains an Exception. + + The established connection will auto close after EXPIRE_THRESHOLD + seconds of inactivity. + """ + # Called by overlay thread + if DEBUG: + print >> sys.stderr,"dlbreturn: connect_dns",dns + # To prevent concurrency problems on sockets the calling thread + # delegates to the network thread. + task = Task(self._connect_dns,dns,callback) + self.rawserver.add_task(task.start, 0) + + + def send(self,dns,msg,callback): + """ Sends a message to the indicated dns. Non-blocking. + + Pre: connection to permid must have been established successfully. + + Network thread calls "callback(exc,dns)" when the message is sent + or when an error occurs during sending. In the former case, exc + is None, otherwise it contains an Exception. + """ + # Called by overlay thread + # To prevent concurrency problems on sockets the calling thread + # delegates to the network thread. + task = Task(self._send,dns,msg,callback) + self.rawserver.add_task(task.start, 0) + + + + def close(self,dns): + """ Closes any connection to indicated permid. Non-blocking. + + Pre: connection to permid must have been established successfully. + + Network thread calls "callback(exc,permid,selver)" when the connection + is closed. + """ + # Called by overlay thread + # To prevent concurrency problems on sockets the calling thread + # delegates to the network thread. + task = Task(self._close,dns) + self.rawserver.add_task(task.start, 0) + + + def register_recv_callback(self,callback): + """ Register a callback to be called when receiving a message from + any permid. Non-blocking. + + Network thread calls "callback(exc,permid,selver,msg)" when a message + is received. The callback is not called on errors e.g. remote + connection close. + """ + self.usermsghandler = callback + + def register_conns_callback(self,callback): + """ Register a callback to be called when receiving a connection from + any permid. Non-blocking. + + Network thread calls "callback(exc,permid,selver,locally_initiated)" + when a connection is established (locally initiated or remote), or + when a connection is closed locally or remotely. In the former case, + exc is None, otherwise it contains an Exception. + + Note that this means that if a callback is registered via this method, + both this callback and the callback passed to a connect() method + will be called. + """ + self.userconnhandler = callback + + + # + # Internal methods + # + def _connect_dns(self,dns,callback): + # Called by network thread + try: + if DEBUG: + print >> sys.stderr,"dlbreturn: actual connect_dns",dns + iplport = ip_and_port2str(dns[0],dns[1]) + oc = None + try: + oc = self.iplport2oc[iplport] + except KeyError: + pass + if oc is None: + oc = self.start_connection(dns) + self.iplport2oc[iplport] = oc + oc.queue_callback(dns,callback) + else: + callback(None,dns) + except Exception,exc: + if DEBUG: + print_exc(file=sys.stderr) + callback(exc,dns) + + def _send(self,dns,message,callback): + # Called by network thread + try: + iplport = ip_and_port2str(dns[0],dns[1]) + oc = None + try: + oc = self.iplport2oc[iplport] + except KeyError: + pass + if oc is None: + callback(KeyError('Not connected to dns'),dns) + else: + oc.send_message(message) + callback(None,dns) + except Exception,exc: + if DEBUG: + print_exc(file=sys.stderr) + callback(exc,dns) + + + def _close(self,dns): + # Called by network thread + if DEBUG: + print >> sys.stderr,"dlbreturn: actual close",dns + try: + iplport = ip_and_port2str(dns[0],dns[1]) + oc = None + try: + oc = self.iplport2oc[iplport] + except KeyError: + pass + if oc is None: + if DEBUG: + print >> sys.stderr,"dlbreturn: error - actual close, but no connection to peer in admin" + else: + oc.close() + except Exception,e: + print_exc(file=sys.stderr) + + # + # Interface for SocketHandler + # + def external_connection_made(self,singsock): + """ incoming connection (never used) """ + if DEBUG: + print >> sys.stderr,"dlbreturn: external_connection_made",singsock.get_ip(),singsock.get_port() + oc = ReturnConnection(self,singsock,self.rawserver) + singsock.set_handler(oc) + + def connection_flushed(self,singsock): + """ sockethandler flushes connection """ + if DEBUG: + print >> sys.stderr,"dlbreturn: connection_flushed",singsock.get_ip(),singsock.get_port() + pass + + # + # Interface for ServerPortHandler + # + def externally_handshaked_connection_made(self, singsock, options, msg_remainder): + """ incoming connection, handshake partially read to identity + as an it as overlay connection (used always) + """ + if DEBUG: + print >> sys.stderr,"dlbreturn: externally_handshaked_connection_made",\ + singsock.get_ip(),singsock.get_port() + oc = ReturnConnection(self,singsock,self.rawserver,ext_handshake = True, options = options) + singsock.set_handler(oc) + if msg_remainder: + oc.data_came_in(singsock,msg_remainder) + return True + + + # + # Interface for ReturnConnection + # + def got_connection(self,oc): + + if DEBUG: + print >>sys.stderr,"dlbreturn: Got connection from",oc.get_ip(),"listen",oc.get_listen_port() + + ret = True + iplport = ip_and_port2str(oc.get_ip(),oc.get_listen_port()) + known = iplport in self.iplport2oc + if not known: + self.iplport2oc[iplport] = oc + elif known and not oc.is_locally_initiated(): + # Locally initiated connections will already be registered, + # so if it's not a local connection and we already have one + # we have a duplicate, and we close the new one. + if DEBUG: + print >> sys.stderr,"dlbreturn: got_connection:", \ + "closing because we already have a connection to",iplport + self.cleanup_admin_and_callbacks(oc, + Exception('closing because we already have a connection to peer')) + ret = False + + if ret: + oc.dequeue_callbacks() + if self.userconnhandler is not None: + try: + self.userconnhandler(None,(oc.get_ip(),oc.get_listen_port()),oc.is_locally_initiated()) + except: + # Catchall + print_exc(file=sys.stderr) + return ret + + def local_close(self,oc): + """ our side is closing the connection """ + if DEBUG: + print >> sys.stderr,"dlbreturn: local_close" + self.cleanup_admin_and_callbacks(oc,Exception('local close')) + + def connection_lost(self,oc): + """ overlay connection telling us to clear admin """ + if DEBUG: + print >> sys.stderr,"dlbreturn: connection_lost" + self.cleanup_admin_and_callbacks(oc,Exception('connection lost')) + + def got_message(self,dns,message): + """ received message from peer, pass to upper layer """ + if DEBUG: + print >> sys.stderr,"dlbreturn: got_message",getMessageName(message[0]) + if self.usermsghandler is None: + if DEBUG: + print >> sys.stderr,"dlbreturn: User receive callback not set" + return + try: + ret = self.usermsghandler(dns,message) + if ret is None: + if DEBUG: + print >> sys.stderr,"dlbreturn: INTERNAL ERROR:", \ + "User receive callback returned None, not True or False" + ret = False + return ret + except: + # Catch all + print_exc(file=sys.stderr) + return False + + + def get_max_len(self): + return self.max_len + + def get_my_peer_id(self): + return self.myid + + def measurefunc(self,length): + pass + + def start_connection(self,dns): + if DEBUG: + print >> sys.stderr,"dlbreturn: Attempt to connect to",dns + singsock = self.sock_hand.start_connection(dns) + oc = ReturnConnection(self,singsock,self.rawserver, + locally_initiated=True,specified_dns=dns) + singsock.set_handler(oc) + return oc + + def cleanup_admin_and_callbacks(self,oc,exc): + oc.cleanup_callbacks(exc) + self.cleanup_admin(oc) + if self.userconnhandler is not None: + self.userconnhandler(exc,(oc.get_ip(),oc.get_listen_port()),oc.is_locally_initiated()) + + def cleanup_admin(self,oc): + iplports = [] + d = 0 + for key in self.iplport2oc.keys(): + #print "***** iplport2oc:", key, self.iplport2oc[key] + if self.iplport2oc[key] == oc: + del self.iplport2oc[key] + #print "*****!!! del", key, oc + d += 1 + + +class Task: + def __init__(self,method,*args, **kwargs): + self.method = method + self.args = args + self.kwargs = kwargs + + def start(self): + if DEBUG: + print >> sys.stderr,"dlbreturn: task: start",self.method + #print_stack(file=sys.stderr) + self.method(*self.args,**self.kwargs) + + +class ReturnConnection: + def __init__(self,handler,singsock,rawserver,locally_initiated = False, + specified_dns = None, ext_handshake = False,options = None): + # Called by network thread + self.handler = handler + self.singsock = singsock # for writing + self.rawserver = rawserver + self.buffer = StringIO() + self.cb_queue = [] + self.listen_port = None + self.options = None + self.locally_initiated = locally_initiated + self.specified_dns = specified_dns + self.last_use = time() + + self.state = STATE_INITIAL + self.write(chr(len(protocol_name)) + protocol_name + + option_pattern + dialback_infohash + self.handler.get_my_peer_id()) + if ext_handshake: + self.state = STATE_HS_PEERID_WAIT + self.next_len = 20 + self.next_func = self.read_peer_id + self.set_options(options) + else: + self.state = STATE_HS_FULL_WAIT + self.next_len = 1 + self.next_func = self.read_header_len + + # Leave autoclose here instead of ReturnConnHandler, as that doesn't record + # remotely-initiated ReturnConnections before authentication is done. + self.rawserver.add_task(self._dlbconn_auto_close, EXPIRE_CHECK_INTERVAL) + + # + # Interface for SocketHandler + # + def data_came_in(self, singsock, data): + """ sockethandler received data """ + # now we got something we can ask for the peer's real port + dummy_port = singsock.get_port(True) + + if DEBUG: + print >> sys.stderr,"dlbconn: data_came_in",singsock.get_ip(),singsock.get_port() + self.handler.measurefunc(len(data)) + self.last_use = time() + while 1: + if self.state == STATE_CLOSED: + return + i = self.next_len - self.buffer.tell() + if i > len(data): + self.buffer.write(data) + return + self.buffer.write(data[:i]) + data = data[i:] + m = self.buffer.getvalue() + self.buffer.reset() + self.buffer.truncate() + try: + #if DEBUG: + # print >> sys.stderr,"dlbconn: Trying to read",self.next_len,"using",self.next_func + x = self.next_func(m) + except: + self.next_len, self.next_func = 1, self.read_dead + if DEBUG: + print_exc(file=sys.stderr) + raise + if x is None: + if DEBUG: + print >> sys.stderr,"dlbconn: next_func returned None",self.next_func + self.close() + return + self.next_len, self.next_func = x + + def connection_lost(self,singsock): + """ kernel or socket handler reports connection lost """ + if DEBUG: + print >> sys.stderr,"dlbconn: connection_lost",singsock.get_ip(),singsock.get_port(),self.state + if self.state != STATE_CLOSED: + self.state = STATE_CLOSED + self.handler.connection_lost(self) + + def connection_flushed(self,singsock): + """ sockethandler flushes connection """ + pass + + # + # Interface for ReturnConnHandler + # + def send_message(self,message): + self.last_use = time() + s = tobinary(len(message))+message + if DEBUG: + print >> sys.stderr,"dlbconn: Sending message",len(message) + self.write(s) + + def is_locally_initiated(self): + return self.locally_initiated + + def get_ip(self): + return self.singsock.get_ip() + + def get_port(self): + return self.singsock.get_port() + + def get_listen_port(self): + return self.listen_port + + def queue_callback(self,dns,callback): + if callback is not None: + self.cb_queue.append(callback) + + def dequeue_callbacks(self): + try: + for callback in self.cb_queue: + callback(None,self.specified_dns) + self.cb_queue = [] + except Exception,e: + print_exc(file=sys.stderr) + + + def cleanup_callbacks(self,exc): + if DEBUG: + print >> sys.stderr,"dlbconn: cleanup_callbacks: #callbacks is",len(self.cb_queue) + try: + for callback in self.cb_queue: + ## Failure connecting + if DEBUG: + print >> sys.stderr,"dlbconn: cleanup_callbacks: callback is",callback + callback(exc,self.specified_dns) + except Exception,e: + print_exc(file=sys.stderr) + + # + # Internal methods + # + def read_header_len(self, s): + if ord(s) != len(protocol_name): + return None + return len(protocol_name), self.read_header + + def read_header(self, s): + if s != protocol_name: + return None + return 8, self.read_reserved + + def read_reserved(self, s): + if DEBUG: + print >> sys.stderr,"dlbconn: Reserved bits:", `s` + self.set_options(s) + return 20, self.read_download_id + + def read_download_id(self, s): + if s != dialback_infohash: + return None + return 20, self.read_peer_id + + def read_peer_id(self, s): + self.unauth_peer_id = s + self.listen_port = decode_listen_port(self.unauth_peer_id) + self.state = STATE_DATA_WAIT + if not self.got_connection(): + self.close() + return + return 4, self.read_len + + + def got_connection(self): + return self.handler.got_connection(self) + + def read_len(self, s): + l = toint(s) + if l > self.handler.get_max_len(): + return None + return l, self.read_message + + def read_message(self, s): + + if DEBUG: + print >>sys.stderr,"dlbconn: read_message len",len(s),self.state + + if s != '': + if self.state == STATE_DATA_WAIT: + if not self.handler.got_message((self.get_ip(),self.get_listen_port()),s): + return None + else: + if DEBUG: + print >> sys.stderr,"dlbconn: Received message while in illegal state, internal error!" + return None + return 4, self.read_len + + def read_dead(self, s): + return None + + def write(self,s): + self.singsock.write(s) + + def set_options(self,options): + self.options = options + + def close(self): + if DEBUG: + print >> sys.stderr,"dlbconn: we close()",self.get_ip(),self.get_port() + self.state_when_error = self.state + if self.state != STATE_CLOSED: + self.state = STATE_CLOSED + self.handler.local_close(self) + self.singsock.close() + return + + def _dlbconn_auto_close(self): + if (time() - self.last_use) > EXPIRE_THRESHOLD: + self.close() + else: + self.rawserver.add_task(self._dlbconn_auto_close, EXPIRE_CHECK_INTERVAL) + +def create_my_peer_id(my_listen_port): + myid = createPeerID() + myid = myid[:14] + pack('> sys.stderr, "TIMEOUTCHECK:", "-> ping" + + # Send the ping to the server specifying the delay of the reply + pingMsg = (str("ping:"+str(ping))) + udpsock.send(pingMsg) + udpsock.send(pingMsg) + udpsock.send(pingMsg) + + # Wait for reply from the server + while True: + + rcvaddr = None + + try: + reply = udpsock.recv(1024) + + except timeout: # No reply from the server: timeout passed + + if udpsock: + udpsock.close() + + if DEBUG: print >> sys.stderr, "TIMEOUTCHECK:", "UDP connection to the pingback server has timed out for ping", ping + + lck.acquire() + evnt.set() + evnt.clear() + lck.release() + break + + if DEBUG: print >> sys.stderr, pingbacksrvr + if DEBUG: print >> sys.stderr, rcvaddr + + if reply: + data = reply.split(':') + if DEBUG: print >> sys.stderr, data, "received from the pingback server" + + if data[0] == "pong": + if DEBUG: print >> sys.stderr, "TIMEOUTCHECK:", "<-", data[0], "after", data[1], "seconds" + to = ping + if int(data[1])==145: + lck.acquire() + evnt.set() + evnt.clear() + lck.release() + return + + return + + +# Main method of the library: launches nat-timeout discovery algorithm +def GetTimeout(pingbacksrvr): + """ + Returns the NAT timeout for UDP traffic + """ + + pings = [25, 35, 55, 85, 115, 145] + + # Send pings and wait for replies + for ping in pings: + thread.start_new_thread(pingback, (ping, pingbacksrvr)) + + global evnt + evnt.wait() + + if DEBUG: print >> sys.stderr, "TIMEOUTCHECK: timeout is", to + return to diff --git a/instrumentation/next-share/BaseLib/Core/NATFirewall/TimeoutFinder.py b/instrumentation/next-share/BaseLib/Core/NATFirewall/TimeoutFinder.py new file mode 100644 index 0000000..cd25024 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/NATFirewall/TimeoutFinder.py @@ -0,0 +1,112 @@ +# Written by Gertjan Halkes +# see LICENSE.txt for license information + + +import struct +import time +import sys + +DEBUG = False + +class TimeoutFinder: + PINGBACK_TIMES = [ 245, 235, 175, 115, 85, 55, 25, 10 ] + PINGBACK_ADDRESS = ("m23trial-udp.tribler.org", 7396) + + def __init__(self, rawserver, initial_ping, reportback = None): + self.sockets = [] + self.rawserver = rawserver + self.timeout_found = -1 + self.initial_ping = initial_ping + self.reportback = reportback + self.timeout_index = 0 + + # Stagger the pings by 1 second to unsure minimum impact on other traffic + rawserver.add_task(self.ping, 1) + rawserver.add_task(self.report_done, TimeoutFinder.PINGBACK_TIMES[0] + 5) + + + def ping(self): + sock = self.rawserver.create_udpsocket(0, "0.0.0.0") + self.sockets.append(sock) + self.rawserver.start_listening_udp(sock, self) + if self.initial_ping: + sock.sendto(struct.pack("!Id", 0, float(TimeoutFinder.PINGBACK_TIMES[self.timeout_index])), + TimeoutFinder.PINGBACK_ADDRESS) + else: + sock.sendto(struct.pack("!Id", TimeoutFinder.PINGBACK_TIMES[self.timeout_index], + time.time()), TimeoutFinder.PINGBACK_ADDRESS) + self.timeout_index += 1 + if self.timeout_index < len(TimeoutFinder.PINGBACK_TIMES): + self.rawserver.add_task(self.ping, 1) + + + def data_came_in(self, address, data): + if len(data) != 12: + return + #FIXME: the address should be checked, but that can only be done if + # the address is in dotted-decimal notation + #~ if address != TimeoutFinder.PINGBACK_ADDRESS: + #~ return + + timeout = struct.unpack("!Id", data) + if timeout[0] == 0: + to_find = int(timeout[1]) + for i in range(0, len(TimeoutFinder.PINGBACK_TIMES)): + if to_find == TimeoutFinder.PINGBACK_TIMES[i]: + self.sockets[i].sendto(struct.pack("!Id", to_find, time.time()), TimeoutFinder.PINGBACK_ADDRESS) + break + else: + if DEBUG: + print >>sys.stderr, ("Received ping with %d delay" % (timeout[0])) + self.timeout_found = timeout[0] + #FIXME: log reception of packet + + def report_done(self): + for i in self.sockets: + self.rawserver.stop_listening_udp(i) + i.close() + + if self.reportback: + self.reportback(self.timeout_found, self.initial_ping) + + +if __name__ == "__main__": + import BaseLib.Core.BitTornado.RawServer as RawServer + from threading import Event + import thread + from traceback import print_exc + import os + + def fail(e): + print "Fatal error: " + str(e) + print_exc() + + def error(e): + print "Non-fatal error: " + str(e) + + def report(timeout, initial_ping): + if initial_ping: + with_ = "with" + else: + with_ = "without" + + if DEBUG: + print >>sys.stderr, ("Timeout %s initial ping: %d" % (with_, timeout)) + + DEBUG = True + + log = open("log-timeout.txt", "w") + + rawserver_ = RawServer.RawServer(Event(), + 60.0, + 300.0, + False, + failfunc = fail, + errorfunc = error) + thread.start_new_thread(rawserver_.listen_forever, (None,)) + time.sleep(0.5) + TimeoutFinder(rawserver_, False, report) + TimeoutFinder(rawserver_, True, report) + + print "TimeoutFinder started, press enter to quit" + sys.stdin.readline() diff --git a/instrumentation/next-share/BaseLib/Core/NATFirewall/UDPPuncture.py b/instrumentation/next-share/BaseLib/Core/NATFirewall/UDPPuncture.py new file mode 100644 index 0000000..9e43f00 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/NATFirewall/UDPPuncture.py @@ -0,0 +1,1065 @@ +# Written by Gertjan Halkes +# see LICENSE.txt for license information +# +# NATSwarm implementation for testing NAT/firewall puncturing +# This module creates UDP "connections" and tries to connect to other +# peers in the NATSwarm. PEX is used to find more peers. + +import guessip +import time +import socket +import sys +import errno +import random +from collections import deque +import TimeoutFinder + +DEBUG = False + +#NOTE: the current implementation allows PEX_ADD and PEX_DEL messages to name +# the same peer. Although these events will be rare, we may want to do something +# about it. + + +# Packet format definitions: +# Each packet starts with a single byte packet type. After this, the contents +# is type dependent: +# Connect: 1 byte version number, 4 byte ID, 1 byte NAT/fw state, +# 1 byte NAT/fw state version. +# Your IP: 4 bytes IPv4 address, 2 bytes port number. +# Forward connection request: 4 bytes ID. +# Reverse connect: 4 bytes ID, 4 bytes IPv4 address, 2 bytes port number, +# 1 byte NAT/fw state, 1 byte NAT/fw state version. +# NAT/fw state may not yet be known through PEX, but we need it for future PEX. +# Furthermore, we may not learn it through the remote peer's connect, as that may +# not reach us due to filtering. +# PEX add: 1 byte number of addresses. Per address: +# 4 bytes ID, 4 bytes IPv4 address, 2 bytes port, 1 byte NAT/fw state, +# 1 byte NAT/fw state version. +# PEX del: 1 byte number of addresses. Per address: +# 4 bytes ID. +# Close: 1 byte reason +# Update NAT/fw state: 1 byte NAT/fw state, 1 byte NAT/fw state version. +# Peer unknown: 4 bytes ID. +# +# NAT/fw state is encoded as follows: the least significant 2 bits (0 and 1) +# encode the NAT state: 0 UNKNOWN, 1 NONE, 2 A(P)DM. Bits 2 and 3 encode +# the filtering state: 0 UNKNOWN, 1 EIF/NONE, 2 A(P)DF + +# Packet sequence for connection setup through rendez-vous: +# A -> B CONNECT (in all likelyhood dropped at NAT/fw) +# A -> R FW_CONNECT_REQ +# R -> B REV_CONNECT +# B -> A CONNECT +# A -> B YOUR_IP +# B -> A YOUR_IP +# +# NOTE: it is important that three packets are exchanged on the connection, +# because some NAT/firewalls (most notably linux based ones) use an increased +# timeout if they detect that the 'connection' is more than a simple +# transaction. + +# Information to keep for each peer: +# - IP/port/NATfw state +# - List of peers through which we heard of this peer +# - Whether a connection attempt was already made +# - To which other peers we have advertised this peer, and the FW state we +# advertised so updates can be sent + +# WARNING: copied from SocketHandler. An import would be better, to keep this +# definition in one place +if sys.platform == 'win32': + SOCKET_BLOCK_ERRORCODE=10035 # WSAEWOULDBLOCK +else: + SOCKET_BLOCK_ERRORCODE=errno.EWOULDBLOCK + + +class UDPHandler: + TRACKER_ADDRESS = "m23trial-udp.tribler.org" + #~ TRACKER_ADDRESS = "localhost" + + # Define message types + CONNECT = chr(0) # Connection request, directly sent to target + YOUR_IP = chr(1) # Information regarding remote ip as seen by local peer + FW_CONNECT_REQ = chr(2) # Request to forward a reverse connection request + REV_CONNECT = chr(3) # Reverse connection request, for NAT/firewall state setup + PEX_ADD = chr(4) # Notify peer of other known peers + PEX_DEL = chr(5) # Notify peer of peers that are no longer available + CLOSE = chr(6) # Close connection + UPDATE_NATFW_STATE = chr(7) # Notify peer of changed NAT state + PEER_UNKNOWN = chr(8) # Response to FW_CONNECT_REQ if the requested peer is unknown + KEEP_ALIVE = chr(9) # Simple keep-alive message + + # Connection reset error codes + CLOSE_NORMAL = chr(0) + CLOSE_TOO_MANY = chr(1) + CLOSE_LEN = chr(2) + CLOSE_PROTO_VER, = chr(3) + CLOSE_GARBAGE = chr(4) + CLOSE_NOT_CONNECTED = chr(5) + CLOSE_STATE_CORRUPT = chr(6) + + # Enumerate NAT states + # Note that the difference EIM and NONE is irrelevant for our purposes, + # as both are connectable if combined with EIF + NAT_UNKNOWN, NAT_NONE, NAT_APDM = range(0, 3) + # There is a distinction between EIF and no filtering, because the latter + # does not require keep-alives. However, we need keep-alives anyway for + # the connections so the distinction is irrelevant. + FILTER_UNKNOWN, FILTER_NONE, FILTER_APDF = range(0, 3) + + # Number of connections to be made before a decision is made about NAT/fw state + RECV_CONNECT_THRESHOLD = 4 + # Number of connections before scaling the numbers (prevent overflow, allow change) + RECV_CONNECT_SCALE_THRESHOLD = 64 + # Fixed threshold above which the filter state is assumed to be FILTER_NONE. This is to + # make sure that a few (or rather quite a few) missing packets or TIVs don't screw up a + # peer's idea of its filtering type. + FIXED_THRESHOLD = 7 + + def __init__(self, rawserver, check_crawler, port = 0): + # initialise connections now because it is used in shutdown which will + # be called for Crawler instances as well + self.connections = {} + if check_crawler: + from BaseLib.Core.Statistics.Crawler import Crawler + crawler = Crawler.get_instance() + if crawler.am_crawler(): + return + + # initialise connections now because it is used in shutdown which will + # be called for Crawler instances as well + self.connections = {} + + # 17/03/10 Boudewijn: obsolete code, see the same code a few + # lines above that include the check_crawler if/else check + # from BaseLib.Core.Statistics.Crawler import Crawler + # crawler = Crawler.get_instance() + # if crawler.am_crawler(): + # return + + self.rawserver = rawserver + self.socket = rawserver.create_udpsocket(port, "0.0.0.0") + self.known_peers = {} + self.nat_type = UDPHandler.NAT_UNKNOWN + self.filter_type = UDPHandler.FILTER_UNKNOWN + self.max_connections = 100 + self.connect_threshold = 75 + self.recv_unsolicited = 0 + self.recv_connect_total = 0 + self.recv_address = 0 + self.recv_different_address = 0 + self.sendqueue = deque([]) + self.last_connect = 0 + self.last_info_dump = time.time() + self.natfw_version = 1 + self.keepalive_intvl = 100 + self.done = False + self.reporter = None + self.last_sends = {} + + rawserver.start_listening_udp(self.socket, self) + + # Contact NATSwarm tracker peer after 5 seconds + if port == 9473: + self.tracker = True + + # Tracker needs a known ID, so set it to all zero + self.id = "\0\0\0\0" + # Tracker should accept many more connections than other nodes + self.max_connections = 1000 + rawserver.add_task(self.check_for_timeouts, 10) + else: + self.tracker = False + + # Create a 4 byte random ID + self.id = (chr(random.getrandbits(8)) + chr(random.getrandbits(8)) + + chr(random.getrandbits(8)) + chr(random.getrandbits(8))) + if DEBUG: + debug("My ID: %s" % self.id.encode('hex')) + rawserver.add_task(self.bootstrap, 5) + TimeoutFinder.TimeoutFinder(rawserver, False, self.timeout_report) + TimeoutFinder.TimeoutFinder(rawserver, True, self.timeout_report) + + if not DEBUG: + if check_crawler: + #~ from BaseLib.Core.Statistics.StatusReporter import get_reporter_instance + from BaseLib.Core.Statistics.PunctureCrawler import get_reporter_instance + self.reporter = get_reporter_instance() + + if self.reporter: + my_wan_ip = guessip.get_my_wan_ip() + if my_wan_ip == None and sys.platform == 'win32': + try: + import os + for line in os.popen("netstat -nr").readlines(): + words = line.split() + if words[0] == '0.0.0.0': + my_wan_ip = words[3] + break + except: + pass + if my_wan_ip == None: + my_wan_ip = 'Unknown' + self.reporter.add_event("UDPPuncture", "ID:%s;IP:%s" % (self.id.encode('hex'), my_wan_ip)) + + def shutdown(self): + self.done = True + for connection in self.connections.values(): + self.sendto(UDPHandler.CLOSE + UDPHandler.CLOSE_NORMAL, connection.address) + self.delete_closed_connection(connection) + + def data_came_in(self, address, data): + if DEBUG: + debug("Data came (%d) in from address %s:%d" % (ord(data[0]), address[0], address[1])) + connection = self.connections.get(address) + if not connection: + if data[0] == UDPHandler.CLOSE: + # Prevent stroms of packets, by not responding to this + return + if data[0] != UDPHandler.CONNECT: + self.sendto(UDPHandler.CLOSE + UDPHandler.CLOSE_NOT_CONNECTED, address) + return + if len(data) != 8: + self.sendto(UDPHandler.CLOSE + UDPHandler.CLOSE_LEN, address) + return + if data[1] != chr(0): + self.sendto(UDPHandler.CLOSE + UDPHandler.CLOSE_PROTO_VER, address) + return + + + if self.check_connection_count(): + if self.reporter: + self.reporter.add_event("UDPPuncture", "OCTM:%s,%d,%s" % (address[0], address[1], data[2:6].encode('hex'))) + + self.sendto(UDPHandler.CLOSE + UDPHandler.CLOSE_TOO_MANY, address) + return + + id = data[2:6] + connection = self.known_peers.get(id) + if not connection: + # Create new connection state and add to table + connection = UDPConnection(address, id, self) + self.known_peers[id] = connection + elif connection.address != address: + if connection.connection_state == UDPConnection.CONNECT_ESTABLISHED: + self.sendto(UDPHandler.CLOSE + UDPHandler.CLOSE_STATE_CORRUPT, address) + return + + # ADPM NAT-boxes will have different address, so if we sent a + # connect already we will have done so to a different address. + try: + del self.connections[connection.address] + except: + pass + # As we knew this peer under a different address, we have to + # set the address to the one we actually use. + connection.address = address + + if not address in self.last_sends: + self.incoming_connect(address, True) # Update NAT and Filter states + self.connections[address] = connection + + if not connection.handle_msg(data): + self.delete_closed_connection(connection) + + def check_connection_count(self): + # If we still have open slots, we can simply connect + if len(self.connections) < self.max_connections: + return False + + if DEBUG: + debug(" Connection threshold reached, trying to find an old connection") + # Find oldest connection, and close if it is older than 5 minutes + oldest = None + oldest_time = 1e308 + for connection in self.connections.itervalues(): + if (not connection.tracker) and connection.connected_since < oldest_time: + oldest_time = connection.connected_since + oldest = connection + + if not oldest: + return True + + if (not self.tracker) and oldest.connected_since > time.time() - 300: + if DEBUG: + debug(" All connections are under 5 minutes old") + return True + + if DEBUG: + debug(" Closing connection to %s %s:%d" % (oldest.id.encode('hex'), oldest.address[0], oldest.address[1])) + oldest.sendto(UDPHandler.CLOSE + UDPHandler.CLOSE_NORMAL) + self.delete_closed_connection(oldest) + return False + + def incoming_connect(self, address, unsolicited): + if self.tracker: + return + + if unsolicited: + self.recv_unsolicited += 1 + self.recv_connect_total += 1 + + if self.recv_connect_total > UDPHandler.RECV_CONNECT_SCALE_THRESHOLD: + self.recv_connect_total >>= 1 + self.recv_unsolicited >>= 1 + # Check if we have enough data-points to say something sensible about + # our NAT/fw state. + if self.recv_connect_total > UDPHandler.RECV_CONNECT_THRESHOLD: + if DEBUG: + debug("Setting filter state (recv total %d, recv unsol %d)" % + (self.recv_connect_total, self.recv_unsolicited)) + update_filter = False + if self.recv_unsolicited > self.recv_connect_total / 2 or self.recv_unsolicited > UDPHandler.FIXED_THRESHOLD: + if self.filter_type != UDPHandler.FILTER_NONE or self.nat_type != UDPHandler.NAT_NONE: + update_filter = True + self.filter_type = UDPHandler.FILTER_NONE + self.nat_type = UDPHandler.NAT_NONE + elif self.filter_type != UDPHandler.FILTER_APDF: + update_filter = True + self.filter_type = UDPHandler.FILTER_APDF + + if update_filter: + self.natfw_version += 1 + if self.natfw_version > 255: + self.natfw_version = 0 + if self.reporter: + self.reporter.add_event("UDPPuncture", "UNAT:%d,%d,%d" % (self.nat_type, + self.filter_type, self.natfw_version)) + map(lambda x: x.readvertise_nat(), self.connections.itervalues()) + + def incoming_ip(self, address): + if self.tracker: + return + + self.recv_address += 1 + if self.recv_address == 1: + self.reported_wan_address = address + return + + if self.recv_address > UDPHandler.RECV_CONNECT_SCALE_THRESHOLD: + self.recv_address >>= 1 + self.recv_different_address >>= 1 + + if self.reported_wan_address != address: + self.reported_wan_address = address + self.recv_different_address += 1 + + # Check if we have enough data-points to say something sensible about + # our NAT/fw state. + if self.recv_address > UDPHandler.RECV_CONNECT_THRESHOLD: + if DEBUG: + debug("Setting nat state (recv addr %d, recv diff %d)" % + (self.recv_address, self.recv_different_address)) + update_nat = False + if self.recv_different_address > self.recv_address / 2: + if self.nat_type != UDPHandler.NAT_APDM: + update_nat = True + self.nat_type = UDPHandler.NAT_APDM + self.filter_type = UDPHandler.FILTER_APDF + elif self.nat_type != UDPHandler.NAT_NONE: + update_nat = True + self.nat_type = UDPHandler.NAT_NONE + + if update_nat: + self.natfw_version += 1 + if self.natfw_version > 255: + self.natfw_version = 0 + if self.reporter: + self.reporter.add_event("UDPPuncture", "UNAT:%d,%d,%d" % (self.nat_type, + self.filter_type, self.natfw_version)) + map(lambda x: x.readvertise_nat(), self.connections.itervalues()) + + def bootstrap(self): + if DEBUG: + debug("Starting bootstrap") + try: + address = socket.gethostbyname(UDPHandler.TRACKER_ADDRESS) + except: + return + if address == '130.161.211.245': + return # Don't connect to catch-all address + tracker = UDPConnection((address, 9473), "\0\0\0\0", self) + # Make sure this is never removed, by setting an address that we will never receive + tracker.advertised_by[("0.0.0.0", 0)] = 1e308 + tracker.nat_type = UDPHandler.NAT_NONE + tracker.filter_type = UDPHandler.FILTER_NONE + tracker.tracker = True + self.known_peers[tracker.id] = tracker + self.check_for_timeouts() + + def sendto(self, data, address): + if DEBUG: + debug("Sending data (%d) to address %s:%d" % (ord(data[0]), address[0], address[1])) + if len(self.sendqueue) > 0: + self.sendqueue.append((data, address)) + return + + try: + self.socket.sendto(data, address) + except socket.error, error: + if error[0] == SOCKET_BLOCK_ERRORCODE: + self.sendqueue.append((data, address)) + self.rawserver.add_task(self.process_sendqueue, 0.1) + + def process_sendqueue(self): + while len(self.sendqueue) > 0: + data, address = self.sendqueue[0] + try: + self.socket.sendto(data, address) + except socket.error, error: + if error[0] == SOCKET_BLOCK_ERRORCODE: + self.rawserver.add_task(self.process_sendqueue, 0.1) + return + self.sendqueue.popleft() + + def check_nat_compatible(self, peer): + #~ if self.filter_type == UDPHandler.FILTER_APDF and peer.nat_type == UDPHandler.NAT_APDM: + #~ return False + if self.nat_type == UDPHandler.NAT_APDM and peer.filter_type == UDPHandler.FILTER_APDF: + return False + return True + + def check_for_timeouts(self): + if self.done: + return + + now = time.time() + + # Remove info about last sends after 5 minutes + close_list = [] + for address in self.last_sends.iterkeys(): + if self.last_sends[address] < now - 300: + close_list.append(address) + for address in close_list: + del self.last_sends[address] + + # Close connections older than 10 minutes, if the number of connections is more + # than the connect threshold. However, only discard upto 1/3 of the connect + # threshold. + if (not self.tracker) and len(self.connections) >= self.connect_threshold: + if DEBUG: + debug("Closing connections older than 10 minutes") + close_list = [] + for connection in self.connections.itervalues(): + if (not connection.tracker) and connection.connected_since < now - 600: + if DEBUG: + debug(" Closing connection to %s %s:%d" % (connection.id.encode('hex'), + connection.address[0], connection.address[1])) + close_list.append(connection) + + for connection in close_list: + connection.sendto(UDPHandler.CLOSE + UDPHandler.CLOSE_NORMAL) + self.delete_closed_connection(connection) + if len(self.connections) < self.connect_threshold / 1.5: + break + + # Check to see if we should try to make new connections + if ((not self.tracker) and len(self.connections) < self.connect_threshold and + self.last_connect < now - 20): + unconnected_peers = list(set(self.known_peers.iterkeys()) - set(ConnectionIteratorByID(self.connections))) + random.shuffle(unconnected_peers) + while len(unconnected_peers) > 0: + peer = self.known_peers[unconnected_peers.pop()] + # Only connect to peers that are not connected (should be all, but just in case) + if peer.connection_state != UDPConnection.CONNECT_NONE: + continue + if not self.check_nat_compatible(peer): + continue + # Don't connect to peers with who we have communicated in the last five minutes + if peer.last_comm > now - 300: + continue + + if not self.try_connect(peer): + continue + self.last_connect = now + break + + need_advert_time = now - self.keepalive_intvl + timeout_time = now - 250 + can_advert_time = now - 30 + + close_list = [] + pex_only = 0 + + # Find all the connections that have timed out and put them in a separate list + for connection in self.connections.itervalues(): + if (connection.connection_state == UDPConnection.CONNECT_SENT and + connection.last_received < can_advert_time): + if connection.connection_tries < 0: + if DEBUG: + debug("Dropping connection with %s:%d (timeout)" % + (connection.address[0], connection.address[1])) + close_list.append(connection) + elif not self.try_connect(connection): + if DEBUG: + debug("Too many retries %s:%d" % (connection.address[0], connection.address[1])) + close_list.append(connection) + elif connection.last_received < timeout_time: + if DEBUG: + debug("Dropping connection with %s:%d (timeout)" % + (connection.address[0], connection.address[1])) + close_list.append(connection) + + # Close all the connections + for connection in close_list: + self.delete_closed_connection(connection) + + # Check whether we need to send keep-alives or PEX messages + for connection in self.connections.itervalues(): + if connection.last_send < need_advert_time: + # If there is a need for a keep-alive, first check if we also + # have PEX info or changed NAT/fw state, because we might as + # well send that instead of an empty keep-alive + if (connection.advertise_nat or len(connection.pex_add) != 0 or len(connection.pex_del) != 0): + connection.send_pex() or connection.sendto(UDPHandler.KEEP_ALIVE) + else: + connection.sendto(UDPHandler.KEEP_ALIVE) + elif (connection.advertise_nat or (len(connection.pex_add) != 0 or len(connection.pex_del) != 0) and + connection.last_advert < can_advert_time and pex_only < 35): + if connection.send_pex(): + pex_only += 1 + + # Reschedule this task in 10 seconds + self.rawserver.add_task(self.check_for_timeouts, 10) + + # Debug info + if DEBUG: + if self.last_info_dump + 60 < now: + self.last_info_dump = now + for connection in self.known_peers.itervalues(): + msg = "Peer %d %s %s:%d,%d,%d: Advertisers:" % (connection.connection_state, + connection.id.encode('hex'), connection.address[0], + connection.address[1], connection.nat_type, connection.filter_type) + for advertiser in connection.advertised_by.iterkeys(): + msg += " %s:%d" % (advertiser[0], advertiser[1]) + debug(msg) + + def try_connect(self, peer): + # Don't try to connect to peers that we can't arange a rendez-vous for + # when we think we need it + if peer.filter_type != UDPHandler.FILTER_NONE and len(peer.advertised_by) == 0: + return False + + if peer.connection_tries > 2: + return False + peer.connection_tries += 1 + + if DEBUG: + debug("Found compatible peer at %s:%d attempt %d" % (peer.address[0], peer.address[1], peer.connection_tries)) + + # Always send connect, to ensure the other peer's idea of its firewall + # is maintained correctly + if self.reporter: + self.reporter.add_event("UDPPuncture", "OCON%d:%s,%d,%s,%d,%d,%d" % (peer.connection_tries, peer.address[0], + peer.address[1], peer.id.encode('hex'), peer.nat_type, peer.filter_type, peer.natfw_version)) + peer.sendto(UDPHandler.CONNECT + chr(0) + self.id + + natfilter_to_byte(self.nat_type, self.filter_type) + chr(self.natfw_version)) + + # Request a rendez-vous + if peer.filter_type != UDPHandler.FILTER_NONE: + if DEBUG: + debug("Rendez-vous needed") + # Pick a random advertising peer for rendez vous + rendezvous_peers = list(peer.advertised_by.iterkeys()) + random.shuffle(rendezvous_peers) + rendezvous_addr = rendezvous_peers[0] + rendezvous = self.connections.get(rendezvous_addr) + if rendezvous: + if self.reporter: + self.reporter.add_event("UDPPuncture", "OFWC:%s,%d,%s,%s" % (rendezvous.address[0], + rendezvous.address[1], rendezvous.id.encode('hex'), peer.id.encode('hex'))) + rendezvous.sendto(UDPHandler.FW_CONNECT_REQ + peer.id) + + peer.connection_state = UDPConnection.CONNECT_SENT + peer.last_received = time.time() + self.connections[peer.address] = peer + return True + + def delete_closed_connection(self, connection): + del self.connections[connection.address] + orig_state = connection.connection_state + connection.connection_state = UDPConnection.CONNECT_NONE + connection.last_comm = time.time() + # Save the fact that we have sent something to this address, to ensure that retries won't be + # counted as proper incomming connects without prior communication + if connection.last_send > time.time() - 300: + self.last_sends[connection.address] = connection.last_send + connection.last_send = 0 + connection.last_received = 0 + connection.last_advert = 0 + if connection.id == "\0\0\0\0": + connection.nat_type = UDPHandler.NAT_NONE + connection.filter_type = UDPHandler.FILTER_NONE + connection.natfw_version = 0 + else: + connection.nat_type = UDPHandler.NAT_UNKNOWN + connection.filter_type = UDPHandler.FILTER_UNKNOWN + connection.natfw_version = 0 + connection.pex_add.clear() + connection.pex_del.clear() + connection.connection_tries = -1 + if len(connection.advertised_by) == 0: + try: + del self.known_peers[connection.id] + except: + pass + map(lambda x: x.remove_advertiser(connection.address), self.known_peers.itervalues()) + if orig_state == UDPConnection.CONNECT_ESTABLISHED: + map(lambda x: x.pex_del.append(connection), self.connections.itervalues()) + + def timeout_report(self, timeout, initial_ping): + if DEBUG: + debug("Timeout reported: %d %d" % (timeout, initial_ping)) + if self.reporter: + self.reporter.add_event("UDPPuncture", "TOUT:%d,%d" % (timeout, initial_ping)) + if initial_ping: + # Don't want to set the timeout too low, even if the firewall is acting funny + if timeout > 45 and timeout - 15 < self.keepalive_intvl: + self.keepalive_intvl = timeout - 15 + +class ConnectionIteratorByID: + def __init__(self, connections): + self.value_iterator = connections.itervalues() + + def __iter__(self): + return self + + def next(self): + value = self.value_iterator.next() + return value.id + +class UDPConnection: + CONNECT_NONE, CONNECT_SENT, CONNECT_ESTABLISHED = range(0, 3) + + def __init__(self, address, id, handler): + self.address = address + self.handler = handler + self.connection_state = UDPConnection.CONNECT_NONE + self.nat_type = UDPHandler.NAT_UNKNOWN + self.filter_type = UDPHandler.FILTER_UNKNOWN + self.natfw_version = 0 + self.advertised_by = {} + self.pex_add = deque([]) + self.pex_del = deque([]) + self.last_comm = 0 + self.last_send = 0 + self.last_advert = 0 + self.last_received = 0 + self.connected_since = 0 + self.advertise_nat = False + self.tracker = False + self.id = id + self.connection_tries = -1 + + def sendto(self, data): + self.handler.sendto(data, self.address) + self.last_send = time.time() + + def handle_msg(self, data): + self.last_received = time.time() + if data[0] == UDPHandler.CONNECT: + if DEBUG: + debug(" Message %d" % ord(data[0])) + if len(data) != 8: + self.sendto(UDPHandler.CLOSE + UDPHandler.CLOSE_LEN) + return False + + if ord(data[1]) != 0: + # Protocol version mismatch + self.sendto(UDPHandler.CLOSE + UDPHandler.CLOSE_PROTO_VER) + return False + + if data[2:6] != self.id or self.connection_state == UDPConnection.CONNECT_ESTABLISHED: + self.sendto(UDPHandler.CLOSE + UDPHandler.CLOSE_STATE_CORRUPT) + return False + + if self.handler.reporter: + self.handler.reporter.add_event("UDPPuncture", "ICON-AC:%s,%d,%s" % (self.address[0], + self.address[1], data[2:6].encode('hex'))) + + if self.handler.tracker: + peers = self.handler.connections.values() + random.shuffle(peers) + self.pex_add.extend(peers) + else: + self.pex_add.extend(self.handler.connections.itervalues()) + + self.connected_since = time.time() + + message = UDPHandler.YOUR_IP + address_to_string(self.address) + message += self.pex_string(self.pex_add, 1024 - len(message), True) + self.sendto(message) + self.last_advert = self.connected_since + self.nat_type, self.filter_type = byte_to_natfilter(data[6]) + self.natfw_version = ord(data[7]) + + self.connection_state = UDPConnection.CONNECT_ESTABLISHED + map(lambda x: x.pex_add.append(self), self.handler.connections.itervalues()) + self.pex_add.pop() # Remove ourselfves from our own pex_add list + return True + + if self.connection_state == UDPConnection.CONNECT_NONE: + # Other messages cannot be the first message in the stream. Drop this connection + return False + + while len(data) > 0: + if DEBUG: + debug(" Message %d len %d" % (ord(data[0]), len(data))) + if data[0] == UDPHandler.YOUR_IP: + if len(data) < 7: + self.sendto(UDPHandler.CLOSE + UDPHandler.CLOSE_LEN) + return False + + my_addres = string_to_address(data[1:7]) + if DEBUG: + debug(" My IP: %s:%d" % (my_addres[0], my_addres[1])) + if self.handler.reporter: + self.handler.reporter.add_event("UDPPuncture", "IYIP:%s,%d,%s" % (my_addres[0], my_addres[1], self.id.encode('hex'))) + + self.handler.incoming_ip(my_addres) + + if self.connection_state == UDPConnection.CONNECT_SENT: + self.pex_add.extend(self.handler.connections.itervalues()) + + message = UDPHandler.YOUR_IP + address_to_string(self.address) + message += self.pex_string(self.pex_add, 1024 - len(message), True) + self.sendto(message) + self.last_advert = time.time() + self.connected_since = time.time() + + self.connection_state = UDPConnection.CONNECT_ESTABLISHED + + map(lambda x: x.pex_add.append(self), self.handler.connections.itervalues()) + self.pex_add.pop() # Remove ourselfves from our own pex_add list + data = data[7:] + + elif data[0] == UDPHandler.FW_CONNECT_REQ: + if len(data) < 5: + self.sendto(UDPHandler.CLOSE + UDPHandler.CLOSE_LEN) + return False + + remote = data[1:5] + connection = self.handler.known_peers.get(remote) + if connection: + if DEBUG: + debug(" Rendez vous requested for peer %s %s:%d" % ( + remote.encode('hex'), connection.address[0], connection.address[1])) + if self.handler.reporter: + self.handler.reporter.add_event("UDPPuncture", "IFRQ:%s,%d,%s,%s,%d,%s" % (self.address[0], + self.address[1], self.id.encode('hex'), connection.address[0], connection.address[1], + remote.encode('hex'))) + else: + if DEBUG: + debug(" Rendez vous requested for peer %s (unknown)" % ( + remote.encode('hex'))) + if self.handler.reporter: + self.handler.reporter.add_event("UDPPuncture", "IFRQ:%s,%d,%s,Unknown,Unknown,%s" % (self.address[0], + self.address[1], self.id.encode('hex'), remote.encode('hex'))) + + if connection: + #FIXME: should we delay this action by some time to ensure the direct connect arives first? + # If we do, we should recheck whether we are connected to the requested peer! + connection.sendto(UDPHandler.REV_CONNECT + self.id + address_to_string(self.address) + + natfilter_to_byte(self.nat_type, self.filter_type) + + chr(self.natfw_version)) + else: + self.sendto(UDPHandler.PEER_UNKNOWN + remote) + + data = data[5:] + + elif data[0] == UDPHandler.REV_CONNECT: + if len(data) < 13: + self.sendto(UDPHandler.CLOSE + UDPHandler.CLOSE_LEN) + return False + + remote = string_to_address(data[5:11]) + if self.handler.reporter: + self.handler.reporter.add_event("UDPPuncture", "IRRQ:%s,%d,%s,%s,%d,%s" % (self.address[0], + self.address[1], self.id.encode('hex'), remote[0], remote[1], data[1:5].encode('hex'))) + connection = self.handler.connections.get(remote) + if connection: + pass + elif self.handler.check_connection_count(): + if self.handler.reporter: + self.handler.reporter.add_event("UDPPuncture", "OCTM-IRRQ:%s,%d,%s" % (connection.address[0], + connection.address[1], connection.id.encode('hex'))) + self.handler.sendto(UDPHandler.CLOSE + UDPHandler.CLOSE_TOO_MANY, remote) + else: + self.handler.incoming_connect(remote, False) # Update NAT and Filter states + remote_id = data[1:5] + connection = self.handler.known_peers.get(remote_id) + if not connection: + connection = UDPConnection(remote, remote_id, self.handler) + self.handler.known_peers[remote_id] = connection + elif connection.address != remote: + self.sendto(UDPHandler.PEER_UNKNOWN + remote_id) + data = data[13:] + continue + + if compare_natfw_version(ord(data[12]), connection.natfw_version): + connection.nat_type, connection.filter_type = byte_to_natfilter(data[11]) + connection.natfw_version = ord(data[12]) + + self.handler.connections[remote] = connection + connection.connection_state = UDPConnection.CONNECT_SENT + if self.handler.reporter: + self.handler.reporter.add_event("UDPPuncture", "OCON-IRRQ:%s,%d,%s" % (connection.address[0], + connection.address[1], connection.id.encode('hex'))) + connection.sendto(UDPHandler.CONNECT + chr(0) + self.handler.id + + natfilter_to_byte(self.handler.nat_type, self.handler.filter_type) + + chr(self.natfw_version)) + data = data[13:] + + elif data[0] == UDPHandler.PEX_ADD: + if len(data) < 2: + self.sendto(UDPHandler.CLOSE + UDPHandler.CLOSE_LEN) + return False + + addresses = ord(data[1]) + if len(data) < 2 + 12 * addresses: + self.sendto(UDPHandler.CLOSE + UDPHandler.CLOSE_LEN) + return False + + for i in range(0, addresses): + id = data[2 + i * 12:2 + i * 12 + 4] + address = string_to_address(data[2 + i * 12 + 4:2 + i * 12 + 10]) + peer = self.handler.known_peers.get(id) + if not peer: + peer = UDPConnection(address, id, self.handler) + peer.natfw_version = ord(data[2 + i * 12 + 11]) + peer.nat_type, peer.filter_type = byte_to_natfilter(data[2 + i * 12 + 10]) + self.handler.known_peers[id] = peer + #FIXME: should we check the received address here as well? + + peer.advertised_by[self.address] = time.time() + if DEBUG: + nat_type, filter_type = byte_to_natfilter(data[2 + i * 12 + 10]) + debug(" Received peer %s %s:%d NAT/fw:%d,%d" % (id.encode('hex'), + address[0], address[1], nat_type, filter_type)) + if compare_natfw_version(ord(data[2 + i * 12 + 11]), peer.natfw_version): + peer.natfw_version = ord(data[2 + i * 12 + 11]) + peer.nat_type, peer.filter_type = byte_to_natfilter(data[2 + i * 12 + 10]) + if peer.connection_state == UDPConnection.CONNECT_ESTABLISHED: + map(lambda x: x.pex_add.append(peer), self.handler.connections.itervalues()) + peer.pex_add.pop() # Remove ourselfves from our own pex_add list + + data = data[2 + addresses * 12:] + + elif data[0] == UDPHandler.PEX_DEL: + if len(data) < 2: + self.sendto(UDPHandler.CLOSE + UDPHandler.CLOSE_LEN) + return False + + addresses = ord(data[1]) + if len(data) < 2 + 4 * addresses: + self.sendto(UDPHandler.CLOSE + UDPHandler.CLOSE_LEN) + return False + + for i in range(0, addresses): + id = data[2 + i * 6:2 + i * 6 + 4] + if DEBUG: + debug(" Received peer %s" % (id.encode('hex'))) + peer = self.handler.known_peers.get(id) + if not peer or not self.address in peer.advertised_by: + continue + + del peer.advertised_by[self.address] + if len(peer.advertised_by) == 0 and peer.connection_state == UDPConnection.CONNECT_NONE: + del self.handler.known_peers[id] + + data = data[2 + addresses * 6:] + + elif data[0] == UDPHandler.CLOSE: + if DEBUG: + debug(" Reason %d" % ord(data[1])) + if len(data) == 2 and data[1] == UDPHandler.CLOSE_TOO_MANY and self.handler.reporter: + self.handler.reporter.add_event("UDPPuncture", "ICLO:%s,%d,%s" % (self.address[0], + self.address[1], self.id.encode('hex'))) + return False + elif data[0] == UDPHandler.UPDATE_NATFW_STATE: + if len(data) < 3: + self.sendto(UDPHandler.CLOSE + UDPHandler.CLOSE_LEN) + return False + if compare_natfw_version(ord(data[2]), self.natfw_version): + self.natfw_version = ord(data[2]) + self.nat_type, self.filter_type = byte_to_natfilter(data[1]) + if DEBUG: + debug(" Type: %d, %d" % (self.nat_type, self.filter_type)) + map(lambda x: x.pex_add.append(self), self.handler.connections.itervalues()) + self.pex_add.pop() # Remove ourselfves from our own pex_add list + data = data[3:] + + elif data[0] == UDPHandler.PEER_UNKNOWN: + # WARNING: there is a big security issue here: we trust the + # remote peer to send us the address that we sent it. However, + # if the peer is malicious it may send us another address. This + # can all be verified, but then we need to keep track of lots + # more state which I don't want to do for the current + # implementation. + if len(data) < 5: + self.sendto(UDPHandler.CLOSE + UDPHandler.CLOSE_LEN) + return False + + remote = data[1:5] + peer = self.handler.known_peers.get(remote) + if not peer: + data = data[5:] + continue + + if self.address in peer.advertised_by: + del peer.advertised_by[self.address] + if len(peer.advertised_by) == 0 and peer.connection_state == UDPConnection.CONNECT_NONE: + del self.handler.known_peers[remote] + data = data[5:] + continue + + if len(peer.advertised_by) > 0 and peer.connection_state == UDPConnection.CONNECT_SENT: + rendezvous_addr = peer.advertised_by.iterkeys().next() + rendezvous = self.handler.connections.get(rendezvous_addr) + #FIXME: handle unconnected peers! I.e. delete from advertised_by list and goto next + if rendezvous: + if self.handler.reporter: + self.handler.reporter.add_event("UDPPuncture", "OFWC-RTR:%s,%d,%s,%s" % (rendezvous.address[0], + rendezvous.address[1], rendezvous.id.encode('hex'), peer.id.encode('hex'))) + rendezvous.sendto(UDPHandler.FW_CONNECT_REQ + remote) + + data = data[5:] + elif data[0] == UDPHandler.KEEP_ALIVE: + data = data[1:] + else: + self.sendto(UDPHandler.CLOSE + UDPHandler.CLOSE_GARBAGE) + return False + + return True + + def readvertise_nat(self): + self.advertise_nat = True + + def remove_advertiser(self, address): + try: + del self.advertised_by[address] + except: + pass + + def send_pex(self): + self.last_advert = time.time() + + message = "" + if self.advertise_nat: + self.advertise_nat = False + message += (UDPHandler.UPDATE_NATFW_STATE + + natfilter_to_byte(self.handler.nat_type, self.handler.filter_type) + + chr(self.handler.natfw_version)) + + if self.tracker: + self.pex_add.clear() + self.pex_del.clear() + else: + if len(self.pex_add) > 0: + message += self.pex_string(self.pex_add, 1023, True) + if len(self.pex_del) > 0: + message += self.pex_string(self.pex_del, 1023 - len(message), False) + if len(message) > 0: + self.sendto(message) + return True + return False + + def pex_string(self, items, max_size, add): + retval = "" + num_added = 0 + added = set() + if add: + max_size = (max_size - 2) / 12 + else: + max_size = (max_size - 2) / 4 + + while len(items) > 0 and max_size > num_added: + connection = items.popleft() + if DEBUG: + debug("- peer %s:%d (%d, %d) state %d" % (connection.address[0], connection.address[1], + connection.nat_type, connection.filter_type, connection.connection_state)) + if connection != self and (not connection.tracker) and (not connection.address in added) and ( + (add and connection.connection_state == UDPConnection.CONNECT_ESTABLISHED) or + ((not add) and connection.connection_state != UDPConnection.CONNECT_ESTABLISHED)): + added.add(connection.address) + if add: + retval += (connection.id + address_to_string(connection.address) + + natfilter_to_byte(connection.nat_type, connection.filter_type) + + chr(connection.natfw_version)) + else: + retval += connection.id + num_added += 1 + + if DEBUG: + debug("- created pex string: " + retval.encode('hex')) + if num_added == 0: + return "" + if add: + return UDPHandler.PEX_ADD + chr(num_added) + retval + else: + return UDPHandler.PEX_DEL + chr(num_added) + retval + +# Utility functions for often used conversions +def address_to_string(address): + return socket.inet_aton(address[0]) + chr(address[1] >> 8) + chr(address[1] & 255) + +def string_to_address(address): + return socket.inet_ntoa(address[0:4]), (ord(address[4]) << 8) + ord(address[5]) + +def natfilter_to_byte(nat_type, filter_type): + return chr((nat_type & 3) + ((filter_type & 3) << 2)) + +def byte_to_natfilter(byte): + return ord(byte) & 3, (ord(byte) >> 2) & 3 + +def compare_natfw_version(a, b): + return ((a - b + 256) % 256) < ((b - a + 256) % 256) + +if __name__ == "__main__": + import BaseLib.Core.BitTornado.RawServer as RawServer + from threading import Event + import thread + from traceback import print_exc + import os + + def fail(e): + print "Fatal error: " + str(e) + print_exc() + + def error(e): + print "Non-fatal error: " + str(e) + + DEBUG = True + def debug(msg): + if 'log' in globals(): + log.write("%.2f: %s\n" % (time.time(), msg)) + log.flush() + print "%.2f: %s" % (time.time(), msg) + sys.stdout.flush() + + if len(sys.argv) == 2: + log = open("log-%s.txt" % sys.argv[1], "w") + else: + log = open("log-%d.txt" % os.getpid(), "w") + + rawserver = RawServer.RawServer(Event(), + 60.0, + 300.0, + False, + failfunc = fail, + errorfunc = error) + thread.start_new_thread(rawserver.listen_forever, (None,)) + if len(sys.argv) < 2: + port = 0 + else: + port = int(sys.argv[1]) + udp_handler = UDPHandler(rawserver, False, port) + + if sys.argv == "12345": + udp_handler.connect_threshold = 0 + + print "UDPHandler started, press enter to quit" + sys.stdin.readline() + udp_handler.shutdown() + print "Log left in " + log.name diff --git a/instrumentation/next-share/BaseLib/Core/NATFirewall/UPnPThread.py b/instrumentation/next-share/BaseLib/Core/NATFirewall/UPnPThread.py new file mode 100644 index 0000000..6bc9c60 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/NATFirewall/UPnPThread.py @@ -0,0 +1,114 @@ +# Written by Arno Bakker +# see LICENSE.txt for license information + +import sys +from threading import Event,Thread +from traceback import print_exc + +from BaseLib.Core.BitTornado.natpunch import UPnPWrapper, UPnPError + +DEBUG = False + + +class UPnPThread(Thread): + """ Thread to run the UPnP code. Moved out of main startup- + sequence for performance. As you can see this thread won't + exit until the client exits. This is due to a funky problem + with UPnP mode 2. That uses Win32/COM API calls to find and + talk to the UPnP-enabled firewall. This mechanism apparently + requires all calls to be carried out by the same thread. + This means we cannot let the final DeletePortMapping(port) + (==UPnPWrapper.close(port)) be done by a different thread, + and we have to make this one wait until client shutdown. + + Arno, 2006-11-12 + """ + + def __init__(self,upnp_type,ext_ip,listen_port,error_func,got_ext_ip_func): + Thread.__init__(self) + self.setDaemon(True) + self.setName( "UPnP"+self.getName() ) + + self.upnp_type = upnp_type + self.locally_guessed_ext_ip = ext_ip + self.listen_port = listen_port + self.error_func = error_func + self.got_ext_ip_func = got_ext_ip_func + self.shutdownevent = Event() + + def run(self): + if self.upnp_type > 0: + self.upnp_wrap = UPnPWrapper.getInstance() + self.upnp_wrap.register(self.locally_guessed_ext_ip) + + # Disabled Gertjan's UPnP logging for m24 + #from BaseLib.Core.Statistics.StatusReporter import get_reporter_instance + #reporter = get_reporter_instance() + + if self.upnp_wrap.test(self.upnp_type): + # Disabled Gertjan's UPnP logging for m24 + #reporter.add_event("UPnP", "Init'ed") + try: + shownerror=False + # Get external IP address from firewall + if self.upnp_type != 1: # Mode 1 doesn't support getting the IP address" + ret = self.upnp_wrap.get_ext_ip() + if ret == None: + shownerror=True + self.error_func(self.upnp_type,self.listen_port,0) + else: + self.got_ext_ip_func(ret) + + # Do open_port irrespective of whether get_ext_ip() + # succeeds, UPnP mode 1 doesn't support get_ext_ip() + # get_ext_ip() must be done first to ensure we have the + # right IP ASAP. + + # Open TCP listen port on firewall + ret = self.upnp_wrap.open(self.listen_port,iproto='TCP') + if ret == False and not shownerror: + self.error_func(self.upnp_type,self.listen_port,0) + + # Open UDP listen port on firewall + ret = self.upnp_wrap.open(self.listen_port,iproto='UDP') + if ret == False and not shownerror: + self.error_func(self.upnp_type,self.listen_port,0,listenproto='UDP') + # Disabled Gertjan's UPnP logging for m24 + #reporter.add_event("UPnP", "UDP:%d" % ret) + + except UPnPError,e: + self.error_func(self.upnp_type,self.listen_port,1,e) + else: + # Disabled Gertjan's UPnP logging for m24 + #reporter.add_event("UPnP", "Init failed") + if self.upnp_type != 3: + self.error_func(self.upnp_type,self.listen_port,2) + elif DEBUG: + print >>sys.stderr,"upnp: thread: Initialization failed, but didn't report error because UPnP mode 3 is now enabled by default" + + # Now that the firewall is hopefully open, activate other services + # here. For Buddycast we don't have an explicit notification that it + # can go ahead. It will start 15 seconds after client startup, which + # is assumed to be sufficient for UPnP to open the firewall. + ## dmh.start_active() + + if self.upnp_type > 0: + if DEBUG: + print >>sys.stderr,"upnp: thread: Waiting till shutdown" + self.shutdownevent.wait() + # Don't write to sys.stderr, that sometimes doesn't seem to exist + # any more?! Python garbage collection funkiness of module sys import? + # The GUI is definitely gone, so don't use self.error_func() + if DEBUG: + print "upnp: thread: Shutting down, closing port on firewall" + try: + self.upnp_wrap.close(self.listen_port,iproto='TCP') + self.upnp_wrap.close(self.listen_port,iproto='UDP') + except Exception,e: + print "upnp: thread: close port at shutdown threw",e + print_exc() + + # End of UPnPThread + + def shutdown(self): + self.shutdownevent.set() diff --git a/instrumentation/next-share/BaseLib/Core/NATFirewall/__init__.py b/instrumentation/next-share/BaseLib/Core/NATFirewall/__init__.py new file mode 100644 index 0000000..395f8fb --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/NATFirewall/__init__.py @@ -0,0 +1,2 @@ +# Written by Arno Bakker +# see LICENSE.txt for license information diff --git a/instrumentation/next-share/BaseLib/Core/NATFirewall/guessip.py b/instrumentation/next-share/BaseLib/Core/NATFirewall/guessip.py new file mode 100644 index 0000000..fea807d --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/NATFirewall/guessip.py @@ -0,0 +1,161 @@ +# Written by Arno Bakker, Jan David Mol +# see LICENSE.txt for license information +# +# Code to guess the IP address of a host by which it is reachable on the +# Internet, given the host is not behind a firewall or NAT. +# +# For all OSes (Linux,Windows,MacOS X) we first look at the routing table to +# see what the gateway for the default route is. We then try to establish +# our IP address that's on the same network as the gateway. That is our +# external/WAN address. +# +# This code does not support IPv6, that is, IPv6 address are ignored. +# +# Arno, Jan David, 2006-06-30 +# +import os +import sys +import socket +from traceback import print_exc + +DEBUG = False + +def get_my_wan_ip(): + try: + if sys.platform == 'win32': + return get_my_wan_ip_win32() + elif sys.platform == 'darwin': + return get_my_wan_ip_darwin() + else: + return get_my_wan_ip_linux() + except: + print_exc() + return None + +def get_my_wan_ip_win32(): + + routecmd = "netstat -nr" + ifcmd = "ipconfig /all" + + gwip = None + for line in os.popen(routecmd).readlines(): + words = line.split() + if len(words) >= 3: + if words[0] == 'Default' and words[1] == 'Gateway:': + gwip = words[-1] + if DEBUG: + print "netstat found default gateway",gwip + break + + myip = None + mywanip = None + ingw = 0 + for line in os.popen(ifcmd).readlines(): + words = line.split() + if len(words) >= 3: + if (words[0] == 'IP' and words[1] == 'Address.') or (words[1] == 'IP' and words[2] == 'Address.'): # Autoconfiguration entry + try: + socket.getaddrinfo(words[-1],None,socket.AF_INET) + myip = words[-1] + if DEBUG: + print "ipconfig found IP address",myip + except socket.gaierror: + if DEBUG: + print "ipconfig ignoring IPv6 address",words[-1] + pass + elif words[0] == 'Default' and words[1] == 'Gateway': + if words[-1] == ':': + if DEBUG: + print "ipconfig ignoring empty default gateway" + pass + else: + ingw = 1 + if ingw >= 1: + # Assumption: the "Default Gateway" list can only have 2 entries, + # one for IPv4, one for IPv6. Since we don't know the order, look + # at both. + gwip2 = None + ingw = (ingw + 1) % 3 + try: + socket.getaddrinfo(words[-1],None,socket.AF_INET) + gwip2 = words[-1] + if DEBUG: + print "ipconfig found default gateway",gwip2 + except socket.gaierror: + if DEBUG: + print "ipconfig ignoring IPv6 default gateway",words[-1] + pass + if gwip == gwip2: + mywanip = myip + break + return mywanip + + +def get_my_wan_ip_linux(): + routecmd = '/bin/netstat -nr' + ifcmd = '/sbin/ifconfig -a' + + gwif = None + gwip = None + for line in os.popen(routecmd).readlines(): + words = line.split() + if len(words) >= 3: + if words[0] == '0.0.0.0': + gwif = words[-1] + gwip = words[1] + if DEBUG: + print "netstat found default gateway",gwip + break + + mywanip = None + for line in os.popen(ifcmd).readlines(): + words = line.split() + if len(words) >= 2: + if words[0] == gwif: + flag = True + elif words[0] == 'inet': + words2 = words[1].split(':') # "inet addr:130.37.192.1" line + if len(words2) == 2: + mywanip = words2[1] + break + else: + flag = False + else: + flag = False + return mywanip + + +def get_my_wan_ip_darwin(): + routecmd = '/usr/sbin/netstat -nr' + ifcmd = '/sbin/ifconfig -a' + + gwif = None + gwip = None + for line in os.popen(routecmd).readlines(): + words = line.split() + if len(words) >= 3: + if words[0] == 'default': + gwif = words[-1] + gwip = words[1] + if DEBUG: + print "netstat found default gateway",gwip + break + + mywanip = None + flag = False + for line in os.popen(ifcmd).readlines(): + words = line.split() + if len(words) >= 2: + if words[0] == "%s:" % gwif: + flag = True + elif words[0] == 'inet' and flag: + mywanip = words[1] # "inet 130.37.192.1" line + break + return mywanip + + + +if __name__ == "__main__": + DEBUG = True + ip = get_my_wan_ip() + print "External IP address is",ip diff --git a/instrumentation/next-share/BaseLib/Core/NATFirewall/upnp.py b/instrumentation/next-share/BaseLib/Core/NATFirewall/upnp.py new file mode 100644 index 0000000..5eb23aa --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/NATFirewall/upnp.py @@ -0,0 +1,300 @@ +# Written by Arno Bakker +# see LICENSE.txt for license information +# +# Platform independent UPnP client +# +# References: +# - UPnP Device Architecture 1.0, www.upnp.org +# - From Internet Gateway Device IGD V1.0: +# * WANIPConnection:1 Service Template Version 1.01 +# + +import sys +import socket +from cStringIO import StringIO +import urllib +import urllib2 +from urlparse import urlparse +import xml.sax as sax +from xml.sax.handler import ContentHandler +from traceback import print_exc + +UPNP_WANTED_SERVICETYPES = ['urn:schemas-upnp-org:service:WANIPConnection:1','urn:schemas-upnp-org:service:WANPPPConnection:1'] + +DEBUG = False + +class UPnPPlatformIndependent: + + def __init__(self): + # Maps location URL to a dict containing servicetype and control URL + self.services = {} + self.lastdiscovertime = 0 + + def discover(self): + """ Attempts to discover any UPnP services for X seconds + If any are found, they are stored in self.services + """ + #if self.lastdiscovertime != 0 and self.lastdiscovertime + DISCOVER_WAIT < time.time(): + # if DEBUG: + # print >> sys.stderr,"upnp: discover: Already did a discovery recently" + # return + + maxwait = 4 + req = 'M-SEARCH * HTTP/1.1\r\n' + req += 'HOST: 239.255.255.250:1900\r\n' + req += 'MAN: "ssdp:discover"\r\n' # double quotes obligatory + req += 'MX: '+str(maxwait)+'\r\n' + req += 'ST: ssdp:all\r\n' # no double quotes + req += '\r\n\r\n' + + try: + self.s = socket.socket(socket.AF_INET,socket.SOCK_DGRAM) + self.s.settimeout(maxwait+2.0) + self.s.sendto(req,('239.255.255.250',1900)) + while True: # exited by socket.timeout exception only + if DEBUG: + print >> sys.stderr,"upnp: discover: Wait 4 reply" + (rep,sender) = self.s.recvfrom(1024) + + if DEBUG: + print >> sys.stderr,"upnp: discover: Got reply from",sender + #print >> sys.stderr,"upnp: discover: Saying:",rep + repio = StringIO(rep) + while True: + line = repio.readline() + #print >> sys.stderr,"LINE",line + if line == '': + break + if line[-2:] == '\r\n': + line = line[:-2] + idx = line.find(':') + if idx == -1: + continue + key = line[:idx] + key = key.lower() + #print >> sys.stderr,"key",key + if key.startswith('location'): + # Careful: MS Internet Connection Sharing returns "Location:http://bla", so no space + location = line[idx+1:].strip() + desc = self.get_description(location) + self.services[location] = self.parse_services(desc) + + except: + if DEBUG: + print_exc() + + def found_wanted_services(self): + """ Return True if WANIPConnection or WANPPPConnection were found by discover() """ + for location in self.services: + for servicetype in UPNP_WANTED_SERVICETYPES: + if self.services[location]['servicetype'] == servicetype: + return True + return False + + + def add_port_map(self,internalip,port,iproto='TCP'): + """ Sends an AddPortMapping request to all relevant IGDs found by discover() + + Raises UPnPError in case the IGD returned an error reply, + Raises Exception in case of any other error + """ + srch = self.do_soap_request('AddPortMapping',port,iproto=iproto,internalip=internalip) + if srch is not None: + se = srch.get_error() + if se is not None: + raise se + + def del_port_map(self,port,iproto='TCP'): + """ Sends a DeletePortMapping request to all relevant IGDs found by discover() + + Raises UPnPError in case the IGD returned an error reply, + Raises Exception in case of any other error + """ + srch = self.do_soap_request('DeletePortMapping',port,iproto=iproto) + if srch is not None: + se = srch.get_error() + if se is not None: + raise se + + def get_ext_ip(self): + """ Sends a GetExternalIPAddress request to all relevant IGDs found by discover() + + Raises UPnPError in case the IGD returned an error reply, + Raises Exception in case of any other error + """ + srch = self.do_soap_request('GetExternalIPAddress') + if srch is not None: + se = srch.get_error() + if se is not None: + raise se + else: + return srch.get_ext_ip() + + # + # Internal methods + # + def do_soap_request(self,methodname,port=-1,iproto='TCP',internalip=None): + for location in self.services: + for servicetype in UPNP_WANTED_SERVICETYPES: + if self.services[location]['servicetype'] == servicetype: + o = urlparse(location) + endpoint = o[0]+'://'+o[1]+self.services[location]['controlurl'] + # test: provoke error + #endpoint = o[0]+'://'+o[1]+'/bla'+self.services[location]['controlurl'] + if DEBUG: + print >> sys.stderr,"upnp: "+methodname+": Talking to endpoint ",endpoint + (headers,body) = self.create_soap_request(methodname,port,iproto=iproto,internalip=internalip) + #print body + try: + req = urllib2.Request(url=endpoint,data=body,headers=headers) + f = urllib2.urlopen(req) + resp = f.read() + except urllib2.HTTPError,e: + resp = e.fp.read() + if DEBUG: + print_exc() + srch = SOAPResponseContentHandler(methodname) + if DEBUG: + print >> sys.stderr,"upnp: "+methodname+": response is",resp + try: + srch.parse(resp) + except sax.SAXParseException,e: + # Our test linux-IGD appears to return an incompete + # SOAP error reply. Handle this. + se = srch.get_error() + if se is None: + raise e + # otherwise we were able to parse the error reply + return srch + + def get_description(self,url): + if DEBUG: + print >> sys.stderr,"upnp: discover: Reading description from",url + f = urllib.urlopen(url) + data = f.read() + #print >> sys.stderr,"upnp: description: Got",data + return data + + def parse_services(self,desc): + dch = DescriptionContentHandler() + dch.parse(desc) + return dch.services + + def create_soap_request(self,methodname,port=-1,iproto="TCP",internalip=None): + headers = {} + #headers['Host'] = endpoint + #headers['Accept-Encoding'] = 'identity' + headers['Content-type'] = 'text/xml; charset="utf-8"' + headers['SOAPAction'] = '"urn:schemas-upnp-org:service:WANIPConnection:1#'+methodname+'"' + headers['User-Agent'] = 'Mozilla/4.0 (compatible; UPnP/1.0; Windows 9x)' + + body = '' + body += '' + body += '' + if methodname == 'AddPortMapping': + externalport = port + internalport = port + internalclient = internalip + body += '' + body += ''+str(externalport)+'' + body += ''+iproto+'' + body += ''+str(internalport)+'' + body += ''+internalclient+'' + body += '1' + body += 'Insert description here' + body += '0' + elif methodname == 'DeletePortMapping': + externalport = port + body += '' + body += ''+str(externalport)+'' + body += ''+iproto+'' + body += '' + body += '' + return (headers,body) + + +class UPnPError(Exception): + def __init__(self,errorcode,errordesc): + Exception.__init__(self) + self.errorcode = errorcode + self.errordesc = errordesc + + def __str__(self): + return 'UPnP Error %d: %s' % (self.errorcode, self.errordesc) + + +# +# Internal classes +# + +class DescriptionContentHandler(ContentHandler): + + def __init__(self): + ContentHandler.__init__(self) + self.services = {} + + def parse(self,desc): + sax.parseString(desc,self) + + def endDocument(self): + if DEBUG: + print >> sys.stderr,"upnp: discover: Services found",self.services + + def endElement(self, name): + #print >> sys.stderr,"endElement",name + n = name.lower() + if n == 'servicetype': + self.services['servicetype'] = self.content + elif n == 'controlurl': + self.services['controlurl'] = self.content + + def characters(self, content): + # print >> sys.stderr,"content",content + self.content = content + + +class SOAPResponseContentHandler(ContentHandler): + + def __init__(self,methodname): + ContentHandler.__init__(self) + self.methodname = methodname + self.ip = None + self.errorset = False + self.errorcode = 0 + self.errordesc = 'No error' + self.content = None + + def parse(self,resp): + sax.parseString(resp,self) + + def get_ext_ip(self): + return self.ip + + def get_error(self): + if self.errorset: + return UPnPError(self.errorcode,self.methodname+": "+self.errordesc) + else: + return None + + def endElement(self, name): + n = name.lower() + if self.methodname == 'GetExternalIPAddress' and n.endswith('newexternalipaddress'): + self.ip = self.content + elif n== 'errorcode': + self.errorset = True + self.errorcode = int(self.content) + elif n == 'errordescription': + self.errorset = True + self.errordesc = self.content + + def characters(self, content): + #print >>sys.stderr,"upnp: GOT CHARACTERS",content + self.content = content + +if __name__ == '__main__': + u = UPnPPlatformIndependent() + u.discover() + print >> sys.stderr,"IGD say my external IP address is",u.get_ext_ip() + #u.add_port_map('130.37.193.64',6881) diff --git a/instrumentation/next-share/BaseLib/Core/Overlay/MetadataHandler.py b/instrumentation/next-share/BaseLib/Core/Overlay/MetadataHandler.py new file mode 100644 index 0000000..b144a0b --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/Overlay/MetadataHandler.py @@ -0,0 +1,633 @@ +# Written by Jie Yang, Arno Bakker +# see LICENSE.txt for license information +import sys +import os +from BaseLib.Core.Utilities.Crypto import sha +from time import time, ctime +from traceback import print_exc, print_stack +from sets import Set +from threading import currentThread + +from BaseLib.Core.simpledefs import * +from BaseLib.Core.BitTornado.bencode import bencode, bdecode +from BaseLib.Core.BitTornado.BT1.MessageID import * +from BaseLib.Core.Utilities.utilities import isValidInfohash, show_permid_short, sort_dictlist, bin2str, get_collected_torrent_filename +from BaseLib.Core.Overlay.SecureOverlay import OLPROTO_VER_FOURTH, OLPROTO_VER_ELEVENTH +from BaseLib.TrackerChecking.TorrentChecking import TorrentChecking +from BaseLib.Core.osutils import getfreespace,get_readable_torrent_name +from BaseLib.Core.CacheDB.CacheDBHandler import BarterCastDBHandler +from BaseLib.Core.CacheDB.SqliteCacheDBHandler import PopularityDBHandler +from BaseLib.Core.TorrentDef import TorrentDef + +DEBUG = False + +BARTERCAST_TORRENTS = False + +# Python no recursive imports? +# from overlayswarm import overlay_infohash +overlay_infohash = '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + +Max_Torrent_Size = 2*1024*1024 # 2MB torrent = 6GB ~ 250GB content + + +class MetadataHandler: + + __single = None + + def __init__(self): + if MetadataHandler.__single: + raise RuntimeError, "MetadataHandler is singleton" + MetadataHandler.__single = self + self.num_torrents = -100 + self.avg_torrent_size = 25*(2**10) + self.initialized = False + self.registered = False + self.popularity_db = PopularityDBHandler.getInstance() + + + def getInstance(*args, **kw): + if MetadataHandler.__single is None: + MetadataHandler(*args, **kw) + return MetadataHandler.__single + getInstance = staticmethod(getInstance) + + def register(self, overlay_bridge, dlhelper, launchmany, config): + self.registered = True + self.overlay_bridge = overlay_bridge + self.dlhelper = dlhelper + self.launchmany = launchmany + self.torrent_db = launchmany.torrent_db + self.config = config + self.min_free_space = self.config['stop_collecting_threshold']*(2**20) + #if self.min_free_space <= 0: + # self.min_free_space = 200*(2**20) # at least 200 MB left on disk + self.config_dir = os.path.abspath(self.config['state_dir']) + self.torrent_dir = os.path.abspath(self.config['torrent_collecting_dir']) + print >>sys.stderr,"metadata: collect dir is",self.torrent_dir + assert os.path.isdir(self.torrent_dir) + self.free_space = self.get_free_space() + print >> sys.stderr, "Available space for database and collecting torrents: %d MB," % (self.free_space/(2**20)), "Min free space", self.min_free_space/(2**20), "MB" + self.max_num_torrents = self.init_max_num_torrents = int(self.config['torrent_collecting_max_torrents']) + self.upload_rate = 1024 * int(self.config['torrent_collecting_rate']) # 5KB/s + self.num_collected_torrents = 0 + self.recently_collected_torrents = [] # list of infohashes + self.upload_queue = [] + self.requested_torrents = Set() + self.next_upload_time = 0 + self.initialized = True + self.rquerytorrenthandler = None + self.delayed_check_overflow(5) + + def register2(self,rquerytorrenthandler): + self.rquerytorrenthandler = rquerytorrenthandler + + + def handleMessage(self,permid,selversion,message): + + t = message[0] + + if t == GET_METADATA: # the other peer requests a torrent + if DEBUG: + print >> sys.stderr,"metadata: Got GET_METADATA",len(message),show_permid_short(permid) + return self.send_metadata(permid, message, selversion) + elif t == METADATA: # the other peer sends me a torrent + if DEBUG: + print >> sys.stderr,"metadata: Got METADATA",len(message),show_permid_short(permid),selversion, currentThread().getName() + return self.got_metadata(permid, message, selversion) + else: + if DEBUG: + print >> sys.stderr,"metadata: UNKNOWN OVERLAY MESSAGE", ord(t) + return False + + def send_metadata_request(self, permid, infohash, selversion=-1, caller="BC"): + assert isinstance(infohash, str), "INFOHASH has invalid type: %s" % type(infohash) + assert len(infohash) == INFOHASH_LENGTH, "INFOHASH has invalid length: %d" % len(infohash) + if DEBUG: + print >> sys.stderr,"metadata: Connect to send GET_METADATA to",show_permid_short(permid) + if not isValidInfohash(infohash): + return False + + filename,metadata = self.torrent_exists(infohash) + if filename is not None: # torrent already exists on disk + if DEBUG: + print >> sys.stderr,"metadata: send_meta_req: Already on disk??!" + self.notify_torrent_is_in(infohash, metadata, filename) + return True + + if caller == "dlhelp": + self.requested_torrents.add(infohash) + + if self.min_free_space != 0 and (self.free_space - self.avg_torrent_size < self.min_free_space): # no space to collect + self.free_space = self.get_free_space() + if self.free_space - self.avg_torrent_size < self.min_free_space: + self.warn_disk_full() + return True + + try: + # Optimization: don't connect if we're connected, although it won't + # do any harm. + if selversion == -1: # not currently connected + self.overlay_bridge.connect(permid,lambda e,d,p,s:self.get_metadata_connect_callback(e,d,p,s,infohash)) + else: + self.get_metadata_connect_callback(None,None,permid,selversion,infohash) + + except: + print_exc() + return False + return True + + def torrent_exists(self, infohash): + # if the torrent is already on disk, put it in db + + file_name = get_collected_torrent_filename(infohash) + torrent_path = os.path.join(self.torrent_dir, file_name) + if not os.path.exists(torrent_path): + return None,None + else: + metadata = self.read_torrent(torrent_path) + if not self.valid_metadata(infohash, metadata): + return None + self.addTorrentToDB(torrent_path, infohash, metadata, source="BC", extra_info={}) + return file_name, metadata + + def get_metadata_connect_callback(self,exc,dns,permid,selversion,infohash): + if exc is None: + if DEBUG: + print >> sys.stderr,"metadata: Sending GET_METADATA to",show_permid_short(permid) + ## Create metadata_request according to protocol version + try: + metadata_request = bencode(infohash) + self.overlay_bridge.send(permid, GET_METADATA + metadata_request,self.get_metadata_send_callback) + self.requested_torrents.add(infohash) + except: + print_exc() + elif DEBUG: + print >> sys.stderr,"metadata: GET_METADATA: error connecting to",show_permid_short(permid) + + def get_metadata_send_callback(self,exc,permid): + if exc is not None: + if DEBUG: + print >> sys.stderr,"metadata: GET_METADATA: error sending to",show_permid_short(permid),exc + pass + else: + pass + + def send_metadata(self, permid, message, selversion): + try: + infohash = bdecode(message[1:]) + except: + print_exc() + if DEBUG: + print >> sys.stderr,"metadata: GET_METADATA: error becoding" + return False + if not isValidInfohash(infohash): + if DEBUG: + print >> sys.stderr,"metadata: GET_METADATA: invalid hash" + return False + + # TODO: + res = self.torrent_db.getOne(('torrent_file_name', 'status_id'), infohash=bin2str(infohash)) + if not res: + if DEBUG: + print >> sys.stderr,"metadata: GET_METADATA: not in database", infohash + return True # don't close connection because I don't have the torrent + torrent_file_name, status_id = res + if status_id == self.torrent_db._getStatusID('dead'): + if DEBUG: + print >> sys.stderr,"metadata: GET_METADATA: Torrent was dead" + return True + if not torrent_file_name: + if DEBUG: + print >> sys.stderr,"metadata: GET_METADATA: no torrent file name" + return True + torrent_path = os.path.join(self.torrent_dir, torrent_file_name) + if not os.path.isfile(torrent_path): + if DEBUG: + print >> sys.stderr,"metadata: GET_METADATA: not existing", res, torrent_path + return True + + task = {'permid':permid, 'infohash':infohash, 'torrent_path':torrent_path, 'selversion':selversion} + self.upload_queue.append(task) + if int(time()) >= self.next_upload_time: + self.checking_upload_queue() + + return True + + def read_and_send_metadata(self, permid, infohash, torrent_path, selversion): + torrent_data = self.read_torrent(torrent_path) + if torrent_data: + # Arno: Don't send private torrents + try: + metainfo = bdecode(torrent_data) + if 'info' in metainfo and 'private' in metainfo['info'] and metainfo['info']['private']: + if DEBUG: + print >> sys.stderr,"metadata: Not sending torrent", `torrent_path`,"because it is private" + return 0 + except: + print_exc() + return 0 + + + if DEBUG: + print >> sys.stderr,"metadata: sending torrent", `torrent_path`, len(torrent_data) + + torrent = {} + torrent['torrent_hash'] = infohash + # P2PURLs: If URL compat then send URL + tdef = TorrentDef.load_from_dict(metainfo) + if selversion >= OLPROTO_VER_ELEVENTH and tdef.get_url_compat(): + torrent['metatype'] = URL_MIME_TYPE + torrent['metadata'] = tdef.get_url() + else: + torrent['metatype'] = TSTREAM_MIME_TYPE + torrent['metadata'] = torrent_data + + if selversion >= OLPROTO_VER_FOURTH: + data = self.torrent_db.getTorrent(infohash) + if data is None: + # DB inconsistency + return 0 + nleechers = data.get('leecher', -1) + nseeders = data.get('seeder', -1) + last_check_ago = int(time()) - data.get('last_check_time', 0) # relative time + if last_check_ago < 0: + last_check_ago = 0 + status = data.get('status', 'unknown') + + torrent.update({'leecher':nleechers, + 'seeder':nseeders, + 'last_check_time':last_check_ago, + 'status':status}) + + + return self.do_send_metadata(permid, torrent, selversion) + else: # deleted before sending it + self.torrent_db.deleteTorrent(infohash, delete_file=True, commit=True) + if DEBUG: + print >> sys.stderr,"metadata: GET_METADATA: no torrent data to send" + return 0 + + def do_send_metadata(self, permid, torrent, selversion): + metadata_request = bencode(torrent) + if DEBUG: + print >> sys.stderr,"metadata: send metadata", len(metadata_request) + ## Optimization: we know we're currently connected + self.overlay_bridge.send(permid,METADATA + metadata_request,self.metadata_send_callback) + + # BarterCast: add bytes of torrent to BarterCastDB + # Save exchanged KBs in BarterCastDB + if permid != None and BARTERCAST_TORRENTS: + self.overlay_bridge.add_task(lambda:self.olthread_bartercast_torrentexchange(permid, 'uploaded'), 0) + + return len(metadata_request) + + def olthread_bartercast_torrentexchange(self, permid, up_or_down): + + if up_or_down != 'uploaded' and up_or_down != 'downloaded': + return + + bartercastdb = BarterCastDBHandler.getInstance() + + torrent_kb = float(self.avg_torrent_size) / 1024 + name = bartercastdb.getName(permid) + my_permid = bartercastdb.my_permid + + if DEBUG: + print >> sys.stderr, "bartercast: Torrent (%d KB) %s to/from peer %s" % (torrent_kb, up_or_down, `name`) + + if torrent_kb > 0: + bartercastdb.incrementItem((my_permid, permid), up_or_down, torrent_kb) + + + def metadata_send_callback(self,exc,permid): + if exc is not None: + if DEBUG: + print >> sys.stderr,"metadata: METADATA: error sending to",show_permid_short(permid),exc + pass + + def read_torrent(self, torrent_path): + try: + f = open(torrent_path, "rb") + torrent_data = f.read() + f.close() + torrent_size = len(torrent_data) + if DEBUG: + print >> sys.stderr,"metadata: read torrent", `torrent_path`, torrent_size + if torrent_size > Max_Torrent_Size: + return None + return torrent_data + except: + print_exc() + return None + + + def addTorrentToDB(self, filename, torrent_hash, metadata, source='BC', extra_info={}, hack=False): + """ Arno: no need to delegate to olbridge, this is already run by OverlayThread """ + # 03/02/10 Boudewijn: addExternalTorrent now requires a + # torrentdef, consequently we provide the filename through the + # extra_info dictionary + torrentdef = TorrentDef.load(filename) + if not 'filename' in extra_info: + extra_info['filename'] = filename + torrent = self.torrent_db.addExternalTorrent(torrentdef, source, extra_info) + if torrent is None: + return + + # Arno, 2008-10-20: XXX torrents are filtered out in the final display stage + self.launchmany.set_activity(NTFY_ACT_GOT_METADATA,unicode('"'+torrent['name']+'"'),torrent['category']) + + if self.initialized: + self.num_torrents += 1 # for free disk limitation + + if not extra_info: + self.refreshTrackerStatus(torrent) + + if len(self.recently_collected_torrents) < 50: # Queue of 50 + self.recently_collected_torrents.append(torrent_hash) + else: + self.recently_collected_torrents.pop(0) + self.recently_collected_torrents.append(torrent_hash) + + + def set_overflow(self, max_num_torrent): + self.max_num_torrents = self.init_max_num_torrents = max_num_torrent + + def delayed_check_overflow(self, delay=2): + if not self.initialized: + return + self.overlay_bridge.add_task(self.check_overflow, delay) + + def delayed_check_free_space(self, delay=2): + self.free_space = self.get_free_space() + + def check_overflow(self): # check if there are too many torrents relative to the free disk space + if self.num_torrents < 0: + self.num_torrents = self.torrent_db.getNumberCollectedTorrents() + #print >> sys.stderr, "**** torrent collectin self.num_torrents=", self.num_torrents + + if DEBUG: + print >>sys.stderr,"metadata: check overflow: current", self.num_torrents, "max", self.max_num_torrents + + if self.num_torrents > self.max_num_torrents: + num_delete = int(self.num_torrents - self.max_num_torrents*0.95) + print >> sys.stderr, "** limit space::", self.num_torrents, self.max_num_torrents, num_delete + self.limit_space(num_delete) + + def limit_space(self, num_delete): + deleted = self.torrent_db.freeSpace(num_delete) + if deleted: + self.num_torrents = self.torrent_db.getNumberCollectedTorrents() + self.free_space = self.get_free_space() + + + def save_torrent(self, infohash, metadata, source='BC', extra_info={}): + # check if disk is full before save it to disk and database + if not self.initialized: + return None + + self.check_overflow() + + if self.min_free_space != 0 and (self.free_space - len(metadata) < self.min_free_space or self.num_collected_torrents % 10 == 0): + self.free_space = self.get_free_space() + if self.free_space - len(metadata) < self.min_free_space: + self.warn_disk_full() + return None + + file_name = get_collected_torrent_filename(infohash) + if DEBUG: + print >> sys.stderr,"metadata: Storing torrent", sha(infohash).hexdigest(),"in",file_name + + save_path = self.write_torrent(metadata, self.torrent_dir, file_name) + if save_path: + self.num_collected_torrents += 1 + self.free_space -= len(metadata) + self.addTorrentToDB(save_path, infohash, metadata, source=source, extra_info=extra_info) + # check if space is enough and remove old torrents + + return file_name + + + def refreshTrackerStatus(self, torrent): + "Upon the reception of a new discovered torrent, directly check its tracker" + if DEBUG: + print >> sys.stderr, "metadata: checking tracker status of new torrent" + check = TorrentChecking(torrent['infohash']) + check.start() + + def write_torrent(self, metadata, dir, name): + try: + if not os.access(dir,os.F_OK): + os.mkdir(dir) + save_path = os.path.join(dir, name) + file = open(save_path, 'wb') + file.write(metadata) + file.close() + if DEBUG: + print >> sys.stderr,"metadata: write torrent", `save_path`, len(metadata), hash(metadata) + return save_path + except: + print_exc() + print >> sys.stderr, "metadata: write torrent failed" + return None + + def valid_metadata(self, infohash, metadata): + try: + metainfo = bdecode(metadata) + tdef = TorrentDef.load_from_dict(metainfo) + got_infohash = tdef.get_infohash() + if infohash != got_infohash: + print >> sys.stderr, "metadata: infohash doesn't match the torrent " + \ + "hash. Required: " + `infohash` + ", but got: " + `got_infohash` + return False + return True + except: + print_exc() + #print >> sys.stderr, "problem metadata:", repr(metadata) + return False + + def got_metadata(self, permid, message, selversion): + """ receive torrent file from others """ + + # Arno, 2007-06-20: Disabled the following code. What's this? Somebody sends + # us something and we refuse? Also doesn't take into account download help + #and remote-query extension. + + #if self.upload_rate <= 0: # if no upload, no download, that's the game + # return True # don't close connection + + try: + message = bdecode(message[1:]) + except: + print_exc() + return False + if not isinstance(message, dict): + return False + try: + infohash = message['torrent_hash'] + if not isValidInfohash(infohash): + # 19/02/10 Boudewijn: isValidInfohash either returns + # True or raises a ValueError. So this part of the + # code will never be reached... + return False + + assert isinstance(infohash, str), "INFOHASH has invalid type: %s" % type(infohash) + assert len(infohash) == INFOHASH_LENGTH, "INFOHASH has invalid length: %d" % len(infohash) + + #print >>sys.stderr,"metadata: got_metadata: hexinfohash: get_collected_torrent_filename(infohash) + + if not infohash in self.requested_torrents: # got a torrent which was not requested + return True + if self.torrent_db.hasMetaData(infohash): + return True + + # P2PURL + goturl = False + if selversion >= OLPROTO_VER_ELEVENTH: + if 'metatype' in message and message['metatype'] == URL_MIME_TYPE: + try: + tdef = TorrentDef.load_from_url(message['metadata']) + # Internal storage format is still .torrent file + metainfo = tdef.get_metainfo() + metadata = bencode(metainfo) + goturl = True + except: + print_exc() + return False + else: + metadata = message['metadata'] + else: + metadata = message['metadata'] + + if not self.valid_metadata(infohash, metadata): + return False + + if DEBUG: + torrent_size = len(metadata) + if goturl: + mdt = "URL" + else: + mdt = "torrent" + print >> sys.stderr,"metadata: Recvd",mdt,`infohash`,sha(infohash).hexdigest(), torrent_size + + extra_info = {} + if selversion >= OLPROTO_VER_FOURTH: + try: + extra_info = {'leecher': message.get('leecher', -1), + 'seeder': message.get('seeder', -1), + 'last_check_time': message.get('last_check_time', -1), + 'status':message.get('status', 'unknown')} + except Exception, msg: + print_exc() + print >> sys.stderr, "metadata: wrong extra info in msg - ", message + extra_info = {} + + filename = self.save_torrent(infohash, metadata, extra_info=extra_info) + self.requested_torrents.remove(infohash) + + #if DEBUG: + # print >>sys.stderr,"metadata: Was I asked to dlhelp someone",self.dlhelper + + if filename is not None: + self.notify_torrent_is_in(infohash,metadata,filename) + + + # BarterCast: add bytes of torrent to BarterCastDB + # Save exchanged KBs in BarterCastDB + if permid is not None and BARTERCAST_TORRENTS: + self.overlay_bridge.add_task(lambda:self.olthread_bartercast_torrentexchange(permid, 'downloaded'), 0) + + + except Exception, e: + print_exc() + print >> sys.stderr,"metadata: Received metadata is broken",e, message.keys() + return False + + return True + + def notify_torrent_is_in(self,infohash,metadata,filename): + assert isinstance(infohash, str), "INFOHASH has invalid type: %s" % type(infohash) + assert len(infohash) == INFOHASH_LENGTH, "INFOHASH has invalid length: %d" % len(infohash) + if self.dlhelper is not None: + self.dlhelper.metadatahandler_received_torrent(infohash, metadata) + if self.rquerytorrenthandler is not None: + self.rquerytorrenthandler.metadatahandler_got_torrent(infohash,metadata,filename) + + def get_num_torrents(self): + return self.num_torrents + + def warn_disk_full(self): + if DEBUG: + print >> sys.stderr,"metadata: send_meta_req: Disk full!" + drive,dir = os.path.splitdrive(os.path.abspath(self.torrent_dir)) + if not drive: + drive = dir + self.launchmany.set_activity(NTFY_ACT_DISK_FULL, drive) + + def get_free_space(self): + if not self.registered: + return 0 + try: + freespace = getfreespace(self.torrent_dir) + return freespace + except: + print >> sys.stderr, "meta: cannot get free space of", self.torrent_dir + print_exc() + return 0 + + def set_rate(self, rate): + self.upload_rate = rate * 1024 + + def set_min_free_space(self, min_free_space): + self.min_free_space = min_free_space*(2**20) + + def checking_upload_queue(self): + """ check the upload queue every 5 seconds, and send torrent out if the queue + is not empty and the max upload rate is not reached. + It is used for rate control + """ + + if DEBUG: + print >> sys.stderr, "metadata: checking_upload_queue, length:", len(self.upload_queue), "now:", ctime(time()), "next check:", ctime(self.next_upload_time) + if self.upload_rate > 0 and int(time()) >= self.next_upload_time and len(self.upload_queue) > 0: + task = self.upload_queue.pop(0) + permid = task['permid'] + infohash = task['infohash'] + torrent_path = task['torrent_path'] + selversion = task['selversion'] + sent_size = self.read_and_send_metadata(permid, infohash, torrent_path, selversion) + idel = sent_size / self.upload_rate + 1 + self.next_upload_time = int(time()) + idel + self.overlay_bridge.add_task(self.checking_upload_queue, idel) + + def getRecentlyCollectedTorrents(self, num, selversion): + """ + This method returns a list of collected torrents. It is called by the + method hat creates BC message. + @change: changed by Rahim. Since overlay version 10, the returned list should contain the swarm size info for the torrents. + @param num: Maximum length of result list. If num=0 it means that the returned list is unlimited. + @param selversion: Version of the overlay protocol that two communication nodes agreed on. + """ + if selversion >= OLPROTO_VER_ELEVENTH: ## Amended list with swarm size info is returned. + if not self.initialized: + return [] + else: + collectedList=self.recently_collected_torrents[-1*num:] # this is list of infohashes + if len(collectedList) >0: + swarmSizeList= self.popularity_db.calculateSwarmSize(collectedList, content='Infohash' , toBC=True) + for index in range(0,len(collectedList)): + collectedList[index]=[collectedList[index]] + collectedList[index].append(swarmSizeList[index][1]) # appends number of seeders + collectedList[index].append(swarmSizeList[index][2]) # appends number of leechers + collectedList[index].append(swarmSizeList[index][3]) # appends current time + collectedList[index].append(swarmSizeList[index][4]) # appends + return collectedList; + + else: + if not self.initialized: + return [] + return self.recently_collected_torrents[-1*num:] # get the last ones + + + + + diff --git a/instrumentation/next-share/BaseLib/Core/Overlay/OverlayApps.py b/instrumentation/next-share/BaseLib/Core/Overlay/OverlayApps.py new file mode 100644 index 0000000..af46bf7 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/Overlay/OverlayApps.py @@ -0,0 +1,367 @@ +# Written by Arno Bakker, George Milescu +# see LICENSE.txt for license information +# +# All applications on top of the SecureOverlay should be started here. +# +from MetadataHandler import MetadataHandler +from threading import Lock +from threading import currentThread +from time import time +from traceback import print_exc +import sys + +from BaseLib.Core.BitTornado.BT1.MessageID import * +from BaseLib.Core.BuddyCast.buddycast import BuddyCastFactory +from BaseLib.Core.ProxyService.CoordinatorMessageHandler import CoordinatorMessageHandler +from BaseLib.Core.ProxyService.HelperMessageHandler import HelperMessageHandler +from BaseLib.Core.NATFirewall.DialbackMsgHandler import DialbackMsgHandler +from BaseLib.Core.NATFirewall.NatCheckMsgHandler import NatCheckMsgHandler +from BaseLib.Core.SocialNetwork.FriendshipMsgHandler import FriendshipMsgHandler +from BaseLib.Core.SocialNetwork.RemoteQueryMsgHandler import RemoteQueryMsgHandler +from BaseLib.Core.SocialNetwork.RemoteTorrentHandler import RemoteTorrentHandler +from BaseLib.Core.SocialNetwork.SocialNetworkMsgHandler import SocialNetworkMsgHandler +from BaseLib.Core.Statistics.Crawler import Crawler +from BaseLib.Core.Statistics.DatabaseCrawler import DatabaseCrawler +from BaseLib.Core.Statistics.FriendshipCrawler import FriendshipCrawler +from BaseLib.Core.Statistics.SeedingStatsCrawler import SeedingStatsCrawler +from BaseLib.Core.Statistics.VideoPlaybackCrawler import VideoPlaybackCrawler +from BaseLib.Core.Statistics.RepexCrawler import RepexCrawler +from BaseLib.Core.Statistics.PunctureCrawler import PunctureCrawler +from BaseLib.Core.Statistics.ChannelCrawler import ChannelCrawler +from BaseLib.Core.Utilities.utilities import show_permid_short +from BaseLib.Core.simpledefs import * +from BaseLib.Core.Subtitles.SubtitlesHandler import SubtitlesHandler +from BaseLib.Core.Subtitles.SubtitlesSupport import SubtitlesSupport +from BaseLib.Core.Subtitles.PeerHaveManager import PeersHaveManager + +DEBUG = False + +class OverlayApps: + # Code to make this a singleton + __single = None + + def __init__(self): + if OverlayApps.__single: + raise RuntimeError, "OverlayApps is Singleton" + OverlayApps.__single = self + self.coord_handler = None + self.help_handler = None + self.metadata_handler = None + self.buddycast = None + self.collect = None + self.dialback_handler = None + self.socnet_handler = None + self.rquery_handler = None + self.chquery_handler = None + self.friendship_handler = None + self.msg_handlers = {} + self.connection_handlers = [] + self.text_mode = None + self.requestPolicyLock = Lock() + + def getInstance(*args, **kw): + if OverlayApps.__single is None: + OverlayApps(*args, **kw) + return OverlayApps.__single + getInstance = staticmethod(getInstance) + + def register(self, overlay_bridge, session, launchmany, config, requestPolicy): + self.overlay_bridge = overlay_bridge + self.launchmany = launchmany + self.requestPolicy = requestPolicy + self.text_mode = config.has_key('text_mode') + + # OverlayApps gets all messages, and demultiplexes + overlay_bridge.register_recv_callback(self.handleMessage) + overlay_bridge.register_conns_callback(self.handleConnection) + + # Arno, 2010-01-28: Start with crawler support, other mods depend on + # that, e.g. BuddyCast + i_am_crawler = False + if config['crawler']: + crawler = Crawler.get_instance(session) + self.register_msg_handler([CRAWLER_REQUEST], crawler.handle_request) + + database_crawler = DatabaseCrawler.get_instance() + crawler.register_message_handler(CRAWLER_DATABASE_QUERY, database_crawler.handle_crawler_request, database_crawler.handle_crawler_reply) + seeding_stats_crawler = SeedingStatsCrawler.get_instance() + crawler.register_message_handler(CRAWLER_SEEDINGSTATS_QUERY, seeding_stats_crawler.handle_crawler_request, seeding_stats_crawler.handle_crawler_reply) + friendship_crawler = FriendshipCrawler.get_instance(session) + crawler.register_message_handler(CRAWLER_FRIENDSHIP_STATS, friendship_crawler.handle_crawler_request, friendship_crawler.handle_crawler_reply) + natcheck_handler = NatCheckMsgHandler.getInstance() + natcheck_handler.register(launchmany) + crawler.register_message_handler(CRAWLER_NATCHECK, natcheck_handler.gotDoNatCheckMessage, natcheck_handler.gotNatCheckReplyMessage) + crawler.register_message_handler(CRAWLER_NATTRAVERSAL, natcheck_handler.gotUdpConnectRequest, natcheck_handler.gotUdpConnectReply) + videoplayback_crawler = VideoPlaybackCrawler.get_instance() + crawler.register_message_handler(CRAWLER_VIDEOPLAYBACK_EVENT_QUERY, videoplayback_crawler.handle_event_crawler_request, videoplayback_crawler.handle_event_crawler_reply) + crawler.register_message_handler(CRAWLER_VIDEOPLAYBACK_INFO_QUERY, videoplayback_crawler.handle_info_crawler_request, videoplayback_crawler.handle_info_crawler_reply) + repex_crawler = RepexCrawler.get_instance(session) + crawler.register_message_handler(CRAWLER_REPEX_QUERY, repex_crawler.handle_crawler_request, repex_crawler.handle_crawler_reply) + puncture_crawler = PunctureCrawler.get_instance() + crawler.register_message_handler(CRAWLER_PUNCTURE_QUERY, puncture_crawler.handle_crawler_request, puncture_crawler.handle_crawler_reply) + channel_crawler = ChannelCrawler.get_instance() + crawler.register_message_handler(CRAWLER_CHANNEL_QUERY, channel_crawler.handle_crawler_request, channel_crawler.handle_crawler_reply) + + if crawler.am_crawler(): + i_am_crawler = True + # we will only accept CRAWLER_REPLY messages when we are actully a crawler + self.register_msg_handler([CRAWLER_REPLY], crawler.handle_reply) + self.register_connection_handler(crawler.handle_connection) + + if "database" in sys.argv: + # allows access to tribler database (boudewijn) + crawler.register_crawl_initiator(database_crawler.query_initiator) + + if "videoplayback" in sys.argv: + # allows access to video-playback statistics (boudewijn) + crawler.register_crawl_initiator(videoplayback_crawler.query_initiator) + + if "seedingstats" in sys.argv: + # allows access to seeding statistics (Boxun) + crawler.register_crawl_initiator(seeding_stats_crawler.query_initiator, frequency=60*30) + + if "friendship" in sys.argv: + # allows access to friendship statistics (Ali) + crawler.register_crawl_initiator(friendship_crawler.query_initiator) + + if "natcheck" in sys.argv: + # allows access to nat-check statistics (Lucia) + crawler.register_crawl_initiator(natcheck_handler.doNatCheck, 3600) + + if "repex" in sys.argv: + # allows access to RePEX log statistics (Raynor Vliegendhart) + crawler.register_crawl_initiator(repex_crawler.query_initiator) + + if "puncture" in sys.argv: + # allows access to UDPPuncture log statistics (Gertjan) + crawler.register_crawl_initiator(puncture_crawler.query_initiator) + + if "channel" in sys.argv: + # allows access to tribler channels' database (nitin) + crawler.register_crawl_initiator(channel_crawler.query_initiator) + else: + self.register_msg_handler([CRAWLER_REQUEST, CRAWLER_REPLY], self.handleDisabledMessage) + + + # Create handler for metadata messages in two parts, as + # download help needs to know the metadata_handler and we need + # to know the download helper handler. + # Part 1: + self.metadata_handler = MetadataHandler.getInstance() + + if config['download_help']: + # Create handler for messages to dlhelp coordinator + self.coord_handler = CoordinatorMessageHandler(launchmany) + self.register_msg_handler(HelpHelperMessages, self.coord_handler.handleMessage) + + # Create handler for messages to dlhelp helper + self.help_handler = HelperMessageHandler() + self.help_handler.register(session,self.metadata_handler,config['download_help_dir'],config.get('coopdlconfig', False)) + self.register_msg_handler(HelpCoordinatorMessages, self.help_handler.handleMessage) + + # Part 2: + self.metadata_handler.register(overlay_bridge, self.help_handler, launchmany, config) + self.register_msg_handler(MetadataMessages, self.metadata_handler.handleMessage) + + + # 13-04-2010 Andrea: subtitles collecting + if not config['subtitles_collecting'] : + self.subtitles_handler = None + else: + self.subtitles_handler = SubtitlesHandler.getInstance() + self.subtitles_handler.register(self.overlay_bridge, self.launchmany.richmetadataDbHandler, self.launchmany.session) + + self.peersHaveManger = PeersHaveManager.getInstance() + if not self.peersHaveManger.isRegistered(): + self.peersHaveManger.register(self.launchmany.richmetadataDbHandler, self.overlay_bridge) + # I'm not sure if this is the best place to init this + self.subtitle_support = SubtitlesSupport.getInstance() + + keypair = self.launchmany.session.keypair + permid = self.launchmany.session.get_permid() + self.subtitle_support._register(self.launchmany.richmetadataDbHandler, + self.subtitles_handler, + self.launchmany.channelcast_db, permid, + keypair, self.peersHaveManger, + self.overlay_bridge) + + # cleanup the subtitles database at the first launch + self.subtitle_support.runDBConsinstencyRoutine() + + + + if not config['torrent_collecting']: + self.torrent_collecting_solution = 0 + else: + self.torrent_collecting_solution = config['buddycast_collecting_solution'] + + if config['buddycast']: + # Create handler for Buddycast messages + + self.buddycast = BuddyCastFactory.getInstance(superpeer=config['superpeer'], log=config['overlay_log']) + # Using buddycast to handle torrent collecting since they are dependent + self.buddycast.register(overlay_bridge, launchmany, + launchmany.rawserver_fatalerrorfunc, + self.metadata_handler, + self.torrent_collecting_solution, + config['start_recommender'],config['buddycast_max_peers'],i_am_crawler) + + self.register_msg_handler(BuddyCastMessages, self.buddycast.handleMessage) + self.register_connection_handler(self.buddycast.handleConnection) + + if config['dialback']: + self.dialback_handler = DialbackMsgHandler.getInstance() + # The Dialback mechanism needs the real rawserver, not the overlay_bridge + self.dialback_handler.register(overlay_bridge, launchmany, launchmany.rawserver, config) + self.register_msg_handler([DIALBACK_REQUEST], + self.dialback_handler.olthread_handleSecOverlayMessage) + self.register_connection_handler(self.dialback_handler.olthread_handleSecOverlayConnection) + else: + self.register_msg_handler([DIALBACK_REQUEST], self.handleDisabledMessage) + + if config['socnet']: + self.socnet_handler = SocialNetworkMsgHandler.getInstance() + self.socnet_handler.register(overlay_bridge, launchmany, config) + self.register_msg_handler(SocialNetworkMessages,self.socnet_handler.handleMessage) + self.register_connection_handler(self.socnet_handler.handleConnection) + + self.friendship_handler = FriendshipMsgHandler.getInstance() + self.friendship_handler.register(overlay_bridge, launchmany.session) + self.register_msg_handler(FriendshipMessages,self.friendship_handler.handleMessage) + self.register_connection_handler(self.friendship_handler.handleConnection) + + if config['rquery']: + self.rquery_handler = RemoteQueryMsgHandler.getInstance() + self.rquery_handler.register(overlay_bridge,launchmany,config,self.buddycast,log=config['overlay_log']) + self.register_msg_handler(RemoteQueryMessages,self.rquery_handler.handleMessage) + self.register_connection_handler(self.rquery_handler.handleConnection) + + if config['subtitles_collecting']: + hndl = self.subtitles_handler.getMessageHandler() + self.register_msg_handler(SubtitleMessages, hndl) + + self.rtorrent_handler = RemoteTorrentHandler.getInstance() + self.rtorrent_handler.register(overlay_bridge,self.metadata_handler,session) + self.metadata_handler.register2(self.rtorrent_handler) + + # Add notifier as connection handler + self.register_connection_handler(self.notifier_handles_connection) + + if config['buddycast']: + # Arno: to prevent concurrency between mainthread and overlay + # thread where BuddyCast schedules tasks + self.buddycast.register2() + + def early_shutdown(self): + """ Called as soon as Session shutdown is initiated. Used to start + shutdown tasks that takes some time and that can run in parallel + to checkpointing, etc. + """ + # Called by OverlayThread + if self.friendship_handler is not None: + self.friendship_handler.shutdown() + + + def register_msg_handler(self, ids, handler): + """ + ids is the [ID1, ID2, ..] where IDn is a sort of message ID in overlay + swarm. Each ID can only be handled by one handler, but a handler can + handle multiple IDs + """ + for id in ids: + if DEBUG: + print >> sys.stderr,"olapps: Message handler registered for",getMessageName(id) + self.msg_handlers[id] = handler + + def register_connection_handler(self, handler): + """ + Register a handler for if a connection is established + handler-function is called like: + handler(exc,permid,selversion,locally_initiated) + """ + assert handler not in self.connection_handlers, 'This connection_handler is already registered' + if DEBUG: + print >> sys.stderr, "olapps: Connection handler registered for", handler + self.connection_handlers.append(handler) + + def handleMessage(self,permid,selversion,message): + """ demultiplex message stream to handlers """ + + # Check auth + if not self.requestAllowed(permid, message[0]): + if DEBUG: + print >> sys.stderr, "olapps: Message not allowed", getMessageName(message[0]) + return False + + if message[0] in self.msg_handlers: + # This is a one byte id. (For instance a regular + # BitTorrent message) + id_ = message[0] + else: + if DEBUG: + print >> sys.stderr, "olapps: No handler found for", getMessageName(message[0:2]) + return False + + if DEBUG: + print >> sys.stderr, "olapps: handleMessage", getMessageName(id_), "v" + str(selversion) + + try: + if DEBUG: + st = time() + ret = self.msg_handlers[id_](permid, selversion, message) + et = time() + diff = et - st + if diff > 0: + print >> sys.stderr,"olapps: ",getMessageName(id_),"returned",ret,"TOOK %.5f" % diff + return ret + else: + return self.msg_handlers[id_](permid, selversion, message) + except: + # Catch all + print_exc() + return False + + def handleDisabledMessage(self, *args): + return True + + def handleConnection(self,exc,permid,selversion,locally_initiated): + """ An overlay-connection was established. Notify interested parties. """ + + if DEBUG: + print >> sys.stderr,"olapps: handleConnection",exc,selversion,locally_initiated,currentThread().getName() + + for handler in self.connection_handlers: + try: + #if DEBUG: + # print >> sys.stderr,"olapps: calling connection handler:",'%s.%s' % (handler.__module__, handler.__name__) + handler(exc,permid,selversion,locally_initiated) + except: + print >> sys.stderr, 'olapps: Exception during connection handler calling' + print_exc() + + def requestAllowed(self, permid, messageType): + self.requestPolicyLock.acquire() + try: + rp = self.requestPolicy + finally: + self.requestPolicyLock.release() + allowed = rp.allowed(permid, messageType) + if DEBUG: + if allowed: + word = 'allowed' + else: + word = 'denied' + print >> sys.stderr, 'olapps: Request type %s from %s was %s' % (getMessageName(messageType), show_permid_short(permid), word) + return allowed + + def setRequestPolicy(self, requestPolicy): + self.requestPolicyLock.acquire() + try: + self.requestPolicy = requestPolicy + finally: + self.requestPolicyLock.release() + + + def notifier_handles_connection(self, exc,permid,selversion,locally_initiated): + # Notify interested parties (that use the notifier/observer structure) about a connection + self.launchmany.session.uch.notify(NTFY_PEERS, NTFY_CONNECTION, permid, True) diff --git a/instrumentation/next-share/BaseLib/Core/Overlay/OverlayThreadingBridge.py b/instrumentation/next-share/BaseLib/Core/Overlay/OverlayThreadingBridge.py new file mode 100644 index 0000000..672b266 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/Overlay/OverlayThreadingBridge.py @@ -0,0 +1,228 @@ +# Written by Arno Bakker, George Milescu +# see LICENSE.txt for license information +# +# This class bridges between the OverlayApps class and the SecureOverlay +# and ensures that all upcalls made by the NetworkThread via the SecureOverlay +# are handed over to a different thread, the OverlayThread that propagates the +# upcall to the OverlayApps. +# + +import sys +from threading import currentThread +from traceback import print_exc + +from BaseLib.Core.Overlay.SecureOverlay import CloseException +from BaseLib.Core.BitTornado.BT1.MessageID import getMessageName +from BaseLib.Core.Utilities.utilities import show_permid_short +from BaseLib.Utilities.TimedTaskQueue import TimedTaskQueue +import threading + +DEBUG = False + +class OverlayThreadingBridge: + + __single = None + lock = threading.Lock() + + def __init__(self): + if OverlayThreadingBridge.__single: + raise RuntimeError, "OverlayThreadingBridge is Singleton" + OverlayThreadingBridge.__single = self + + self.secover = None + self.olapps = None + self.olappsmsghandler = None + self.olappsconnhandler = None + + # Current impl of wrapper: single thread + self.tqueue = TimedTaskQueue(nameprefix="Overlay") + + def getInstance(*args, **kw): + # Singleton pattern with double-checking + if OverlayThreadingBridge.__single is None: + OverlayThreadingBridge.lock.acquire() + try: + if OverlayThreadingBridge.__single is None: + OverlayThreadingBridge(*args, **kw) + finally: + OverlayThreadingBridge.lock.release() + return OverlayThreadingBridge.__single + getInstance = staticmethod(getInstance) + + def resetSingleton(self): + """ For testing purposes """ + OverlayThreadingBridge.__single = None + + def register_bridge(self,secover,olapps): + """ Called by MainThread """ + self.secover = secover + self.olapps = olapps + + secover.register_recv_callback(self.handleMessage) + secover.register_conns_callback(self.handleConnection) + + # + # SecOverlay interface + # + def register(self,launchmanycore,max_len): + """ Called by MainThread """ + self.secover.register(launchmanycore,max_len) + + # FOR TESTING ONLY + self.iplport2oc = self.secover.iplport2oc + + def get_handler(self): + return self.secover + + def start_listening(self): + """ Called by MainThread """ + self.secover.start_listening() + + def register_recv_callback(self,callback): + """ Called by MainThread """ + self.olappsmsghandler = callback + + def register_conns_callback(self,callback): + """ Called by MainThread """ + self.olappsconnhandler = callback + + def handleConnection(self,exc,permid,selversion,locally_initiated,hisdns): + """ Called by NetworkThread """ + # called by SecureOverlay.got_auth_connection() or cleanup_admin_and_callbacks() + if DEBUG: + print >>sys.stderr,"olbridge: handleConnection",exc,show_permid_short(permid),selversion,locally_initiated,hisdns,currentThread().getName() + + def olbridge_handle_conn_func(): + # Called by OverlayThread + + if DEBUG: + print >>sys.stderr,"olbridge: handle_conn_func",exc,show_permid_short(permid),selversion,locally_initiated,hisdns,currentThread().getName() + + try: + if hisdns: + self.secover.add_peer_to_db(permid,hisdns,selversion) + + if self.olappsconnhandler is not None: # self.olappsconnhandler = OverlayApps.handleConnection + self.olappsconnhandler(exc,permid,selversion,locally_initiated) + except: + print_exc() + + if isinstance(exc,CloseException): + self.secover.update_peer_status(permid,exc.was_auth_done()) + + self.tqueue.add_task(olbridge_handle_conn_func,0) + + def handleMessage(self,permid,selversion,message): + """ Called by NetworkThread """ + #ProxyService_ + # + # DEBUG + #print "### olbridge: handleMessage", show_permid_short(permid), selversion, getMessageName(message[0]), currentThread().getName() + # + #_ProxyService + + if DEBUG: + print >>sys.stderr,"olbridge: handleMessage",show_permid_short(permid),selversion,getMessageName(message[0]),currentThread().getName() + + def olbridge_handle_msg_func(): + # Called by OverlayThread + + if DEBUG: + print >>sys.stderr,"olbridge: handle_msg_func",show_permid_short(permid),selversion,getMessageName(message[0]),currentThread().getName() + + try: + if self.olappsmsghandler is None: + ret = True + else: + ret = self.olappsmsghandler(permid,selversion,message) + except: + print_exc() + ret = False + if ret == False: + if DEBUG: + print >>sys.stderr,"olbridge: olbridge_handle_msg_func closing!",show_permid_short(permid),selversion,getMessageName(message[0]),currentThread().getName() + self.close(permid) + + self.tqueue.add_task(olbridge_handle_msg_func,0) + return True + + + def connect_dns(self,dns,callback): + """ Called by OverlayThread/NetworkThread """ + + if DEBUG: + print >>sys.stderr,"olbridge: connect_dns",dns + + def olbridge_connect_dns_callback(cexc,cdns,cpermid,cselver): + # Called by network thread + + if DEBUG: + print >>sys.stderr,"olbridge: connect_dns_callback",cexc,cdns,show_permid_short(cpermid),cselver + + olbridge_connect_dns_callback_lambda = lambda:callback(cexc,cdns,cpermid,cselver) + self.add_task(olbridge_connect_dns_callback_lambda,0) + + self.secover.connect_dns(dns,olbridge_connect_dns_callback) + + + def connect(self,permid,callback): + """ Called by OverlayThread """ + + if DEBUG: + print >>sys.stderr,"olbridge: connect",show_permid_short(permid), currentThread().getName() + + def olbridge_connect_callback(cexc,cdns,cpermid,cselver): + # Called by network thread + + if DEBUG: + print >>sys.stderr,"olbridge: connect_callback",cexc,cdns,show_permid_short(cpermid),cselver, callback, currentThread().getName() + + + olbridge_connect_callback_lambda = lambda:callback(cexc,cdns,cpermid,cselver) + # Jie: postpone to call this callback to schedule it after the peer has been added to buddycast connection list + # Arno, 2008-09-15: No-no-no + self.add_task(olbridge_connect_callback_lambda,0) + + self.secover.connect(permid,olbridge_connect_callback) + + + def send(self,permid,msg,callback): + """ Called by OverlayThread """ + + if DEBUG: + print >>sys.stderr,"olbridge: send",show_permid_short(permid),len(msg) + + def olbridge_send_callback(cexc,cpermid): + # Called by network thread + + if DEBUG: + print >>sys.stderr,"olbridge: send_callback",cexc,show_permid_short(cpermid) + + + olbridge_send_callback_lambda = lambda:callback(cexc,cpermid) + self.add_task(olbridge_send_callback_lambda,0) + + self.secover.send(permid,msg,olbridge_send_callback) + + def close(self,permid): + """ Called by OverlayThread """ + self.secover.close(permid) + + def add_task(self,task,t=0,ident=None): + """ Called by OverlayThread """ + self.tqueue.add_task(task,t,ident) + +#=============================================================================== +# # Jie: according to Arno's suggestion, commit on demand instead of periodically +# def periodic_commit(self): +# period = 5*60 # commit every 5 min +# try: +# db = SQLiteCacheDB.getInstance() +# db.commit() +# except: +# period = period*2 +# self.add_task(self.periodic_commit, period) +# +#=============================================================================== + + diff --git a/instrumentation/next-share/BaseLib/Core/Overlay/SecureOverlay.py b/instrumentation/next-share/BaseLib/Core/Overlay/SecureOverlay.py new file mode 100644 index 0000000..f8014fa --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/Overlay/SecureOverlay.py @@ -0,0 +1,950 @@ +# Written by Arno Bakker, Bram Cohen, Jie Yang, George Milescu +# see LICENSE.txt for license information +# +# Please apply networking code fixes also to DialbackConnHandler.py + +from cStringIO import StringIO +from struct import pack,unpack +from threading import currentThread +from time import time +from traceback import print_exc,print_stack +import sys + +from BaseLib.Core.BitTornado.BT1.MessageID import protocol_name,option_pattern,getMessageName +from BaseLib.Core.BitTornado.BT1.convert import tobinary,toint +from BaseLib.Core.BitTornado.__init__ import createPeerID +from BaseLib.Core.CacheDB.sqlitecachedb import safe_dict,bin2str +from BaseLib.Core.Overlay.permid import ChallengeResponse +from BaseLib.Core.Utilities.utilities import show_permid_short,hostname_or_ip2ip +from BaseLib.Core.simpledefs import * + +DEBUG = False + +# +# Public definitions +# +overlay_infohash = '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + +# Overlay-protocol version numbers in use in the wild +OLPROTO_VER_FIRST = 1 # Internally used only. +OLPROTO_VER_SECOND = 2 # First public release, >= 3.3.4 +OLPROTO_VER_THIRD = 3 # Second public release, >= 3.6.0, Dialback, BuddyCast2 +OLPROTO_VER_FOURTH = 4 # Third public release, >= 3.7.0, BuddyCast3 +OLPROTO_VER_FIFTH = 5 # Fourth public release, >= 4.0.0, SOCIAL_OVERLAP +OLPROTO_VER_SIXTH = 6 # Fifth public release, >= 4.1.0, extra BC fields, remote query +OLPROTO_VER_SEVENTH = 7 # Sixth public release, >= 4.5.0, supports CRAWLER_REQUEST and CRAWLER_REPLY messages +OLPROTO_VER_EIGHTH = 8 # Seventh public release, >= 5.0, supporting BuddyCast with clicklog info. +OLPROTO_VER_NINETH = 9 # Eighth public release, >= 5.1, additional torrent_size in remote search query reply. +OLPROTO_VER_TENTH = 10 # Nineth public release, M18, simplified the VOD statistics (this code is not likely to be used in public, but still). +OLPROTO_VER_ELEVENTH = 11 # Tenth public release, trial M23, swarm size info part of BC message +OLPROTO_VER_TWELFTH = 12 # 11th public release M24, SIMPLE+METADATA query + ChannelCast BASE64. +OLPROTO_VER_THIRTEENTH = 13 # 12th public release >= 5.2, ChannelCast binary. +OLPROTO_VER_FOURTEENTH = 14 # 13th public release >= M30, ProxyService + Subtitle dissemination through ChannelCast + SUBS and GET_SUBS messages + +# Overlay-swarm protocol version numbers +OLPROTO_VER_CURRENT = OLPROTO_VER_FOURTEENTH + +OLPROTO_VER_LOWEST = OLPROTO_VER_SECOND +SupportedVersions = range(OLPROTO_VER_LOWEST, OLPROTO_VER_CURRENT+1) + +# +# Private definitions +# + +# States for overlay connection +STATE_INITIAL = 0 +STATE_HS_FULL_WAIT = 1 +STATE_HS_PEERID_WAIT = 2 +STATE_AUTH_WAIT = 3 +STATE_DATA_WAIT = 4 +STATE_CLOSED = 5 + +# Misc +EXPIRE_THRESHOLD = 300 # seconds:: keep consistent with sockethandler +EXPIRE_CHECK_INTERVAL = 60 # seconds +NO_REMOTE_LISTEN_PORT_KNOWN = -481 + + +class SecureOverlay: + __single = None + + def __init__(self): + if SecureOverlay.__single: + raise RuntimeError, "SecureOverlay is Singleton" + SecureOverlay.__single = self + self.olproto_ver_current = OLPROTO_VER_CURRENT + self.usermsghandler = None + self.userconnhandler = None + # ARNOCOMMENT: Remove this, DB should be fast enough. Don't want caches allover + self.dns = safe_dict() + + + # + # Interface for upper layer + # + def getInstance(*args, **kw): + if SecureOverlay.__single is None: + SecureOverlay(*args, **kw) + return SecureOverlay.__single + getInstance = staticmethod(getInstance) + + def register(self,launchmanycore, max_len): + self.lm = launchmanycore + self.rawserver = self.lm.rawserver + self.sock_hand = self.rawserver.sockethandler + self.multihandler = self.lm.multihandler + self.overlay_rawserver = self.multihandler.newRawServer(overlay_infohash, + self.rawserver.doneflag, + protocol_name) + self.max_len = max_len + self.iplport2oc = {} # (IP,listen port) -> OverlayConnection + self.peer_db = self.lm.peer_db + self.mykeypair = self.lm.session.keypair + self.permid = self.lm.session.get_permid() + self.myip = self.lm.get_ext_ip() + self.myport = self.lm.session.get_listen_port() + self.myid = create_my_peer_id(self.myport) + + # 25/01/10 boudewijn: because there is no 'keep alive' message + # the last_activity check is prone to get false positives. + # The higher-ups decided that this feature should be removed + # entirely. + # self.last_activity = time() + + def resetSingleton(self): + """ For testing purposes """ + SecureOverlay.__single = None + + def start_listening(self): + self.overlay_rawserver.start_listening(self) + # self.overlay_rawserver.add_task(self.secover_mon_netwact, 2) + + # 25/01/10 boudewijn: because there is no 'keep alive' message the + # last_activity check is prone to get false positives. The + # higher-ups decided that this feature should be removed entirely. + # def secover_mon_netwact(self): + # """ + # periodically notify the network status + # """ + # diff = time() - self.last_activity + # if diff > 120 + 1: + # # 120 is set as the check_period for buddycast until a + # # KEEP_ALIVE message is send + # msg = "no network" + # else: + # msg = "network active" + # self.lm.set_activity(NTFY_ACT_ACTIVE, msg, diff) + # self.overlay_rawserver.add_task(self.secover_mon_netwact, 2) + + def connect_dns(self,dns,callback): + """ Connects to the indicated endpoint and determines the permid + at that endpoint. Non-blocking. + + Pre: "dns" must be an IP address, not a hostname. + + Network thread calls "callback(exc,dns,permid,selver)" when the connection + is established or when an error occurs during connection + establishment. In the former case, exc is None, otherwise + it contains an Exception. + + The established connection will auto close after EXPIRE_THRESHOLD + seconds of inactivity. + """ + if DEBUG: + print >> sys.stderr,"secover: connect_dns",dns + # To prevent concurrency problems on sockets the calling thread + # delegates to the network thread. + task = Task(self._connect_dns,dns,callback) + self.rawserver.add_task(task.start, 0) + + + def connect(self,permid,callback): + """ Connects to the indicated permid. Non-blocking. + + Network thread calls "callback(exc,dns,permid,selver)" when the connection + is established or when an error occurs during connection + establishment. In the former case, exc is None, otherwise + it contains an Exception. + + The established connection will auto close after EXPIRE_THRESHOLD + seconds of inactivity. + """ + if DEBUG: + print >> sys.stderr,"secover: connect",show_permid_short(permid), currentThread().getName() + # To prevent concurrency problems on sockets the calling thread + # delegates to the network thread. + + dns = self.get_dns_from_peerdb(permid) + task = Task(self._connect,permid,dns,callback) + + if DEBUG: + print >> sys.stderr,"secover: connect",show_permid_short(permid),"currently at",dns + + self.rawserver.add_task(task.start, 0) + + + def send(self,permid,msg,callback): + """ Sends a message to the indicated permid. Non-blocking. + + Pre: connection to permid must have been established successfully. + + Network thread calls "callback(exc,permid)" when the message is sent + or when an error occurs during sending. In the former case, exc + is None, otherwise it contains an Exception. + """ + # To prevent concurrency problems on sockets the calling thread + # delegates to the network thread. + dns = self.get_dns_from_peerdb(permid) + task = Task(self._send,permid,dns,msg,callback) + self.rawserver.add_task(task.start, 0) + + + + def close(self,permid): + """ Closes any connection to indicated permid. Non-blocking. + + Pre: connection to permid must have been established successfully. + + Network thread calls "callback(exc,permid,selver)" when the connection + is closed. + """ + # To prevent concurrency problems on sockets the calling thread + # delegates to the network thread. + task = Task(self._close,permid) + self.rawserver.add_task(task.start, 0) + + + def register_recv_callback(self,callback): + """ Register a callback to be called when receiving a message from + any permid. Non-blocking. + + Network thread calls "callback(exc,permid,selver,msg)" when a message + is received. The callback is not called on errors e.g. remote + connection close. + + The callback must return True to keep the connection open. + """ + self.usermsghandler = callback + + def register_conns_callback(self,callback): + """ Register a callback to be called when receiving a connection from + any permid. Non-blocking. + + Network thread calls "callback(exc,permid,selver,locally_initiated)" + when a connection is established (locally initiated or remote), or + when a connection is closed locally or remotely. In the former case, + exc is None, otherwise it contains an Exception. + + Note that this means that if a callback is registered via this method, + both this callback and the callback passed to a connect() method + will be called. + """ + self.userconnhandler = callback + + + # + # Internal methods + # + def _connect_dns(self,dns,callback): + try: + if DEBUG: + print >> sys.stderr,"secover: actual connect_dns",dns + if dns[0] == self.myip and int(dns[1]) == self.myport: + callback(KeyError('IP and port of the target is the same as myself'),dns,None,0) + iplport = ip_and_port2str(dns[0],dns[1]) + oc = None + try: + oc = self.iplport2oc[iplport] + except KeyError: + pass + if oc is None: + oc = self.start_connection(dns) + self.iplport2oc[iplport] = oc + if not oc.is_auth_done(): + oc.queue_callback(dns,callback) + else: + callback(None,dns,oc.get_auth_permid(),oc.get_sel_proto_ver()) + except Exception,exc: + if DEBUG: + print_exc() + callback(exc,dns,None,0) + + def _connect(self,expectedpermid,dns,callback): + if DEBUG: + print >> sys.stderr,"secover: actual connect",show_permid_short(expectedpermid), currentThread().getName() + if expectedpermid == self.permid: + callback(KeyError('The target permid is the same as my permid'),None,expectedpermid,0) + try: + oc = self.get_oc_by_permid(expectedpermid) + if oc is None: + if dns is None: + callback(KeyError('IP address + port for permid unknown'),dns,expectedpermid,0) + else: + self._connect_dns(dns,lambda exc,dns2,peerpermid,selver:\ + self._whoishe_callback(exc,dns2,peerpermid,selver,expectedpermid,callback)) + else: + # We already have a connection to this permid + self._whoishe_callback(None,(oc.get_ip(),oc.get_auth_listen_port()),expectedpermid,oc.get_sel_proto_ver(),expectedpermid,callback) + except Exception,exc: + if DEBUG: + print_exc() + callback(exc,None,expectedpermid,0) + + def _whoishe_callback(self,exc,dns,peerpermid,selver,expectedpermid,callback): + """ Called by network thread after the permid on the other side is known + or an error occured + """ + try: + if exc is None: + # Connect went OK + if peerpermid == expectedpermid: + callback(None,dns,expectedpermid,selver) + else: + # Someone else answered the phone + callback(KeyError('Recorded IP address + port now of other permid'), + dns,expectedpermid,0) + else: + callback(exc,dns,expectedpermid,0) + except Exception,exc: + if DEBUG: + print_exc() + callback(exc,dns,expectedpermid,0) + + def _send(self,permid,dns,message,callback): + if DEBUG: + print >> sys.stderr,"secover: actual send",getMessageName(message[0]),\ + "to",show_permid_short(permid), currentThread().getName() + try: + if dns is None: + callback(KeyError('IP address + port for permid unknown'),permid) + else: + iplport = ip_and_port2str(dns[0],dns[1]) + oc = None + try: + oc = self.iplport2oc[iplport] + except KeyError: + pass + if oc is None: + callback(KeyError('Not connected to permid'),permid) + elif oc.is_auth_done(): + if oc.get_auth_permid() == permid: + oc.send_message(message) + callback(None,permid) + else: + callback(KeyError('Recorded IP address + port now of other permid'),permid) + else: + callback(KeyError('Connection not yet established'),permid) + except Exception,exc: + if DEBUG: + print_exc() + callback(exc,permid) + + + def _close(self,permid): + if DEBUG: + print >> sys.stderr,"secover: actual close",show_permid_short(permid) + try: + oc = self.get_oc_by_permid(permid) + if not oc: + if DEBUG: + print >> sys.stderr,"secover: error - actual close, but no connection to peer in admin" + else: + oc.close() + except Exception,e: + print_exc() + + # + # Interface for SocketHandler + # + def get_handler(self): + return self + + def external_connection_made(self,singsock): + """ incoming connection (never used) """ + if DEBUG: + print >> sys.stderr,"secover: external_connection_made",singsock.get_ip(),singsock.get_port() + # self.last_activity = time() + oc = OverlayConnection(self,singsock,self.rawserver) + singsock.set_handler(oc) + + def connection_flushed(self,singsock): + """ sockethandler flushes connection """ + if DEBUG: + print >> sys.stderr,"secover: connection_flushed",singsock.get_ip(),singsock.get_port() + + # + # Interface for ServerPortHandler + # + def externally_handshaked_connection_made(self, singsock, options, msg_remainder): + """ incoming connection, handshake partially read to identity + as an it as overlay connection (used always) + """ + if DEBUG: + print >> sys.stderr,"secover: externally_handshaked_connection_made",\ + singsock.get_ip(),singsock.get_port() + oc = OverlayConnection(self,singsock,self.rawserver,ext_handshake = True, options = options) + singsock.set_handler(oc) + if msg_remainder: + oc.data_came_in(singsock,msg_remainder) + return True + + + # + # Interface for OverlayConnection + # + def got_auth_connection(self,oc): + """ authentication of peer via identity protocol succesful """ + if DEBUG: + print >> sys.stderr,"secover: got_auth_connection", \ + show_permid_short(oc.get_auth_permid()),oc.get_ip(),oc.get_auth_listen_port(), currentThread().getName() + + if oc.is_locally_initiated() and oc.get_port() != oc.get_auth_listen_port(): + if DEBUG: + print >> sys.stderr,"secover: got_auth_connection: closing because auth", \ + "listen port not as expected",oc.get_port(),oc.get_auth_listen_port() + self.cleanup_admin_and_callbacks(oc,Exception('closing because auth listen port not as expected')) + return False + + # self.last_activity = time() + + ret = True + iplport = ip_and_port2str(oc.get_ip(),oc.get_auth_listen_port()) + known = iplport in self.iplport2oc + if not known: + self.iplport2oc[iplport] = oc + elif known and not oc.is_locally_initiated(): + # Locally initiated connections will already be registered, + # so if it's not a local connection and we already have one + # we have a duplicate, and we close the new one. + if DEBUG: + print >> sys.stderr,"secover: got_auth_connection:", \ + "closing because we already have a connection to",iplport + self.cleanup_admin_and_callbacks(oc, + Exception('closing because we already have a connection to peer')) + ret = False + + if ret: + if oc.is_auth_done(): + hisdns = (oc.get_ip(),oc.get_auth_listen_port()) + else: + hisdns = None + + #if DEBUG: + # print >>sys.stderr,"secover: userconnhandler is",self.userconnhandler + + if self.userconnhandler is not None: + try: + self.userconnhandler(None,oc.get_auth_permid(),oc.get_sel_proto_ver(),oc.is_locally_initiated(),hisdns) + except: + # Catch all + print_exc() + oc.dequeue_callbacks() + return ret + + def local_close(self,oc): + """ our side is closing the connection """ + if DEBUG: + print >> sys.stderr,"secover: local_close" + self.cleanup_admin_and_callbacks(oc,CloseException('local close',oc.is_auth_done())) + + def connection_lost(self,oc): + """ overlay connection telling us to clear admin """ + if DEBUG: + print >> sys.stderr,"secover: connection_lost" + self.cleanup_admin_and_callbacks(oc,CloseException('connection lost',oc.is_auth_done())) + + + def got_message(self,permid,message,selversion): + """ received message from authenticated peer, pass to upper layer """ + if DEBUG: + print >> sys.stderr,"secover: got_message",getMessageName(message[0]),\ + "v"+str(selversion) + # self.last_activity = time() + if self.usermsghandler is None: + if DEBUG: + print >> sys.stderr,"secover: User receive callback not set" + return + try: + + #if DEBUG: + # print >>sys.stderr,"secover: usermsghandler is",self.usermsghandler + + ret = self.usermsghandler(permid,selversion,message) + if ret is None: + if DEBUG: + print >> sys.stderr,"secover: INTERNAL ERROR:", \ + "User receive callback returned None, not True or False" + ret = False + elif DEBUG: + print >> sys.stderr,"secover: message handler returned",ret + return ret + except: + # Catch all + print_exc() + return False + + + def get_max_len(self): + return self.max_len + + def get_my_peer_id(self): + return self.myid + + def get_my_keypair(self): + return self.mykeypair + + def measurefunc(self,length): + pass + + # + # Interface for OverlayThreadingBridge + # + def get_dns_from_peerdb(self,permid,use_cache=True): + # Called by any thread, except NetworkThread + + if currentThread().getName().startswith("NetworkThread"): + print >>sys.stderr,"secover: get_dns_from_peerdb: called by NetworkThread!" + print_stack() + + dns = self.dns.get(permid, None) + + if not dns: + values = ('ip', 'port') + peer = self.peer_db.getOne(values, permid=bin2str(permid)) + if peer and peer[0] and peer[1]: + ip = hostname_or_ip2ip(peer[0]) + dns = (ip, int(peer[1])) + return dns + + def add_peer_to_db(self,permid,dns,selversion): + """ add a connected peer to database """ + # Called by OverlayThread + + if currentThread().getName().startswith("NetworkThread"): + print >>sys.stderr,"secover: add_peer_to_peerdb: called by NetworkThread!" + print_stack() + if DEBUG: + print >>sys.stderr,"secover: add_peer_to_peerdb: called by",currentThread().getName() + + self.dns[permid] = dns # cache it to avoid querying db later + now = int(time()) + peer_data = {'permid':permid, 'ip':dns[0], 'port':dns[1], 'oversion':selversion, 'last_seen':now, 'last_connected':now} + self.peer_db.addPeer(permid, peer_data, update_dns=True, update_connected=True, commit=True) + #self.peer_db.updateTimes(permid, 'connected_times', 1, commit=True) + + + def update_peer_status(self,permid,authwasdone): + """ update last_seen and last_connected in peer db when close """ + # Called by OverlayThread + + if currentThread().getName().startswith("NetworkThread"): + print >>sys.stderr,"secover: update_peer_status: called by NetworkThread!" + print_stack() + + now = int(time()) + if authwasdone: + self.peer_db.updatePeer(permid, last_seen=now, last_connected=now) + self.lm.session.uch.notify(NTFY_PEERS, NTFY_CONNECTION, permid, False) + # + # Interface for debugging + # + def debug_get_live_connections(self): + """ return a list of (permid,dns) tuples of the peers with which we + are connected. Like all methods here it must be called by the network thread + """ + live_conn = [] + for iplport in self.iplport2oc: + oc = self.iplport2oc[iplport] + if oc: + peer_permid = oc.get_auth_permid() + if peer_permid: + live_conn.append((peer_permid,(oc.get_ip(),oc.get_port()))) + return live_conn + + + # + # Internal methods + # + def start_connection(self,dns): + if DEBUG: + print >> sys.stderr,"secover: Attempt to connect to",dns + singsock = self.sock_hand.start_connection(dns) + oc = OverlayConnection(self,singsock,self.rawserver, + locally_initiated=True,specified_dns=dns) + singsock.set_handler(oc) + return oc + + def cleanup_admin_and_callbacks(self,oc,exc): + oc.cleanup_callbacks(exc) + self.cleanup_admin(oc) + if oc.is_auth_done() and self.userconnhandler is not None: + self.userconnhandler(exc,oc.get_auth_permid(),oc.get_sel_proto_ver(), + oc.is_locally_initiated(),None) + + def cleanup_admin(self,oc): + iplports = [] + d = 0 + for key in self.iplport2oc.keys(): + #print "***** iplport2oc:", key, self.iplport2oc[key] + if self.iplport2oc[key] == oc: + del self.iplport2oc[key] + #print "*****!!! del", key, oc + d += 1 + + def get_oc_by_permid(self, permid): + """ return the OverlayConnection instance given a permid """ + + for iplport in self.iplport2oc: + oc = self.iplport2oc[iplport] + if oc.get_auth_permid() == permid: + return oc + return None + + + +class Task: + def __init__(self,method,*args, **kwargs): + self.method = method + self.args = args + self.kwargs = kwargs + + def start(self): + if DEBUG: + print >> sys.stderr,"secover: task: start",self.method + #print_stack() + self.method(*self.args,**self.kwargs) + + +class CloseException(Exception): + def __init__(self,msg=None,authdone=False): + Exception.__init__(self,msg) + self.authdone= authdone + + def __str__(self): + return str(self.__class__)+': '+Exception.__str__(self) + + def was_auth_done(self): + return self.authdone + + +class OverlayConnection: + def __init__(self,handler,singsock,rawserver,locally_initiated = False, + specified_dns = None, ext_handshake = False,options = None): + self.handler = handler + self.singsock = singsock # for writing + self.rawserver = rawserver + self.buffer = StringIO() + self.cb_queue = [] + self.auth_permid = None + self.unauth_peer_id = None + self.auth_peer_id = None + self.auth_listen_port = None + self.low_proto_ver = 0 + self.cur_proto_ver = 0 + self.sel_proto_ver = 0 + self.options = None + self.locally_initiated = locally_initiated + self.specified_dns = specified_dns + self.last_use = time() + + self.state = STATE_INITIAL + self.write(chr(len(protocol_name)) + protocol_name + + option_pattern + overlay_infohash + self.handler.get_my_peer_id()) + if ext_handshake: + self.state = STATE_HS_PEERID_WAIT + self.next_len = 20 + self.next_func = self.read_peer_id + self.set_options(options) + else: + self.state = STATE_HS_FULL_WAIT + self.next_len = 1 + self.next_func = self.read_header_len + + # Leave autoclose here instead of SecureOverlay, as that doesn't record + # remotely-initiated OverlayConnections before authentication is done. + self.rawserver.add_task(self._olconn_auto_close, EXPIRE_CHECK_INTERVAL) + + # + # Interface for SocketHandler + # + def data_came_in(self, singsock, data): + """ sockethandler received data """ + # now we got something we can ask for the peer's real port + dummy_port = singsock.get_port(True) + + if DEBUG: + print >> sys.stderr,"olconn: data_came_in",singsock.get_ip(),singsock.get_port() + self.handler.measurefunc(len(data)) + self.last_use = time() + while 1: + if self.state == STATE_CLOSED: + return + i = self.next_len - self.buffer.tell() + if i > len(data): + self.buffer.write(data) + return + self.buffer.write(data[:i]) + data = data[i:] + m = self.buffer.getvalue() + self.buffer.reset() + self.buffer.truncate() + try: + if DEBUG: + print >> sys.stderr,"olconn: Trying to read",self.next_len #,"using",self.next_func + x = self.next_func(m) + except: + self.next_len, self.next_func = 1, self.read_dead + if DEBUG: + print_exc() + raise + if x is None: + if DEBUG: + print >> sys.stderr,"olconn: next_func returned None",self.next_func + self.close() + return + self.next_len, self.next_func = x + + def connection_lost(self,singsock): + """ kernel or socket handler reports connection lost """ + if DEBUG: + print >> sys.stderr,"olconn: connection_lost",singsock.get_ip(),singsock.get_port(),self.state + if self.state != STATE_CLOSED: + self.state = STATE_CLOSED + self.handler.connection_lost(self) + + def connection_flushed(self,singsock): + """ sockethandler flushes connection """ + pass + + # + # Interface for SecureOverlay + # + def send_message(self,message): + self.last_use = time() + s = tobinary(len(message))+message + self.write(s) + + def is_locally_initiated(self): + return self.locally_initiated + + def get_ip(self): + return self.singsock.get_ip() + + def get_port(self): + return self.singsock.get_port() + + def is_auth_done(self): + return self.auth_permid is not None + + def get_auth_permid(self): + return self.auth_permid + + def get_auth_listen_port(self): + return self.auth_listen_port + + def get_remote_listen_port(self): + if self.is_auth_done(): + return self.auth_listen_port + elif self.is_locally_initiated(): + return self.specified_dns[1] + else: + return NO_REMOTE_LISTEN_PORT_KNOWN + + def get_low_proto_ver(self): + return self.low_proto_ver + + def get_cur_proto_ver(self): + return self.cur_proto_ver + + def get_sel_proto_ver(self): + return self.sel_proto_ver + + def queue_callback(self,dns,callback): + if callback is not None: + self.cb_queue.append(callback) + + def dequeue_callbacks(self): + try: + permid = self.get_auth_permid() + for callback in self.cb_queue: + callback(None,self.specified_dns,permid,self.get_sel_proto_ver()) + self.cb_queue = [] + except Exception,e: + print_exc() + + + def cleanup_callbacks(self,exc): + if DEBUG: + print >> sys.stderr,"olconn: cleanup_callbacks: #callbacks is",len(self.cb_queue) + try: + for callback in self.cb_queue: + ## Failure connecting + if DEBUG: + print >> sys.stderr,"olconn: cleanup_callbacks: callback is",callback + callback(exc,self.specified_dns,self.get_auth_permid(),0) + except Exception,e: + print_exc() + + # + # Interface for ChallengeResponse + # + def get_unauth_peer_id(self): + return self.unauth_peer_id + + def got_auth_connection(self,singsock,permid,peer_id): + """ authentication of peer via identity protocol succesful """ + self.auth_permid = str(permid) + self.auth_peer_id = peer_id + self.auth_listen_port = decode_auth_listen_port(peer_id) + + self.state = STATE_DATA_WAIT + + if not self.handler.got_auth_connection(self): + self.close() + return + + # + # Internal methods + # + def read_header_len(self, s): + if ord(s) != len(protocol_name): + return None + return len(protocol_name), self.read_header + + def read_header(self, s): + if s != protocol_name: + return None + return 8, self.read_reserved + + def read_reserved(self, s): + if DEBUG: + print >> sys.stderr,"olconn: Reserved bits:", `s` + self.set_options(s) + return 20, self.read_download_id + + def read_download_id(self, s): + if s != overlay_infohash: + return None + return 20, self.read_peer_id + + def read_peer_id(self, s): + self.unauth_peer_id = s + + [self.low_proto_ver,self.cur_proto_ver] = get_proto_version_from_peer_id(self.unauth_peer_id) + self.sel_proto_ver = select_supported_protoversion(self.low_proto_ver,self.cur_proto_ver) + if not self.sel_proto_ver: + if DEBUG: + print >> sys.stderr,"olconn: We don't support peer's version of the protocol" + return None + elif DEBUG: + print >> sys.stderr,"olconn: Selected protocol version",self.sel_proto_ver + + if self.cur_proto_ver <= 2: + # Arno, 2010-02-04: Kick TorrentSwapper clones, still around + print >>sys.stderr,"olconn: Kicking ancient peer",`self.unauth_peer_id`,self.get_ip() + return None + + self.state = STATE_AUTH_WAIT + self.cr = ChallengeResponse(self.handler.get_my_keypair(),self.handler.get_my_peer_id(),self) + if self.locally_initiated: + self.cr.start_cr(self) + return 4, self.read_len + + + def read_len(self, s): + l = toint(s) + if l > self.handler.get_max_len(): + return None + return l, self.read_message + + def read_message(self, s): + if s != '': + if self.state == STATE_AUTH_WAIT: + if not self.cr.got_message(self,s): + return None + elif self.state == STATE_DATA_WAIT: + if not self.handler.got_message(self.auth_permid,s,self.sel_proto_ver): + return None + else: + if DEBUG: + print >> sys.stderr,"olconn: Received message while in illegal state, internal error!" + return None + return 4, self.read_len + + def read_dead(self, s): + return None + + def write(self,s): + self.singsock.write(s) + + def set_options(self,options): + self.options = options + + def close(self): + if DEBUG: + print >> sys.stderr,"olconn: we close()",self.get_ip(),self.get_port() + #print_stack() + self.state_when_error = self.state + if self.state != STATE_CLOSED: + self.state = STATE_CLOSED + self.handler.local_close(self) + self.singsock.close() + return + + def _olconn_auto_close(self): + if (time() - self.last_use) > EXPIRE_THRESHOLD: + self.close() + else: + self.rawserver.add_task(self._olconn_auto_close, EXPIRE_CHECK_INTERVAL) + + +# +# Internal functions +# +def create_my_peer_id(my_listen_port): + myid = createPeerID() + myid = myid[:16] + pack(' OLPROTO_VER_CURRENT: # the other's version is too high + return False + if cur_ver < OLPROTO_VER_LOWEST: # the other's version is too low + return False + if cur_ver < OLPROTO_VER_CURRENT and \ + cur_ver not in SupportedVersions: # the other's version is not supported + return False + return True + +def select_supported_protoversion(his_low_ver,his_cur_ver): + selected = None + if his_cur_ver != OLPROTO_VER_CURRENT: + if his_low_ver > OLPROTO_VER_CURRENT: # the other's low version is too high + return selected + if his_cur_ver < OLPROTO_VER_LOWEST: # the other's current version is too low + return selected + if his_cur_ver < OLPROTO_VER_CURRENT and \ + his_cur_ver not in SupportedVersions: # the other's current version is not supported (peer of this version is abondoned) + return selected + + selected = min(his_cur_ver,OLPROTO_VER_CURRENT) + return selected + +def decode_auth_listen_port(peerid): + bin = peerid[14:16] + tup = unpack('> sys.stderr,"permid: Exception in verify_torrent_signature:",str(e) + return False + + +# Exported classes +class PermIDException(Exception): pass + +class ChallengeResponse: + """ Exchange Challenge/Response via Overlay Swarm """ + + def __init__(self, my_keypair, my_id, secure_overlay): + self.my_keypair = my_keypair + self.permid = str(my_keypair.pub().get_der()) + self.my_id = my_id + self.secure_overlay = secure_overlay + + self.my_random = None + self.peer_id = None + self.peer_random = None + self.peer_pub = None + self.state = STATE_INITIAL + # Calculate message limits: + [dummy_random,cdata] = generate_challenge() + [dummy_random1,rdata1] = generate_response1(dummy_random,my_id,self.my_keypair) + rdata2 = generate_response2(dummy_random,my_id,dummy_random,self.my_keypair) + self.minchal = 1+len(cdata) # 1+ = message type + self.minr1 = 1+len(rdata1) - 1 # Arno: hack, also here, just to be on the safe side + self.minr2 = 1+len(rdata2) - 1 # Arno: hack, sometimes the official minimum is too big + + def starting_party(self,locally_initiated): + if self.state == STATE_INITIAL and locally_initiated: + self.state = STATE_AWAIT_R1 + return True + else: + return False + + def create_challenge(self): + [self.my_random,cdata] = generate_challenge() + return cdata + + def got_challenge_event(self,cdata,peer_id): + if self.state != STATE_INITIAL: + self.state = STATE_FAILED + if DEBUG: + print >> sys.stderr, "Got unexpected CHALLENGE message" + raise PermIDException + self.peer_random = check_challenge(cdata) + if self.peer_random is None: + self.state = STATE_FAILED + if DEBUG: + print >> sys.stderr,"Got bad CHALLENGE message" + raise PermIDException + self.peer_id = peer_id + [self.my_random,rdata1] = generate_response1(self.peer_random,peer_id,self.my_keypair) + self.state = STATE_AWAIT_R2 + return rdata1 + + def got_response1_event(self,rdata1,peer_id): + if self.state != STATE_AWAIT_R1: + self.state = STATE_FAILED + if DEBUG: + print >> sys.stderr,"Got unexpected RESPONSE1 message" + raise PermIDException + [randomA,peer_pub] = check_response1(rdata1,self.my_random,self.my_id) + + if randomA is None or peer_pub is None: + self.state = STATE_FAILED + if DEBUG: + print >> sys.stderr,"Got bad RESPONSE1 message" + raise PermIDException + + # avoid being connected by myself + peer_permid = str(peer_pub.get_der()) + if self.permid == peer_permid: + self.state = STATE_FAILED + if DEBUG: + print >> sys.stderr,"Got the same Permid as myself" + raise PermIDException + + self.peer_id = peer_id + self.peer_random = randomA + self.peer_pub = peer_pub + self.set_peer_authenticated() + rdata2 = generate_response2(self.peer_random,self.peer_id,self.my_random,self.my_keypair) + return rdata2 + + def got_response2_event(self,rdata2): + if self.state != STATE_AWAIT_R2: + self.state = STATE_FAILED + if DEBUG: + print >> sys.stderr,"Got unexpected RESPONSE2 message" + raise PermIDException + self.peer_pub = check_response2(rdata2,self.my_random,self.my_id,self.peer_random,self.peer_id) + if self.peer_pub is None: + self.state = STATE_FAILED + if DEBUG: + print >> sys.stderr,"Got bad RESPONSE2 message, authentication failed." + raise PermIDException + else: + # avoid being connected by myself + peer_permid = str(self.peer_pub.get_der()) + if self.permid == peer_permid: + self.state = STATE_FAILED + if DEBUG: + print >> sys.stderr,"Got the same Permid as myself" + raise PermIDException + else: + self.set_peer_authenticated() + + def set_peer_authenticated(self): + if DEBUG: + print >> sys.stderr,"permid: Challenge response succesful!" + self.state = STATE_AUTHENTICATED + + def get_peer_authenticated(self): + return self.state == STATE_AUTHENTICATED + + def get_peer_permid(self): + if self.state != STATE_AUTHENTICATED: + raise PermIDException + return self.peer_pub.get_der() + + def get_auth_peer_id(self): + if self.state != STATE_AUTHENTICATED: + raise PermIDException + return self.peer_id + + def get_challenge_minlen(self): + return self.minchal + + def get_response1_minlen(self): + return self.minr1 + + def get_response2_minlen(self): + return self.minr2 + +#--------------------------------------- + + def start_cr(self, conn): + if not self.get_peer_authenticated() and self.starting_party(conn.is_locally_initiated()): + self.send_challenge(conn) + + def send_challenge(self, conn): + cdata = self.create_challenge() + conn.send_message(CHALLENGE + str(cdata) ) + + def got_challenge(self, cdata, conn): + rdata1 = self.got_challenge_event(cdata, conn.get_unauth_peer_id()) + conn.send_message(RESPONSE1 + rdata1) + + def got_response1(self, rdata1, conn): + rdata2 = self.got_response1_event(rdata1, conn.get_unauth_peer_id()) + conn.send_message(RESPONSE2 + rdata2) + # get_peer_permid() throws exception if auth has failed + self.secure_overlay.got_auth_connection(conn,self.get_peer_permid(),self.get_auth_peer_id()) + + def got_response2(self, rdata2, conn): + self.got_response2_event(rdata2) + if self.get_peer_authenticated(): + #conn.send_message('') # Send KeepAlive message as reply + self.secure_overlay.got_auth_connection(conn,self.get_peer_permid(),self.get_auth_peer_id()) + + + def got_message(self, conn, message): + """ Handle message for PermID exchange and return if the message is valid """ + + if not conn: + return False + t = message[0] + if message[1:]: + msg = message[1:] + + if t == CHALLENGE: + if len(message) < self.get_challenge_minlen(): + if DEBUG: + print >> sys.stderr,"permid: Close on bad CHALLENGE: msg len",len(message) + self.state = STATE_FAILED + return False + try: + self.got_challenge(msg, conn) + except Exception,e: + if DEBUG: + print >> sys.stderr,"permid: Close on bad CHALLENGE: exception",str(e) + traceback.print_exc() + return False + elif t == RESPONSE1: + if len(message) < self.get_response1_minlen(): + if DEBUG: + print >> sys.stderr,"permid: Close on bad RESPONSE1: msg len",len(message) + self.state = STATE_FAILED + return False + try: + self.got_response1(msg, conn) + except Exception,e: + if DEBUG: + print >> sys.stderr,"permid: Close on bad RESPONSE1: exception",str(e) + traceback.print_exc() + return False + elif t == RESPONSE2: + if len(message) < self.get_response2_minlen(): + if DEBUG: + print >> sys.stderr,"permid: Close on bad RESPONSE2: msg len",len(message) + self.state = STATE_FAILED + return False + try: + self.got_response2(msg, conn) + except Exception,e: + if DEBUG: + print >> sys.stderr,"permid: Close on bad RESPONSE2: exception",str(e) + traceback.print_exc() + return False + else: + return False + return True + +if __name__ == '__main__': + init() +# ChallengeResponse(None, None) diff --git a/instrumentation/next-share/BaseLib/Core/ProxyService/Coordinator.py b/instrumentation/next-share/BaseLib/Core/ProxyService/Coordinator.py new file mode 100644 index 0000000..4d0cfa7 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/ProxyService/Coordinator.py @@ -0,0 +1,802 @@ +# Written by Pawel Garbacki, Arno Bakker, George Milescu +# see LICENSE.txt for license information +# +# TODO: when ASK_FOR_HELP cannot be sent, mark this in the interface + +from traceback import print_exc +import copy +import sys +from threading import Lock + +from BaseLib.Core.Overlay.OverlayThreadingBridge import OverlayThreadingBridge +from BaseLib.Core.Utilities.utilities import show_permid_short +from BaseLib.Core.CacheDB.CacheDBHandler import PeerDBHandler, TorrentDBHandler +#from BaseLib.Core.Session import Session +from BaseLib.Core.Overlay.SecureOverlay import OverlayConnection +from BaseLib.Core.BitTornado.bencode import bencode +from BaseLib.Core.BitTornado.bitfield import Bitfield +from BaseLib.Core.BitTornado.BT1.MessageID import ASK_FOR_HELP, STOP_HELPING, REQUEST_PIECES, CANCEL_PIECE, JOIN_HELPERS, RESIGN_AS_HELPER, DROPPED_PIECE +from BaseLib.Core.ProxyService.ProxyServiceUtil import * +from mailcap import show + +# Print debug messages +DEBUG = False +# ??? +MAX_ROUNDS = 137 + + +class Coordinator: + + def __init__(self, infohash, num_pieces): + # Number of pieces in the torrent + self.num_pieces = num_pieces + + # Vector for reserved-state infromation per piece + self.reserved_pieces = [False] * num_pieces + # Torrent infohash + self.infohash = infohash # readonly so no locking on this + + # List of sent challenges + self.sent_challenges_by_challenge = {} + self.sent_challenges_by_permid = {} + + # List of asked helpers + self.asked_helpers_lock = Lock() + self.asked_helpers = [] # protected by asked_helpers_lock + + # List of confirmed helpers + self.confirmed_helpers_lock = Lock() + self.confirmed_helpers = [] # protected by confirmed_helpers_lock + + # Dictionary for keeping evidence of helpers and the pieces requested to them + # Key: permid of a helper + # Value: list of pieces requested to the helper + self.requested_pieces = {} + + # optimization + # List of reserved pieces ??? + self.reserved = [] + + # Tribler overlay warm + self.overlay_bridge = OverlayThreadingBridge.getInstance() + + # BT1Download object + self.downloader = None + + + # + # Send messages + # + + # + # Interface for Core API. + # + def send_ask_for_help(self, peerList, force = False): + """ Asks for help to all the peers in peerList that have not been asked before + + Called by ask_coopdl_helpers in SingleDownload + + @param peerList: A list of peer objects for the peers that will be contacted for helping, containing ['permid','ip','port'] + @param force: If True, all the peers in peerList will be contacted for help, regardless of previous help requests being sent to them + """ + if DEBUG: + for peer in peerList: + print >> sys.stderr, "coordinator: i was requested to send help request to", show_permid_short(peer['permid']) + + try: + # List of helpers to be contacted for help + newly_asked_helpers = [] + if force: + # Contact all peers for help, regardless of previous help requests being sent to them + newly_asked_helpers = peerList + else: + # TODO: optimize the search below + # TODO: if a candidate is in the asked_helpers list, remember the last time it was asked for help + # and wait for a timeout before asking it again + # Check which of the candidate helpers is already a helper + self.confirmed_helpers_lock.acquire() + try: + for candidate in peerList: + flag = 0 + for confirmed_helper in self.confirmed_helpers: + if self.samePeer(candidate,confirmed_helper): + # the candidate is already a helper + flag = 1 + break + + if flag == 0: + # candidate has never been asked for help + newly_asked_helpers.append(candidate) + # Extend the list of asked helpers + # The list is extended and not appended because the candidate might already be in + # this list from previous attempts to contact it for helping + self.asked_helpers.append(candidate) + finally: + self.confirmed_helpers_lock.release() + + # List of permid's for the peers to be asked for help + permidlist = [] + for peer in newly_asked_helpers: + # ??? + peer['round'] = 0 + permidlist.append(peer['permid']) + + # Generate a random challenge - random number on 8 bytes (62**8 possible combinations) + challenge = generate_proxy_challenge() + + # Save permid - challenge pair + self.sent_challenges_by_challenge[challenge] = peer['permid'] + self.sent_challenges_by_permid[peer['permid']] = challenge + + # Send the help request + olthread_send_request_help_lambda = lambda:self.olthread_send_ask_for_help(permidlist) + self.overlay_bridge.add_task(olthread_send_request_help_lambda,0) + except Exception,e: + print_exc() + print >> sys.stderr, "coordinator: Exception while requesting help",e + + + def olthread_send_ask_for_help(self,permidlist): + """ Creates a bridge connection for the help request to be sent + + Called by the overlay thread. + + @param permidlist: A list of permids for the peers that will be contacted for helping + """ + for permid in permidlist: + if DEBUG: + print >> sys.stderr, "coordinator: olthread_send_ask_for_help connecting to",show_permid_short(permid) + + # Connect to the peer designated by permid + self.overlay_bridge.connect(permid,self.olthread_ask_for_help_connect_callback) + + + def olthread_ask_for_help_connect_callback(self,exc,dns,permid,selversion): + """ Sends the help request message on the connection with the peer + + Called by the overlay thread. + + @param exc: Peer reachable/unreachable information. None = peer reachable + @param dns: + @param permid: the permid of the peer that is contacted for helping + @param selversion + """ + if exc is None: + # Peer is reachable + if DEBUG: + print >> sys.stderr, "coordinator: olthread_ask_for_help_connect_callback sending help request to",show_permid_short(permid) + + # get the peer challenge + challenge = self.sent_challenges_by_permid[permid] + + # Create message according to protocol version + message = ASK_FOR_HELP + self.infohash + bencode(challenge) + + # Connect using Tribler Ovrlay Swarm + self.overlay_bridge.send(permid, message, self.olthread_ask_for_help_send_callback) + else: + # Peer is unreachable + if DEBUG: + print >> sys.stderr, "coordinator: olthread_ask_for_help_connect_callback: error connecting to",show_permid_short(permid),exc + # Remove peer from the list of asked peers + self.remove_unreachable_helper(permid) + + + def olthread_ask_for_help_send_callback(self,exc,permid): + """ Callback function for error checking in network communication + + Called by the overlay thread. + + @param exc: Peer reachable/unreachable information. None = peer reachable + @param permid: the permid of the peer that is contacted for helping + """ + if exc is not None: + # Peer is unreachable + if DEBUG: + print >> sys.stderr, "coordinator: olthread_ask_for_help_send_callback: error sending to",show_permid_short(permid),exc + # Remove peer from the list of asked peers + self.remove_unreachable_helper(permid) + + + def remove_unreachable_helper(self,permid): + """ Remove a peer from the list of asked helpers + + Called by the overlay thread. + + @param permid: the permid of the peer to be removed from the list + """ + self.asked_helpers_lock.acquire() + try: + # Search the peers with permid != from the given permid + new_asked_helpers = [] + for peer in self.asked_helpers: + if peer['permid'] != permid: + new_asked_helpers.append(peer) + self.asked_helpers = new_asked_helpers + except Exception,e: + print_exc() + print >> sys.stderr, "coordinator: Exception in remove_unreachable_helper",e + finally: + self.asked_helpers_lock.release() + + + + + + def send_stop_helping(self,peerList, force = False): + """ Asks for all the peers in peerList to stop helping + + Called by stop_coopdl_helpers in SingleDownload + + @param peerList: A list of peer objects (containing ['permid','ip','port']) for the peers that will be asked to stop helping + @param force: If True, all the peers in peerList will be asked to stop helping for help, regardless of previous help requests being sent to them + """ + if DEBUG: + for peer in peerList: + print >> sys.stderr, "coordinator: i was requested to send a stop helping request to", show_permid_short(peer) + + + # TODO: optimize the search below + try: + if force: + # Tell all peers in the peerList to stop helping, regardless of previous help requests being sent to them + to_stop_helpers = peerList + else: + # Who in the peerList is actually a helper currently? + # List of peers that will be asked to stop helping + to_stop_helpers = [] + + + # Searchv and update the confirmed_helpers list + self.confirmed_helpers_lock.acquire() + try: + for candidate in peerList: + # For each candidate + # Search the candidate in the confirmed_helpers list + for confirmed_helper in self.confirmed_helpers: + if self.samePeer(candidate, confirmed_helper): + # candidate was asked for help + to_stop_helpers.append(candidate) + break + + # Who of the confirmed helpers gets to stay? + to_keep_helpers = [] + for confirmed_helper in self.confirmed_helpers: + flag = 0 + for candidate in to_stop_helpers: + if self.samePeer(candidate,confirmed_helper): + # candidate was asked for help + flag = 1 + break + if flag == 0: + # candidate was not asked for help + to_keep_helpers.append(confirmed_helper) + + # Update confirmed_helpers + self.confirmed_helpers = to_keep_helpers + finally: + self.confirmed_helpers_lock.release() + + + # Search and update the asked_helpers list + self.asked_helpers_lock.acquire() + try: + for candidate in peerList: + # Search the candidate in the asked_helpers list + # TODO: if the same helper is both in confirmed_helpers and asked_helepers + # than it will be added twice to the to_stop_helpers list + for asked_helper in self.asked_helpers: + if self.samePeer(candidate, asked_helper): + # candidate was asked for help + to_stop_helpers.append(candidate) + break + # Who of the confirmed helpers gets to stay? + to_keep_helpers = [] + for asked_helper in self.asked_helpers: + flag = 0 + for candidate in to_stop_helpers: + if self.samePeer(candidate,asked_helper): + # candidate was asked for help + flag = 1 + break + if flag == 0: + # candidate was not asked for help + to_keep_helpers.append(asked_helper) + + # Update confirmed_helpers + self.asked_helpers = to_keep_helpers + finally: + self.asked_helpers_lock.release() + + # List of permid's for the peers that are asked to stop helping + permidlist = [] + for peer in to_stop_helpers: + permidlist.append(peer['permid']) + + # Ask peers to stop helping + olthread_send_stop_help_lambda = lambda:self.olthread_send_stop_help(permidlist) + self.overlay_bridge.add_task(olthread_send_stop_help_lambda,0) + except Exception,e: + print_exc() + print >> sys.stderr, "coordinator: Exception in send_stop_helping",e + + + def olthread_send_stop_help(self,permidlist): + """ Creates a bridge connection for the stop helping request to be sent + + Called by the overlay thread. + + @param permidlist: list of the peer permid's to be asked to stop helping + """ + for permid in permidlist: + if DEBUG: + print >> sys.stderr, "coordinator: error connecting to", show_permid_short(permid), "for stopping help" + self.overlay_bridge.connect(permid,self.olthread_stop_help_connect_callback) + + + def olthread_stop_help_connect_callback(self,exc,dns,permid,selversion): + """ Sends the help request message on the connection with the peer + + Called by the overlay thread. + + @param exc: Peer reachable/unreachable information. None = peer reachable + @param dns: + @param permid: the permid of the peer that is contacted to stop helping + @param selversion: + """ + if exc is None: + # Peer is reachable + ## Create message according to protocol version + message = STOP_HELPING + self.infohash + self.overlay_bridge.send(permid, message, self.olthread_stop_help_send_callback) + elif DEBUG: + # Peer is not reachable + print >> sys.stderr, "coordinator: olthread_stop_help_connect_callback: error connecting to",show_permid_short(permid),exc + + + def olthread_stop_help_send_callback(self,exc,permid): + """ Callback function for error checking in network communication + + Called by the overlay thread. + + @param exc: Peer reachable/unreachable information. None = peer reachable + @param permid: the permid of the peer that is contacted to stop helping + """ + if exc is not None: + # Peer is unreachable + if DEBUG: + print >> sys.stderr, "coordinator: STOP_HELPING: error sending to",show_permid_short(permid),exc + + + + + + def send_request_pieces(self, piece, peerid): + """ Send messages to helpers to request the pieces in pieceList + + Called by next() in PiecePicker + + @param piece: The piece that will be requested to one of the helpers + @param peerid: The peerid of the helper that will be requested for the piece + """ + if DEBUG: + print >>sys.stderr, "coordinator: send_request_pieces: will send requests for piece", piece + + try: + # Choose one of the confirmed helpers + chosen_permid = self.choose_helper(peerid); + + # Store the helper identification data and the piece requested to it + if chosen_permid in self.requested_pieces: + # The peer is already in the dictionary: a previous request was sent to it + current_requested_pieces = self.requested_pieces.get(chosen_permid) + # Check if the piece was not requested before + if piece in current_requested_pieces: + # The piece has already been requested to that helper. No re-requests in this version + if DEBUG: + print >> sys.stderr, "coordinator: send_request_pieces: piece", piece, "was already requested to another helper" + return + current_requested_pieces.append(piece) + self.requested_pieces[chosen_permid] = current_requested_pieces + else: + # The peer is not in the dictionary: no previous requests were sent to it + self.requested_pieces[chosen_permid] = [piece] + + # Sent the request message to the helper + olthread_send_request_help_lambda = lambda:self.olthread_send_request_pieces(chosen_permid, piece) + self.overlay_bridge.add_task(olthread_send_request_help_lambda,0) + except Exception,e: + print_exc() + print >> sys.stderr, "coordinator: Exception while requesting piece",piece,e + + + def olthread_send_request_pieces(self, permid, piece): + """ Creates a bridge connection for the piece request message to be sent + + Called by the overlay thread. + + @param permid: The permid of the peer that will be contacted + @param piece: The piece that will be requested + """ + if DEBUG: + print >> sys.stderr, "coordinator: olthread_send_request_pieces connecting to", show_permid_short(permid), "to request piece", piece + # Connect to the peer designated by permid + olthread_reserve_pieces_connect_callback_lambda = lambda e,d,p,s:self.olthread_request_pieces_connect_callback(e,d,p,s,piece) + self.overlay_bridge.connect(permid, olthread_reserve_pieces_connect_callback_lambda) + + + def olthread_request_pieces_connect_callback(self, exc, dns, permid, selversion, piece): + """ Sends the join_helpers message on the connection with the coordinator + + Called by the overlay thread. + + @param exc: Peer reachable/unreachable information. None = peer reachable + @param dns: + @param permid: the permid of the helper that is requested a piece + @param peice: the requested piece + @param selversion: + """ + if exc is None: + # Peer is reachable + if DEBUG: + print >> sys.stderr, "coordinator: olthread_request_pieces_connect_callback sending help request to", show_permid_short(permid), "for piece", piece + + # Create message according to protocol version + message = REQUEST_PIECES + self.infohash + bencode(piece) + + # Connect using Tribler Ovrlay Swarm + self.overlay_bridge.send(permid, message, self.olthread_request_pieces_send_callback) + else: + # Peer is unreachable + if DEBUG: + print >> sys.stderr, "coordinator: olthread_request_pieces_connect_callback: error connecting to",show_permid_short(permid),exc + # Remove peer from the list of asked peers + self.remove_unreachable_helper(permid) + + + def olthread_request_pieces_send_callback(self,exc,permid): + """ Callback function for error checking in network communication + + Called by the overlay thread. + + @param exc: Peer reachable/unreachable information. None = peer reachable + @param permid: the permid of the peer that is contacted for helping + """ + if exc is not None: + # Peer is unreachable + if DEBUG: + print >> sys.stderr, "coordinator: olthread_request_pieces_send_callback: error sending to",show_permid_short(permid),exc + # Remove peer from the list of asked peers + self.remove_unreachable_helper(permid) + + + def choose_helper(self, peerid): + """ The method returns one of the confirmed helpers, to be contacted for help for a specific piece + + Called by send_request_pieces + @param peerid: The peerid of the helper that will be requested to download a piece + @return: the permid of that peer + """ + + helper_challenge = decode_challenge_from_peerid(peerid) + chosen_helper = self.sent_challenges_by_challenge[helper_challenge] + + # Current proxy selection policy: choose a random helper from the confirmed helper list +# chosen_helper = random.choice(self.confirmed_helpers) + + return chosen_helper + + + + + + def send_cancel_piece(self, piece): + """ Send a cancel message for the specified piece + + Called by TODO + + @param piece: The piece that will be canceled to the respective helper + """ + if DEBUG: + print >> sys.stderr, "coordinator: i will cancel the request for piece", piece + + try: + # Check if the piece was reserved before + all_requested_pieces = self.requested_pieces.values() + if piece not in all_requested_pieces: + if DEBUG: + print >> sys.stderr, "coordinator: piece", piece, "was not requested to any peer" + return + + # Find the peer that was requested to download the piece + for helper in self.requested_pieces.keys(): + his_pieces = self.requested_pieces[helper] + if piece in his_pieces: + if DEBUG: + print >> sys.stderr, "coordinator: canceling piece", piece, "to peer", show_permid_short(helper) + # Sent the cancel message to the helper + olthread_send_cancel_piece_lambda = lambda:self.olthread_send_cancel_piece(chosen_permid, piece) + self.overlay_bridge.add_task(olthread_send_cancel_piece_lambda,0) + except Exception,e: + print_exc() + print >> sys.stderr, "coordinator: Exception while requesting piece",piece,e + + + def olthread_send_cancel_piece(self, permid, piece): + """ Creates a bridge connection for the piece cancel message to be sent + + Called by the overlay thread. + + @param permid: The permid of the peer that will be contacted + @param piece: The piece that will be canceled + """ + if DEBUG: + print >> sys.stderr, "coordinator: olthread_send_cancel_piece connecting to", show_permid_short(permid), "to cancel piece", piece + # Connect to the peer designated by permid + self.overlay_bridge.connect(permid, piece, self.olthread_cancel_piece_connect_callback) + + + def olthread_cancel_piece_connect_callback(self, exc, dns, permid, piece, selversion): + """ Sends the cancel piece message on the connection with the peer + + Called by the overlay thread. + + @param exc: Peer reachable/unreachable information. None = peer reachable + @param dns: + @param permid: the permid of the helper that is requested a piece + @param peice: the canceled piece + @param selversion: + """ + if exc is None: + # Peer is reachable + if DEBUG: + print >> sys.stderr, "coordinator: olthread_cancel_piece_connect_callback sending a cancel request to", show_permid_short(permid), "for piece", piece + + # Create message according to protocol version + message = CANCEL_PIECE + self.infohash + bencode(piece) + + # Connect using Tribler Ovrlay Swarm + self.overlay_bridge.send(permid, message, self.olthread_cancel_piece_send_callback) + else: + # Peer is unreachable + if DEBUG: + print >> sys.stderr, "coordinator: olthread_cancel_piece_connect_callback: error connecting to",show_permid_short(permid),exc + # Remove peer from the list of asked peers + self.remove_unreachable_helper(permid) + + + def olthread_cancel_piece_send_callback(self,exc,permid): + """ Callback function for error checking in network communication + + Called by the overlay thread. + + @param exc: Peer reachable/unreachable information. None = peer reachable + @param permid: the permid of the peer that is contacted for helping + """ + if exc is not None: + # Peer is unreachable + if DEBUG: + print >> sys.stderr, "coordinator: olthread_cancel_piece_send_callback: error sending to",show_permid_short(permid),exc + # Remove peer from the list of asked peers + self.remove_unreachable_helper(permid) + + + + + + # + # Got (received) messages + # + def got_join_helpers(self,permid,selversion): + """ Mark the peer as an active helper + + @param permid: The permid of the node sending the message + @param selversion: + """ + if DEBUG: + print >> sys.stderr, "coordinator: received a JOIN_HELPERS message from", show_permid_short(permid) + + #Search the peer in the asked_helpers list, remove it from there, and put it in the confirmed_helpers list. + self.asked_helpers_lock.acquire() + try: + # Search the peers with permid != from the given permid + new_asked_helpers = [] + for peer in self.asked_helpers: + if peer['permid'] != permid: + new_asked_helpers.append(peer) + else: + # Keep a reference to the peer, to add it to the confirmed_helpers list + # + # If there are more than one peer with the same peerid in the asked_helpers list + # than only add the last one to the confirmed_helpers list. + confirmed_helper = peer + self.asked_helpers = new_asked_helpers + finally: + self.asked_helpers_lock.release() + + self.confirmed_helpers_lock.acquire() + self.confirmed_helpers.append(confirmed_helper) + self.confirmed_helpers_lock.release() + + + + + + def got_resign_as_helper(self,permid,selversion): + """ Remove the peer from the list of active helpers (and form the list of asked helpers) + + @param permid: The permid of the node sending the message + @param selversion: + """ + if DEBUG: + print >> sys.stderr, "coordinator: received a RESIGN_AS_HELPER message from", show_permid_short(permid) + + #Search the peer in the asked_helpers list and remove it from there + self.asked_helpers_lock.acquire() + try: + # Search the peers with permid != from the given permid + new_asked_helpers = [] + for peer in self.asked_helpers: + if peer['permid'] != permid: + new_asked_helpers.append(peer) + self.asked_helpers = new_asked_helpers + finally: + self.asked_helpers_lock.release() + + #Search the peer in the confirmed_helpers list and remove it from there + self.confirmed_helpers_lock.acquire() + try: + # Search the peers with permid != from the given permid + new_confirmed_helpers = [] + for peer in self.confirmed_helpers: + if peer['permid'] != permid: + new_confirmed_helpers.append(peer) + self.confirmed_helpers = new_confirmed_helpers + finally: + self.confirmed_helpers_lock.release() + + + + + + def got_dropped_piece(self, permid, piece, selversion): + """ TODO + + @param permid: The permid of the node sending the message + @param peice: The piece that are dropped + @param selversion: + """ + if DEBUG: + print >> sys.stderr, "coordinator: received a DROPPED_PIECE message from", show_permid_short(permid) + + pass + + + + + + def got_proxy_have(self,permid,selversion, aggregated_string): + """ Take the list of pieces the helper sent and combine it with the numhaves in the piece picker + + @param permid: The permid of the node sending the message + @param selversion: + @param aggregated_string: a bitstring of available pieces built by the helper based on HAVE messages it received + """ + if DEBUG: + print >> sys.stderr, "coordinator: received a PROXY_HAVE message from", show_permid_short(permid) + +# if len(aggregated_string) != self.num_pieces: +# print >> sys.stderr, "coordinator: got_proxy_have: invalid payload in received PROXY_HAVE message. self.num_pieces=", self.num_pieces, "len(aggregated_string)=", len(aggregated_string) + + # Get the recorded peer challenge + peer_challenge = self.sent_challenges_by_permid[permid] + + # Search for the connection that has this challenge + for d in self.downloader.downloads: + peer_id = d.connection.get_id() + if peer_challenge == decode_challenge_from_peerid(peer_id): + # If the connection is found, add the piece_list information to the d.have information + #new_have_list = map(sum, zip(d.have, piece_list)) + d.proxy_have = Bitfield(length=self.downloader.numpieces, bitstring=aggregated_string) + break + + + + + + # Returns a copy of the asked helpers lit + def network_get_asked_helpers_copy(self): + """ Returns a COPY of the list. We need 'before' and 'after' info here, + so the caller is not allowed to update the current confirmed_helpers """ + if DEBUG: + print >> sys.stderr, "coordinator: network_get_asked_helpers_copy: Number of helpers:",len(self.confirmed_helpers) + self.confirmed_helpers_lock.acquire() + try: + return copy.deepcopy(self.confirmed_helpers) + finally: + self.confirmed_helpers_lock.release() + + # Compares peers a and b + def samePeer(self,a,b): + """ Compares peers a and b + + @param a: First peer to compare + @param b: Second peer to compare + @return: True, if the peers are identical. False, if the peers are different + """ + if a.has_key('permid'): + if b.has_key('permid'): + if a['permid'] == b['permid']: + return True + # TODO: Why, if permid's are different, the function returns True ??? + if a['ip'] == b['ip'] and a['port'] == b['port']: + return True + else: + return False + + + + + + # + # Interface for Encrypter.Connection + # + # TODO: rename this function + # TODO: change ip param to permid + # Returns true if the peer with the IP ip is a helper + def is_helper_ip(self, ip): + """ Used by Coordinator's Downloader (via Encrypter) to see what + connections are helpers """ + # called by network thread + self.confirmed_helpers_lock.acquire() + try: + for peer in self.confirmed_helpers: + if peer['ip'] == ip: + return True + return False + finally: + self.confirmed_helpers_lock.release() + + + + + + # + # Interface for CoordinatorMessageHandler + # + # TOSO: rename this function + # Return True if the peer is a helper + # permid = permid of the peer + def network_is_helper_permid(self, permid): + """ Used by CoordinatorMessageHandler to check if RESERVE_PIECES is from good source (if the permid is a helper) """ + # called by overlay thread + for peer in self.confirmed_helpers: + if peer['permid'] == permid: + return True + return False + + + + + + + + + # TODO: rename this function + # Returns the list of reserved pieces + def network_get_reserved(self): + return self.reserved + + + + + + # Set download object + def set_downloader(self, downloader): + """ Method used to set a reference to the downloader object + + Called by BT1Download, after it creates the Downloader object + + @param downloader: A reference to the downloader object for the current download + """ + self.downloader = downloader diff --git a/instrumentation/next-share/BaseLib/Core/ProxyService/CoordinatorMessageHandler.py b/instrumentation/next-share/BaseLib/Core/ProxyService/CoordinatorMessageHandler.py new file mode 100644 index 0000000..e052c93 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/ProxyService/CoordinatorMessageHandler.py @@ -0,0 +1,256 @@ +# Written by Arno Bakker, George Milescu +# see LICENSE.txt for license information +# +# SecureOverlay message handler for a Coordinator +# +import sys + +from BaseLib.Core.BitTornado.bencode import bdecode +from BaseLib.Core.BitTornado.BT1.MessageID import * +from BaseLib.Core.Utilities.utilities import show_permid_short +from BaseLib.Core.simpledefs import * + +DEBUG = False + +class CoordinatorMessageHandler: + def __init__(self, launchmany): + # Launchmany ??? + self.launchmany = launchmany + + def handleMessage(self, permid, selversion, message): + """ Handle the received message and call the appropriate function to solve it. + + As there are multiple coordinator instances, one for each download/upload, the right coordinator instance must be found prior to making a call to it's methods. + + @param permid: The permid of the peer who sent the message + @param selversion: + @param message: The message received + """ + + type = message[0] + if DEBUG: + print >> sys.stderr, "coordinator message handler: received the message", getMessageName(type), "from", show_permid_short(permid) + + # Call the appropriate function + if type == JOIN_HELPERS: + return self.got_join_helpers(permid, message, selversion) + elif type == RESIGN_AS_HELPER: + return self.got_resign_as_helper(permid, message, selversion) + elif type == DROPPED_PIECE: + return self.got_dropped_piece(permid, message, selversion) + elif type == PROXY_HAVE: + return self.got_proxy_have(permid, message, selversion) + + + + + + def got_join_helpers(self, permid, message, selversion): + """ Handle the JOIN_HELPERS message. + + @param permid: The permid of the peer who sent the message + @param message: The message received + @param selversion: + """ + + if DEBUG: + print >> sys.stderr, "coordinator: network_got_join_helpers: got_join_helpers" + + try: + infohash = message[1:21] + except: + print >> sys.stderr, "coordinator: network_got_join_helpers: warning - bad data in JOIN_HELPERS" + return False + + # Add a task to find the appropriate Coordinator object method + network_got_join_helpers_lambda = lambda:self.network_got_join_helpers(permid, infohash, selversion) + self.launchmany.rawserver.add_task(network_got_join_helpers_lambda, 0) + + return True + + + def network_got_join_helpers(self, permid, infohash, selversion): + """ Find the appropriate Coordinator object and call it's method. + + Called by the network thread. + + @param permid: The permid of the peer who sent the message + @param infohash: The infohash sent by the remote peer + @param selversion: + """ + + if DEBUG: + print >> sys.stderr, "coordinator: network_got_join_helpers: network_got_join_helpers" + + # Get coordinator object + coord_obj = self.launchmany.get_coopdl_role_object(infohash, COOPDL_ROLE_COORDINATOR) + if coord_obj is None: + # There is no coordinator object associated with this infohash + if DEBUG: + print >> sys.stderr, "coordinator: network_got_join_helpers: There is no coordinator object associated with this infohash" + return + + # Call the coordinator method + coord_obj.got_join_helpers(permid, selversion) + + + + + + def got_resign_as_helper(self, permid, message, selversion): + """ Handle the RESIGN_AS_HELPER message. + + @param permid: The permid of the peer who sent the message + @param message: The message received + @param selversion: + """ + + if DEBUG: + print >> sys.stderr, "coordinator: got_resign_as_helper" + + try: + infohash = message[1:21] + except: + print >> sys.stderr, "coordinator warning: bad data in RESIGN_AS_HELPER" + return False + + # Add a task to find the appropriate Coordinator object method + network_got_resign_as_helper_lambda = lambda:self.network_got_resign_as_helper(permid, infohash, selversion) + self.launchmany.rawserver.add_task(network_got_resign_as_helper_lambda, 0) + + return True + + + def network_got_resign_as_helper(self, permid, infohash, selversion): + """ Find the appropriate Coordinator object and call it's method. + + Called by the network thread. + + @param permid: The permid of the peer who sent the message + @param infohash: The infohash sent by the remote peer + @param selversion: + """ + + if DEBUG: + print >> sys.stderr, "coordinator: network_got_resign_as_helper" + + # Get coordinator object + coord_obj = self.launchmany.get_coopdl_role_object(infohash, COOPDL_ROLE_COORDINATOR) + if coord_obj is None: + # There is no coordinator object associated with this infohash + if DEBUG: + print >> sys.stderr, "coordinator: network_got_resign_as_helper: There is no coordinator object associated with this infohash" + return + + # Call the coordinator method + coord_obj.got_resign_as_helper(permid, selversion) + + + + + + def got_dropped_piece(self, permid, message, selversion): + """ Handle the DROPPED_PIECE message. + + @param permid: The permid of the peer who sent the message + @param message: The message received + @param selversion: + """ + + if DEBUG: + print >> sys.stderr, "coordinator: got_dropped_piece" + + try: + infohash = message[1:21] + piece = bdecode(message[22:]) + except: + print >> sys.stderr, "coordinator warning: bad data in DROPPED_PIECE" + return False + + # Add a task to find the appropriate Coordinator object method + network_got_dropped_piece_lambda = lambda:self.network_got_dropped_piece(permid, infohash, peice, selversion) + self.launchmany.rawserver.add_task(network_got_dropped_piece_lambda, 0) + + return True + + + def network_got_dropped_piece(self, permid, infohash, piece, selversion): + """ Find the appropriate Coordinator object and call it's method. + + Called by the network thread. + + @param permid: The permid of the peer who sent the message + @param infohash: The infohash sent by the remote peer + @param piece: The piece that is dropped + @param selversion: + """ + + if DEBUG: + print >> sys.stderr, "coordinator: network_got_dropped_piece" + + # Get coordinator object + coord_obj = self.launchmany.get_coopdl_role_object(infohash, COOPDL_ROLE_COORDINATOR) + if coord_obj is None: + # There is no coordinator object associated with this infohash + if DEBUG: + print >> sys.stderr, "coordinator: network_got_dropped_piece: There is no coordinator object associated with this infohash" + return + + # Call the coordinator method + coord_obj.got_dropped_piece_(permid, piece, selversion) + + + + + + def got_proxy_have(self, permid, message, selversion): + """ Handle the PROXY_HAVE message. + + @param permid: The permid of the peer who sent the message + @param message: The message received + @param selversion: + """ + + if DEBUG: + print >> sys.stderr, "coordinator: network_got_proxy_have: got_proxy_have" + + try: + infohash = message[1:21] + aggregated_string = bdecode(message[21:]) + except: + print >> sys.stderr, "coordinator: network_got_proxy_have: warning - bad data in PROXY_HAVE" + return False + + # Add a task to find the appropriate Coordinator object method + network_got_proxy_have_lambda = lambda:self.network_got_proxy_have(permid, infohash, selversion, aggregated_string) + self.launchmany.rawserver.add_task(network_got_proxy_have_lambda, 0) + + return True + + + def network_got_proxy_have(self, permid, infohash, selversion, aggregated_string): + """ Find the appropriate Coordinator object and call it's method. + + Called by the network thread. + + @param permid: The permid of the peer who sent the message + @param infohash: The infohash sent by the remote peer + @param selversion: + @param aggregated_string: a bitstring of pieces the helper built based on HAVE messages + """ + + if DEBUG: + print >> sys.stderr, "coordinator: network_got_proxy_have: network_got_proxy_have" + + # Get coordinator object + coord_obj = self.launchmany.get_coopdl_role_object(infohash, COOPDL_ROLE_COORDINATOR) + if coord_obj is None: + # There is no coordinator object associated with this infohash + if DEBUG: + print >> sys.stderr, "coordinator: network_got_proxy_have: There is no coordinator object associated with this infohash" + return + + # Call the coordinator method + coord_obj.got_proxy_have(permid, selversion, aggregated_string) + + diff --git a/instrumentation/next-share/BaseLib/Core/ProxyService/Helper.py b/instrumentation/next-share/BaseLib/Core/ProxyService/Helper.py new file mode 100644 index 0000000..1a66f9e --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/ProxyService/Helper.py @@ -0,0 +1,543 @@ +# Written by Pawel Garbacki, George Milescu +# see LICENSE.txt for license information + +import sys +from traceback import print_exc +from time import time +from collections import deque +from threading import Lock + +from BaseLib.Core.BitTornado.bencode import bencode +from BaseLib.Core.BitTornado.BT1.MessageID import ASK_FOR_HELP, STOP_HELPING, REQUEST_PIECES, CANCEL_PIECE, JOIN_HELPERS, RESIGN_AS_HELPER, DROPPED_PIECE, PROXY_HAVE, PROXY_UNHAVE + +from BaseLib.Core.Overlay.OverlayThreadingBridge import OverlayThreadingBridge +from BaseLib.Core.CacheDB.CacheDBHandler import PeerDBHandler, TorrentDBHandler +from BaseLib.Core.Utilities.utilities import show_permid_short + +# ??? +MAX_ROUNDS = 200 +# ??? +DEBUG = False + +class SingleDownloadHelperInterface: + """ This interface should contain all methods that the PiecePiecker/Helper + calls on the SingleDownload class. + """ + def __init__(self): + self.frozen_by_helper = False + + def helper_set_freezing(self,val): + self.frozen_by_helper = val + + def is_frozen_by_helper(self): + return self.frozen_by_helper + + def is_choked(self): + pass + + def helper_forces_unchoke(self): + pass + + def _request_more(self, new_unchoke = False): + pass + + +class Helper: + def __init__(self, torrent_hash, num_pieces, coordinator_permid, coordinator = None): + + self.torrent_hash = torrent_hash + self.coordinator = coordinator + + if coordinator_permid is not None and coordinator_permid == '': + self.coordinator_permid = None + else: + self.coordinator_permid = coordinator_permid + + # Get coordinator ip and address + self.coordinator_ip = None # see is_coordinator() + self.coordinator_port = -1 + if self.coordinator_permid is not None: + peerdb = PeerDBHandler.getInstance() + peer = peerdb.getPeer(coordinator_permid) + if peer is not None: + self.coordinator_ip = peer['ip'] + self.coordinator_port = peer['port'] + + self.overlay_bridge = OverlayThreadingBridge.getInstance() + + self.reserved_pieces = [False] * num_pieces + self.ignored_pieces = [False] * num_pieces + self.distr_reserved_pieces = [False] * num_pieces + + self.requested_pieces = deque() + self.requested_pieces_lock = Lock() + + self.counter = 0 + self.completed = False + self.marker = [True] * num_pieces + self.round = 0 + self.encoder = None + self.continuations = [] + self.outstanding = None + self.last_req_time = 0 + + # The challenge sent by the coordinator + self.challenge = None + + + def test(self): + result = self.reserve_piece(10,None) + print >> sys.stderr,"reserve piece returned: " + str(result) + print >> sys.stderr,"Test passed" + + + + + + def notify(self): + """ Called by HelperMessageHandler to "wake up" the download that's + waiting for its coordinator to reserve it a piece + """ + if self.outstanding is None: + if DEBUG: + print >> sys.stderr,"helper: notify: No continuation waiting?" + else: + if DEBUG: + print >> sys.stderr,"helper: notify: Waking downloader" + sdownload = self.outstanding + self.outstanding = None # must be not before calling self.restart! + self.restart(sdownload) + + #self.send_reservation() + l = self.continuations[:] # copy just to be sure + self.continuations = [] + for sdownload in l: + self.restart(sdownload) + + def restart(self,sdownload): + """ TODO ??? + """ + # Chokes can get in while we're waiting for reply from coordinator. + # But as we were called from _request_more() we were not choked + # just before, so pretend we didn't see the message yet. + if sdownload.is_choked(): + sdownload.helper_forces_unchoke() + sdownload.helper_set_freezing(False) + sdownload._request_more() + + + + + + # + # Send messages + # + + def send_join_helpers(self, permid): + """ Send a confirmation to the coordinator that the current node will provide proxy services + + Called by self.got_ask_for_help() + + @param permid: The permid of the node that will become coordinator + """ + + if DEBUG: + print "helper: send_join_helpers: sending a join_helpers message to", show_permid_short(permid) + + olthread_send_join_helpers_lambda = lambda:self.olthread_send_join_helpers() + self.overlay_bridge.add_task(olthread_send_join_helpers_lambda,0) + + + def olthread_send_join_helpers(self): + """ Creates a bridge connection for the join helpers message to be sent + + Called by the overlay thread. + """ + # TODO: ??? We need to create the message according to protocol version, so need to pass all info. + olthread_join_helpers_connect_callback_lambda = lambda e,d,p,s:self.olthread_join_helpers_connect_callback(e,d,p,s) + self.overlay_bridge.connect(self.coordinator_permid,olthread_join_helpers_connect_callback_lambda) + + + def olthread_join_helpers_connect_callback(self,exc,dns,permid,selversion): + """ Sends the join helpers message on the connection with the coordinator + + Called by the overlay thread. + + @param exc: Peer reachable/unreachable information. None = peer reachable + @param dns: + @param permid: the permid of the coordinator + @param selversion: + """ + if exc is None: + # Create message according to protocol version + message = JOIN_HELPERS + self.torrent_hash + + if DEBUG: + print >> sys.stderr,"helper: olthread_join_helpers_connect_callback: Sending JOIN_HELPERS to",show_permid_short(permid) + + self.overlay_bridge.send(permid, message, self.olthread_join_helpers_send_callback) + elif DEBUG: + # The coordinator is unreachable + print >> sys.stderr,"helper: olthread_join_helpers_connect_callback: error connecting to",show_permid_short(permid),exc + + + def olthread_join_helpers_send_callback(self, exc, permid): + """ Callback function for error checking in network communication + + Called by the overlay thread. + + @param exc: Peer reachable/unreachable information. None = peer reachable + @param permid: the permid of the peer that is contacted for helping + """ + + if exc is not None: + if DEBUG: + print >> sys.stderr,"helper: olthread_join_helpers_send_callback: error sending message to",show_permid_short(permid),exc + + pass + + + + + + def send_proxy_have(self, aggregated_haves): + """ Send a list of aggregated have and bitfield information + + Called by Downloader.aggregate_and_send_haves + + @param aggregated_haves: A Bitfield object, containing an aggregated list of stored haves + """ + + if DEBUG: + print "helper: send_proxy_have: sending a proxy_have message to", show_permid_short(self.coordinator_permid) + + aggregated_string = aggregated_haves.tostring() + olthread_send_proxy_have_lambda = lambda:self.olthread_send_proxy_have(aggregated_string) + self.overlay_bridge.add_task(olthread_send_proxy_have_lambda,0) + + + def olthread_send_proxy_have(self, aggregated_string): + """ Creates a bridge connection for the proxy_have message to be sent + + Called by the overlay thread. + + @param aggregated_string: a bitstring of available piesces + """ + # TODO: ??? We need to create the message according to protocol version, so need to pass all info. + olthread_proxy_have_connect_callback_lambda = lambda e,d,p,s:self.olthread_proxy_have_connect_callback(e,d,p,s,aggregated_string) + self.overlay_bridge.connect(self.coordinator_permid,olthread_proxy_have_connect_callback_lambda) + + + def olthread_proxy_have_connect_callback(self,exc,dns,permid,selversion, aggregated_string): + """ Sends the proxy_have message on the connection with the coordinator + + Called by the overlay thread. + + @param exc: Peer reachable/unreachable information. None = peer reachable + @param dns: + @param permid: the permid of the coordinator + @param selversion: selected (buddycast?) version + @param aggregated_string: a bitstring of available pieces + """ + if exc is None: + # Create message according to protocol version + message = PROXY_HAVE + self.torrent_hash + bencode(aggregated_string) + + if DEBUG: + print >> sys.stderr,"helper: olthread_proxy_have_connect_callback: Sending PROXY_HAVE to",show_permid_short(permid) + + self.overlay_bridge.send(permid, message, self.olthread_proxy_have_send_callback) + elif DEBUG: + # The coordinator is unreachable + print >> sys.stderr,"helper: olthread_proxy_have_connect_callback: error connecting to",show_permid_short(permid),exc + + + def olthread_proxy_have_send_callback(self, exc, permid): + """ Callback function for error checking in network communication + + Called by the overlay thread. + + @param exc: Peer reachable/unreachable information. None = peer reachable + @param permid: the permid of the peer that is contacted for helping + """ + + if exc is not None: + if DEBUG: + print >> sys.stderr,"helper: olthread_proxy_have_send_callback: error sending message to",show_permid_short(permid),exc + + pass + + + + + + def send_resign_as_helper(self, permid): + """ Send a message to the coordinator that the current node will stop providing proxy services + + Called by TODO + + @param permid: The permid of the coordinator + """ + + if DEBUG: + print "helper: send_resign_as_helper: sending a resign_as_helper message to", permid + + olthread_send_resign_as_helper_lambda = lambda:self.olthread_send_resign_as_helper() + self.overlay_bridge.add_task(olthread_send_resign_as_helper_lambda,0) + + + def olthread_send_resign_as_helper(self): + """ Creates a bridge connection for the resign_as_helper message to be sent + + Called by the overlay thread. + """ + olthread_resign_as_helper_connect_callback_lambda = lambda e,d,p,s:self.olthread_resign_as_helper_connect_callback(e,d,p,s) + self.overlay_bridge.connect(self.coordinator_permid,olthread_resign_as_helper_connect_callback_lambda) + + + def olthread_resign_as_helper_connect_callback(self,exc,dns,permid,selversion): + """ Sends the resign_as_helper message on the connection with the coordinator + + Called by the overlay thread. + + @param exc: Peer reachable/unreachable information. None = peer reachable + @param dns: + @param permid: the permid of the coordinator + @param selversion: + """ + if exc is None: + # Create message according to protocol version + message = RESIGN_AS_HELPER + self.torrent_hash + + if DEBUG: + print >> sys.stderr,"helper: olthread_resign_as_helper_connect_callback: Sending RESIGN_AS_HELPER to",show_permid_short(permid) + + self.overlay_bridge.send(permid, message, self.olthread_resign_as_helper_send_callback) + elif DEBUG: + # The coordinator is unreachable + print >> sys.stderr,"helper: olthread_resign_as_helper_connect_callback: error connecting to",show_permid_short(permid),exc + + + def olthread_resign_as_helper_send_callback(self,exc,permid): + """ Callback function for error checking in network communication + + Called by the overlay thread. + + @param exc: Peer reachable/unreachable information. None = peer reachable + @param permid: the permid of the peer that is contacted for helping + """ + + if exc is not None: + if DEBUG: + print >> sys.stderr,"helper: olthread_resign_as_helper_send_callback: error sending message to",show_permid_short(permid),exc + + pass + + + + + + # + # Got (received) messages + # + def got_ask_for_help(self, permid, infohash, challenge): + """ Start helping a coordinator or reply with an resign_as_helper message + + @param permid: The permid of the node sending the help request message + @param infohash: the infohash of the torrent for which help is requested + @param challenge: The challenge sent by the coordinator + """ + if DEBUG: + print >>sys.stderr,"helper: got_ask_for_help: will answer to the help request from", show_permid_short(permid) + if self.can_help(infohash): + # Send JOIN_HELPERS + if DEBUG: + print >>sys.stderr,"helper: got_ask_for_help: received a help request, going to send join_helpers" + self.send_join_helpers(permid) + self.challenge = challenge + else: + # Send RESIGN_AS_HELPER + if DEBUG: + print >>sys.stderr,"helper: got_ask_for_help: received a help request, going to send resign_as_helper" + self.send_resign_as_helper(permid) + return False + + return True + + + def can_help(self, infohash): + """ Decide if the current node can help a coordinator for the current torrent + + @param infohash: the infohash of the torrent for which help is requested + """ + #TODO: test if I can help the cordinator to download this file + #Future support: make the decision based on my preference + return True + + + + + + def got_stop_helping(self, permid, infohash): + """ Stop helping a coordinator + + @param permid: The permid of the node sending the message + @param infohash: the infohash of the torrent for which help is released + """ + #TODO: decide what to do here + return True + + + + + + def got_request_pieces(self, permid, piece): + """ Start downloading the pieces that the coordinator requests + + @param permid: The permid of the node requesting the pieces + @param piece: a piece number, that is going to be downloaded + """ + if DEBUG: + print "helper: got_request_pieces: received request_pieces for piece", piece + + # Mark the piece as requested in the local data structures + self.reserved_pieces[piece] = True +# if self.distr_reserved_pieces[piece] == True: + # if the piece was previously requested by the same coordinator, don't do anything + #self.distr_reserved_pieces[piece] = True +# print "Received duplicate proxy request for", piece +# return + + self.distr_reserved_pieces[piece] = True + self.ignored_pieces[piece] = False + + self.requested_pieces_lock.acquire() + self.requested_pieces.append(piece) + self.requested_pieces_lock.release() + + # Start data connection + self.start_data_connection() + + def start_data_connection(self): + """ Start a data connection with the coordinator + + @param permid: The permid of the coordinator + """ + # Do this always, will return quickly when connection already exists + dns = (self.coordinator_ip, self.coordinator_port) + if DEBUG: + print >> sys.stderr,"helper: start_data_connection: Starting data connection to coordinator at", dns + + self.encoder.start_connection(dns, id = None, coord_con = True, challenge = self.challenge) + + + + # + # Util functions + # + def is_coordinator(self, permid): + """ Check if the permid is the current coordinator + + @param permid: The permid to be checked if it is the coordinator + @return: True, if the permid is the current coordinator; False, if the permid is not the current coordinator + """ + # If we could get coordinator_ip, don't help + if self.coordinator_ip is None: + return False + + if self.coordinator_permid == permid: + return True + else: + return False + + + def next_request(self): + """ Returns the next piece in the list of coordinator-requested pieces + + Called by the PiecePicker + + @return: a piece number, if there is a requested piece pending download; None, if there is no pending piece + """ + self.requested_pieces_lock.acquire() + if len(self.requested_pieces) == 0: + self.requested_pieces_lock.release() + if DEBUG: + print >>sys.stderr,"helper: next_request: no requested pieces yet. Returning None" + return None + else: + next_piece = self.requested_pieces.popleft() + self.requested_pieces_lock.release() + if DEBUG: + print >>sys.stderr,"helper: next_request: Returning", next_piece + return next_piece + + + def set_encoder(self, encoder): + """ Sets the current encoder. + + Called from download_bt1.py + + @param encoder: the new encoder that will be set + """ + self.encoder = encoder + self.encoder.set_coordinator_ip(self.coordinator_ip) + # To support a helping user stopping and restarting a torrent + if self.coordinator_permid is not None: + self.start_data_connection() + + + def get_coordinator_permid(self): + """ Returns the coordinator permid + + Called from SingleDownload.py + + @return: Coordinator permid + """ + return self.coordinator_permid + + + def is_reserved(self, piece): + """ Check if a piece is reserved (requested) by a coordinator + + Called by the network thread (Interface for PiecePicker and Downloader) + + @param piece: the piece whose status is to be checked + @return: True, if the piece is reqested by a coordinator; False, otherwise. + """ + if self.reserved_pieces[piece] or (self.coordinator is not None and self.is_complete()): + return True + return self.reserved_pieces[piece] + + + def is_ignored(self, piece): + """ Check if a piece is ignored by a coordinator + + Called by the network thread (Interface for PiecePicker and Downloader) + + @param piece: the piece whose status is to be checked + @return: True, if the piece is ignored by a coordinator; False, otherwise. + """ + if not self.ignored_pieces[piece] or (self.coordinator is not None and self.is_complete()): + return False + return self.ignored_pieces[piece] + + + def is_complete(self): + """ Check torrent is completely downloaded + + Called by the network thread (Interface for PiecePicker and Downloader) + + @return: True, all the pieces are downloaded; False, otherwise. + """ + if self.completed: + return True + + self.round = (self.round + 1) % MAX_ROUNDS + + if self.round != 0: + return False + if self.coordinator is not None: + self.completed = (self.coordinator.reserved_pieces == self.marker) + else: + self.completed = (self.distr_reserved_pieces == self.marker) + return self.completed \ No newline at end of file diff --git a/instrumentation/next-share/BaseLib/Core/ProxyService/HelperMessageHandler.py b/instrumentation/next-share/BaseLib/Core/ProxyService/HelperMessageHandler.py new file mode 100644 index 0000000..fa327e4 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/ProxyService/HelperMessageHandler.py @@ -0,0 +1,419 @@ +# Written by Pawel Garbacki, Arno Bakker, George Milescu +# see LICENSE.txt for license information +# +# SecureOverlay message handler for a Helper +# + +import sys, os +import binascii +from threading import Lock +from time import sleep + +from BaseLib.Core.TorrentDef import * +from BaseLib.Core.Session import * +from BaseLib.Core.simpledefs import * +from BaseLib.Core.DownloadConfig import DownloadStartupConfig +from BaseLib.Core.Utilities.utilities import show_permid_short +from BaseLib.Core.BitTornado.bencode import bencode, bdecode +from BaseLib.Core.BitTornado.BT1.MessageID import * +from BaseLib.Core.CacheDB.CacheDBHandler import PeerDBHandler, TorrentDBHandler + +from BaseLib.Core.Overlay.OverlayThreadingBridge import OverlayThreadingBridge + +DEBUG = False + +class HelperMessageHandler: + def __init__(self): + self.metadata_queue = {} + self.metadata_queue_lock = Lock() + self.overlay_bridge = OverlayThreadingBridge.getInstance() + self.received_challenges = {} + + def register(self,session,metadata_handler,helpdir,dlconfig): + self.session = session + self.helpdir = helpdir + # The default DownloadStartupConfig dict as set in the Session + self.dlconfig = dlconfig + + self.metadata_handler = metadata_handler + self.torrent_db = TorrentDBHandler.getInstance() + + def handleMessage(self,permid,selversion,message): + """ Handle the received message and call the appropriate function to solve it. + + As there are multiple helper instances, one for each download/upload, the right helper instance must be found prior to making a call to it's methods. + + @param permid: The permid of the peer who sent the message + @param selversion: + @param message: The message received + """ + + t = message[0] + if DEBUG: + print >> sys.stderr, "helper: received the message", getMessageName(t), "from", show_permid_short(permid) + + #if ProxyService is not turned on, return + session_config = self.session.get_current_startup_config_copy() + if session_config.get_proxyservice_status() == PROXYSERVICE_OFF: + if DEBUG: + print >> sys.stderr, "helper: ProxyService not active, ignoring message" + + return + + if t == ASK_FOR_HELP: + return self.got_ask_for_help(permid, message, selversion) + elif t == STOP_HELPING: + return self.got_stop_helping(permid, message, selversion) + elif t == REQUEST_PIECES: + return self.got_request_pieces(permid, message, selversion) + + + + + + def got_ask_for_help(self, permid, message, selversion): + """ Handle the ASK_FOR_HELP message. + + @param permid: The permid of the peer who sent the message + @param message: The message received + @param selversion: + """ + try: + infohash = message[1:21] + challenge = bdecode(message[21:]) + except: + if DEBUG: + print >> sys.stderr, "helper: got_ask_for_help: bad data in ask_for_help" + return False + + if len(infohash) != 20: + if DEBUG: + print >> sys.stderr, "helper: got_ask_for_help: bad infohash in ask_for_help" + return False + + if DEBUG: + print >> sys.stderr, "helper: got_ask_for_help: received a help request from",show_permid_short(permid) + + + # Save the challenge + self.received_challenges[permid] = challenge + + # Find the appropriate Helper object. If no helper object is associated with the requested infohash, than start a new download for it + helper_obj = self.session.lm.get_coopdl_role_object(infohash, COOPDL_ROLE_HELPER) + if helper_obj is None: + if DEBUG: + print >> sys.stderr, "helper: got_ask_for_help: There is no current download for this infohash. A new download must be started." + + self.start_helper_download(permid, infohash, selversion) + # start_helper_download will make, indirectly, a call to the network_got_ask_for_help method of the helper, + # in a similar fashion as the one below + return + + # Call the helper object got_ask_for_help method + # If the object was created with start_helepr_download, an amount of time is required + # before the download is fully operational, so the call to the the helper object got_ask_for_help method + # is made using the network thread (the network thread executes tasks sequentially, so the start_download task should + # be executed before the network_got_ask_for_help) + network_got_ask_for_help_lambda = lambda:self.network_got_ask_for_help(permid, infohash) + self.session.lm.rawserver.add_task(network_got_ask_for_help_lambda, 0) + + return True + + + def network_got_ask_for_help(self, permid, infohash): + """ Find the appropriate Helper object and call it's method. If no helper object is associated with the requested + infohash, than start a new download for it + + Called by the network thread. + + @param permid: The permid of the peer who sent the message + @param infohash: The infohash sent by the remote peer + @param challenge: The challenge sent by the coordinator + """ + + helper_obj = self.session.lm.get_coopdl_role_object(infohash, COOPDL_ROLE_HELPER) + if helper_obj is None: + if DEBUG: + print >> sys.stderr, "helper: network_got_ask_for_help: There is no current download for this infohash. Try again later..." + return + + # At this point, a previous download existed + # A node can not be a helper and a coordinator at the same time + if not helper_obj.is_coordinator(permid): + if DEBUG: + print >> sys.stderr, "helper: network_got_ask_for_help: The node asking for help is not the current coordinator" + return + + challenge = self.received_challenges[permid] + helper_obj.got_ask_for_help(permid, infohash, challenge) + # Wake up download thread + helper_obj.notify() + + + def start_helper_download(self, permid, infohash, selversion): + """ Start a new download, as a helper, for the requested infohash + + @param permid: the coordinator permid requesting help + @param infohash: the infohash of the .torrent + @param selversion: + @param challenge: The challenge sent by the coordinator + """ + + # Getting .torrent information + torrent_data = self.find_torrent(infohash) + if torrent_data: + # The .torrent was already in the local cache + self.new_download(infohash, torrent_data, permid) + else: + # The .torrent needs to be downloaded + # new_download will be called at the end of get_torrent_metadata + self.get_torrent_metadata(permid, infohash, selversion) + + + # It is very important here that we create safe filenames, i.e., it should + # not be possible for a coordinator to send a METADATA message that causes + # important files to be overwritten + # + def new_download(self, infohash, torrent_data, permid): + """ Start a new download in order to get the pieces that will be requested by the coordinator. + After the download is started, find the appropriate Helper object and call it's method. + + @param infohash: the infohash of the torrent for which help is requested + @param torrent_data: the content of the .torrent file + @param permid: the permid of the coordonator + @param challenge: The challenge sent by the coordinator + """ + + # Create the name for the .torrent file in the helper cache + basename = binascii.hexlify(infohash)+'.torrent' # ignore .tribe stuff, not vital + torrentfilename = os.path.join(self.helpdir,basename) + + # Write the .torrent information in the .torrent helper cache file + tfile = open(torrentfilename, "wb") + tfile.write(torrent_data) + tfile.close() + + if DEBUG: + print >> sys.stderr, "helper: new_download: Got metadata required for helping",show_permid_short(permid) + print >> sys.stderr, "helper: new_download: torrent: ",torrentfilename + + tdef = TorrentDef.load(torrentfilename) + if self.dlconfig is None: + dscfg = DownloadStartupConfig() + else: + dscfg = DownloadStartupConfig(self.dlconfig) + dscfg.set_coopdl_coordinator_permid(permid) + dscfg.set_dest_dir(self.helpdir) + dscfg.set_proxy_mode(PROXY_MODE_OFF) # a helper does not use other helpers for downloading data + + # Start new download + if DEBUG: + print >> sys.stderr, "helper: new_download: Starting a new download" + d=self.session.start_download(tdef,dscfg) + d.set_state_callback(self.state_callback, getpeerlist=False) + + # Call the helper object got_ask_for_help method + # If the object was created with start_helepr_download, an amount of time is required + # before the download is fully operational, so the call to the the helper object got_ask_for_help method + # is made using the network thread (the network thread executes tasks sequentially, so the start_download task should + # be executed before the network_got_ask_for_help) + network_got_ask_for_help_lambda = lambda:self.network_got_ask_for_help(permid, infohash) + self.session.lm.rawserver.add_task(network_got_ask_for_help_lambda, 0) + + # Print torrent statistics + def state_callback(self, ds): + d = ds.get_download() + # print >>sys.stderr,`d.get_def().get_name()`,dlstatus_strings[ds.get_status()],ds.get_progress(),"%",ds.get_error(),"up",ds.get_current_speed(UPLOAD),"down",ds.get_current_speed(DOWNLOAD) + print >>sys.stderr, '%s %s %5.2f%% %s up %8.2fKB/s down %8.2fKB/s' % \ + (d.get_def().get_name(), \ + dlstatus_strings[ds.get_status()], \ + ds.get_progress() * 100, \ + ds.get_error(), \ + ds.get_current_speed(UPLOAD), \ + ds.get_current_speed(DOWNLOAD)) + + return (1.0, False) + + + + + def get_torrent_metadata(self, permid, infohash, selversion): + """ Get the .torrent file from the coordinator requesting help for it + + @param permid: the permid of the coordinator + @param infihash: the infohash of the .torrent + @param selversion: + """ + if DEBUG: + print >> sys.stderr, "helper: get_torrent_metadata: Asking coordinator for the .torrent" + self.metadata_queue_lock.acquire() + try: + if not self.metadata_queue.has_key(infohash): + self.metadata_queue[infohash] = [] + self.metadata_queue[infohash].append(permid) + finally: + self.metadata_queue_lock.release() + + self.metadata_handler.send_metadata_request(permid, infohash, selversion, caller="dlhelp") + + + def metadatahandler_received_torrent(self, infohash, torrent_data): + """ The coordinator sent the .torrent file. + """ + # TODO: Where is this handler registered ? + # TODO: Is this handler actually called by the network thread ? + if DEBUG: + print >> sys.stderr, "helper: metadatahandler_received_torrent: the .torrent is in." + + self.metadata_queue_lock.acquire() + try: + if not self.metadata_queue.has_key(infohash) or not self.metadata_queue[infohash]: + if DEBUG: + print >> sys.stderr, "helper: metadatahandler_received_torrent: a .torrent was received that we are not waiting for." + return + + infohash_queue = self.metadata_queue[infohash] + del self.metadata_queue[infohash] + finally: + self.metadata_queue_lock.release() + for permid in infohash_queue: + # only ask for metadata once + self.new_download(infohash, torrent_data, permid) + + + def find_torrent(self, infohash): + """ Find the .torrent for the required infohash. + + @param infohash: the infohash of the .torrent that must be returned + """ + torrent = self.torrent_db.getTorrent(infohash) + if torrent is None: + # The .torrent file is not in the local cache + if DEBUG: + print >> sys.stderr, "helper: find_torrent: The .torrent file is not in the local cache" + return None + elif 'torrent_dir' in torrent: + fn = torrent['torrent_dir'] + if os.path.isfile(fn): + f = open(fn,"rb") + data = f.read() + f.close() + return data + else: + # The .torrent file path does not exist or the path is not for a file + if DEBUG: + print >> sys.stderr, "helper: find_torrent: The .torrent file path does not exist or the path is not for a file" + return None + else: + # The torrent dictionary does not contain a torrent_dir field + if DEBUG: + print >> sys.stderr, "helper: find_torrent: The torrent dictionary does not contain a torrent_dir field" + return None + + + + + + def got_stop_helping(self, permid, message, selversion): + """ Handle the STOP_HELPING message. + + @param permid: The permid of the peer who sent the message + @param message: The message received + @param selversion: + """ + try: + infohash = message[1:] + except: + if DEBUG: + print >> sys.stderr, "helper: got_stop_helping: bad data in STOP_HELPING" + return False + + if len(infohash) != 20: + if DEBUG: + print >> sys.stderr, "helper: got_stop_helping: bad infohash in STOP_HELPING" + return False + + network_got_stop_helping_lambda = lambda:self.network_got_stop_helping(permid, infohash, selversion) + self.session.lm.rawserver.add_task(network_got_stop_helping_lambda, 0) + + # If the request is from a unauthorized peer, we close + # If the request is from an authorized peer (=coordinator) we close as well. So return False + return False + + + def network_got_stop_helping(self, permid, infohash, selversion): + """ Find the appropriate Helper object and call it's method. + + Called by the network thread. + + @param permid: The permid of the peer who sent the message + @param infohash: The infohash sent by the remote peer + @param selversion: + """ + helper_obj = self.session.lm.get_coopdl_role_object(infohash, COOPDL_ROLE_HELPER) + if helper_obj is None: + if DEBUG: + print >> sys.stderr, "helper: network_got_stop_helping: There is no helper object associated with this infohash" + return + + if not helper_obj.is_coordinator(permid): + if DEBUG: + print >> sys.stderr, "helper: network_got_stop_helping: The node asking for help is not the current coordinator" + return + +# helper_obj.got_stop_helping(permid, infohash) +# # Wake up download thread +# helper_obj.notify() + # Find and remove download + dlist = self.session.get_downloads() + for d in dlist: + if d.get_def().get_infohash() == infohash: + self.session.remove_download(d) + break + + + + + + def got_request_pieces(self,permid, message, selversion): + """ Handle the REQUEST_PIECES message. + + @param permid: The permid of the peer who sent the message + @param message: The message received + @param selversion: + """ + try: + infohash = message[1:21] + pieces = bdecode(message[21:]) + except: + print >> sys.stderr, "helper: got_request_pieces: bad data in REQUEST_PIECES" + return False + + network_got_request_pieces_lambda = lambda:self.network_got_request_pieces(permid, message, selversion, infohash, pieces) + self.session.lm.rawserver.add_task(network_got_request_pieces_lambda, 0) + + return True + + def network_got_request_pieces(self, permid, message, selversion, infohash, pieces): + """ Find the appropriate Helper object and call it's method. + + Called by the network thread. + + @param permid: The permid of the peer who sent the message + @param infohash: The infohash sent by the remote peer + @param selversion: + """ + helper_obj = self.session.lm.get_coopdl_role_object(infohash, COOPDL_ROLE_HELPER) + if helper_obj is None: + if DEBUG: + print >> sys.stderr, "helper: network_got_request_pieces: There is no helper object associated with this infohash" + return + + if not helper_obj.is_coordinator(permid): + if DEBUG: + print >> sys.stderr, "helper: network_got_request_pieces: The node asking for help is not the current coordinator" + return + + helper_obj.got_request_pieces(permid, pieces) + # Wake up download thread + helper_obj.notify() diff --git a/instrumentation/next-share/BaseLib/Core/ProxyService/ProxyServiceUtil.py b/instrumentation/next-share/BaseLib/Core/ProxyService/ProxyServiceUtil.py new file mode 100644 index 0000000..6ff3467 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/ProxyService/ProxyServiceUtil.py @@ -0,0 +1,46 @@ +# Written George Milescu +# see LICENSE.txt for license information + +# This class contains all util methods related to the ProxyService + +import string +import random + +def generate_proxy_challenge(): + """ Generates a challenge (8 byte long random number) that a doe sends to a proxy during the handshake + + @return: an 8 byte log string + """ + # Generate a random challenge - random number on 8 bytes (62**8 possible combinations) + chars = string.letters + string.digits #len(chars)=62 + challenge = '' + for i in range(8): + challenge = challenge + random.choice(chars) + + return challenge + + +def decode_challenge_from_peerid(peerid): + """ Method used to retrieve (decode) a challenge from a peerid + + @param peerid: the peerid of the peer whose challenge is to be retrieved + + @return: a number, the challenge previously send to that peer, and encoded by the peer in its peerid + """ + + return peerid[12:20] + + +def encode_challenge_in_peerid(peerid, challenge): + """ Method used to insert (encode) a challenge into a peerid + + @param peerid: the regular peerid, into which the challenge will be encoded + @param challenge: an 8 byte long number, to be encoded in the peerid + + @return: a new peerid, with the challenge encoded in it + """ + + # proxy_peer_id = | regular_peer_id[1:12] | challenge[1:8] | + proxy_peer_id = peerid[:12] + challenge # len(self.challenge) = 8 + + return proxy_peer_id diff --git a/instrumentation/next-share/BaseLib/Core/ProxyService/RatePredictor.py b/instrumentation/next-share/BaseLib/Core/ProxyService/RatePredictor.py new file mode 100644 index 0000000..ddd8673 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/ProxyService/RatePredictor.py @@ -0,0 +1,59 @@ +# Written by Pawel Garbacki, George Milescu +# see LICENSE.txt for license information + +import sys +from BaseLib.Core.BitTornado.clock import clock + +MIN_CAPACITY = 0.75 +DEBUG = False + +class RatePredictor: + def __init__(self, raw_server, rate_measure, max_rate, probing_period = 2): + self.raw_server = raw_server + self.rate_measure = rate_measure + if max_rate == 0: + self.max_rate = 2147483647 + else: + self.max_rate = max_rate + self.probing_period = probing_period # in seconds + +class ExpSmoothRatePredictor(RatePredictor): + def __init__(self, raw_server, rate_measure, max_rate, alpha = 0.5, max_period = 30, probing_period = 2): + RatePredictor.__init__(self, raw_server, rate_measure, max_rate, probing_period) + if DEBUG: print >>sys.stderr, "RatePredictor:__init__" + self.alpha = alpha + self.max_period = max_period + self.value = None + self.timestamp = None + + def update(self): + if DEBUG: print >>sys.stderr, "RatePredictor:update" + self.raw_server.add_task(self.update, self.probing_period) + current_value = self.rate_measure.get_rate() / 1000. + current_time = clock() + if self.value is None or current_time - self.timestamp > self.max_period: + self.value = current_value + else: + self.value = self.alpha * current_value + (1 - self.alpha) * self.value + if self.max_rate > 0 and self.value > self.max_rate: + self.value = self.max_rate + self.timestamp = current_time + + # exponential smoothing prediction + def predict(self): + if DEBUG: print >>sys.stderr, "RatePredictor:predict" + # self.update() + if self.value is None: + return 0 + return self.value + + def has_capacity(self): + if DEBUG: print >>sys.stderr, "RatePredictor:has_capacity" +# return False + # self.update() + result = None + if self.value is None: + result = True + else: + result = (1. - float(self.value) / self.max_rate) > MIN_CAPACITY + return result diff --git a/instrumentation/next-share/BaseLib/Core/ProxyService/__init__.py b/instrumentation/next-share/BaseLib/Core/ProxyService/__init__.py new file mode 100644 index 0000000..afcd0f2 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/ProxyService/__init__.py @@ -0,0 +1,2 @@ +# Written by Arno Bakker, George Milescu +# see LICENSE.txt for license information diff --git a/instrumentation/next-share/BaseLib/Core/RequestPolicy.py b/instrumentation/next-share/BaseLib/Core/RequestPolicy.py new file mode 100644 index 0000000..58ffcb8 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/RequestPolicy.py @@ -0,0 +1,138 @@ +# Written by Jelle Roozenburg +# see LICENSE.txt for license information +""" Controls the authorization of messages received via the Tribler Overlay """ + +from BaseLib.Core.simpledefs import * +from BaseLib.Core.exceptions import * +from BaseLib.Core.BitTornado.BT1.MessageID import * + +DEBUG = False + +MAX_QUERIES_FROM_RANDOM_PEER = 1000 + + +class AbstractRequestPolicy: + """ Superclass for all Tribler RequestPolicies. A RequestPolicy controls + the authorization of messages received via the Tribler Overlay, such + as distributed recommendations, remote queries, etc. + """ + def __init__(self): + """ Constructor """ + + def allowed(self, permid, messageID): + """ Returns whether or not the peer identified by permid is allowed to + send us a message of type messageID. + @param permid The permid of the sending peer. + @param messageID A integer messageID, see BaseLib.Core.BitTornado.BT1.MessageID + @returns A boolean indicating whether the message is authorized. + """ + raise NotYetImplementedException() + + +class AllowAllRequestPolicy(AbstractRequestPolicy): + """ A RequestPolicy that allows all messages to be sent by all peers. """ + + def allowed(self, permid, messageID): + return self.allowAllRequestsAllPeers(permid, messageID) + + def allowAllRequestsAllPeers(self, permid, messageID): + return True + + +class CommonRequestPolicy(AbstractRequestPolicy): + """ A base class implementing some methods that can be used as building + blocks for RequestPolicies. + """ + def __init__(self,session): + """ Constructor """ + self.session = session + self.friendsdb = session.open_dbhandler(NTFY_FRIENDS) + self.peerdb = session.open_dbhandler(NTFY_PEERS) + AbstractRequestPolicy.__init__(self) + + def isFriend(self, permid): + """ + @param permid The permid of the sending peer. + @return Whether or not the specified permid is a friend. + """ + fs = self.friendsdb.getFriendState(permid) + return (fs == FS_MUTUAL or fs == FS_I_INVITED) + + def isSuperPeer(self, permid): + """ + @param permid The permid of the sending peer. + @return Whether of not the specified permid is a superpeer. + """ + return permid in self.session.lm.superpeer_db.getSuperPeers() + + def isCrawler(self, permid): + """ + @param permid The permid of the sending peer. + @return Whether of not the specified permid is a crawler. + """ + return permid in self.session.lm.crawler_db.getCrawlers() + + def benign_random_peer(self,permid): + """ + @param permid The permid of the sending peer. + @return Whether or not the specified permid has exceeded his + quota of remote query messages. + """ + if MAX_QUERIES_FROM_RANDOM_PEER > 0: + nqueries = self.get_peer_nqueries(permid) + return nqueries < MAX_QUERIES_FROM_RANDOM_PEER + else: + return True + + def get_peer_nqueries(self, permid): + """ + @param permid The permid of the sending peer. + @return The number of remote query messages already received from + this peer. + """ + peer = self.peerdb.getPeer(permid) + #print >>sys.stderr,"CommonRequestPolicy: get_peer_nqueries: getPeer",`permid`,peer + #print >>sys.stderr,"CommonRequestPolicy: get_peer_nqueries: called by",currentThread().getName() + if peer is None: + return 0 + else: + return peer['num_queries'] + +class AllowFriendsRequestPolicy(CommonRequestPolicy): + """ + A RequestPolicy that allows all non-crawler messages to be sent by + friends only. Crawler messages are allowed from Crawlers only. + """ + + def allowed(self, permid, messageID): + if messageID in (CRAWLER_REQUEST, CRAWLER_REPLY): + return self.isCrawler(permid) + else: + return self.allowAllRequestsFromFriends(permid, messageID) + + def allowAllRequestsFromFriends(self, permid, messageID): + # Access control + return self.isFriend(permid) + + +class FriendsCoopDLOtherRQueryQuotumCrawlerAllowAllRequestPolicy(CommonRequestPolicy): + """ + Allows friends to send all messages related to cooperative + downloads, subjects all other peers to a remote query quotum of + 100, and allows all peers to send all other non-crawler + messages. Crawler messages are allowed from Crawlers only. + """ + + def allowed(self, permid, messageID): + """ Returns whether or not the peer identified by permid is allowed to + send us a message of type messageID. + @return Boolean. """ + if messageID == CRAWLER_REQUEST: + return self.isCrawler(permid) + elif (messageID in HelpCoordinatorMessages or messageID in HelpHelperMessages) and not self.isFriend(permid): + return False + elif messageID == QUERY and not (self.isFriend(permid) or self.benign_random_peer(permid)): + return False + else: + return True + diff --git a/instrumentation/next-share/BaseLib/Core/Search/KeywordSearch.py b/instrumentation/next-share/BaseLib/Core/Search/KeywordSearch.py new file mode 100644 index 0000000..3421a84 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/Search/KeywordSearch.py @@ -0,0 +1,107 @@ +# written by Jelle Roozenburg +# see LICENSE.txt for license information + +import re +import sys + +DEBUG = False + +class KeywordSearch: + """ + Tribler keywordsearch now has the following features: + 1. All items with one of the keywords in the 'name' field are returned (self.simpleSearch() ) + 2. The sorting of the results is based on: + a) The number of matching keywords + b) The length of the matching keywords + c) If the keywords matched a whole word (search for 'cat' find 'category') + (done in self.search() ) + 3. Searching is case insensitive + """ + def search(self, haystack, needles, haystackismatching=False): + if DEBUG: + print >>sys.stderr,'kws: unprocessed keywords: %s' % needles + needles = self.unRegExpifySearchwords(needles) + if DEBUG: + print >>sys.stderr,'kws: Searching for %s in %d items' % (repr(needles), len(haystack)) + + if not haystackismatching: + searchspace = self.simpleSearch(haystack, needles) + if DEBUG: + print >>sys.stderr,'kws: Found %s items using simple search' % len(searchspace) + else: + searchspace = haystack + results = [] + wbsearch = [] + + for needle in needles: + wbsearch.append(re.compile(r'\b%s\b' % needle)) + + for item in searchspace: + title = item['name'].lower() + score = 0 + for i in xrange(len(needles)): + wb = wbsearch[i].findall(title) + score += len(wb) * 2 * len(needles[i]) + if len(wb) == 0: + if title.find(needles[i].lower()) != -1: + score += len(needles[i]) + + results.append((score, item)) + + results.sort(reverse=True) + if DEBUG: + print >>sys.stderr,'kws: Found %d items eventually' % len(results) + #for r in results: + # print r + return [r[1] for r in results] + + + def unRegExpifySearchwords(self, needles): + replaceRegExpChars = re.compile(r'(\\|\*|\.|\+|\?|\||\(|\)|\[|\]|\{|\})') + new_needles = [] + for needle in needles: + needle = needle.strip() + if len(needle)== 0: + continue + new_needle = re.sub(replaceRegExpChars, r'\\\1', needle.lower()) + new_needles.append(new_needle) + return new_needles + + def simpleSearch(self, haystack, needles, searchtype='AND'): + "Can do both OR or AND search" + hits = [] + if searchtype == 'OR': + searchRegexp = r'' + for needle in needles: + searchRegexp+= needle+'|' + searchRegexp = re.compile(searchRegexp[:-1]) + for item in haystack: + title = item['name'].lower() + if len(searchRegexp.findall(title)) > 0: + hits.append(item) + elif searchtype == 'AND': + for item in haystack: + title = item['name'].lower() + foundAll = True + for needle in needles: + if title.find(needle) == -1: + foundAll = False + break + if foundAll: + hits.append(item) + return hits + + +def test(): + data = [{'name':'Fedoras 3.10'}, + {'name':'Fedora 2.10'}, + {'name':'Movie 3.10'}, + {'name':'fedora_2'}, + {'name':'movie_theater.avi'} + ] + words = ['fedora', '1'] + #print KeywordSearch().simpleSearch(data, words) + print KeywordSearch().search(data, words) +if __name__ == '__main__': + test() + diff --git a/instrumentation/next-share/BaseLib/Core/Search/Reranking.py b/instrumentation/next-share/BaseLib/Core/Search/Reranking.py new file mode 100644 index 0000000..2df031e --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/Search/Reranking.py @@ -0,0 +1,97 @@ +# written by Nicolas Neubauer +# see LICENSE.txt for license information + +import sys, time + +DEBUG = False + +class Reranker: + def getID(self): + """the ID that is stored in the clicklog 'reranking_strategy' field for later comparison""" + return 0 + + def rerank(self, hits, keywords, torrent_db, pref_db, mypref_db, search_db): + """takes hits and reorders them given the current keywords""" + return hits + +class DefaultTorrentReranker(Reranker): + """ just leave the hits alone """ + def getID(self): + return 1 + def rerank(self, hits, keywords, torrent_db, pref_db, mypref_db, search_db): + return hits + +class TestReranker(Reranker): + """ for testing purposes only """ + def getID(self): + return 2 + def rerank(self, hits, keywords, torrent_db, pref_db, mypref_db, search_db): + if len(hits)>1: + h = hits[0] + hits[0] = hits[1] + hits[1] = h + return hits + +class SwapFirstTwoReranker(Reranker): + """ swaps first and second place if second place has been frequently selected from bad position """ + + def __init__(self): + self.MAX_SEEN_BEFORE_RERANK = 5 + self.MAX_POPULAR_RATIO = 5 + + def getID(self): + return 2 + + def rerank(self, hits, keywords, torrent_db, pref_db, mypref_db, search_db): + if len(hits)<2: + return hits + + torrent_id_0 = hits[0].get('torrent_id',0) + torrent_id_1 = hits[1].get('torrent_id',0) + if torrent_id_0 == 0 or torrent_id_1 == 0: + if DEBUG: + print >> sys.stderr, "reranking: torrent_id=0 in hits, exiting" + # we got some problems elsewhere, don't add to it + return hits + + (num_hits_0, position_score_0) = pref_db.getPositionScore(torrent_id_0, keywords) + (num_hits_1, position_score_1) = pref_db.getPositionScore(torrent_id_1, keywords) + if DEBUG: + print >> sys.stderr, "reranking: first torrent (%d): (num, score)= (%s, %s)" % (torrent_id_0, num_hits_0, position_score_0) + print >> sys.stderr, "reranking: second torrent (%d): (num, score)= (%s, %s)" % (torrent_id_1, num_hits_1, position_score_1) + + if (num_hits_0 < self.MAX_SEEN_BEFORE_RERANK or num_hits_1 < self.MAX_SEEN_BEFORE_RERANK): + # only start thinking about reranking if we have seen enough samples + if DEBUG: + print >> sys.stderr, "reranking: not enough samples, not reranking" + return hits + + if (num_hits_0/num_hits_1 > self.MAX_POPULAR_RATIO): + # if number one is much more popular, keep everything as it is + if DEBUG: + print >> sys.stderr, "reranking: first torrent is too popular, not reranking" + return hits + + # if all these tests are successful, we may swap first and second if second + # has gotten hits from worse positions than first + + if position_score_0> sys.stderr, "reranking: second torrent has better position score, reranking!" + h = hits[0] + hits[0] = hits[1] + hits[1] = h + else: + if DEBUG: + print >> sys.stderr, "reranking: second torrent does not have better position score, reranking!" + + return hits + +_rerankers = [DefaultTorrentReranker(), SwapFirstTwoReranker()] + + +def getTorrentReranker(): + global _rerankers + index = int(time.strftime("%H")) % (len(_rerankers)) + return _rerankers[index] + diff --git a/instrumentation/next-share/BaseLib/Core/Search/SearchManager.py b/instrumentation/next-share/BaseLib/Core/Search/SearchManager.py new file mode 100644 index 0000000..06792f4 --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/Search/SearchManager.py @@ -0,0 +1,51 @@ +# Written by Jelle Roozenburg, Arno Bakker +# see LICENSE.txt for license information + +# ARNOCOMMENT: remove this now it doesn't use KeywordSearch anymore? + +import re +import sys + +#from BaseLib.Core.Search.KeywordSearch import KeywordSearch + +DEBUG = False + +re_keywordsplit = re.compile(r"[\W_]", re.UNICODE) +def split_into_keywords(string): + """ + Takes a (unicode) string and returns a list of (unicode) lowercase + strings. No empty strings are returned. + + We currently split on non-alphanumeric characters and the + underscore. This ensures that the keywords are SQL insertion + proof. + """ + return [keyword for keyword in re_keywordsplit.split(string.lower()) if len(keyword) > 0] + + +class SearchManager: + """ Arno: This is DB neutral. All it assumes is a DBHandler with + a searchNames() method that returns records with at least a 'name' field + in them. + """ + + def __init__(self,dbhandler): + self.dbhandler = dbhandler + # self.keywordsearch = KeywordSearch() + + def search(self,kws,maxhits=None): + """ Called by any thread """ + if DEBUG: + print >>sys.stderr,"SearchManager: search",kws + + hits = self.dbhandler.searchNames(kws) + if maxhits is None: + return hits + else: + return hits[:maxhits] + + def searchChannels(self, query): ## + data = self.dbhandler.searchChannels(query) + return data + + diff --git a/instrumentation/next-share/BaseLib/Core/Search/__init__.py b/instrumentation/next-share/BaseLib/Core/Search/__init__.py new file mode 100644 index 0000000..395f8fb --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/Search/__init__.py @@ -0,0 +1,2 @@ +# Written by Arno Bakker +# see LICENSE.txt for license information diff --git a/instrumentation/next-share/BaseLib/Core/Session.py b/instrumentation/next-share/BaseLib/Core/Session.py new file mode 100644 index 0000000..840507b --- /dev/null +++ b/instrumentation/next-share/BaseLib/Core/Session.py @@ -0,0 +1,914 @@ +# Written by Arno Bakker +# see LICENSE.txt for license information +""" A Session is a running instance of the Tribler Core and the Core's central class. """ + +import os +import sys +import copy +import binascii +from traceback import print_exc +from threading import RLock + +from BaseLib.__init__ import LIBRARYNAME +from BaseLib.Core.simpledefs import * +from BaseLib.Core.defaults import sessdefaults +from BaseLib.Core.Base import * +from BaseLib.Core.SessionConfig import * +from BaseLib.Core.DownloadConfig import get_default_dest_dir +from BaseLib.Core.Utilities.utilities import find_prog_in_PATH +from BaseLib.Core.APIImplementation.SessionRuntimeConfig import SessionRuntimeConfig +from BaseLib.Core.APIImplementation.LaunchManyCore import TriblerLaunchMany +from BaseLib.Core.APIImplementation.UserCallbackHandler import UserCallbackHandler +from BaseLib.Core.osutils import get_appstate_dir +GOTM2CRYPTO=False +try: + import M2Crypto + import BaseLib.Core.Overlay.permid as permidmod + GOTM2CRYPTO=True +except ImportError: + pass + +DEBUG = False + +class Session(SessionRuntimeConfig): + """ + + A Session is a running instance of the Tribler Core and the Core's central + class. It implements the SessionConfigInterface which can be used to change + session parameters at runtime (for selected parameters). + + cf. libtorrent session + """ + __single = None + + + def __init__(self,scfg=None,ignore_singleton=False): + """ + A Session object is created which is configured following a copy of the + SessionStartupConfig scfg. (copy constructor used internally) + + @param scfg SessionStartupConfig object or None, in which case we + look for a saved session in the default location (state dir). If + we can't find it, we create a new SessionStartupConfig() object to + serve as startup config. Next, the config is saved in the directory + indicated by its 'state_dir' attribute. + + In the current implementation only a single session instance can exist + at a time in a process. The ignore_singleton flag is used for testing. + """ + if not ignore_singleton: + if Session.__single: + raise RuntimeError, "Session is singleton" + Session.__single = self + + self.sesslock = RLock() + + # Determine startup config to use + if scfg is None: # If no override + try: + # Then try to read from default location + state_dir = Session.get_default_state_dir() + cfgfilename = Session.get_default_config_filename(state_dir) + scfg = SessionStartupConfig.load(cfgfilename) + except: + # If that fails, create a fresh config with factory defaults + print_exc() + scfg = SessionStartupConfig() + self.sessconfig = scfg.sessconfig + else: # overrides any saved config + # Work from copy + self.sessconfig = copy.copy(scfg.sessconfig) + + # Create dir for session state, if not exist + state_dir = self.sessconfig['state_dir'] + if state_dir is None: + state_dir = Session.get_default_state_dir() + self.sessconfig['state_dir'] = state_dir + + if not os.path.isdir(state_dir): + os.makedirs(state_dir) + + collected_torrent_dir = self.sessconfig['torrent_collecting_dir'] + if not collected_torrent_dir: + collected_torrent_dir = os.path.join(self.sessconfig['state_dir'], STATEDIR_TORRENTCOLL_DIR) + self.sessconfig['torrent_collecting_dir'] = collected_torrent_dir + + collected_subtitles_dir = self.sessconfig.get('subtitles_collecting_dir',None) + if not collected_subtitles_dir: + collected_subtitles_dir = os.path.join(self.sessconfig['state_dir'], STATEDIR_SUBSCOLL_DIR) + self.sessconfig['subtitles_collecting_dir'] = collected_subtitles_dir + + if not os.path.exists(collected_torrent_dir): + os.makedirs(collected_torrent_dir) + + if not self.sessconfig['peer_icon_path']: + self.sessconfig['peer_icon_path'] = os.path.join(self.sessconfig['state_dir'], STATEDIR_PEERICON_DIR) + + # PERHAPS: load default TorrentDef and DownloadStartupConfig from state dir + # Let user handle that, he's got default_state_dir, etc. + + # Core init + #print >>sys.stderr,'Session: __init__ config is', self.sessconfig + + if GOTM2CRYPTO: + permidmod.init() + + # + # Set params that depend on state_dir + # + # 1. keypair + # + pairfilename = os.path.join(self.sessconfig['state_dir'],'ec.pem') + if self.sessconfig['eckeypairfilename'] is None: + self.sessconfig['eckeypairfilename'] = pairfilename + + if os.access(self.sessconfig['eckeypairfilename'],os.F_OK): + # May throw exceptions + self.keypair = permidmod.read_keypair(self.sessconfig['eckeypairfilename']) + else: + self.keypair = permidmod.generate_keypair() + + # Save keypair + pubfilename = os.path.join(self.sessconfig['state_dir'],'ecpub.pem') + permidmod.save_keypair(self.keypair,pairfilename) + permidmod.save_pub_key(self.keypair,pubfilename) + + # 2. Downloads persistent state dir + dlpstatedir = os.path.join(self.sessconfig['state_dir'],STATEDIR_DLPSTATE_DIR) + if not os.path.isdir(dlpstatedir): + os.mkdir(dlpstatedir) + + # 3. tracker + trackerdir = self.get_internal_tracker_dir() + if not os.path.isdir(trackerdir): + os.mkdir(trackerdir) + + if self.sessconfig['tracker_dfile'] is None: + self.sessconfig['tracker_dfile'] = os.path.join(trackerdir,'tracker.db') + + if self.sessconfig['tracker_allowed_dir'] is None: + self.sessconfig['tracker_allowed_dir'] = trackerdir + + if self.sessconfig['tracker_logfile'] is None: + if sys.platform == "win32": + # Not "Nul:" but "nul" is /dev/null on Win32 + sink = 'nul' + else: + sink = '/dev/null' + self.sessconfig['tracker_logfile'] = sink + + # 4. superpeer.txt and crawler.txt + if self.sessconfig['superpeer_file'] is None: + self.sessconfig['superpeer_file'] = os.path.join(self.sessconfig['install_dir'],LIBRARYNAME,'Core','superpeer.txt') + if 'crawler_file' not in self.sessconfig or self.sessconfig['crawler_file'] is None: + self.sessconfig['crawler_file'] = os.path.join(self.sessconfig['install_dir'], LIBRARYNAME,'Core','Statistics','crawler.txt') + + # 5. download_help_dir + if self.sessconfig['overlay'] and self.sessconfig['download_help']: + if self.sessconfig['download_help_dir'] is None: + self.sessconfig['download_help_dir'] = os.path.join(get_default_dest_dir(),DESTDIR_COOPDOWNLOAD) + # Jelle: under linux, default_dest_dir can be /tmp. Then download_help_dir can be deleted inbetween + # sessions. + if not os.path.isdir(self.sessconfig['download_help_dir']): + os.makedirs(self.sessconfig['download_help_dir']) + + # 6. peer_icon_path + if self.sessconfig['peer_icon_path'] is None: + self.sessconfig['peer_icon_path'] = os.path.join(self.sessconfig['state_dir'],STATEDIR_PEERICON_DIR) + if not os.path.isdir(self.sessconfig['peer_icon_path']): + os.mkdir(self.sessconfig['peer_icon_path']) + + # 7. Poor man's versioning of SessionConfig, add missing + # default values. Really should use PERSISTENTSTATE_CURRENTVERSION + # and do conversions. + for key,defvalue in sessdefaults.iteritems(): + if key not in self.sessconfig: + self.sessconfig[key] = defvalue + + if not 'live_aux_seeders' in self.sessconfig: + # Poor man's versioning, really should update PERSISTENTSTATE_CURRENTVERSION + self.sessconfig['live_aux_seeders'] = sessdefaults['live_aux_seeders'] + + if not 'nat_detect' in self.sessconfig: + self.sessconfig['nat_detect'] = sessdefaults['nat_detect'] + if not 'puncturing_internal_port' in self.sessconfig: + self.sessconfig['puncturing_internal_port'] = sessdefaults['puncturing_internal_port'] + if not 'stun_servers' in self.sessconfig: + self.sessconfig['stun_servers'] = sessdefaults['stun_servers'] + if not 'pingback_servers' in self.sessconfig: + self.sessconfig['pingback_servers'] = sessdefaults['pingback_servers'] + if not 'mainline_dht' in self.sessconfig: + self.sessconfig['mainline_dht'] = sessdefaults['mainline_dht'] + + # Checkpoint startup config + self.save_pstate_sessconfig() + + # Create handler for calling back the user via separate threads + self.uch = UserCallbackHandler(self) + + # Create engine with network thread + self.lm = TriblerLaunchMany() + self.lm.register(self,self.sesslock) + self.lm.start() + + + # + # Class methods + # + def get_instance(*args, **kw): + """ Returns the Session singleton if it exists or otherwise + creates it first, in which case you need to pass the constructor + params. + @return Session.""" + if Session.__single is None: + Session(*args, **kw) + return Session.__single + get_instance = staticmethod(get_instance) + + def get_default_state_dir(homedirpostfix='.Tribler'): + """ Returns the factory default directory for storing session state + on the current platform (Win32,Mac,Unix). + @return An absolute path name. """ + + # Allow override + statedirvar = '${TSTATEDIR}' + statedir = os.path.expandvars(statedirvar) + if statedir and statedir != statedirvar: + return statedir + + appdir = get_appstate_dir() + statedir = os.path.join(appdir, homedirpostfix) + return statedir + + get_default_state_dir = staticmethod(get_default_state_dir) + + + # + # Public methods + # + def start_download(self,tdef,dcfg=None,initialdlstatus=None): + """ + Creates a Download object and adds it to the session. The passed + TorrentDef and DownloadStartupConfig are copied into the new Download + object. The Download is then started and checkpointed. + + If a checkpointed version of the Download is found, that is restarted + overriding the saved DownloadStartupConfig if "dcfg" is not None. + + @param tdef A finalized TorrentDef + @param dcfg DownloadStartupConfig or None, in which case + a new DownloadStartupConfig() is created with its default settings + and the result becomes the runtime config of this Download. + @param initialdlstatus The initial download status of this Download + or None. This enables the caller to create a Download in e.g. + DLSTATUS_REPEXING state instead. + @return Download + """ + # locking by lm + return self.lm.add(tdef,dcfg,initialdlstatus=initialdlstatus) + + def resume_download_from_file(self,filename): + """ + Recreates Download from resume file + + @return a Download object. + + Note: this cannot be made into a method of Download, as the Download + needs to be bound to a session, it cannot exist independently. + """ + raise NotYetImplementedException() + + def get_downloads(self): + """ + Returns a copy of the list of Downloads. + @return A list of Download objects. + """ + # locking by lm + return self.lm.get_downloads() + + + def remove_download(self,d,removecontent=False): + """ + Stops the download and removes it from the session. + @param d The Download to remove + @param removecontent Whether to delete the already downloaded content + from disk. + """ + # locking by lm + self.lm.remove(d,removecontent=removecontent) + + + def set_download_states_callback(self,usercallback,getpeerlist=False): + """ + See Download.set_state_callback. Calls usercallback with a list of + DownloadStates, one for each Download in the Session as first argument. + The usercallback must return a tuple (when,getpeerlist) that indicates + when to reinvoke the callback again (as a number of seconds from now, + or < 0.0 if not at all) and whether to also include the details of + the connected peers in the DownloadStates on that next call. + + The callback will be called by a popup thread which can be used + indefinitely (within reason) by the higher level code. + + @param usercallback A function adhering to the above spec. + """ + self.lm.set_download_states_callback(usercallback,getpeerlist) + + + # + # Config parameters that only exist at runtime + # + def get_permid(self): + """ Returns the PermID of the Session, as determined by the + SessionConfig.set_permid() parameter. A PermID is a public key + @return The PermID encoded in a string in DER format. """ + self.sesslock.acquire() + try: + return str(self.keypair.pub().get_der()) + finally: + self.sesslock.release() + + def get_external_ip(self): + """ Returns the external IP address of this Session, i.e., by which + it is reachable from the Internet. This address is determined via + various mechanisms such as the UPnP protocol, our dialback mechanism, + and an inspection of the local network configuration. + @return A string. """ + # locking done by lm + return self.lm.get_ext_ip() + + + def get_externally_reachable(self): + """ Returns whether the Session is externally reachable, i.e., its + listen port is not firewalled. Use add_observer() with NTFY_REACHABLE + to register to the event of detecting reachablility. Note that due to + the use of UPnP a Session may become reachable some time after + startup and due to the Dialback mechanism, this method may return + False while the Session is actually already reachable. Note that True + doesn't mean the Session is reachable from the open Internet, could just + be from the local (otherwise firewalled) LAN. + @return A boolean. """ + + # Arno, LICHT: make it throw exception when used in LITE versie. + from BaseLib.Core.NATFirewall.DialbackMsgHandler import DialbackMsgHandler + + return DialbackMsgHandler.getInstance().isConnectable() + + + def get_current_startup_config_copy(self): + """ Returns a SessionStartupConfig that is a copy of the current runtime + SessionConfig. + @return SessionStartupConfig + """ + # Called by any thread + self.sesslock.acquire() + try: + sessconfig = copy.copy(self.sessconfig) + return SessionStartupConfig(sessconfig=sessconfig) + finally: + self.sesslock.release() + + # + # Internal tracker + # + def get_internal_tracker_url(self): + """ Returns the announce URL for the internal tracker. + @return URL """ + # Called by any thread + self.sesslock.acquire() + try: + url = None + if 'tracker_url' in self.sessconfig: + url = self.sessconfig['tracker_url'] # user defined override, e.g. specific hostname + if url is None: + ip = self.lm.get_ext_ip() + port = self.get_listen_port() + url = 'http://'+ip+':'+str(port)+'/announce/' + return url + finally: + self.sesslock.release() + + + def get_internal_tracker_dir(self): + """ Returns the directory containing the torrents tracked by the internal + tracker (and associated databases). + @return An absolute path. """ + # Called by any thread + self.sesslock.acquire() + try: + if self.sessconfig['state_dir'] is None: + return None + else: + return os.path.join(self.sessconfig['state_dir'],STATEDIR_ITRACKER_DIR) + finally: + self.sesslock.release() + + + def add_to_internal_tracker(self,tdef): + """ Add a torrent def to the list of torrents tracked by the internal + tracker. Use this method to use the Session as a standalone tracker. + @param tdef A finalized TorrentDef. + """ + # Called by any thread + self.sesslock.acquire() + try: + infohash = tdef.get_infohash() + filename = self.get_internal_tracker_torrentfilename(infohash) + tdef.save(filename) + + print >>sys.stderr,"Session: add_to_int_tracker: saving to",filename,"url-compat",tdef.get_url_compat() + + # Bring to attention of Tracker thread + self.lm.tracker_rescan_dir() + finally: + self.sesslock.release() + + def remove_from_internal_tracker(self,tdef): + """ Remove a torrent def from the list of torrents tracked by the + internal tracker. Use this method to use the Session as a standalone + tracker. + @param tdef A finalized TorrentDef. + """ + infohash = tdef.get_infohash() + self.remove_from_internal_tracker_by_infohash(infohash) + + def remove_from_internal_tracker_by_infohash(self,infohash): + """ Remove a torrent def from the list of torrents tracked by the + internal tracker. Use this method to use the Session as a standalone + tracker. + @param infohash Identifier of the torrent def to remove. + """ + # Called by any thread + self.sesslock.acquire() + try: + filename = self.get_internal_tracker_torrentfilename(infohash) + if DEBUG: + print >>sys.stderr,"Session: removing itracker entry",filename + if os.access(filename,os.F_OK): + os.remove(filename) + # Bring to attention of Tracker thread + self.lm.tracker_rescan_dir() + finally: + self.sesslock.release() + + # + # Notification of events in the Session + # + def add_observer(self, func, subject, changeTypes = [NTFY_UPDATE, NTFY_INSERT, NTFY_DELETE], objectID = None): + """ Add an observer function function to the Session. The observer + function will be called when one of the specified events (changeTypes) + occurs on the specified subject. + + The function will be called by a popup thread which can be used + indefinitely (within reason) by the higher level code. + + @param func The observer function. It should accept as its first argument + the subject, as second argument the changeType, as third argument an + objectID (e.g. the primary key in the observed database) and an + optional list of arguments. + @param subject The subject to observe, one of NTFY_* subjects (see + simpledefs). + @param changeTypes The list of events to be notified of one of NTFY_* + events. + @param objectID The specific object in the subject to monitor (e.g. a + specific primary key in a database to monitor for updates.) + + + TODO: Jelle will add per-subject/event description here ;o) + + """ + #Called by any thread + self.uch.notifier.add_observer(func, subject, changeTypes, objectID) # already threadsafe + + def remove_observer(self, func): + """ Remove observer function. No more callbacks will be made. + @param func The observer function to remove. """ + #Called by any thread + self.uch.notifier.remove_observer(func) # already threadsafe + + def open_dbhandler(self,subject): + """ Opens a connection to the specified database. Only the thread + calling this method may use this connection. The connection must be + closed with close_dbhandler() when this thread exits. + + @param subject The database to open. Must be one of the subjects + specified here. + @return A reference to a DBHandler class for the specified subject or + None when the Session was not started with megacaches enabled. + 
 NTFY_PEERS -> PeerDBHandler
+        NTFY_TORRENTS -> TorrentDBHandler
+        NTFY_PREFERENCES -> PreferenceDBHandler
+        NTFY_SUPERPEERS -> SuperpeerDBHandler
+        NTFY_FRIENDS -> FriendsDBHandler
+        NTFY_MYPREFERENCES -> MyPreferenceDBHandler
+        NTFY_BARTERCAST -> BartercastDBHandler
+        NTFY_SEARCH -> SearchDBHandler
+        NTFY_TERM -> TermDBHandler
+        NTFY_VOTECAST -> VotecastDBHandler
+        NTFY_CHANNELCAST -> ChannelCastDBHandler
+        NTFY_RICH_METADATA -> MetadataDBHandler
+
+ """ + # Called by any thread + self.sesslock.acquire() + try: + if subject == NTFY_PEERS: + return self.lm.peer_db + elif subject == NTFY_TORRENTS: + return self.lm.torrent_db + elif subject == NTFY_PREFERENCES: + return self.lm.pref_db + elif subject == NTFY_SUPERPEERS: + return self.lm.superpeer_db + elif subject == NTFY_FRIENDS: + return self.lm.friend_db + elif subject == NTFY_MYPREFERENCES: + return self.lm.mypref_db + elif subject == NTFY_BARTERCAST: + return self.lm.bartercast_db + elif subject == NTFY_SEEDINGSTATS: + return self.lm.seedingstats_db + elif subject == NTFY_SEEDINGSTATSSETTINGS: + return self.lm.seedingstatssettings_db + elif subject == NTFY_VOTECAST: + return self.lm.votecast_db + elif subject == NTFY_SEARCH: + return self.lm.search_db + elif subject == NTFY_TERM: + return self.lm.term_db + elif subject == NTFY_CHANNELCAST: + return self.lm.channelcast_db + elif subject == NTFY_RICH_METADATA: + return self.lm.richmetadataDbHandler + else: + raise ValueError('Cannot open DB subject: '+subject) + finally: + self.sesslock.release() + + + def close_dbhandler(self,dbhandler): + """ Closes the given database connection """ + dbhandler.close() + + + # + # Access control + # + def set_overlay_request_policy(self, reqpol): + """ + Set a function which defines which overlay requests (e.g. dl_helper, rquery msg) + will be answered or will be denied. + + The function will be called by a network thread and must return + as soon as possible to prevent performance problems. + + @param reqpol is a BaseLib.Core.RequestPolicy.AbstractRequestPolicy + object. + """ + # Called by any thread + # to protect self.sessconfig + self.sesslock.acquire() + try: + overlay_loaded = self.sessconfig['overlay'] + finally: + self.sesslock.release() + if overlay_loaded: + self.lm.overlay_apps.setRequestPolicy(reqpol) # already threadsafe + elif DEBUG: + print >>sys.stderr,"Session: overlay is disabled, so no overlay request policy needed" + + + # + # Persistence and shutdown + # + def load_checkpoint(self,initialdlstatus=None): + """ Restart Downloads from checkpoint, if any. + + This method allows the API user to manage restoring downloads. + E.g. a video player that wants to start the torrent the user clicked + on first, and only then restart any sleeping torrents (e.g. seeding). + The optional initialdlstatus parameter can be set to DLSTATUS_STOPPED + to restore all the Downloads in DLSTATUS_STOPPED state. + """ + self.lm.load_checkpoint(initialdlstatus) + + + def checkpoint(self): + """ Saves the internal session state to the Session's state dir. """ + #Called by any thread + self.checkpoint_shutdown(stop=False,checkpoint=True,gracetime=None,hacksessconfcheckpoint=False) + + def shutdown(self,checkpoint=True,gracetime=2.0,hacksessconfcheckpoint=True): + """ Checkpoints the session and closes it, stopping the download engine. + @param checkpoint Whether to checkpoint the Session state on shutdown. + @param gracetime Time to allow for graceful shutdown + signoff (seconds). + """ + # Called by any thread + self.lm.early_shutdown() + self.checkpoint_shutdown(stop=True,checkpoint=checkpoint,gracetime=gracetime,hacksessconfcheckpoint=hacksessconfcheckpoint) + # Arno, 2010-08-09: now shutdown after gracetime + #self.uch.shutdown() + + def has_shutdown(self): + """ Whether the Session has completely shutdown, i.e., its internal + threads are finished and it is safe to quit the process the Session + is running in. + @return A Boolean. + """ + return self.lm.sessdoneflag.isSet() + + def get_downloads_pstate_dir(self): + """ Returns the directory in which to checkpoint the Downloads in this + Session. """ + # Called by network thread + self.sesslock.acquire() + try: + return os.path.join(self.sessconfig['state_dir'],STATEDIR_DLPSTATE_DIR) + finally: + self.sesslock.release() + + # + # Tribler Core special features + # + def query_connected_peers(self,query,usercallback,max_peers_to_query=None): + """ Ask all Tribler peers we're currently connected to resolve the + specified query and return the hits. For each peer that returns + hits the usercallback method is called with first parameter the + permid of the peer, as second parameter the query string and + as third parameter a dictionary of hits. The number of times the + usercallback method will be called is undefined. + + The callback will be called by a popup thread which can be used + indefinitely (within reason) by the higher level code. + + At the moment we support three types of query, which are all queries for + torrent files that match a set of keywords. The format of the + query string is "SIMPLE kw1 kw2 kw3" (type 1) or "SIMPLE+METADATA kw1 kw2 + kw3" (type 3). In the future we plan to support full SQL queries. + + For SIMPLE queries the dictionary of hits consists of + (infohash,torrentrecord) pairs. The torrentrecord is a + dictionary that contains the following keys: + 
+        * 'content_name': The 'name' field of the torrent as Unicode string.
+        * 'length': The total size of the content in the torrent.
+        * 'leecher': The currently known number of downloaders.
+        * 'seeder': The currently known number of seeders.
+        * 'category': A list of category strings the torrent was classified into
+          by the remote peer.
+
+ + From Session API version 1.0.2 the following keys were added + to the torrentrecord: + 
+        * 'torrent_size': The size of the .torrent file.
+
+ + From Session API version 1.0.4 the following keys were added + to the torrentrecord: + 
+        * 'channel_permid': PermID of the channel this torrent belongs to (or '')
+        * 'channel_name': channel name as Unicode string (or '').
+       
+
+        For SIMPLE+METADATA queries there is an extra field
+        
+        * 'torrent_file': Bencoded contents of the .torrent file. 
+        

+        The torrents *not* be automatically added to the TorrentDBHandler 
+        (if enabled) at the time of the call.
+
+        
+        The third type of query: search for channels. It is used to query for 
+        channels: either a particular channel matching the permid in the query, 
+        or a list of channels whose names match the keywords in the query
+        by sending the query to connected peers. 
+        
+        The format of the query in the corresponding scenarios should be: 
+        a. keyword-based query: "CHANNEL k bbc"     
+            ('k' stands for keyword-based and ' '{space} is a separator followed by 
+            the keywords)
+        b. permid-based query: "CHANNEL p f34wrf2345wfer2345wefd3r34r54" 
+            ('p' stands for permid-based and ' '{space} is a separator followed by
+            the permid)
+        
+        In each of the above 2 cases, the format of the hits that is returned 
+        by the queried peer is a dictionary of hits of (signature,channelrecord). 
+        The channelrecord is a dictionary the contains following keys: 
+        
+        * 'publisher_id': a PermID
+        * 'publisher_name': as Unicode string
+        * 'infohash': 20-byte SHA1 hash
+        * 'torrenthash': 20-byte SHA1 hash
+        * 'torrentname': as Unicode string
+        * 'time_stamp': as integer
+        

+
+        
+        @param query A Unicode query string adhering to the above spec.
+        @param usercallback A function adhering to the above spec.
+        """
+        self.sesslock.acquire()
+        try:
+            if self.sessconfig['overlay']:
+                if not (query.startswith('SIMPLE ') or query.startswith('SIMPLE+METADATA ')) and not query.startswith('CHANNEL '):
+                    raise ValueError('Query does not start with SIMPLE or SIMPLE+METADATA or CHANNEL')
+                
+                from BaseLib.Core.SocialNetwork.RemoteQueryMsgHandler import RemoteQueryMsgHandler
+                
+                rqmh = RemoteQueryMsgHandler.getInstance()
+                rqmh.send_query(query,usercallback,max_peers_to_query=max_peers_to_query)
+            else:
+                raise OperationNotEnabledByConfigurationException("Overlay not enabled")
+        finally:
+            self.sesslock.release()
+            
+    
+    def download_torrentfile_from_peer(self,permid,infohash,usercallback):
+        """ Ask the designated peer to send us the torrentfile for the torrent
+        identified by the passed infohash. If the torrent is succesfully 
+        received, the usercallback method is called with the infohash as first
+        and the contents of the torrentfile (bencoded dict) as second parameter.
+        If the torrent could not be obtained, the callback is not called.
+        The torrent will have been added to the TorrentDBHandler (if enabled)
+        at the time of the call.
+        
+        @param permid The PermID of the peer to query.
+        @param infohash The infohash of the torrent.
+        @param usercallback A function adhering to the above spec.
+        """
+        # ARNOCOMMENT: Perhaps make save to database optional.
+        self.sesslock.acquire()
+        try:
+            if self.sessconfig['overlay']:
+                from BaseLib.Core.SocialNetwork.RemoteTorrentHandler import RemoteTorrentHandler
+                
+                rtorrent_handler = RemoteTorrentHandler.getInstance()
+                rtorrent_handler.download_torrent(permid,infohash,usercallback)
+            else:
+                raise OperationNotEnabledByConfigurationException("Overlay not enabled")
+        finally:
+            self.sesslock.release()
+        
+
+    #
+    # Internal persistence methods
+    #
+    def checkpoint_shutdown(self,stop,checkpoint,gracetime,hacksessconfcheckpoint):
+        """ Checkpoints the Session and optionally shuts down the Session.
+        @param stop Whether to shutdown the Session as well.
+        @param checkpoint Whether to checkpoint at all, or just to stop.
+        @param gracetime Time to allow for graceful shutdown + signoff (seconds). 
+        """
+        # Called by any thread
+        self.sesslock.acquire()
+        try:
+            # Arno: Make checkpoint optional on shutdown. At the moment setting 
+            # the config at runtime is not possible (see SessionRuntimeConfig)
+            # so this has little use, and interferes with our way of
+            # changing the startup config, which is to write a new
+            # config to disk that will be read at start up.
+            if hacksessconfcheckpoint:
+                try:
+                    self.save_pstate_sessconfig()
+                except Exception,e:
+                    self.lm.rawserver_nonfatalerrorfunc(e)
+
+            # Checkpoint all Downloads and stop NetworkThread
+            if DEBUG:
+                print >>sys.stderr,"Session: checkpoint_shutdown"
+            self.lm.checkpoint(stop=stop,checkpoint=checkpoint,gracetime=gracetime)
+        finally:
+            self.sesslock.release()
+
+    def save_pstate_sessconfig(self):
+        """ Save the runtime SessionConfig to disk """
+        # Called by any thread
+        sscfg = self.get_current_startup_config_copy()
+        cfgfilename = Session.get_default_config_filename(sscfg.get_state_dir())
+        sscfg.save(cfgfilename)
+
+
+    def get_default_config_filename(state_dir):
+        """ Return the name of the file where a session config is saved by default. 
+        @return A filename 
+        """
+        return os.path.join(state_dir,STATEDIR_SESSCONFIG)
+    get_default_config_filename = staticmethod(get_default_config_filename)
+
+
+    def get_internal_tracker_torrentfilename(self,infohash):
+        """ Return the absolute pathname of the torrent file used by the
+        internal tracker.
+        @return A filename
+        """
+        trackerdir = self.get_internal_tracker_dir()
+        basename = binascii.hexlify(infohash)+'.torrent' # ignore .tribe stuff, not vital
+        return os.path.join(trackerdir,basename)
+
+    def get_nat_type(self, callback=None):
+        """ Return the type of Network Address Translator (NAT) detected.
+
+        When a callback parameter is supplied it will always be
+        called. When the NAT-type is already known the callback will
+        be made instantly. Otherwise, the callback will be made when
+        the NAT discovery has finished.
+
+        The callback will be called by a popup thread which can be used
+        indefinitely (within reason) by the higher level code.
+
+        Return values:
+        "Blocked"
+        "Open Internet"
+        "Restricted Cone Firewall"
+        "Port Restricted Cone Firewall"
+        "Full Cone NAT"
+        "Restricted Cone NAT"
+        "Port Restricted Cone NAT"
+        "Symmetric NAT"
+        "Unknown NAT/Firewall"
+
+        @param callback Optional callback used to notify the NAT type
+        @return String 
+        """
+        # TODO: define constants in simpledefs for these
+        # Called by any thread
+        self.sesslock.acquire()
+        try:
+            from BaseLib.Core.NATFirewall.ConnectionCheck import ConnectionCheck
+            
+            return ConnectionCheck.getInstance(self).get_nat_type(callback=callback)
+        finally:
+            self.sesslock.release()
+
+    #
+    # Friendship functions
+    #
+    def send_friendship_message(self,permid,mtype,approved=None):
+        """ Send friendship msg to the specified peer 
+        
+        F_REQUEST_MSG:
+        
+        F_RESPONSE_MSG:
+        @param approved Whether you want him as friend or not.
+        
+        """
+        self.sesslock.acquire()
+        try:
+            if self.sessconfig['overlay']:
+                if mtype == F_FORWARD_MSG:
+                    raise ValueError("User cannot send FORWARD messages directly")
+                
+                from BaseLib.Core.SocialNetwork.FriendshipMsgHandler import FriendshipMsgHandler
+                
+                fmh = FriendshipMsgHandler.getInstance()
+                params = {}
+                if approved is not None:
+                    params['response'] = int(approved)
+                fmh.anythread_send_friendship_msg(permid,mtype,params)
+            else:
+                raise OperationNotEnabledByConfigurationException("Overlay not enabled")
+        finally:
+            self.sesslock.release()
+
+
+    def set_friendship_callback(self,usercallback):
+        """ When a new friendship request is received the given
+        callback function is called with as first parameter the
+        requester's permid and as second parameter a dictionary of
+        request arguments:
+            callback(requester_permid,params)
+
+        The callback is called by a popup thread which can be used
+        indefinitely (within reason) by the higher level code.
+        
+        @param usercallback A callback function adhering to the above spec.
+        """
+        self.sesslock.acquire()
+        try:
+            if self.sessconfig['overlay']:
+                from BaseLib.Core.SocialNetwork.FriendshipMsgHandler import FriendshipMsgHandler
+                
+                fmh = FriendshipMsgHandler.getInstance()
+                fmh.register_usercallback(usercallback)
+            else:
+                raise OperationNotEnabledByConfigurationException("Overlay not enabled")
+        finally:
+            self.sesslock.release()
+            
+            
+    # 02-06-2010 Andrea: returns a reference to SubtitleSupport instance, which
+    # is the facade (i.e. acts as the entry point) of the Subtitles subsystem
+    def get_subtitles_support_facade(self):
+        '''
+        Returns an instance of SubtitleSupport, which is intended to be used
+        by clients to interact with the subtitles subsystem.
+        
+        Subsequent calls to this method should always return the same instance.
+        
+        If the instance is not available, None will be returned
+        '''
+        try:
+            return self.lm.overlay_apps.subtitle_support
+        except:
+            return None
diff --git a/instrumentation/next-share/BaseLib/Core/SessionConfig.py b/instrumentation/next-share/BaseLib/Core/SessionConfig.py
new file mode 100644
index 0000000..ff2c368
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/SessionConfig.py
@@ -0,0 +1,1319 @@
+# Written by Arno Bakker, George Milescu 
+# see LICENSE.txt for license information
+""" Controls the operation of a Session """
+
+#
+# WARNING: When extending this class:
+#
+# 1. Add a JavaDoc description for each method you add.
+# 2. Also add the methods to APIImplementation/SessionRuntimeConfig.py  
+# 3. Document your changes in API.py
+#
+#
+
+import sys
+import copy
+import pickle
+
+from BaseLib.Core.simpledefs import *
+from BaseLib.Core.defaults import sessdefaults
+from BaseLib.Core.Base import *
+from BaseLib.Core.BitTornado.RawServer import autodetect_socket_style
+from BaseLib.Core.Utilities.utilities import find_prog_in_PATH
+
+
+class SessionConfigInterface:
+    """ 
+    (key,value) pair config of global parameters, 
+    e.g. PermID keypair, listen port, max upload speed, etc.
+    
+    Use SessionStartupConfig from creating and manipulation configurations
+    before session startup time. This is just a parent class.
+    """
+    def __init__(self,sessconfig=None):
+        """ Constructor. 
+        @param sessconfig Optional dictionary used internally 
+        to make this a copy constructor.
+        """
+
+        if sessconfig is not None: # copy constructor
+            self.sessconfig = sessconfig
+            return
+        
+        self.sessconfig = {}
+        
+        # Define the built-in default here
+        self.sessconfig.update(sessdefaults)
+        
+        # Set video_analyser_path
+        if sys.platform == 'win32':
+            ffmpegname = "ffmpeg.exe"
+        else:
+            ffmpegname = "ffmpeg"
+    
+        ffmpegpath = find_prog_in_PATH(ffmpegname)
+        if ffmpegpath is None:
+            if sys.platform == 'win32':
+                self.sessconfig['videoanalyserpath'] = ffmpegname
+            elif sys.platform == 'darwin':
+                self.sessconfig['videoanalyserpath'] = "macbinaries/ffmpeg"
+            else:
+                self.sessconfig['videoanalyserpath'] = ffmpegname
+        else:
+            self.sessconfig['videoanalyserpath'] = ffmpegpath
+
+        self.sessconfig['ipv6_binds_v4'] = autodetect_socket_style()
+
+
+
+    def set_state_dir(self,statedir):
+        """ Set the directory to store the Session's state in.
+        @param statedir  A preferably absolute path name. If the directory
+        does not yet exist it will be created at Session create time.
+        """
+        self.sessconfig['state_dir'] = statedir
+    
+    def get_state_dir(self):
+        """ Returns the directory the Session stores its state in. 
+        @return An absolute path name. """
+        return self.sessconfig['state_dir']
+    
+    def set_install_dir(self,installdir):
+        """ Set the directory in which the Tribler Core software is installed. 
+        @param installdir An absolute path name
+        """
+        self.sessconfig['install_dir'] = installdir
+    
+    def get_install_dir(self):
+        """ Returns the directory the Tribler Core software is installed in.
+        @return An absolute path name. """
+        return self.sessconfig['install_dir']
+    
+    
+    def set_permid_keypair_filename(self,keypairfilename):
+        """ Set the filename containing the Elliptic Curve keypair to use for 
+        PermID-based authentication in this Session. 
+        
+        Note: if a Session is started with a SessionStartupConfig that
+        points to an existing state dir and that state dir contains a saved
+        keypair, that keypair will be used unless a different keypair is
+        explicitly configured via this method.
+        """
+        self.sessconfig['eckeypairfilename'] = keypairfilename
+
+    def get_permid_keypair_filename(self):
+        """ Returns the filename of the Session's keypair.
+        @return An absolute path name. """
+        return self.sessconfig['eckeypairfilename']
+    
+
+    def set_listen_port(self,port):
+        """ Set the UDP and TCP listen port for this Session.
+        @param port A port number.
+        """
+        self.sessconfig['minport'] = port
+        self.sessconfig['maxport'] = port
+
+    def get_listen_port(self):
+        """ Returns the current UDP/TCP listen port.
+        @return Port number. """
+        return self.sessconfig['minport']
+        
+    #
+    # Advanced network settings
+    #
+    def set_ip_for_tracker(self,value):
+        """ IP address to report to the tracker (default = set automatically).
+        @param value  An IP address as string. """
+        self.sessconfig['ip'] = value
+
+    def get_ip_for_tracker(self):
+        """ Returns the IP address being reported to the tracker.
+        @return String """
+        return self.sessconfig['ip']
+
+    def set_bind_to_addresses(self,value):
+        """ Set the list of IP addresses/hostnames to bind to locally.
+        @param value A list of IP addresses as strings. """
+        self.sessconfig['bind'] = value
+
+    def get_bind_to_addresses(self):
+        """ Returns the list of IP addresses bound to.
+        @return list """
+        return self.sessconfig['bind']
+
+    def set_upnp_mode(self,value):
+        """ Use to autoconfigure a UPnP router to forward the UDP/TCP listen 
+        port to this host:
+        
+         * UPNPMODE_DISABLED: Autoconfigure turned off.
+         * UPNPMODE_WIN32_HNetCfg_NATUPnP: Use Windows COM interface (slow)
+         * UPNPMODE_WIN32_UPnP_UPnPDeviceFinder: Use Windows COM interface (faster)
+         * UPNPMODE_UNIVERSAL_DIRECT: Talk UPnP directly to the network (best)
+        

+        @param value UPNPMODE_* 
+        """
+        self.sessconfig['upnp_nat_access'] = value
+
+    def get_upnp_mode(self):
+        """ Returns the UPnP mode set. 
+        @return UPNPMODE_* """
+        return self.sessconfig['upnp_nat_access']
+
+    def set_autoclose_timeout(self,value):
+        """ Time to wait between closing sockets which nothing has been received
+        on.
+        @param value A number of seconds.
+        """
+        self.sessconfig['timeout'] = value
+
+    def get_autoclose_timeout(self):
+        """ Returns the autoclose timeout.
+        @return A number of seconds. """
+        return self.sessconfig['timeout']
+
+    def set_autoclose_check_interval(self,value):
+        """ Time to wait between checking if any connections have timed out.
+        @param value A number of seconds.
+        """
+        self.sessconfig['timeout_check_interval'] = value
+
+    def get_autoclose_check_interval(self):
+        """ Returns the autoclose check interval.
+        @return A number of seconds. """
+        return self.sessconfig['timeout_check_interval']
+
+    #
+    # Enable/disable Tribler features 
+    #
+    def set_megacache(self,value):
+        """ Enable megacache databases to cache peers, torrent files and 
+        preferences (default = True).
+        @param value Boolean. """
+        self.sessconfig['megacache'] = value
+
+    def get_megacache(self):
+        """ Returns whether Megacache is enabled.
+        @return Boolean. """
+        return self.sessconfig['megacache']
+
+    #
+    # Secure Overlay
+    #
+    def set_overlay(self,value):
+        """ Enable overlay swarm to enable Tribler's special features 
+        (default = True).
+        @param value Boolean. 
+        """
+        self.sessconfig['overlay'] = value
+
+    def get_overlay(self):
+        """ Returns whether overlay-swarm extension is enabled. The overlay
+        swarm allows strong authentication of peers and is used for all
+        Tribler-specific messages.
+        @return Boolean. """
+        return self.sessconfig['overlay']
+
+    def set_overlay_max_message_length(self,value):
+        """ Maximal message length for messages sent over the secure overlay.
+        @param value A number of bytes.
+        """
+        self.sessconfig['overlay_max_message_length'] = value
+
+    def get_overlay_max_message_length(self):
+        """ Returns the maximum overlay-message length.
+        @return A number of bytes. """
+        return self.sessconfig['overlay_max_message_length']
+
+
+    #
+    # Buddycast
+    #
+    def set_buddycast(self,value):
+        """ Enable buddycast recommendation system at startup (default = True)
+        @param value Boolean.
+        """
+        self.sessconfig['buddycast'] = value
+
+    def get_buddycast(self):
+        """ Returns whether buddycast is enabled at startup. 
+        @return Boolean."""
+        return self.sessconfig['buddycast']
+
+    def set_start_recommender(self,value):
+        """ Buddycast can be temporarily disabled via this parameter 
+        (default = True). Must have been enabled at startup, see
+        set_buddycast().
+        @param value Boolean. 
+        """
+        self.sessconfig['start_recommender'] = value
+
+    def get_start_recommender(self):
+        """ Returns whether Buddycast is temporarily enabled. 
+        @return Boolean."""
+        return self.sessconfig['start_recommender']
+
+    def set_buddycast_interval(self,value):
+        """ Number of seconds to pause between exchanging preference with a 
+        peer in Buddycast.
+        @param value A number of seconds.
+        """
+        self.sessconfig['buddycast_interval'] = value
+
+    def get_buddycast_interval(self):
+        """ Returns the number of seconds between Buddycast pref. exchanges. 
+        @return A number of seconds. """
+        return self.sessconfig['buddycast_interval']
+
+    def set_buddycast_collecting_solution(self,value):
+        """ Set the Buddycast collecting solution. Only one policy implemented
+        at the moment:
+        
+         * BCCOLPOLICY_SIMPLE: Simplest solution: per torrent/buddycasted peer/4 hours,
+         

+        @param value BCCOLPOLICY_* 
+        """
+        self.sessconfig['buddycast_collecting_solution'] = value
+
+    def get_buddycast_collecting_solution(self):
+        """ Returns the Buddycast collecting solution. 
+        @return BCOLPOLICY_* """
+        return self.sessconfig['buddycast_collecting_solution']
+
+    def set_buddycast_max_peers(self,value):
+        """ Set max number of peers to use for Buddycast recommendations """
+        self.sessconfig['buddycast_max_peers'] = value
+
+    def get_buddycast_max_peers(self):
+        """ Return the max number of peers to use for Buddycast recommendations.
+        @return A number of peers.
+        """
+        return self.sessconfig['buddycast_max_peers']
+
+    #
+    # ProxyService_ parameters
+    #
+    def set_download_help(self,value):
+        """ Enable download helping/cooperative download (default = True).
+        @param value Boolean. """
+        self.sessconfig['download_help'] = value
+
+    def get_download_help(self):
+        """ Returns whether download help is enabled. 
+        @return Boolean. """
+        return self.sessconfig['download_help']
+
+    def set_download_help_dir(self,value):
+        """ Set the directory for storing state and content for download
+        helping (default = Default destination dir (see get_default_dest_dir()
+        +'downloadhelp'.
+        @param value An absolute path. """
+        self.sessconfig['download_help_dir'] = value
+
+    def get_download_help_dir(self):
+        """ Returns the directory for download helping storage. 
+        @return An absolute path name. """
+        return self.sessconfig['download_help_dir']
+
+    def set_proxyservice_status(self,value):
+        """ Set the status of the proxyservice (on or off).
+        
+        ProxyService off means the current node could not be used as a proxy. ProxyService on means other nodes will be able to use it as a proxy.
+        
+        @param value: one of the possible two values: PROXYSERVICE_OFF, PROXYSERVICE_ON
+        """
+        if value == PROXYSERVICE_OFF or value == PROXYSERVICE_ON:
+            self.sessconfig['proxyservice_status'] = value
+        else:
+            # If the method is called with an incorrect value, turn off the ProxyService
+            self.sessconfig['proxyservice_status'] = PROXYSERVICE_OFF
+
+    def get_proxyservice_status(self):
+        """ Returns the status of the proxyservice (on or off).
+        @return: one of the possible two values: PROXYSERVICE_OFF, PROXYSERVICE_ON
+        """
+        return self.sessconfig['proxyservice_status']
+    #
+    # _ProxyService
+    #
+
+    #
+    # Torrent file collecting
+    #
+    def set_torrent_collecting(self,value):
+        """ Automatically collect torrents from peers in the network (default = 
+        True).
+        @param value Boolean. 
+        """
+        self.sessconfig['torrent_collecting'] = value
+
+    def get_torrent_collecting(self):
+        """ Returns whether to automatically collect torrents.
+        @return Boolean. """
+        return self.sessconfig['torrent_collecting']
+
+    def set_torrent_collecting_max_torrents(self,value):
+        """ Set the maximum number of torrents to collect from other peers.
+        @param value A number of torrents.
+        """
+        self.sessconfig['torrent_collecting_max_torrents'] = value
+
+    def get_torrent_collecting_max_torrents(self):
+        """ Returns the maximum number of torrents to collect.
+        @return A number of torrents. """
+        return self.sessconfig['torrent_collecting_max_torrents']
+
+    def set_torrent_collecting_dir(self,value):
+        """ Where to place collected torrents? (default is state_dir + 'collected_torrent_files')
+        @param value An absolute path.
+        """
+        self.sessconfig['torrent_collecting_dir'] = value
+
+    def get_torrent_collecting_dir(self):
+        """ Returns the directory to save collected torrents.
+        @return An absolute path name. """
+        return self.sessconfig['torrent_collecting_dir']
+    
+    def set_torrent_collecting_rate(self,value):
+        """ Maximum download rate to use for torrent collecting.
+        @param value A rate in KB/s. """
+        self.sessconfig['torrent_collecting_rate'] = value
+
+    def get_torrent_collecting_rate(self):
+        """ Returns the download rate to use for torrent collecting.
+        @return A rate in KB/s. """
+        return self.sessconfig['torrent_collecting_rate']
+
+    def set_torrent_checking(self,value):
+        """ Whether to automatically check the health of collected torrents by
+        contacting their trackers (default = True).
+        @param value Boolean 
+        """
+        self.sessconfig['torrent_checking'] = value
+
+    def get_torrent_checking(self):
+        """ Returns whether to check health of collected torrents.
+        @return Boolean. """
+        return self.sessconfig['torrent_checking']
+
+    def set_torrent_checking_period(self,value):
+        """ Interval between automatic torrent health checks.
+        @param value An interval in seconds.
+        """
+        self.sessconfig['torrent_checking_period'] = value
+
+    def get_torrent_checking_period(self):
+        """ Returns the check interval.
+        @return A number of seconds. """
+        return self.sessconfig['torrent_checking_period']
+
+    def set_stop_collecting_threshold(self,value):
+        """ Stop collecting more torrents if the disk has less than this limit 
+        @param value A limit in MB.
+        """
+        self.sessconfig['stop_collecting_threshold'] = value
+
+    def get_stop_collecting_threshold(self):
+        """ Returns the disk-space limit when to stop collecting torrents. 
+        @return A number of megabytes. """
+        return self.sessconfig['stop_collecting_threshold']
+
+    #
+    # The Tribler dialback mechanism is used to test whether a Session is
+    # reachable from the outside and what its external IP address is.
+    #
+    def set_dialback(self,value):
+        """ Use other peers to determine external IP address (default = True)
+        @param value Boolean 
+        """
+        self.sessconfig['dialback'] = value
+
+    def get_dialback(self):
+        """ Returns whether to use the dialback mechanism. 
+        @return Boolean. """
+        return self.sessconfig['dialback']
+
+    #
+    # Tribler's social networking feature transmits a nickname and picture
+    # to all Tribler peers it meets.
+    #
+    def set_social_networking(self,value):
+        """ Enable social networking. If enabled, a message containing the
+        user's nickname and icon is sent to each Tribler peer met
+        (default = True).
+        @param value Boolean 
+        """
+        self.sessconfig['socnet'] = value
+
+    def get_social_networking(self):
+        """ Returns whether social network is enabled.
+        @return Boolean. """
+        return self.sessconfig['socnet']
+
+    def set_nickname(self,value):
+        """ The nickname you want to show to others.
+        @param value A Unicode string.
+        """
+        self.sessconfig['nickname'] = value
+
+    def get_nickname(self):
+        """ Returns the set nickname.
+        @return A Unicode string. """
+        return self.sessconfig['nickname']
+
+    def set_mugshot(self,value, mime = 'image/jpeg'):
+        """ The picture of yourself you want to show to others.
+        @param value A string of binary data of your image.
+        @param mime A string of the mimetype of the data
+        """
+        self.sessconfig['mugshot'] = (mime, value)
+
+    def get_mugshot(self):
+        """ Returns binary image data and mime-type of your picture.
+        @return (String, String) value and mimetype. """
+        if self.sessconfig['mugshot'] is None:
+            return None, None
+        else:
+            return self.sessconfig['mugshot']
+    
+    def set_peer_icon_path(self,value):
+        """ Directory to store received peer icons (Default is statedir +
+        STATEDIR_PEERICON_DIR).
+        @param value An absolute path. """
+        self.sessconfig['peer_icon_path'] = value
+
+    def get_peer_icon_path(self):
+        """ Returns the directory to store peer icons.
+        @return An absolute path name. """
+        return self.sessconfig['peer_icon_path']
+
+    #
+    # Tribler remote query: ask other peers when looking for a torrent file 
+    # or peer
+    #
+    def set_remote_query(self,value):
+        """ Enable queries from other peers. At the moment peers can ask
+        whether this Session has collected or opened a torrent that matches
+        a specified keyword query. (default = True)
+        @param value Boolean"""
+        self.sessconfig['rquery'] = value
+
+    def get_remote_query(self):
+        """ Returns whether remote query is enabled. 
+        @return Boolean. """
+        return self.sessconfig['rquery']
+
+    #
+    # BarterCast
+    #
+    def set_bartercast(self,value):
+        """ Exchange upload/download statistics with peers (default = True)
+        @param value Boolean
+        """
+        self.sessconfig['bartercast'] = value
+
+    def get_bartercast(self):
+        """ Returns to exchange statistics with peers.
+        @return Boolean. """
+        return self.sessconfig['bartercast']
+
+
+    #
+    # For Tribler Video-On-Demand
+    #
+    def set_video_analyser_path(self,value):
+        """ Path to video analyser FFMPEG. The analyser is used to guess the
+        bitrate of a video if that information is not present in the torrent
+        definition. (default = look for it in $PATH)
+        @param value An absolute path name.
+        """
+        self.sessconfig['videoanalyserpath'] = value
+    
+    def get_video_analyser_path(self):
+        """ Returns the path of the FFMPEG video analyser.
+        @return An absolute path name. """
+        return self.sessconfig['videoanalyserpath'] # strings immutable
+    
+
+    #
+    # Tribler's internal tracker
+    #
+    def set_internal_tracker(self,value):
+        """ Enable internal tracker (default = True)
+        @param value Boolean.
+        """
+        self.sessconfig['internaltracker'] = value
+
+    def get_internal_tracker(self):
+        """ Returns whether the internal tracker is enabled.
+        @return Boolean. """
+        return self.sessconfig['internaltracker']
+
+    def set_internal_tracker_url(self,value):
+        """ Set the internal tracker URL (default = determined dynamically
+        from Session's IP+port)
+        @param value URL.
+        """
+        self.sessconfig['tracker_url'] = value
+
+    def get_internal_tracker_url(self):
+        """ Returns the URL of the tracker as set by set_internal_tracker_url().
+        Overridden at runtime by Session class.
+        @return URL. """
+        return self.sessconfig['tracker_url']
+
+
+    def set_mainline_dht(self,value):
+        """ Enable mainline DHT support (default = True)
+        @param value Boolean.
+        """
+        self.sessconfig['mainline_dht'] = value
+
+    def get_mainline_dht(self):
+        """ Returns whether mainline DHT support is enabled.
+        @return Boolean. """
+        return self.sessconfig['mainline_dht']
+
+
+    #
+    # Internal tracker access control settings
+    #
+    def set_tracker_allowed_dir(self,value):
+        """ Only accept tracking requests for torrent in this dir (default is
+        Session state-dir + STATEDIR_ITRACKER_DIR
+        @param value An absolute path name.
+        """
+        self.sessconfig['tracker_allowed_dir'] = value
+
+    def get_tracker_allowed_dir(self):
+        """ Returns the internal tracker's directory of allowed torrents.
+        @return An absolute path name. """
+        return self.sessconfig['tracker_allowed_dir']
+
+    def set_tracker_allowed_list(self,value):
+        """ Only allow peers to register for torrents that appear in the
+        specified file. Cannot be used in combination with set_tracker_allowed_dir()
+        @param value An absolute filename containing a list of torrent infohashes in hex format, one per 
+        line. """
+        self.sessconfig['tracker_allowed_list'] = value
+
+    def get_tracker_allowed_list(self):
+        """ Returns the filename of the list of allowed torrents.
+        @return An absolute path name. """
+        return self.sessconfig['tracker_allowed_list']
+
+    def set_tracker_allowed_controls(self,value):
+        """ Allow special keys in torrents in the allowed_dir to affect tracker
+        access.
+        @param value Boolean
+        """
+        self.sessconfig['tracker_allowed_controls'] = value
+
+    def get_tracker_allowed_controls(self):
+        """ Returns whether to allow allowed torrents to control tracker access.
+        @return Boolean. """
+        return self.sessconfig['tracker_allowed_controls']
+
+    def set_tracker_allowed_ips(self,value):
+        """ Only allow connections from IPs specified in the given file; file 
+        contains subnet data in the format: aa.bb.cc.dd/len.
+        @param value An absolute path name.
+        """
+        self.sessconfig['tracker_allowed_ips'] = value
+
+    def get_tracker_allowed_ips(self):
+        """ Returns the filename containing allowed IP addresses. 
+        @return An absolute path name."""
+        return self.sessconfig['tracker_allowed_ips']
+
+    def set_tracker_banned_ips(self,value):
+        """ Don't allow connections from IPs specified in the given file; file
+        contains IP range data in the format: xxx:xxx:ip1-ip2
+        @param value An absolute path name.
+        """
+        self.sessconfig['tracker_banned_ips'] = value
+
+    def get_tracker_banned_ips(self):
+        """ Returns the filename containing banned IP addresses. 
+        @return An absolute path name. """
+        return self.sessconfig['tracker_banned_ips']
+
+    def set_tracker_only_local_override_ip(self,value):
+        """ Ignore the 'ip' parameter in the GET announce from machines which 
+        aren't on local network IPs.
+        
+         * ITRACK_IGNORE_ANNOUNCEIP_NEVER
+         * ITRACK_IGNORE_ANNOUNCEIP_ALWAYS
+         * ITRACK_IGNORE_ANNOUNCEIP_IFNONATCHECK
+        

+        @param value ITRACK_IGNORE_ANNOUNCEIP*
+        """
+        self.sessconfig['tracker_only_local_override_ip'] = value
+
+    def get_tracker_only_local_override_ip(self):
+        """ Returns the ignore policy for 'ip' parameters in announces. 
+        @return ITRACK_IGNORE_ANNOUNCEIP_* """
+        return self.sessconfig['tracker_only_local_override_ip']
+
+    def set_tracker_parse_dir_interval(self,value):
+        """ Seconds between reloading of allowed_dir or allowed_file and 
+        allowed_ips and banned_ips lists.
+        @param value A number of seconds.
+        """
+        self.sessconfig['tracker_parse_dir_interval'] = value
+
+    def get_tracker_parse_dir_interval(self):
+        """ Returns the number of seconds between refreshes of access control
+        info.
+        @return A number of seconds. """
+        return self.sessconfig['tracker_parse_dir_interval']
+
+    def set_tracker_scrape_allowed(self,value):
+        """ Allow scrape access on the internal tracker (with a scrape request
+        a BitTorrent client can retrieve information about how many peers are
+        downloading the content.
+        
+        * ITRACKSCRAPE_ALLOW_NONE: Don't allow scrape requests.
+        * ITRACKSCRAPE_ALLOW_SPECIFIC: Allow scrape requests for a specific torrent.
+        * ITRACKSCRAPE_ALLOW_FULL: Allow scrape of all torrents at once.
+        

+        @param value ITRACKSCRAPE_* 
+        """
+        self.sessconfig['tracker_scrape_allowed'] = value
+
+    def get_tracker_scrape_allowed(self):
+        """ Returns the scrape access policy.
+        @return ITRACKSCRAPE_ALLOW_* """
+        return self.sessconfig['tracker_scrape_allowed']
+
+    def set_tracker_allow_get(self,value):
+        """ Setting this parameter adds a /file?hash={hash} links to the
+        overview page that the internal tracker makes available via HTTP
+        at hostname:listenport. These links allow users to download the 
+        torrent file from the internal tracker. Use with 'allowed_dir' parameter.
+        @param value Boolean.
+        """
+        self.sessconfig['tracker_allow_get'] = value
+
+    def get_tracker_allow_get(self):
+        """ Returns whether to allow HTTP torrent-file downloads from the
+        internal tracker.
+        @return Boolean. """
+        return self.sessconfig['tracker_allow_get']
+
+
+    #
+    # Controls for internal tracker's output as Web server
+    #
+    def set_tracker_favicon(self,value):
+        """ File containing image/x-icon data to return when browser requests 
+        favicon.ico from the internal tracker. (Default = Tribler/Images/tribler.ico)
+        @param value An absolute filename. 
+        """
+        self.sessconfig['tracker_favicon'] = value
+
+    def get_tracker_favicon(self):
+        """ Returns the filename of the internal tracker favicon. 
+        @return An absolute path name. """
+        return self.sessconfig['tracker_favicon']
+
+    def set_tracker_show_infopage(self,value):
+        """ Whether to display an info page when the tracker's root dir is 
+        requested via HTTP.
+        @param value Boolean
+        """
+        self.sessconfig['tracker_show_infopage'] = value
+
+    def get_tracker_show_infopage(self):
+        """ Returns whether to show an info page on the internal tracker. 
+        @return Boolean. """
+        return self.sessconfig['tracker_show_infopage']
+
+    def set_tracker_infopage_redirect(self,value):
+        """ A URL to redirect the request for an info page to.
+        @param value URL.
+        """
+        self.sessconfig['tracker_infopage_redirect'] = value
+
+    def get_tracker_infopage_redirect(self):
+        """ Returns the URL to redirect request for info pages to. 
+        @return URL """
+        return self.sessconfig['tracker_infopage_redirect']
+
+    def set_tracker_show_names(self,value):
+        """ Whether to display names from the 'allowed dir'.
+        @param value Boolean.
+        """
+        self.sessconfig['tracker_show_names'] = value
+
+    def get_tracker_show_names(self):
+        """ Returns whether the tracker displays names from the 'allowed dir'. 
+        @return Boolean. """
+        return self.sessconfig['tracker_show_names']
+
+    def set_tracker_keep_dead(self,value):
+        """ Keep dead torrents after they expire (so they still show up on your
+        /scrape and web page)
+        @param value Boolean.
+        """
+        self.sessconfig['tracker_keep_dead'] = value
+
+    def get_tracker_keep_dead(self):
+        """ Returns whether to keep dead torrents for statistics. 
+        @return Boolean. """
+        return self.sessconfig['tracker_keep_dead']
+
+    #
+    # Controls for internal tracker replies
+    #
+    def set_tracker_reannounce_interval(self,value):
+        """ Seconds downloaders should wait between reannouncing themselves
+        to the internal tracker.
+        @param value A number of seconds.
+        """
+        self.sessconfig['tracker_reannounce_interval'] = value
+
+    def get_tracker_reannounce_interval(self):
+        """ Returns the reannounce interval for the internal tracker. 
+        @return A number of seconds. """
+        return self.sessconfig['tracker_reannounce_interval']
+
+    def set_tracker_response_size(self,value):
+        """ Number of peers to send to a peer in a reply to its announce
+        at the internal tracker (i.e., in the info message)
+        @param value A number of peers.
+        """
+        self.sessconfig['tracker_response_size'] = value
+
+    def get_tracker_response_size(self):
+        """ Returns the number of peers to send in a tracker reply. 
+        @return A number of peers. """
+        return self.sessconfig['tracker_response_size']
+
+    def set_tracker_nat_check(self,value):
+        """ How many times the internal tracker should attempt to check if a 
+        downloader is behind a  Network Address Translator (NAT) or firewall.
+        If it is, the downloader won't be registered at the tracker, as other
+        peers can probably not contact it. 
+        @param value A number of times, 0 = don't check.
+        """
+        self.sessconfig['tracker_nat_check'] = value
+
+    def get_tracker_nat_check(self):
+        """ Returns the number of times to check for a firewall.
+        @return A number of times. """
+        return self.sessconfig['tracker_nat_check']
+
+
+    #
+    # Internal tracker persistence
+    #
+    def set_tracker_dfile(self,value):
+        """ File to store recent downloader info in (default = Session state 
+        dir + STATEDIR_ITRACKER_DIR + tracker.db
+        @param value An absolute path name.
+        """
+        self.sessconfig['tracker_dfile'] = value
+
+    def get_tracker_dfile(self):
+        """ Returns the tracker database file. 
+        @return An absolute path name. """
+        return self.sessconfig['tracker_dfile']
+
+    def set_tracker_dfile_format(self,value):
+        """ Format of the tracker database file. *_PICKLE is needed when Unicode
+        filenames may appear in the tracker's state (=default).
+        
+         * ITRACKDBFORMAT_BENCODE: Use BitTorrent bencoding to store records.
+         * ITRACKDBFORMAT_PICKLE: Use Python pickling to store records.
+        

+        @param value ITRACKDBFORFMAT_* 
+        """
+        self.sessconfig['tracker_dfile_format'] = value
+
+    def get_tracker_dfile_format(self):
+        """ Returns the format of the tracker database file. 
+        @return ITRACKDBFORMAT_* """
+        return self.sessconfig['tracker_dfile_format']
+
+    def set_tracker_save_dfile_interval(self,value):
+        """ The interval between saving the internal tracker's state to
+        the tracker database (see set_tracker_dfile()).
+        @param value A number of seconds.
+        """
+        self.sessconfig['tracker_save_dfile_interval'] = value
+
+    def get_tracker_save_dfile_interval(self):
+        """ Returns the tracker-database save interval. 
+        @return A number of seconds. """
+        return self.sessconfig['tracker_save_dfile_interval']
+
+    def set_tracker_logfile(self,value):
+        """ File to write the tracker logs to (default is NIL: or /dev/null).
+        @param value A device name.
+        """
+        self.sessconfig['tracker_logfile'] = value
+
+    def get_tracker_logfile(self):
+        """ Returns the device name to write log messages to. 
+        @return A device name. """
+        return self.sessconfig['tracker_logfile']
+
+    def set_tracker_min_time_between_log_flushes(self,value):
+        """ Minimum time between flushes of the tracker log.
+        @param value A number of seconds.
+        """
+        self.sessconfig['tracker_min_time_between_log_flushes'] = value
+
+    def get_tracker_min_time_between_log_flushes(self):
+        """ Returns time between tracker log flushes. 
+        @return A number of seconds. """
+        return self.sessconfig['tracker_min_time_between_log_flushes']
+
+    def set_tracker_log_nat_checks(self,value):
+        """ Whether to add entries to the tracker log for NAT-check results.
+        @param value Boolean
+        """
+        self.sessconfig['tracker_log_nat_checks'] = value
+
+    def get_tracker_log_nat_checks(self):
+        """ Returns whether to log NAT-check attempts to the tracker log. 
+        @return Boolean. """
+        return self.sessconfig['tracker_log_nat_checks']
+
+    def set_tracker_hupmonitor(self,value):
+        """ Whether to reopen the tracker log file upon receipt of a SIGHUP 
+        signal (Mac/UNIX only).
+        @param value Boolean.
+        """
+        self.sessconfig['tracker_hupmonitor'] = value
+
+    def get_tracker_hupmonitor(self):
+        """ Returns whether to reopen the tracker log file upon receipt of a 
+        SIGHUP signal. 
+        @return Boolean. """
+        return self.sessconfig['tracker_hupmonitor']
+
+
+    #
+    # Esoteric tracker config parameters 
+    #
+    def set_tracker_socket_timeout(self,value):
+        """ Set timeout for closing connections to trackers.
+        @param value A number of seconds.
+        """
+        self.sessconfig['tracker_socket_timeout'] = value
+
+    def get_tracker_socket_timeout(self):
+        """ Returns the tracker socket timeout. 
+        @return A number of seconds. """
+        return self.sessconfig['tracker_socket_timeout']
+
+    def set_tracker_timeout_downloaders_interval(self,value):
+        """ Interval between checks for expired downloaders, i.e., peers
+        no longer in the swarm because they did not reannounce themselves.
+        @param value A number of seconds.
+        """
+        self.sessconfig['tracker_timeout_downloaders_interval'] = value
+
+    def get_tracker_timeout_downloaders_interval(self):
+        """ Returns the number of seconds between checks for expired peers. 
+        @return A number of seconds. """
+        return self.sessconfig['tracker_timeout_downloaders_interval']
+
+    def set_tracker_timeout_check_interval(self,value):
+        """ Time to wait between checking if any connections to the internal
+        tracker have timed out.
+        @param value A number of seconds.
+        """
+        self.sessconfig['tracker_timeout_check_interval'] = value
+
+    def get_tracker_timeout_check_interval(self):
+        """ Returns timeout for connections to the internal tracker. 
+        @return A number of seconds. """
+        return self.sessconfig['tracker_timeout_check_interval']
+
+    def set_tracker_min_time_between_cache_refreshes(self,value):
+        """ Minimum time before a cache is considered stale and is 
+        flushed.
+        @param value A number of seconds.
+        """
+        self.sessconfig['tracker_min_time_between_cache_refreshes'] = value
+
+    def get_tracker_min_time_between_cache_refreshes(self):
+        """ Return the minimum time between cache refreshes.
+        @return A number of seconds. """
+        return self.sessconfig['tracker_min_time_between_cache_refreshes']
+
+
+    #
+    # BitTornado's Multitracker feature
+    #
+    def set_tracker_multitracker_enabled(self,value):
+        """ Whether to enable multitracker operation in which multiple
+        trackers are used to register the peers for a specific torrent.
+        @param value Boolean.
+        """
+        self.sessconfig['tracker_multitracker_enabled'] = value
+
+    def get_tracker_multitracker_enabled(self):
+        """ Returns whether multitracking is enabled. 
+        @return Boolean. """
+        return self.sessconfig['tracker_multitracker_enabled']
+
+    def set_tracker_multitracker_allowed(self,value):
+        """ Whether to allow incoming tracker announces.
+        
+         * ITRACKMULTI_ALLOW_NONE: Don't allow.
+         * ITRACKMULTI_ALLOW_AUTODETECT: Allow for allowed torrents (see set_tracker_allowed_dir())
+         * ITRACKMULTI_ALLOW_ALL: Allow for all. 
+        

+        @param value ITRACKMULTI_ALLOW_*
+        """
+        self.sessconfig['tracker_multitracker_allowed'] = value
+
+    def get_tracker_multitracker_allowed(self):
+        """ Returns the multitracker allow policy of the internal tracker. 
+        @return ITRACKMULTI_ALLOW_* """
+        return self.sessconfig['tracker_multitracker_allowed']
+
+    def set_tracker_multitracker_reannounce_interval(self,value):
+        """ Seconds between outgoing tracker announces to the other trackers in
+        a multi-tracker setup.
+        @param value A number of seconds. 
+        """
+        self.sessconfig['tracker_multitracker_reannounce_interval'] = value
+
+    def get_tracker_multitracker_reannounce_interval(self):
+        """ Returns the multitracker reannouce interval. 
+        @return A number of seconds. """
+        return self.sessconfig['tracker_multitracker_reannounce_interval']
+
+    def set_tracker_multitracker_maxpeers(self,value):
+        """ Number of peers to retrieve from the other trackers in a tracker
+         announce in a multi-tracker setup. 
+         @param value A number of peers.
+         """
+        self.sessconfig['tracker_multitracker_maxpeers'] = value
+
+    def get_tracker_multitracker_maxpeers(self):
+        """ Returns the number of peers to retrieve from another tracker. 
+        @return A number of peers. """
+        return self.sessconfig['tracker_multitracker_maxpeers']
+
+    def set_tracker_aggregate_forward(self,value):
+        """ Set an URL to which, if set, all non-multitracker requests are
+        forwarded, with a password added (optional).
+        @param value A 2-item list with format: [,|None]
+        """
+        self.sessconfig['tracker_aggregate_forward'] = value
+
+    def get_tracker_aggregate_forward(self):
+        """ Returns the aggregate forward URL and optional password as a 2-item 
+        list. 
+        @return URL """
+        return self.sessconfig['tracker_aggregate_forward']
+
+    def set_tracker_aggregator(self,value):
+        """ Whether to act as a data aggregator rather than a tracker. 
+        To enable, set to True or ; if password is set, then an 
+        incoming password is required for access.
+        @param value Boolean or string.
+        """
+        self.sessconfig['tracker_aggregator'] = value
+
+    def get_tracker_aggregator(self):
+        """ Returns the tracker aggregator parameter. 
+        @return Boolean or string. """
+        return self.sessconfig['tracker_aggregator']
+
+    def set_tracker_multitracker_http_timeout(self,value):
+        """ Time to wait before assuming that an HTTP connection
+        to another tracker in a multi-tracker setup has timed out. 
+        @param value A number of seconds.
+        """
+        self.sessconfig['tracker_multitracker_http_timeout'] = value
+
+    def get_tracker_multitracker_http_timeout(self):
+        """ Returns timeout for inter-multi-tracker HTTP connections. 
+        @return A number of seconds. """
+        return self.sessconfig['tracker_multitracker_http_timeout']
+
+
+    #
+    # For Tribler superpeer servers
+    #
+    def set_superpeer(self,value):
+        """ Run Session in super peer mode (default = disabled).
+        @param value Boolean.
+        """
+        self.sessconfig['superpeer'] = value
+
+    def get_superpeer(self):
+        """ Returns whether the Session runs in superpeer mode. 
+        @return Boolean. """
+        return self.sessconfig['superpeer']
+
+    def set_superpeer_file(self,value):
+        """ File with addresses of superpeers (default = install_dir+
+        Tribler/Core/superpeer.txt).
+        @param value An absolute path name.
+        """
+        self.sessconfig['superpeer_file'] = value
+
+    def get_superpeer_file(self):
+        """ Returns the superpeer file.
+        @return An absolute path name. """
+        return self.sessconfig['superpeer_file']
+
+    def set_overlay_log(self,value):
+        """ File to log message to in super peer mode (default = No logging)
+        @param value An absolute path name.
+        """
+        self.sessconfig['overlay_log'] = value
+
+    def get_overlay_log(self):
+        """ Returns the file to log messages to or None.
+        @return An absolute path name. """
+        return self.sessconfig['overlay_log']
+
+    def set_coopdlconfig(self,dscfg):
+        """ Sets the DownloadStartupConfig with which to start Downloads
+        when you are asked to help in a cooperative download.
+        """
+        c = dscfg.copy()
+        self.sessconfig['coopdlconfig'] = c.dlconfig # copy internal dict
+        
+    def get_coopdlconfig(self):
+        """ Return the DownloadStartupConfig that is used when helping others
+        in a cooperative download.
+        @return DownloadStartupConfig
+        """
+        dlconfig = self.sessconfig['coopdlconfig']
+        if dlconfig is None:
+            return None
+        else:
+            from BaseLib.Core.DownloadConfig import DownloadStartupConfig 
+            return DownloadStartupConfig(dlconfig)
+        
+
+    #
+    # NAT Puncturing servers information setting
+    #
+    def set_nat_detect(self,value):
+        """ Whether to try to detect the type of Network Address Translator
+        in place.
+        @param value Boolean.
+        """
+        self.sessconfig['nat_detect'] = value
+    
+    def set_puncturing_internal_port(self, puncturing_internal_port):
+        """ The listening port of the puncturing module.
+        @param puncturing_internal_port integer. """
+        self.sessconfig['puncturing_internal_port'] = puncturing_internal_port
+
+    def set_stun_servers(self, stun_servers):
+        """ The addresses of the STUN servers (at least 2)
+        @param stun_servers List of (hostname/ip,port) tuples. """
+        self.sessconfig['stun_servers'] = stun_servers
+
+    def set_pingback_servers(self, pingback_servers):
+        """ The addresses of the pingback servers (at least 1)
+        @param pingback_servers List of (hostname/ip,port) tuples. """
+        self.sessconfig['pingback_servers'] = pingback_servers
+
+    # Puncturing servers information retrieval
+    def get_nat_detect(self):
+        """ Whether to try to detect the type of Network Address Translator
+        in place.
+        @return Boolean
+        """
+        return self.sessconfig['nat_detect']
+    
+    def get_puncturing_internal_port(self):
+        """ Returns the listening port of the puncturing module.
+        @return integer. """
+        return self.sessconfig['puncturing_internal_port']
+
+    def get_stun_servers(self):
+        """ Returns the addresses of the STUN servers.
+        @return List of (hostname/ip,port) tuples. """
+        return self.sessconfig['stun_servers']
+
+    def get_pingback_servers(self):
+        """ Returns the addresses of the pingback servers.
+        @return List of (hostname/ip,port) tuples. """
+        return self.sessconfig['pingback_servers']
+
+    #
+    # Crawler
+    #
+    def set_crawler(self, value):
+        """ Handle crawler messages when received (default = True)
+        @param value Boolean
+        """
+        self.sessconfig['crawler'] = value
+
+    def get_crawler(self):
+        """ Whether crawler messages are processed
+        @return Boolean. """
+        return self.sessconfig['crawler']
+    
+    # 
+    # Local Peer Discovery using IP Multicast
+    #
+    def set_multicast_local_peer_discovery(self,value):
+        """ Set whether the Session tries to detect local peers
+        using a local IP multicast. Overlay swarm (set_overlay()) must
+        be enabled as well.
+        @param value Boolean
+        """
+        self.sessconfig['multicast_local_peer_discovery'] = value
+        
+    def get_multicast_local_peer_discovery(self):
+        """
+        Returns whether local peer discovery is enabled.
+        @return Boolean
+        """
+        return self.sessconfig['multicast_local_peer_discovery']
+
+    #
+    # VoteCast
+    #
+    def set_votecast_recent_votes(self, value):
+        """ Sets the maximum limit for the recent votes by the user, 
+        that will be forwarded to connected peers 
+        @param value int 
+        """
+        self.sessconfig['votecast_recent_votes'] = value 
+
+    def get_votecast_recent_votes(self):
+        """ Returns the maximum limit for the recent votes by the user, 
+        that will be forwarded to connected peers 
+        @return int 
+        """
+        return self.sessconfig['votecast_recent_votes']
+    
+    def set_votecast_random_votes(self, value):
+        """ Sets the maximum limit for the user's votes that are different from recent ones
+        but selected randomly; these votes will be forwarded to connected peers along with recent votes 
+        @param value int 
+        """
+        self.sessconfig['votecast_random_votes'] = value
+
+    def get_votecast_random_votes(self):
+        """ Returns the maximum limit for the user's votes that are different from recent ones
+        but selected randomly; these votes will be forwarded to connected peers along with recent votes 
+        @return int 
+        """        
+        return self.sessconfig['votecast_random_votes']
+
+    #
+    # ChannelCast
+    #
+    def set_channelcast_recent_own_subscriptions(self, value):
+        """ Sets the maximum limit for the recent subscriptions by the user, 
+        that will be forwarded to connected peers 
+        @param value int 
+        """
+        self.sessconfig['channelcast_recent_own_subscriptions'] = value
+
+    def get_channelcast_recent_own_subscriptions(self):
+        """ Returns the maximum limit for the recent subscriptions by the user, 
+        that will be forwarded to connected peers 
+        @return int 
+        """
+        return self.sessconfig['channelcast_recent_own_subscriptions']
+    
+    def set_channelcast_random_own_subscriptions(self, value):
+        """ Sets the maximum limit for the user's subscriptions that are different from recent ones
+        but selected randomly; these subscriptions will be forwarded to connected peers 
+        @param value int 
+        """
+        self.sessconfig['channelcast_random_own_subscriptions'] = value
+
+    def get_channelcast_random_own_subscriptions(self):
+        """ Returns the maximum limit for the user's subscriptions that are different from recent ones
+        but selected randomly; these subscriptions will be forwarded to connected peers 
+        @return int 
+        """
+        return self.sessconfig['channelcast_random_own_subscriptions']
+    
+    #
+    # Subtitle collection via Andrea Reale's extension
+    #
+    def set_subtitles_collecting(self,value):
+        """ Automatically collect subtitles from peers in the network (default = 
+        False).
+        @param value Boolean. 
+        """
+        self.sessconfig['subtitles_collecting'] = value
+
+    def get_subtitles_collecting(self):
+        """ Returns whether to automatically collect subtitles.
+        @return Boolean. """
+        return self.sessconfig['subtitles_collecting']
+
+    def set_subtitles_collecting_dir(self,value):
+        """
+        Where to place collected subtitles? (default is state_dir + 'collected_subtitles_files')
+        @param value An absolute path.
+        """
+        self.sessconfig['subtitles_collecting_dir'] = value
+        
+    def get_subtitles_collecting_dir(self):
+        """ Returns the directory to save collected subtitles.
+        @return An absolute path name. """
+        return self.sessconfig['subtitles_collecting_dir']
+        
+    def set_subtitles_upload_rate(self,value):
+        """ Maximum upload rate to use for subtitles collecting.
+        @param value A rate in KB/s. """
+        self.sessconfig['subtitles_upload_rate'] = value
+    
+    def get_subtitles_upload_rate(self):
+        """ Returns the upload rate to use for subtitle collecting.
+        @return A rate in KB/s. """
+        return self.sessconfig['subtitles_upload_rate']
+
+
+
+class SessionStartupConfig(SessionConfigInterface,Copyable,Serializable):  
+    """ Class to configure a Session """
+    
+    def __init__(self,sessconfig=None):
+        SessionConfigInterface.__init__(self,sessconfig)
+
+    #
+    # Class method
+    #
+    def load(filename):
+        """
+        Load a saved SessionStartupConfig from disk.
+        
+        @param filename  An absolute Unicode filename
+        @return SessionStartupConfig object
+        """
+        # Class method, no locking required
+        f = open(filename,"rb")
+        sessconfig = pickle.load(f)
+        sscfg = SessionStartupConfig(sessconfig)
+        f.close()
+        return sscfg
+    load = staticmethod(load)
+
+    def save(self,filename):
+        """ Save the SessionStartupConfig to disk.
+        @param filename  An absolute Unicode filename
+        """
+        # Called by any thread
+        f = open(filename,"wb")
+        pickle.dump(self.sessconfig,f)
+        f.close()
+
+    #
+    # Copyable interface
+    # 
+    def copy(self):
+        config = copy.copy(self.sessconfig)
+        return SessionStartupConfig(config)
diff --git a/instrumentation/next-share/BaseLib/Core/SocialNetwork/FriendshipMsgHandler.py b/instrumentation/next-share/BaseLib/Core/SocialNetwork/FriendshipMsgHandler.py
new file mode 100644
index 0000000..672a831
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/SocialNetwork/FriendshipMsgHandler.py
@@ -0,0 +1,874 @@
+# Written by Ali Abbas, Arno Bakker
+# see LICENSE.txt for license information
+
+# TODO: either maintain connections to friends always or supplement the
+# list of friends with a number of on-line taste buddies.
+#
+# TODO: at least add fifo order to msgs, otherwise clicking 
+# "make friend", "delete friend", "make friend" could arive in wrong order
+# due to forwarding.
+#
+
+import threading
+import sys
+import os
+import random
+import cPickle
+from time import time
+from types import DictType
+from traceback import print_exc
+from sets import Set
+
+from BaseLib.Core.simpledefs import *
+from BaseLib.Core.BitTornado.bencode import bencode, bdecode
+
+from BaseLib.Core.BitTornado.BT1.MessageID import *
+from BaseLib.Core.CacheDB.CacheDBHandler import PeerDBHandler, FriendDBHandler
+from BaseLib.Core.CacheDB.SqliteFriendshipStatsCacheDB import FriendshipStatisticsDBHandler
+from BaseLib.Core.CacheDB.sqlitecachedb import bin2str
+from BaseLib.Core.Utilities.utilities import *
+
+from BaseLib.Core.Overlay.SecureOverlay import OLPROTO_VER_SEVENTH
+
+DEBUG = False
+
+"""
+State diagram:
+
+NOFRIEND -> I_INVITED or HE_INVITED
+I_INVITED -> APPROVED or HE_DENIED
+HE_INVITED -> APPROVED
+HE_INVITED -> I_DENIED
+
+In theory it could happen that he sends an response=1 RESP, in which case
+he approved us. I consider that an HE_INIVITE
+"""
+
+RESCHEDULE_INTERVAL = 60
+RESEND_INTERVAL = 5*60
+
+
+class FriendshipMsgHandler:
+    __singleton = None
+    __lock = threading.Lock()
+
+    @classmethod
+    def getInstance(cls, *args, **kargs):
+        if not cls.__singleton:
+            cls.__lock.acquire()
+            try:
+                if not cls.__singleton:
+                    cls.__singleton = cls(*args, **kargs)
+            finally:
+                cls.__lock.release()
+        return cls.__singleton
+    
+    def __init__(self):
+        if FriendshipMsgHandler.__singleton:
+            raise RuntimeError, "FriendshipMsgHandler is singleton"
+        self.overlay_bridge = None
+        self.currmsgs = {}
+        self.online_fsext_peers = Set() # online peers that speak FRIENDSHIP ext
+        self.peerdb = PeerDBHandler.getInstance()
+        self.frienddb = FriendDBHandler.getInstance()
+        self.friendshipStatistics_db = FriendshipStatisticsDBHandler.getInstance()
+        self.list_no_of_conn_attempts_per_target= {}
+        self.usercallback = None
+    
+    def register(self, overlay_bridge, session):
+        if DEBUG:
+            print >> sys.stderr, "friendship: register"
+        self.overlay_bridge = overlay_bridge
+        self.session = session
+        try:
+            self.load_checkpoint()
+        except:
+            print_exc()
+        self.overlay_bridge.add_task(self.reschedule_connects,RESCHEDULE_INTERVAL)
+     
+     
+    def shutdown(self):
+        """
+        Delegate all outstanding messages to others
+        """
+        # Called by OverlayThread
+        self.delegate_friendship_making()
+        self.checkpoint()
+        
+
+    def register_usercallback(self,usercallback):
+        self.usercallback = usercallback
+    
+    def anythread_send_friendship_msg(self,permid,type,params):
+        """ Called when user adds someone from the person found, or by 
+        explicity adding someone with her credentials
+        It establishes overlay connection with the target peer """
+        # Called by any thread
+        
+        olthread_func = lambda:self.send_friendship_msg(permid,type,params,submit=True)
+        self.overlay_bridge.add_task(olthread_func,0)
+        
+        
+    def send_friendship_msg(self,permid,type,params,submit=False):
+        # Called by overlay thread 
+        
+        if submit:
+            if DEBUG:
+                print >>sys.stderr,"friendship: send_friendship_msg: Saving msg",show_permid_short(permid)
+            self.save_msg(permid,type,params)
+            
+            if type == F_REQUEST_MSG:
+                # Make him my friend, pending his approval
+                self.frienddb.setFriendState(permid, commit=True,state=FS_I_INVITED)
+            elif type == F_RESPONSE_MSG:
+                # Mark response in DB
+                if params['response']:
+                    state = FS_MUTUAL
+                else:
+                    state = FS_I_DENIED
+                self.frienddb.setFriendState(permid, commit=True,state=state)
+
+        func = lambda exc,dns,permid,selversion:self.fmsg_connect_callback(exc, dns, permid, selversion, type)
+        self.overlay_bridge.connect(permid,self.fmsg_connect_callback)
+        
+        
+    def fmsg_connect_callback(self,exc,dns,permid,selversion, type = None):
+        """ Callback function for the overlay connect function """
+        # Called by OverlayThread
+
+        if exc is None:
+            if selversion < OLPROTO_VER_SEVENTH:
+                self.remove_msgs_for_ltv7_peer(permid)
+                return
+            
+            # Reached him
+            sendlist = self.get_msgs_as_sendlist(targetpermid=permid)
+            if DEBUG:
+                print >> sys.stderr, 'friendship: fmsg_connect_callback: sendlist len',len(sendlist)
+                #print_stack()
+            
+            for i in range(0,len(sendlist)):
+                tuple = sendlist[i]
+                
+                permid,msgid,msg = tuple
+                send_callback = lambda exc,permid:self.fmsg_send_callback(exc,permid,msgid)
+                
+                if DEBUG:
+                    print >>sys.stderr,"friendship: fmsg_connect_callback: Sending",`msg`,msgid
+                
+                mypermid = self.session.get_permid()
+                
+                commit = (i == len(sendlist)-1)
+                isForwarder = 0
+                no_of_helpers = 0
+#                if type == F_REQUEST_MSG:
+#                    print
+#                elif type == F_RESPONSE_MSG:
+#                    print
+                #Set forwarder to True and also no of helpers to 10
+                if type == F_FORWARD_MSG:
+                    isForwarder = 1
+                    no_of_helpers = 10
+                    
+                  
+                no_of_attempts = 0
+                if permid in self.currmsgs:
+                    msgid2rec = self.currmsgs[permid]
+                    if msgid in msgid2rec:
+                        msgrec = msgid2rec[msgid]
+                        no_of_attempts = msgrec['attempt']
+                
+#                insertFriendshipStatistics(self, my_permid, target_permid, current_time, isForwarder = 0, no_of_attempts = 0, no_of_helpers = 0, commit = True):
+                
+                self.friendshipStatistics_db.insertOrUpdateFriendshipStatistics( bin2str(mypermid), 
+                                                                         bin2str(permid), 
+                                                                         int(time()),
+                                                                         isForwarder, 
+                                                                         no_of_attempts ,
+                                                                         no_of_helpers, 
+                                                                         commit=commit)
+                
+                self.overlay_bridge.send(permid, FRIENDSHIP + bencode(msg), send_callback)
+                
+                
+        else:
+            if DEBUG:
+                peer = self.peerdb.getPeer(permid)
+                if peer is None:
+                    print >>sys.stderr, 'friendship: Could not connect to peer', show_permid_short(permid),peer
+                else:
+                    print >>sys.stderr, 'friendship: Could not connect to peer', show_permid_short(permid),peer['name']
+                print >>sys.stderr,exc
+            
+            mypermid = self.session.get_permid()
+            
+            isForwarder = 0
+            no_of_helpers = 0
+            if type == F_FORWARD_MSG:
+                isForwarder = 1
+                no_of_helpers = 10
+                    
+                 
+            no_of_attempts = 0
+            if permid in self.currmsgs:
+                msgid2rec = self.currmsgs[permid]
+                for msgid in msgid2rec:
+                    msgrec = msgid2rec[msgid]
+                    no_of_attempts = msgrec['attempt']
+                
+                
+            self.friendshipStatistics_db.insertOrUpdateFriendshipStatistics( bin2str(mypermid), 
+                                                                         bin2str(permid), 
+                                                                         int(time()),
+                                                                         isForwarder, 
+                                                                         no_of_attempts ,
+                                                                         no_of_helpers)
+        
+        
+
+
+    def fmsg_send_callback(self,exc,permid,msgid):
+        
+        # If an exception arises
+        if exc is None:
+            self.delete_msg(permid,msgid)
+        else:
+            if DEBUG:
+                print >> sys.stderr, 'friendship: Could not send to ',show_permid_short(permid)  
+                print_exc()
+            
+        mypermid = self.session.get_permid()
+        
+        no_of_attempts = 0
+        no_of_helpers = 10
+        isForwarder = False
+        if permid in self.currmsgs:
+            msgid2rec = self.currmsgs[permid]
+            for msgid in msgid2rec:
+                msgrec = msgid2rec[msgid]
+                no_of_attempts = msgrec['attempt']
+                if msgrec['forwarded'] == True:
+                    isForwarder = 1
+            
+            
+        self.friendshipStatistics_db.insertOrUpdateFriendshipStatistics( bin2str(mypermid), 
+                                                                         bin2str(permid), 
+                                                                         int(time()),
+                                                                         isForwarder, 
+                                                                         no_of_attempts ,
+                                                                         no_of_helpers)
+                    
+
+    def remove_msgs_for_ltv7_peer(self,permid):
+        """ Remove messages destined for a peer that does not speak >= v7 of
+        the overlay protocol
+        """
+        sendlist = self.get_msgs_as_sendlist(targetpermid=permid)
+        if DEBUG:
+            print >> sys.stderr, 'friendship: remove_msgs_for_ltv7_peer: sendlist len',len(sendlist)
+        
+        for i in range(0,len(sendlist)):
+            tuple = sendlist[i]
+            
+            permid,msgid,msg = tuple
+            self.delete_msg(permid,msgid)
+
+
+    #
+    # Incoming connections
+    #
+    def handleConnection(self, exc, permid, selversion, locally_initiated):
+
+        if selversion < OLPROTO_VER_SEVENTH:
+            return True
+
+        if exc is None:
+            self.online_fsext_peers.add(permid)
+
+            # if we meet peer otherwise, dequeue messages
+            if DEBUG:
+                print >> sys.stderr,"friendship: Met peer, attempting to deliver msgs",show_permid_short(permid)
+            
+            # If we're initiating the connection from this handler, the 
+            # fmsg_connect_callback will get called twice:
+            # 1. here
+            # 2. just a bit later when the callback for a successful connect()
+            #    is called.
+            # Solution: we delay this call, which should give 2. the time to
+            # run and remove msgs from the queue.
+            #
+            # Better: remove msgs from queue when sent and reinsert if send fails
+            #
+            friendship_delay_func = lambda:self.fmsg_connect_callback(None,None,permid,selversion)
+            self.overlay_bridge.add_task(friendship_delay_func,4)
+        else:
+            try:
+                self.online_fsext_peers.remove(permid)
+            except:
+                pass
+            
+        return True        
+
+
+    #
+    # Incoming messages
+    # 
+    def handleMessage(self, permid, selversion, message):
+        """ Handle incoming Friend Request, and their response"""
+
+        if selversion < OLPROTO_VER_SEVENTH:
+            if DEBUG:
+                print >> sys.stderr,"friendship: Got FRIENDSHIP msg from peer with old protocol",show_permid_short(permid)
+            return False
+        
+        try:
+            d = bdecode(message[1:])
+        except:
+            print_exc()
+            return False
+        
+        return self.process_message(permid,selversion,d)
+    
+    
+    def process_message(self,permid,selversion,d):
+        
+        if self.isValidFriendMsg(d):
+
+            if DEBUG:
+                print >> sys.stderr,"friendship: Got FRIENDSHIP msg",d['msg type']
+        
+            # If the message is to become a friend, i.e., a friendship request
+            if d['msg type'] == F_REQUEST_MSG:
+                self.process_request(permid,d)                  
+                        
+            # If the message is to have a response on friend request    
+            elif d['msg type'] == F_RESPONSE_MSG: 
+                self.process_response(permid,d)
+                    
+            # If the receiving message is to delegate the Friendship request to the target peer
+            elif d['msg type'] == F_FORWARD_MSG:
+                return self.process_forward(permid,selversion,d)
+            else:
+                if DEBUG:
+                    print >>sys.stderr,"friendship: Got unknown msg type",d['msg type']
+                return False
+            
+            return True
+        else:
+            if DEBUG:
+                print >>sys.stderr,"friendship: Got bad FRIENDSHIP message"
+            return False
+                
+    def process_request(self,permid,d):
+        # to see that the following peer is already a friend, or not
+        fs = self.frienddb.getFriendState(permid) 
+
+        if DEBUG:
+            print >>sys.stderr,"friendship: process_request: Got request, fs",show_permid_short(permid),fs
+
+        
+        if fs == FS_NOFRIEND or fs == FS_HE_DENIED:
+            # not on HE_INVITED, to filter out duplicates
+            
+            # And if that peer is not already added as a friend, either approved, or unapproved
+            # call friend dialog
+            self.frienddb.setFriendState(permid, commit=True, state = FS_HE_INVITED)
+            
+            # FUTURE: always do callback, such that we also know about failed
+            # attempts
+            if self.usercallback is not None:
+                friendship_usercallback = lambda:self.usercallback(permid,[])
+                self.session.uch.perform_usercallback(friendship_usercallback)
+        elif fs == FS_I_INVITED: 
+            # In case, requestee is already added as friend, just make this 
+            # requestee as an approved friend
+
+            if DEBUG:
+                print >>sys.stderr,"friendship: process_request: Got request but I already invited him"
+            
+            self.frienddb.setFriendState(permid, commit=True, state = FS_MUTUAL)
+
+            if DEBUG:
+                print >>sys.stderr,"friendship: process_request: Got request but I already invited him: sending reply"
+
+            self.send_friendship_msg(permid,F_RESPONSE_MSG,{'response':1},submit=True)
+        elif fs == FS_MUTUAL:
+            if DEBUG:
+                print >>sys.stderr,"friendship: process_request: Got request but already approved"
+        elif fs == FS_I_DENIED:
+            if DEBUG:
+                print >>sys.stderr,"friendship: process_request: Got request but I already denied"
+        elif DEBUG:
+            print >>sys.stderr,"friendship: process_request: Got request, but fs is",fs
+
+    def process_response(self,permid,d):
+
+        mypermid = self.session.get_permid()
+                     
+                
+        self.friendshipStatistics_db.updateFriendshipResponseTime( bin2str(mypermid), 
+                                                                         bin2str(permid), 
+                                                                         int(time()))
+
+        
+        fs = self.frienddb.getFriendState(permid)
+         
+        # If the request to add has been approved
+        if d['response'] == 1:
+            if fs == FS_I_INVITED:
+                self.frienddb.setFriendState(permid, commit=True, state = FS_MUTUAL)
+            elif fs != FS_MUTUAL:
+                # Unsollicited response, consider this an invite, if not already friend
+                self.frienddb.setFriendState(permid, commit=True, state = FS_HE_INVITED)
+        else:
+            # He denied our friendship
+            self.frienddb.setFriendState(permid, commit=True, state = FS_HE_DENIED)
+
+                
+    def process_forward(self,permid,selversion,d):
+        
+        mypermid = self.session.get_permid()
+        if d['dest']['permid'] == mypermid:
+            # This is a forward containing a message meant for me
+            
+            # First add original sender to DB so we can connect back to it
+            self.addPeerToDB(d['source'])
+            
+            self.process_message(d['source']['permid'],selversion,d['msg'])
+            
+            return True
+        
+            
+        else:
+            # Queue and forward
+            if DEBUG:
+                print >>sys.stderr,"friendship: process_fwd: Forwarding immediately to",show_permid_short(d['dest']['permid'])
+
+            if permid != d['source']['permid']:
+                if DEBUG:
+                    print >>sys.stderr,"friendship: process_fwd: Forwarding: Illegal, source is not sender, and dest is not me"
+                return False
+            # First add dest to DB so we can connect to it
+            
+            # FUTURE: don't let just any peer overwrite the IP+port of a peer
+            # if self.peer_db.hasPeer(d['dest']['permid']):
+            self.addPeerToDB(d['dest'])
+            
+            self.send_friendship_msg(d['dest']['permid'],d['msg type'],d,submit=True)
+            return True
+
+    def addPeerToDB(self,mpeer):
+        peer = {}
+        peer['permid'] = mpeer['permid']
+        peer['ip'] = mpeer['ip']
+        peer['port'] = mpeer['port']
+        peer['last_seen'] = 0
+        self.peerdb.addPeer(mpeer['permid'],peer,update_dns=True,commit=True)
+        
+
+    def create_friendship_msg(self,type,params):
+        
+        if DEBUG:
+            print >>sys.stderr,"friendship: create_fs_msg:",type,`params`
+        
+        mypermid = self.session.get_permid()
+        myip = self.session.get_external_ip()
+        myport = self.session.get_listen_port()
+        
+        d ={'msg type':type}     
+        if type == F_RESPONSE_MSG:
+            d['response'] = params['response']
+        elif type == F_FORWARD_MSG:
+            
+            if DEBUG:
+                print >>sys.stderr,"friendship: create: fwd: params",`params`
+            peer = self.peerdb.getPeer(params['destpermid']) # ,keys=['ip', 'port']) 
+            if peer is None:
+                if DEBUG:
+                    print >> sys.stderr, "friendship: create msg: Don't know IP + port of peer", show_permid_short(params['destpermid'])
+                return
+            #if DEBUG:
+            #    print >> sys.stderr, "friendship: create msg: Peer at",peer
+            
+            # FUTURE: add signatures on ip+port
+            src = {'permid':mypermid,'ip':myip,'port':myport}
+            dst = {'permid':params['destpermid'],'ip':str(peer['ip']),'port':peer['port']}
+            d.update({'source':src,'dest':dst,'msg':params['msg']})
+        return d
+
+
+
+    def isValidFriendMsg(self,d):
+
+        if DEBUG:
+            print >>sys.stderr,"friendship: msg: payload is",`d`    
+
+        
+        if type(d) != DictType:
+            if DEBUG:
+                print >>sys.stderr,"friendship: msg: payload is not bencoded dict"    
+            return False
+        if not 'msg type' in d:
+            if DEBUG:
+                print >>sys.stderr,"friendship: msg: dict misses key",'msg type'    
+            return False
+        
+        if d['msg type'] == F_REQUEST_MSG:
+            keys = d.keys()[:]
+            if len(keys)-1 != 0:
+                if DEBUG:
+                    print >>sys.stderr,"friendship: msg: REQ: contains superfluous keys",keys    
+                return False
+            return True
+            
+        if d['msg type'] == F_RESPONSE_MSG:
+            if (d.has_key('response') and (d['response'] == 1 or d['response'] == 0)):
+                return True
+            else:
+                if DEBUG:
+                    print >>sys.stderr,"friendship: msg: RESP: something wrong",`d`    
+                return False
+            
+        if d['msg type'] == F_FORWARD_MSG:
+            if not self.isValidPeer(d['source']):
+                if DEBUG:
+                    print >>sys.stderr,"friendship: msg: FWD: source bad",`d`    
+                return False
+            if not self.isValidPeer(d['dest']):
+                if DEBUG:
+                    print >>sys.stderr,"friendship: msg: FWD: dest bad",`d`    
+                return False
+            if not 'msg' in d:
+                if DEBUG:
+                    print >>sys.stderr,"friendship: msg: FWD: no msg",`d`    
+                return False
+            if not self.isValidFriendMsg(d['msg']):
+                if DEBUG:
+                    print >>sys.stderr,"friendship: msg: FWD: bad msg",`d`    
+                return False
+            if d['msg']['msg type'] == F_FORWARD_MSG:
+                if DEBUG:
+                    print >>sys.stderr,"friendship: msg: FWD: cannot contain fwd",`d`    
+                return False
+            return True
+        
+        return False
+
+
+    def isValidPeer(self,d):
+        if (d.has_key('ip') and d.has_key('port') and d.has_key('permid') 
+            and validPermid(d['permid'])
+            and validIP(d['ip'])and validPort(d['port'])):
+            return True
+        else:
+            return False                 
+
+
+    def save_msg(self,permid,type,params):
+        
+        if not permid in self.currmsgs:
+            self.currmsgs[permid] = {}
+        
+        mypermid = self.session.get_permid()    
+        now = time()
+        attempt = 1
+        
+        base = mypermid+permid+str(now)+str(random.random())
+        msgid = sha(base).hexdigest()
+        msgrec = {'permid':permid,'type':type,'params':params,'attempt':attempt,'t':now,'forwarded':False}
+        
+        msgid2rec = self.currmsgs[permid]
+        msgid2rec[msgid] = msgrec
+        
+    def delete_msg(self,permid,msgid):
+        try: 
+            if DEBUG:
+                print >>sys.stderr,"friendship: Deleting msg",show_permid_short(permid),msgid
+            msgid2rec = self.currmsgs[permid]
+            del msgid2rec[msgid]
+        except:
+            #print_exc()
+            pass
+
+    def set_msg_forwarded(self,permid,msgid):
+        try: 
+            msgid2rec = self.currmsgs[permid]
+            msgid2rec[msgid]['forwarded'] = True
+        except:
+            print_exc()
+
+    def reschedule_connects(self):
+        """ This function is run periodically and reconnects to peers when
+        messages meant for it are due to be retried
+        """
+        now = time()
+        delmsgids = []
+        reconnectpermids = Set()
+        for permid in self.currmsgs:
+            msgid2rec = self.currmsgs[permid]
+            for msgid in msgid2rec:
+                msgrec = msgid2rec[msgid]
+
+                eta = self.calc_eta(msgrec)
+                
+                if DEBUG:
+                    diff = None
+                    if eta is not None:
+                        diff = eta - now
+                        
+                    if DEBUG:
+                        peer = self.peerdb.getPeer(permid)
+                        if peer is None:
+                            print >>sys.stderr,"friendship: reschedule: ETA: wtf, peer not in DB!",show_permid_short(permid)
+                        else:
+                            print >>sys.stderr,"friendship: reschedule: ETA",show_permid_short(permid),peer['name'],diff
+                
+                if eta is None: 
+                    delmsgids.append((permid,msgid))
+                elif now > eta-1.0: # -1 for round off
+                    # reconnect
+                    reconnectpermids.add(permid)
+                    msgrec['attempt'] = msgrec['attempt'] + 1
+                    
+                    # Delegate 
+                    if msgrec['type'] == F_REQUEST_MSG and msgrec['attempt'] == 2:
+                        self.delegate_friendship_making(targetpermid=permid,targetmsgid=msgid)
+        
+        # Remove timed out messages
+        for permid,msgid in delmsgids:
+            if DEBUG:
+                print >>sys.stderr,"friendship: reschedule: Deleting",show_permid_short(permid),msgid
+            self.delete_msg(permid,msgid)
+        
+        # Initiate connections to peers for which we have due messages    
+        for permid in reconnectpermids:
+            if DEBUG:
+                print >>sys.stderr,"friendship: reschedule: Reconnect to",show_permid_short(permid)
+
+            self.overlay_bridge.connect(permid,self.fmsg_connect_callback)
+        
+        # Reschedule this periodic task
+        self.overlay_bridge.add_task(self.reschedule_connects,RESCHEDULE_INTERVAL)
+        
+        
+    def calc_eta(self,msgrec):
+        if msgrec['type'] == F_FORWARD_MSG:
+            if msgrec['attempt'] >= 10:
+                # Stop trying to forward after a given period 
+                return None
+            # exponential backoff, on 10th attempt we would wait 24hrs
+            eta = msgrec['t'] + pow(3.116,msgrec['attempt'])
+        else:
+            if msgrec['attempt'] >= int(7*24*3600/RESEND_INTERVAL):
+                # Stop trying to forward after a given period = 1 week 
+                return None
+
+            eta = msgrec['t'] + msgrec['attempt']*RESEND_INTERVAL 
+        return eta
+        
+
+    def get_msgs_as_sendlist(self,targetpermid=None):
+
+        sendlist = []
+        if targetpermid is None:
+            permids = self.currmsgs.keys()
+        else:
+            permids = [targetpermid] 
+        
+        for permid in permids:
+            msgid2rec = self.currmsgs.get(permid,{})
+            for msgid in msgid2rec:
+                msgrec = msgid2rec[msgid]
+                
+                if DEBUG:
+                    print >>sys.stderr,"friendship: get_msgs: Creating",msgrec['type'],`msgrec['params']`,msgid
+                if msgrec['type'] == F_FORWARD_MSG:
+                    msg = msgrec['params']
+                else:
+                    msg = self.create_friendship_msg(msgrec['type'],msgrec['params'])
+                tuple = (permid,msgid,msg)
+                sendlist.append(tuple)
+        return sendlist
+
+
+    def get_msgs_as_fwd_sendlist(self,targetpermid=None,targetmsgid=None):
+
+        sendlist = []
+        if targetpermid is None:
+            permids = self.currmsgs.keys()
+        else:
+            permids = [targetpermid] 
+        
+        for permid in permids:
+            msgid2rec = self.currmsgs.get(permid,{})
+            for msgid in msgid2rec:
+                if targetmsgid is None or msgid == targetmsgid:
+                    msgrec = msgid2rec[msgid]
+                    if msgrec['type'] != F_FORWARD_MSG and msgrec['forwarded'] == False:
+                        # Don't forward forwards, or messages already forwarded
+                    
+                        # Create forward message for original
+                        params = {}
+                        params['destpermid'] = permid
+                        params['msg'] = self.create_friendship_msg(msgrec['type'],msgrec['params'])
+                    
+                        msg = self.create_friendship_msg(F_FORWARD_MSG,params)
+                        tuple = (permid,msgid,msg)
+                        sendlist.append(tuple)
+        return sendlist
+        
+
+                
+    def delegate_friendship_making(self,targetpermid=None,targetmsgid=None):
+        if DEBUG:
+            print >>sys.stderr,"friendship: delegate:",show_permid_short(targetpermid),targetmsgid
+
+        # 1. See if there are undelivered msgs
+        sendlist = self.get_msgs_as_fwd_sendlist(targetpermid=targetpermid,targetmsgid=targetmsgid)
+        if DEBUG:
+            print >>sys.stderr,"friendship: delegate: Number of messages queued",len(sendlist)
+        
+        if len(sendlist) == 0:
+            return
+  
+        # 2. Get friends, not necess. online
+        friend_permids = self.frienddb.getFriends()
+        
+        if DEBUG:
+            l = len(friend_permids)
+            print >>sys.stderr,"friendship: delegate: friend helpers",l
+            for permid in friend_permids:
+                print >>sys.stderr,"friendship: delegate: friend helper",show_permid_short(permid)
+        
+        # 3. Sort online peers on similarity, highly similar should be tastebuddies
+        if DEBUG:
+            print >>sys.stderr,"friendship: delegate: Number of online v7 peers",len(self.online_fsext_peers)
+        tastebuddies = self.peerdb.getPeers(list(self.online_fsext_peers),['similarity','name']) 
+        tastebuddies.sort(sim_desc_cmp)
+
+        if DEBUG:
+            print >>sys.stderr,"friendship: delegate: Sorted tastebuddies",`tastebuddies`
+
+        tastebuddies_permids = []
+        size = min(10,len(tastebuddies))
+        for i in xrange(0,size):
+            peer = tastebuddies[i]
+            if DEBUG:
+                print >>sys.stderr,"friendship: delegate: buddy helper",show_permid_short(peer['permid'])
+            tastebuddies_permids.append(peer['permid'])
+
+        # 4. Create list of helpers:
+        #
+        # Policy: Helpers are a mix of friends and online tastebuddies
+        # with 70% friends (if avail) and 30% tastebuddies
+        #
+        # I chose this policy because friends are not guaranteed to be online
+        # and waiting to see if we can connect to them before switching to
+        # the online taste buddies is complex code-wise and time-consuming.
+        # We don't have a lot of time when this thing is called by Session.shutdown()
+        #
+        nwant = 10
+        nfriends = int(nwant * .7)
+        nbuddies = int(nwant * .3)
+        
+        part1 = sampleorlist(friend_permids,nfriends)
+        fill = nfriends-len(part1) # if no friends, use tastebuddies
+        part2 = sampleorlist(tastebuddies_permids,nbuddies+fill)
+        helpers = part1 + part2
+
+        if DEBUG:
+            l = len(helpers)
+            print >>sys.stderr,"friendship: delegate: end helpers",l
+            for permid in helpers:
+                print >>sys.stderr,"friendship: delegate: end helper",show_permid_short(permid),self.frienddb.getFriendState(permid),self.peerdb.getPeers([permid],['similarity','name'])
+
+
+        for tuple in sendlist:
+            destpermid,msgid,msg = tuple
+            for helperpermid in helpers:
+                if destpermid != helperpermid:
+                    connect_callback = lambda exc,dns,permid,selversion:self.forward_connect_callback(exc,dns,permid,selversion,destpermid,msgid,msg)
+                    
+                    if DEBUG:
+                        print >>sys.stderr,"friendship: delegate: Connecting to",show_permid_short(helperpermid)
+                     
+                    self.overlay_bridge.connect(helperpermid, connect_callback)
+
+
+    def forward_connect_callback(self,exc,dns,permid,selversion,destpermid,msgid,msg):
+        if exc is None:
+            
+            if selversion < OLPROTO_VER_SEVENTH:
+                return
+            
+            send_callback = lambda exc,permid:self.forward_send_callback(exc,permid,destpermid,msgid)
+            if DEBUG:
+                print >>sys.stderr,"friendship: forward_connect_callback: Sending",`msg`
+            self.overlay_bridge.send(permid, FRIENDSHIP + bencode(msg), send_callback)
+        elif DEBUG:
+            print >>sys.stderr,"friendship: forward: Could not connect to helper",show_permid_short(permid)
+
+
+    def forward_send_callback(self,exc,permid,destpermid,msgid):
+        if DEBUG:
+            if exc is None:
+                if DEBUG:
+                    print >>sys.stderr,"friendship: forward: Success forwarding to helper",show_permid_short(permid)
+                self.set_msg_forwarded(destpermid,msgid)
+            else:
+                if DEBUG:
+                    print >>sys.stderr,"friendship: forward: Failed to forward to helper",show_permid_short(permid)
+        
+    def checkpoint(self):
+        statedir = self.session.get_state_dir()
+        newfilename = os.path.join(statedir,'new-friendship-msgs.pickle')
+        finalfilename = os.path.join(statedir,'friendship-msgs.pickle')
+        try:
+            f = open(newfilename,"wb")
+            cPickle.dump(self.currmsgs,f)
+            f.close()
+            try:
+                os.remove(finalfilename)
+            except:
+                # If first time, it doesn't exist
+                print_exc()
+            os.rename(newfilename,finalfilename)
+        except:
+            print_exc()
+        
+    def load_checkpoint(self):
+        statedir = self.session.get_state_dir()
+        finalfilename = os.path.join(statedir,'friendship-msgs.pickle')
+        try:
+            f = open(finalfilename,"rb")
+            self.currmsgs = cPickle.load(f)
+        except:
+            print >>sys.stderr, "friendship: could not read previous messages from", finalfilename
+
+        # Increase # attempts till current time
+        now = time()
+        for permid in self.currmsgs:
+            msgid2rec = self.currmsgs[permid]
+            for msgid in msgid2rec:
+                msgrec = msgid2rec[msgid]
+                diff = now - msgrec['t']
+                a = int(diff/RESEND_INTERVAL)
+                a += 1
+                if DEBUG:
+                    print >>sys.stderr,"friendship: load_checkp: Changing #attempts from",msgrec['attempt'],a
+                msgrec['attempt'] = a
+
+        
+def sim_desc_cmp(peera,peerb):
+    if peera['similarity'] < peerb['similarity']:
+        return 1
+    elif peera['similarity'] > peerb['similarity']:
+        return -1
+    else:
+        return 0
+    
+def sampleorlist(z,k):
+    if len(z) < k:
+        return z
+    else:
+        return random.sample(k)
diff --git a/instrumentation/next-share/BaseLib/Core/SocialNetwork/OverlapMsgHandler.py b/instrumentation/next-share/BaseLib/Core/SocialNetwork/OverlapMsgHandler.py
new file mode 100644
index 0000000..615a63f
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/SocialNetwork/OverlapMsgHandler.py
@@ -0,0 +1,277 @@
+# Written by Arno Bakker
+# see LICENSE.txt for license information
+
+import sys
+from time import time
+from traceback import print_exc
+
+from BaseLib.Core.BitTornado.bencode import bencode, bdecode
+from BaseLib.Core.BitTornado.BT1.MessageID import *
+
+from BaseLib.Core.Utilities.utilities import *
+from BaseLib.Core.Utilities.unicode import str2unicode
+
+DEBUG = False
+
+MIN_OVERLAP_WAIT = 12.0*3600.0 # half a day in seconds
+
+ICON_MAX_SIZE = 10*1024
+
+class OverlapMsgHandler:
+    
+    def __init__(self):
+        
+        self.recentpeers = {}
+
+    def register(self, overlay_bridge, launchmany):
+        if DEBUG:
+            print >> sys.stderr,"socnet: bootstrap: overlap"
+        self.mypermid = launchmany.session.get_permid()
+        self.session = launchmany.session
+        self.peer_db = launchmany.peer_db 
+        self.superpeer_db = launchmany.superpeer_db
+        self.overlay_bridge = overlay_bridge
+
+    #
+    # Incoming SOCIAL_OVERLAP
+    # 
+    def recv_overlap(self,permid,message,selversion):
+        # 1. Check syntax
+        try:
+            oldict = bdecode(message[1:])
+        except:
+            print_exc()
+            if DEBUG:
+                print >> sys.stderr,"socnet: SOCIAL_OVERLAP: error becoding"
+            return False
+
+        if not isValidDict(oldict,permid):
+            return False
+
+        # 2. Process
+        self.process_overlap(permid,oldict)
+        return True
+
+    def process_overlap(self,permid,oldict):
+        #self.print_hashdict(oldict['hashnetwork'])
+
+        # 1. Clean recently contacted admin
+        self.clean_recentpeers()
+
+        # 3. Save persinfo + hrwidinfo + ipinfo
+        if self.peer_db.hasPeer(permid):
+            save_ssocnet_peer(self,permid,oldict,False,False,False)
+        elif DEBUG:
+            print >> sys.stderr,"socnet: overlap: peer unknown?! Weird, we just established connection"
+
+        # 6. Reply
+        if not (permid in self.recentpeers.keys()):
+            self.recentpeers[permid] = time()
+            self.reply_to_overlap(permid)
+
+    def clean_recentpeers(self):
+        newdict = {}
+        for permid2,t in self.recentpeers.iteritems():
+            if (t+MIN_OVERLAP_WAIT) > time():
+                newdict[permid2] = t
+            #elif DEBUG:
+            #    print >> sys.stderr,"socnet: overlap: clean recent: not keeping",show_permid_short(permid2)
+                
+        self.recentpeers = newdict
+
+    def reply_to_overlap(self,permid):
+        oldict = self.create_oldict()
+        self.send_overlap(permid,oldict)
+
+    #
+    # At overlay-connection establishment time.
+    #
+    def initiate_overlap(self,permid,locally_initiated):
+        self.clean_recentpeers()
+        if not (permid in self.recentpeers.keys() or permid in self.superpeer_db.getSuperPeers()):
+            if locally_initiated:
+                # Make sure only one sends it
+                self.recentpeers[permid] = time()
+                self.reply_to_overlap(permid)
+            elif DEBUG:
+                print >> sys.stderr,"socnet: overlap: active: he should initiate"
+        elif DEBUG:
+            print >> sys.stderr,"socnet: overlap: active: peer recently contacted already"
+
+    #
+    # General
+    #
+    def create_oldict(self):
+        """
+        Send:
+        * Personal info: name, picture, rwidhashes
+        * IP info: IP + port
+        Both are individually signed by us so dest can safely 
+        propagate. We distinguish between what a peer said
+        is his IP+port and the information obtained from the network
+        or from other peers (i.e. BUDDYCAST)
+        """
+
+        nickname = self.session.get_nickname().encode("UTF-8")
+        persinfo = {'name':nickname}
+        # See if we can find icon
+        iconmime, icondata = self.session.get_mugshot()
+        if icondata:
+            persinfo.update({'icontype':iconmime, 'icondata':icondata})
+        
+        oldict = {}
+        oldict['persinfo'] = persinfo
+
+        #print >> sys.stderr, 'Overlap: Sending oldict: %s' % `oldict`
+                            
+        #if DEBUG:
+        #    print >> sys.stderr,"socnet: overlap: active: sending hashdict"
+        #    self.print_hashdict(oldict['hashnetwork'])
+
+        return oldict
+
+
+    def send_overlap(self,permid,oldict):
+        try:
+            body = bencode(oldict)
+            ## Optimization: we know we're currently connected
+            self.overlay_bridge.send(permid, SOCIAL_OVERLAP + body,self.send_callback)
+        except:
+            if DEBUG:
+                print_exc(file=sys.stderr)
+
+    
+    def send_callback(self,exc,permid):
+        if exc is not None:
+            if DEBUG:
+                print >> sys.stderr,"socnet: SOCIAL_OVERLAP: error sending to",show_permid_short(permid),exc
+
+    #
+    # Internal methods
+    #
+
+
+def isValidDict(oldict,source_permid):
+    if not isinstance(oldict, dict):
+        if DEBUG:
+            print >> sys.stderr,"socnet: SOCIAL_OVERLAP: not a dict"
+        return False
+    k = oldict.keys()        
+
+    if DEBUG:
+        print >> sys.stderr,"socnet: SOCIAL_OVERLAP: keys",k
+
+    if not ('persinfo' in k) or not isValidPersinfo(oldict['persinfo'],False):
+        if DEBUG:
+            print >> sys.stderr,"socnet: SOCIAL_OVERLAP: key 'persinfo' missing or value wrong type in dict"
+        return False
+
+    for key in k:
+        if key not in ['persinfo']:
+            if DEBUG:
+                print >> sys.stderr,"socnet: SOCIAL_OVERLAP: unknown key",key,"in dict"
+            return False
+
+    return True
+
+
+
+def isValidPersinfo(persinfo,signed):
+    if not isinstance(persinfo,dict):
+        if DEBUG:
+            print >> sys.stderr,"socnet: SOCIAL_*: persinfo: not a dict"
+        return False
+
+    k = persinfo.keys()
+    #print >> sys.stderr,"socnet: SOCIAL_*: persinfo: keys are",k
+    if not ('name' in k) or not isinstance(persinfo['name'],str):
+        if DEBUG:
+            print >> sys.stderr,"socnet: SOCIAL_*: persinfo: key 'name' missing or value wrong type"
+        return False
+
+    if 'icontype' in k and not isValidIconType(persinfo['icontype']):
+        if DEBUG:
+            print >> sys.stderr,"socnet: SOCIAL_*: persinfo: key 'icontype' value wrong type"
+        return False
+
+    if 'icondata' in k and not isValidIconData(persinfo['icondata']):
+        if DEBUG:
+            print >> sys.stderr,"socnet: SOCIAL_*: persinfo: key 'icondata' value wrong type"
+        return False
+
+    if ('icontype' in k and not ('icondata' in k)) or ('icondata' in k and not ('icontype' in k)):
+        if DEBUG:
+            print >> sys.stderr,"socnet: SOCIAL_*: persinfo: key 'icontype' without 'icondata' or vice versa"
+        return False
+
+    if signed:
+        if not ('insert_time' in k) or not isinstance(persinfo['insert_time'],int):
+            if DEBUG:
+                print >> sys.stderr,"socnet: SOCIAL_*: persinfo: key 'insert_time' missing or value wrong type"
+            return False
+
+    for key in k:
+        if key not in ['name','icontype','icondata','insert_time']:
+            if DEBUG:
+                print >> sys.stderr,"socnet: SOCIAL_*: persinfo: unknown key",key,"in dict"
+            return False
+
+    return True
+
+
+def isValidIconType(type):
+    """ MIME-type := type "/" subtype ... """
+    if not isinstance(type,str):
+        return False
+    idx = type.find('/')
+    ridx = type.rfind('/')
+    return idx != -1 and idx == ridx
+
+def isValidIconData(data):
+    if not isinstance(data,str):
+        return False
+    
+#    if DEBUG:
+#        print >>sys.stderr,"socnet: SOCIAL_*: persinfo: IconData length is",len(data)
+    
+    return len(data) <= ICON_MAX_SIZE
+
+
+
+def save_ssocnet_peer(self,permid,record,persinfo_ignore,hrwidinfo_ignore,ipinfo_ignore):
+    """ This function is used by both BootstrapMsgHandler and 
+        OverlapMsgHandler, and uses their database pointers. Hence the self 
+        parameter. persinfo_ignore and ipinfo_ignore are booleans that
+        indicate whether to ignore the personal info, resp. ip info in
+        this record, because they were unsigned in the message and
+        we already received signed versions before.
+    """
+    if permid == self.mypermid:
+        return
+    
+    # 1. Save persinfo
+    if not persinfo_ignore:
+        persinfo = record['persinfo']
+        
+        if DEBUG:
+            print >>sys.stderr,"socnet: Got persinfo",persinfo.keys()
+            if len(persinfo.keys()) > 1:
+                print >>sys.stderr,"socnet: Got persinfo THUMB THUMB THUMB THUMB"
+        
+        # Arno, 2008-08-22: to avoid UnicodeDecode errors when commiting 
+        # on sqlite
+        name = str2unicode(persinfo['name'])
+
+        if DEBUG:
+            print >> sys.stderr,"socnet: SOCIAL_OVERLAP",show_permid_short(permid),`name`
+        
+        if self.peer_db.hasPeer(permid):
+            self.peer_db.updatePeer(permid, name=name)
+        else:
+            self.peer_db.addPeer(permid,{'name':name})
+    
+        # b. Save icon
+        if 'icontype' in persinfo and 'icondata' in persinfo: 
+            if DEBUG:
+                print >> sys.stderr,"socnet: saving icon for",show_permid_short(permid),`name`
+            self.peer_db.updatePeerIcon(permid, persinfo['icontype'],persinfo['icondata'])    
diff --git a/instrumentation/next-share/BaseLib/Core/SocialNetwork/RemoteQueryMsgHandler.py b/instrumentation/next-share/BaseLib/Core/SocialNetwork/RemoteQueryMsgHandler.py
new file mode 100644
index 0000000..c56e1f5
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/SocialNetwork/RemoteQueryMsgHandler.py
@@ -0,0 +1,728 @@
+# Written by Arno Bakker, Jie Yang
+# see LICENSE.txt for license information
+#
+# Send free-form queries to all the peers you are connected to.
+#
+# TODO: make sure we return also items from download history, but need to verify if 
+# their status is still checked.
+#
+#
+
+import os
+import sys
+import re
+from time import time
+from sets import Set
+from traceback import print_stack, print_exc
+import datetime
+import time as T
+
+from M2Crypto import Rand
+
+from BaseLib.Core.simpledefs import *
+from BaseLib.Core.BitTornado.bencode import bencode,bdecode
+from BaseLib.Core.CacheDB.sqlitecachedb import bin2str, str2bin
+from BaseLib.Core.CacheDB.CacheDBHandler import ChannelCastDBHandler,PeerDBHandler
+from BaseLib.Core.BitTornado.BT1.MessageID import *
+from BaseLib.Core.BuddyCast.moderationcast_util import *
+from BaseLib.Core.TorrentDef import TorrentDef
+from BaseLib.Core.Overlay.SecureOverlay import OLPROTO_VER_SIXTH, OLPROTO_VER_NINETH, OLPROTO_VER_ELEVENTH, OLPROTO_VER_TWELFTH, OLPROTO_VER_THIRTEENTH
+from BaseLib.Core.Utilities.utilities import show_permid_short,show_permid
+from BaseLib.Core.Statistics.Logger import OverlayLogger
+from BaseLib.Core.Utilities.unicode import dunno2unicode
+from BaseLib.Core.Search.SearchManager import split_into_keywords
+
+MAX_RESULTS = 20
+QUERY_ID_SIZE = 20
+MAX_QUERY_REPLY_LEN = 100*1024    # 100K
+MAX_PEERS_TO_QUERY = 20
+
+DEBUG = False
+
+class FakeUtility:
+    
+    def __init__(self,config_path):
+        self.config_path = config_path
+        
+    def getConfigPath(self):
+        return self.config_path
+
+
+class RemoteQueryMsgHandler:
+    
+    __single = None
+    
+    def __init__(self):
+        if RemoteQueryMsgHandler.__single:
+            raise RuntimeError, "RemoteQueryMsgHandler is singleton"
+        RemoteQueryMsgHandler.__single = self
+
+        self.connections = {}    # only connected remote_search_peers -> selversion
+        self.query_ids2rec = {}    # ARNOCOMMENT: TODO: purge old entries...
+        self.overlay_log = None
+        self.registered = False
+        self.logfile = None 
+
+    def getInstance(*args, **kw):
+        if RemoteQueryMsgHandler.__single is None:
+            RemoteQueryMsgHandler(*args, **kw)
+        return RemoteQueryMsgHandler.__single
+    getInstance = staticmethod(getInstance)
+        
+
+    def register(self,overlay_bridge,launchmany,config,bc_fac,log=''):
+        if DEBUG:
+            print >> sys.stderr,"rquery: register"
+        self.overlay_bridge = overlay_bridge
+        self.session =  launchmany.session
+        self.torrent_db = launchmany.torrent_db
+        self.peer_db = launchmany.peer_db
+        self.channelcast_db = launchmany.channelcast_db
+        # debug
+        # self.superpeer_db = launchmany.superpeer_db
+        
+        self.config = config
+        self.bc_fac = bc_fac # May be None
+        if log:
+            self.overlay_log = OverlayLogger.getInstance(log)
+        self.torrent_dir = os.path.abspath(self.config['torrent_collecting_dir'])
+        self.registered = True
+        
+        # 14-04-2010, Andrea: limit the size of channel query results.
+        # see create_channel_query_reply (here) and process_query_reply
+        # for other details. (The whole thing is done to avoid freezes in the GUI
+        # when there are too many results)
+        self.max_channel_query_results = self.config['max_channel_query_results']
+        
+        
+
+    #
+    # Incoming messages
+    # 
+    def handleMessage(self,permid,selversion,message):
+        if not self.registered:
+            return True
+        
+        t = message[0]
+        if t == QUERY:
+            if DEBUG:
+                print >> sys.stderr,"rquery: Got QUERY",len(message)
+            return self.recv_query(permid,message,selversion)
+        if t == QUERY_REPLY:
+            if DEBUG:
+                print >> sys.stderr,"rquery: Got QUERY_REPLY",len(message)
+            return self.recv_query_reply(permid,message,selversion)
+        else:
+            if DEBUG:
+                print >> sys.stderr,"rquery: UNKNOWN OVERLAY MESSAGE", ord(t)
+            return False
+
+    #
+    # Incoming connections
+    #
+    def handleConnection(self,exc,permid,selversion,locally_initiated):
+        if not self.registered:
+            return True
+        
+        if DEBUG:
+            print >> sys.stderr,"rquery: handleConnection",exc,"v",selversion,"local",locally_initiated, ";#conn:", len(self.connections)
+        
+        if selversion < OLPROTO_VER_SIXTH:
+            return True
+
+        if exc is None:
+            self.connections[permid] = selversion
+            #superpeers = self.superpeer_db.getSuperPeers()
+            #if permid in superpeers:
+             #   print >> sys.stderr,"rquery: handleConnection: Connect to superpeer"
+        else:
+            try:
+                del self.connections[permid]
+            except:
+                pass
+                #print_exc()
+
+        return True
+
+    #
+    # Send query
+    # 
+    def send_query(self,query,usercallback,max_peers_to_query=MAX_PEERS_TO_QUERY):
+        """ Called by GUI Thread """
+        if max_peers_to_query is None or max_peers_to_query > MAX_PEERS_TO_QUERY:
+            max_peers_to_query = MAX_PEERS_TO_QUERY
+        if DEBUG:
+            print >>sys.stderr,"rquery: send_query",`query`,max_peers_to_query
+        if max_peers_to_query > 0:
+            send_query_func = lambda:self.network_send_query_callback(query,usercallback,max_peers_to_query)
+            self.overlay_bridge.add_task(send_query_func,0)
+
+
+    def network_send_query_callback(self,query,usercallback,max_peers_to_query):
+        """ Called by overlay thread """
+        p = self.create_query(query,usercallback)
+        m = QUERY+p
+        query_conn_callback_lambda = lambda exc,dns,permid,selversion:self.conn_callback(exc,dns,permid,selversion,m)
+        
+        if query.startswith("CHANNEL"):
+            wantminoversion = OLPROTO_VER_THIRTEENTH  # channel queries and replies only for the latest version (13) 
+        elif query.startswith("SIMPLE+METADATA"):
+            wantminoversion = OLPROTO_VER_TWELFTH
+        else:
+            wantminoversion =  OLPROTO_VER_SIXTH
+            
+        if DEBUG:
+            print >>sys.stderr,"rquery: send_query: Connected",len(self.connections),"peers; minoversion=", wantminoversion
+        
+        #print "******** send query net cb:", query, len(self.connections), self.connections
+        
+        # 1. See how many peers we already know about from direct connections
+        peers_to_query = 0
+        for permid,selversion in self.connections.iteritems():
+            if selversion >= wantminoversion:
+                self.overlay_bridge.connect(permid,query_conn_callback_lambda)
+                peers_to_query += 1
+        
+        # 2. If not enough, get some remote-search capable peers from BC
+        if peers_to_query < max_peers_to_query and self.bc_fac and self.bc_fac.buddycast_core:
+            query_cand = self.bc_fac.buddycast_core.getRemoteSearchPeers(max_peers_to_query-peers_to_query,wantminoversion)
+            for permid in query_cand:
+                if permid not in self.connections:    # don't call twice
+                    self.overlay_bridge.connect(permid,query_conn_callback_lambda)
+                    peers_to_query += 1
+        
+        if DEBUG:
+            print >>sys.stderr,"rquery: send_query: Sent to",peers_to_query,"peers; query=", query
+        
+    def create_query(self,query,usercallback):
+        d = {}
+        d['q'] = query.strip().encode("UTF-8")
+        d['id'] = self.create_and_register_query_id(query,usercallback)
+        return bencode(d)
+        
+    def create_and_register_query_id(self,query,usercallback):
+        id = Rand.rand_bytes(QUERY_ID_SIZE)
+        queryrec = {'query':query,'usercallback':usercallback}
+        self.query_ids2rec[id] = queryrec
+        return id
+        
+    def is_registered_query_id(self,id):
+        if id in self.query_ids2rec:
+            return self.query_ids2rec[id]
+        else:
+            return None
+        
+    def conn_callback(self,exc,dns,permid,selversion,message):
+        if exc is None and selversion >= OLPROTO_VER_SIXTH:
+            self.overlay_bridge.send(permid,message,self.send_callback)
+            
+    def send_callback(self,exc,permid):
+        #print "******* query was sent to", show_permid_short(permid), exc
+        pass
+    
+    
+    #
+    # Receive query
+    # 
+    
+    def recv_query(self,permid,message,selversion):
+        if selversion < OLPROTO_VER_SIXTH:
+            return False
+
+        # Unpack
+        try:
+            d = bdecode(message[1:])
+        except:
+            if DEBUG:
+                print >>sys.stderr,"rquery: Cannot bdecode QUERY message"
+            #print_exc()
+            return False
+        
+        if not isValidQuery(d,selversion):
+            if DEBUG:
+                print >>sys.stderr,"rquery: QUERY invalid",`d`
+            return False
+
+        # ACCESS CONTROL, INCLUDING CHECKING IF PEER HAS NOT EXCEEDED
+        # QUERY QUOTUM IS DONE in Tribler/Core/RequestPolicy.py
+        #
+
+        # Process
+        self.process_query(permid, d, selversion)
+        
+        return True
+
+    def set_log_file(self, logfile):
+        self.logfile = open(logfile, "a") 
+   
+   
+    def log(self, permid, decoded_message):        
+        lt = T.localtime(T.time())
+        timestamp = "%04d-%02d-%02d %02d:%02d:%02d" % (lt[0], lt[1], lt[2], lt[3], lt[4], lt[5])
+        ip = self.peer_db.getPeer(permid, "ip")
+        #ip = "x.y.z.1"
+        s = "%s\t%s\t%s\t%s\n"% (timestamp, bin2str(permid), ip, decoded_message)
+        
+        print dunno2unicode(s)
+        self.logfile.write(dunno2unicode(s)) # bin2str(
+        self.logfile.flush()
+    
+    
+    #
+    # Send query reply
+    #
+    def process_query(self, permid, d, selversion):
+        hits = None
+        p = None
+        sendtorrents = False
+
+        netwq = d['q']
+        if netwq.startswith("SIMPLE"): # remote query
+            # Format: 'SIMPLE '+string of space separated keywords or
+            #         'SIMPLE+METADATA' +string of space separated keywords
+            #
+            # In the future we could support full SQL queries:
+            # SELECT infohash,torrent_name FROM torrent_db WHERE status = ALIVE
+            
+            if netwq.startswith('SIMPLE+METADATA'):
+                q = d['q'][len('SIMPLE+METADATA '):]
+                sendtorrents = True
+            else:
+                q = d['q'][len('SIMPLE '):]
+                    
+            uq = self.clean_netwq(q)
+            kws = split_into_keywords(uq)
+            hits = self.search_torrents(kws, maxhits=MAX_RESULTS,sendtorrents=sendtorrents)
+            p = self.create_remote_query_reply(d['id'],hits,selversion)
+            
+        elif netwq.startswith("CHANNEL"): # channel query
+            if DEBUG:
+                print>>sys.stderr, "Incoming channel query", d['q']
+            q = d['q'][len('CHANNEL '):]
+            uq = self.clean_netwq(q,channelquery=True)
+            hits = self.channelcast_db.searchChannels(uq)
+            p = self.create_channel_query_reply(d['id'],hits,selversion)
+
+        # log incoming query, if logfile is set
+        if self.logfile:
+            self.log(permid, q)        
+     
+        m = QUERY_REPLY+p
+
+        if self.overlay_log:
+            nqueries = self.get_peer_nqueries(permid)
+            # RECV_MSG PERMID OVERSION NUM_QUERIES MSG
+            self.overlay_log('RECV_QRY', show_permid(permid), selversion, nqueries, repr(d))
+
+            # RPLY_QRY PERMID NUM_HITS MSG
+            self.overlay_log('RPLY_QRY', show_permid(permid), len(hits), repr(p))
+
+        self.overlay_bridge.send(permid, m, self.send_callback)
+        
+        self.inc_peer_nqueries(permid)
+
+ # This function need not be used, since it is handled quite well by split_into_keywords 
+    def clean_netwq(self,q,channelquery=False):
+        # Filter against bad input
+        uq = q.decode("UTF-8")
+        newq = u''
+        for i in range(0,len(uq)):
+            if uq[i].isalnum() or uq[i] == ' ' or (channelquery and uq[i] == '+') or (channelquery and uq[i] == '/'):
+                newq += uq[i]
+        return newq
+            
+        
+    def create_remote_query_reply(self,id,hits,selversion):
+        getsize = os.path.getsize
+        join = os.path.join
+        d = {}
+        d['id'] = id
+        d2 = {}
+        for torrent in hits:
+            r = {}
+            # NEWDBSTANDARD. Do not rename r's fields: they are part of the 
+            # rquery protocol spec.
+            # Arno, 2010-01-28: name DB record contains the Unicode object
+            r['content_name'] = torrent['name'].encode("UTF-8")  
+            r['length'] = torrent['length']
+            r['leecher'] = torrent['num_leechers']
+            r['seeder'] = torrent['num_seeders']
+            # Arno: TODO: sending category doesn't make sense as that's user-defined
+            # leaving it now because of time constraints
+            r['category'] = torrent['category']
+            if selversion >= OLPROTO_VER_NINETH:
+                r['torrent_size'] = getsize(join(self.torrent_dir, torrent['torrent_file_name']))
+            if selversion >= OLPROTO_VER_ELEVENTH:
+                r['channel_permid'] = torrent['channel_permid']
+                # Arno, 2010-01-28: name DB record contains the Unicode object
+                r['channel_name'] = torrent['channel_name'].encode("UTF-8")
+            if selversion >= OLPROTO_VER_TWELFTH and 'metadata' in torrent:
+                if DEBUG:
+                    print >>sys.stderr,"rqmh: create_query_reply: Adding torrent file"
+                r['metatype'] = torrent['metatype']
+                r['metadata'] = torrent['metadata']
+                
+            d2[torrent['infohash']] = r
+        d['a'] = d2
+        return bencode(d)
+
+    def create_channel_query_reply(self,id,hits,selversion):
+        d = {}
+        d['id'] = id
+        
+        # 14-04-2010, Andrea: sometimes apperently trivial queries like 'a' can produce
+        # enormouse amounts of hit that will keep the receiver busy in processing them.
+        # I made an "hack" in 'gotMessage" in ChannelSearchGridManager that drops results
+        # when they are more then a threshold. At this point is better to limit the results
+        # from the source to use less network bandwidth
+        hitslen = len(hits)
+        if hitslen > self.max_channel_query_results:
+            if DEBUG:
+                print >> sys.stderr, "Too many results for query (%d). Dropping to %d." % \
+                    (hitslen,self.max_channel_query_results) 
+            hits = hits[:self.max_channel_query_results] #hits are ordered by timestampe descending
+       
+        # 09-04-2010 Andrea: this code was exactly a duplicate copy of some
+        # code in channelcast module. Refactoring performed
+#        d2 = {}
+#        for hit in hits:
+#            r = {}
+#            r['publisher_id'] = str(hit[0]) # ARNOUNICODE: must be str
+#            r['publisher_name'] = hit[1].encode("UTF-8")  # ARNOUNICODE: must be explicitly UTF-8 encoded
+#            r['infohash'] = str(hit[2])     # ARNOUNICODE: must be str
+#            r['torrenthash'] = str(hit[3])  # ARNOUNICODE: must be str
+#            r['torrentname'] = hit[4].encode("UTF-8") # ARNOUNICODE: must be explicitly UTF-8 encoded
+#            r['time_stamp'] = int(hit[5])
+#            # hit[6]: signature, which is unique for any torrent published by a user
+#            signature = hit[6]
+#            d2[signature] = r
+        if self.bc_fac.channelcast_core is not None:
+            d2 = self.bc_fac.channelcast_core.buildChannelcastMessageFromHits(hits,selversion,fromQuery=True)
+            d['a'] = d2
+        else:
+            d['a'] = {}
+        return bencode(d)
+    
+    #
+    # Receive query reply
+    #
+    def recv_query_reply(self,permid,message,selversion):
+        
+        #print "****** recv query reply", len(message)
+        
+        if selversion < OLPROTO_VER_SIXTH:
+            return False
+        
+        #if len(message) > MAX_QUERY_REPLY_LEN:
+        #    return True    # don't close
+
+        # Unpack
+        try:
+            d = bdecode(message[1:])
+        except:
+            if DEBUG:
+                print >>sys.stderr,"rquery: Cannot bdecode QUERY_REPLY message", selversion
+            return False
+        
+        if not isValidQueryReply(d,selversion):
+            if DEBUG:
+                print >>sys.stderr,"rquery: not valid QUERY_REPLY message", selversion
+            return False
+
+        
+        # Check auth
+        queryrec = self.is_registered_query_id(d['id'])
+        if not queryrec:
+            if DEBUG:
+                print >>sys.stderr,"rquery: QUERY_REPLY has unknown query ID", selversion
+            return False
+
+        if selversion >= OLPROTO_VER_TWELFTH:
+            if queryrec['query'].startswith('SIMPLE+METADATA'):
+                for infohash,torrentrec in d['a'].iteritems():
+                    if not 'metatype' in torrentrec:
+                        if DEBUG:
+                            print >>sys.stderr,"rquery: QUERY_REPLY has no metatype field", selversion
+                        return False
+
+                    if not 'metadata' in torrentrec:
+                        if DEBUG:
+                            print >>sys.stderr,"rquery: QUERY_REPLY has no metadata field", selversion
+                        return False
+                    if torrentrec['torrent_size'] != len(torrentrec['metadata']):
+                        if DEBUG:
+                            print >>sys.stderr,"rquery: QUERY_REPLY torrent_size != len metadata", selversion
+                        return False
+                    try:
+                        # Validity test
+                        if torrentrec['metatype'] == URL_MIME_TYPE:
+                            tdef = TorrentDef.load_from_url(torrentrec['metadata'])
+                        else:
+                            metainfo = bdecode(torrentrec['metadata'])
+                            tdef = TorrentDef.load_from_dict(metainfo)
+                    except:
+                        if DEBUG:
+                            print_exc()
+                        return False
+                        
+
+        # Process
+        self.process_query_reply(permid,queryrec['query'],queryrec['usercallback'],d)
+        return True
+
+
+    def process_query_reply(self,permid,query,usercallback,d):
+        
+        if DEBUG:
+            print >>sys.stderr,"rquery: process_query_reply:",show_permid_short(permid),query,d
+        
+        if len(d['a']) > 0:
+            self.unidecode_hits(query,d)
+            if query.startswith("CHANNEL"):
+                # 13-04-2010 Andrea: The gotRemoteHits in SearchGridManager is too slow.
+                # Since it is run by the GUIThread when there are too many hits the GUI
+                # gets freezed.
+                # dropping some random results if they are too many. 
+                # It is just an hack, a better method to improve performance should be found.
+                
+                if len(d['a']) > self.max_channel_query_results:
+                    if DEBUG:
+                        print >> sys.stderr, "DROPPING some answers: they where %d" % len(d['a'])
+                    newAnswers = {}
+                    newKeys = d['a'].keys()[:self.max_channel_query_results] 
+                    for key in newKeys:
+                        newAnswers[key] = d['a'][key]
+                    d['a'] = newAnswers
+                    
+                # Andrea 05-06-2010: updates the database through channelcast. Before this was
+                # done by the GUIThread in SearchGridManager    
+                self.bc_fac.channelcast_core.updateChannel(permid,query,d['a'])
+
+                # Inform user of remote channel hits
+                remote_query_usercallback_lambda = lambda:usercallback(permid,query,d['a'])
+            else:
+                remote_query_usercallback_lambda = lambda:usercallback(permid,query,d['a'])
+            
+            self.session.uch.perform_usercallback(remote_query_usercallback_lambda)
+        elif DEBUG:
+            print >>sys.stderr,"rquery: QUERY_REPLY: no results found"
+
+
+    def unidecode_hits(self,query,d):
+        if query.startswith("SIMPLE"):
+            for infohash,r in d['a'].iteritems():
+                r['content_name'] = r['content_name'].decode("UTF-8")
+        elif query.startswith("CHANNEL"):
+            for signature,r in d['a'].iteritems():
+                r['publisher_name'] = r['publisher_name'].decode("UTF-8")
+                r['torrentname'] = r['publisher_name'].decode("UTF-8")
+            
+
+    def inc_peer_nqueries(self, permid):
+        peer = self.peer_db.getPeer(permid)
+        try:
+            if peer is not None:
+                nqueries = peer['num_queries']
+                if nqueries is None:
+                    nqueries = 0
+                self.peer_db.updatePeer(permid, num_queries=nqueries+1)
+        except:
+            print_exc()
+
+    def get_peer_nqueries(self, permid):
+        peer = self.peer_db.getPeer(permid)
+        if peer is None:
+            return 0
+        else:
+            return peer['num_queries']
+
+
+    def search_torrents(self,kws,maxhits=None,sendtorrents=False):
+        
+        if DEBUG:
+            print >>sys.stderr,"rquery: search for torrents matching",`kws`
+        
+        allhits = self.torrent_db.searchNames(kws,local=False)
+        if maxhits is None:
+            hits = allhits
+        else:
+            hits = allhits[:maxhits]
+            
+        colltorrdir = self.session.get_torrent_collecting_dir()
+        if sendtorrents:
+            
+            print >>sys.stderr,"rqmh: search_torrents: adding torrents"
+            for hit in hits:
+                filename = os.path.join(colltorrdir,hit['torrent_file_name'])
+                try:
+                    tdef = TorrentDef.load(filename)
+                    if tdef.get_url_compat():
+                        metatype = URL_MIME_TYPE
+                        metadata = tdef.get_url()
+                    else:
+                        metatype = TSTREAM_MIME_TYPE
+                        metadata = bencode(tdef.get_metainfo())
+                except:
+                    print_exc()
+                    metadata = None
+                hit['metatype'] = metatype
+                hit['metadata'] = metadata
+                
+            # Filter out hits for which we could not read torrent file (rare)
+            newhits = []
+            for hit in hits:
+                if hit['metadata'] is not None:
+                    newhits.append(hit)
+            hits = newhits
+            
+        return hits
+
+
+def isValidQuery(d,selversion):
+    if not isinstance(d,dict):
+        if DEBUG:
+            print >> sys.stderr, "rqmh: not dict"
+        return False
+    if not ('q' in d and 'id' in d):
+        if DEBUG:
+            print >> sys.stderr, "rqmh: some keys are missing", d.keys()
+        return False
+    if not (isinstance(d['q'],str) and isinstance(d['id'],str)):
+        if DEBUG:
+            print >> sys.stderr, "rqmh: d['q'] or d['id'] are not of string format", d['q'], d['id']
+        return False
+    if len(d['q']) == 0:
+        if DEBUG:
+            print >> sys.stderr, "rqmh: len(d['q']) == 0"
+        return False
+    if selversion < OLPROTO_VER_TWELFTH and d['q'].startswith('SIMPLE+METADATA'):
+        if DEBUG:
+            print >>sys.stderr,"rqmh: SIMPLE+METADATA but old olversion",`d['q']`
+        return False
+    idx = d['q'].find(' ')
+    if idx == -1:
+        if DEBUG:
+            print >>sys.stderr,"rqmh: no space in q",`d['q']`
+        return False
+    try:
+        keyws = d['q'][idx+1:]
+        ukeyws = keyws.decode("UTF-8").strip().split()
+        for ukeyw in ukeyws:
+            if not ukeyw.isalnum():
+                # Arno, 2010-02-09: Allow for BASE64-encoded permid in CHANNEL queries
+                rep = ukeyw.replace("+","p").replace("/","s")
+                if not rep.isalnum():
+                    if DEBUG:
+                        print >>sys.stderr,"rqmh: not alnum",`ukeyw`
+                    return False
+    except:
+        print_exc()
+        if DEBUG:
+            print >>sys.stderr,"rqmh: not alnum query",`d['q']`
+        return False
+    if len(d) > 2: # no other keys
+        if DEBUG:
+            print >> sys.stderr, "rqmh: d has more than 2 keys"
+        return False
+    return True
+
+
+def isValidQueryReply(d,selversion):
+    if not isinstance(d,dict):
+        if DEBUG:
+            print >>sys.stderr,"rqmh: reply: not dict"
+        return False
+    if not ('a' in d and 'id' in d):
+        if DEBUG:
+            print >>sys.stderr,"rqmh: reply: a or id key missing"
+        return False
+    if not (isinstance(d['a'],dict) and isinstance(d['id'],str)):
+        if DEBUG:
+            print >>sys.stderr,"rqmh: reply: a or id key not dict/str"
+        return False
+    if not isValidHits(d['a'],selversion):
+        return False
+    if len(d) > 2: # no other keys
+        if DEBUG:
+            print >>sys.stderr,"rqmh: reply: too many keys, got",d.keys()
+        return False
+    return True
+
+def isValidHits(d,selversion):
+    if not isinstance(d,dict):
+        return False
+    ls = d.values()
+    if len(ls)>0:
+        l = ls[0]
+        if 'publisher_id' in l: # channel search result
+            if not validChannelCastMsg(d):
+                return False
+        elif 'content_name' in l: # remote search
+            for key in d.keys():
+        #        if len(key) != 20:
+        #            return False
+                val = d[key]
+                if not isValidRemoteVal(val,selversion):
+                    return False
+    return True
+
+def isValidChannelVal(d, selversion):
+    if not isinstance(d,dict):
+        if DEBUG:
+            print >>sys.stderr,"rqmh: reply: torrentrec: value not dict"
+        return False
+    if not ('publisher_id' in d and 'publisher_name' in d and 'infohash' in d and 'torrenthash' in d and 'torrentname' in d and 'time_stamp' in d):
+        if DEBUG:
+            print >>sys.stderr,"rqmh: reply: a: key missing, got",d.keys()
+        return False
+    return True
+
+def isValidRemoteVal(d,selversion):
+    if not isinstance(d,dict):
+        if DEBUG:
+            print >>sys.stderr,"rqmh: reply: a: value not dict"
+        return False
+    if selversion >= OLPROTO_VER_TWELFTH:
+        if not ('content_name' in d and 'length' in d and 'leecher' in d and 'seeder' in d and 'category' in d and 'torrent_size' in d and 'channel_permid' in d and 'channel_name' in d):
+            if DEBUG:
+                print >>sys.stderr,"rqmh: reply: torrentrec12: key missing, got",d.keys()
+            return False
+        if 'metatype' in d and 'metadata' in d:
+            try:
+                metatype = d['metatype']
+                metadata = d['metadata']
+                if metatype == URL_MIME_TYPE:
+                    tdef = TorrentDef.load_from_url(metadata)
+                else:
+                    metainfo = bdecode(metadata)
+                    tdef = TorrentDef.load_from_dict(metainfo)
+            except:
+                if DEBUG:
+                    print >>sys.stderr,"rqmh: reply: torrentrec12: metadata invalid"
+                    print_exc()
+                return False
+
+    elif selversion >= OLPROTO_VER_ELEVENTH:
+        if not ('content_name' in d and 'length' in d and 'leecher' in d and 'seeder' in d and 'category' in d and 'torrent_size' in d and 'channel_permid' in d and 'channel_name' in d):
+            if DEBUG:
+                print >>sys.stderr,"rqmh: reply: torrentrec11: key missing, got",d.keys()
+            return False
+        
+    elif selversion >= OLPROTO_VER_NINETH:
+        if not ('content_name' in d and 'length' in d and 'leecher' in d and 'seeder' in d and 'category' in d and 'torrent_size' in d):
+            if DEBUG:
+                print >>sys.stderr,"rqmh: reply: torrentrec9: key missing, got",d.keys()
+            return False
+    else:
+        if not ('content_name' in d and 'length' in d and 'leecher' in d and 'seeder' in d and 'category' in d):
+            if DEBUG:
+                print >>sys.stderr,"rqmh: reply: torrentrec6: key missing, got",d.keys()
+            return False
+        
+#    if not (isinstance(d['content_name'],str) and isinstance(d['length'],int) and isinstance(d['leecher'],int) and isinstance(d['seeder'],int)):
+#        return False
+#    if len(d) > 4: # no other keys
+#        return False
+    return True
+
+        
diff --git a/instrumentation/next-share/BaseLib/Core/SocialNetwork/RemoteTorrentHandler.py b/instrumentation/next-share/BaseLib/Core/SocialNetwork/RemoteTorrentHandler.py
new file mode 100644
index 0000000..5fbb053
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/SocialNetwork/RemoteTorrentHandler.py
@@ -0,0 +1,80 @@
+# Written by Arno Bakker
+# see LICENSE.txt for license information
+#
+# Handles the case where the user did a remote query and now selected one of the
+# returned torrents for download. 
+#
+
+import sys
+
+from BaseLib.Core.simpledefs import INFOHASH_LENGTH
+from BaseLib.Core.CacheDB.CacheDBHandler import TorrentDBHandler
+
+DEBUG = False
+
+class RemoteTorrentHandler:
+    
+    __single = None
+    
+    def __init__(self):
+        if RemoteTorrentHandler.__single:
+            raise RuntimeError, "RemoteTorrentHandler is singleton"
+        RemoteTorrentHandler.__single = self
+        self.torrent_db = TorrentDBHandler.getInstance()
+        self.requestedtorrents = {}
+
+    def getInstance(*args, **kw):
+        if RemoteTorrentHandler.__single is None:
+            RemoteTorrentHandler(*args, **kw)
+        return RemoteTorrentHandler.__single
+    getInstance = staticmethod(getInstance)
+
+
+    def register(self,overlay_bridge,metadatahandler,session):
+        self.overlay_bridge = overlay_bridge
+        self.metadatahandler = metadatahandler
+        self.session = session
+    
+    def download_torrent(self,permid,infohash,usercallback):
+        """ The user has selected a torrent referred to by a peer in a query 
+        reply. Try to obtain the actual .torrent file from the peer and then 
+        start the actual download. 
+        """
+        assert isinstance(infohash, str), "INFOHASH has invalid type: %s" % type(infohash)
+        assert len(infohash) == INFOHASH_LENGTH, "INFOHASH has invalid length: %d" % len(infohash)
+        # Called by GUI thread 
+
+        olthread_remote_torrent_download_lambda = lambda:self.olthread_download_torrent_callback(permid,infohash,usercallback)
+        self.overlay_bridge.add_task(olthread_remote_torrent_download_lambda,0)
+        
+    def olthread_download_torrent_callback(self,permid,infohash,usercallback):
+        """ Called by overlay thread """
+        assert isinstance(infohash, str), "INFOHASH has invalid type: %s" % type(infohash)
+        assert len(infohash) == INFOHASH_LENGTH, "INFOHASH has invalid length: %d" % len(infohash)
+    
+        #if infohash in self.requestedtorrents:
+        #    return    # TODO RS:the previous request could have failed
+              
+        self.requestedtorrents[infohash] = usercallback
+        
+        self.metadatahandler.send_metadata_request(permid,infohash,caller="rquery")
+        if DEBUG:
+            print >>sys.stderr,'rtorrent: download: Requested torrent: %s' % `infohash`
+       
+    def metadatahandler_got_torrent(self,infohash,metadata,filename):
+        """ Called by MetadataHandler when the requested torrent comes in """
+        assert isinstance(infohash, str), "INFOHASH has invalid type: %s" % type(infohash)
+        assert len(infohash) == INFOHASH_LENGTH, "INFOHASH has invalid length: %d" % len(infohash)
+
+        #Called by overlay thread
+
+        if DEBUG:
+            print >>sys.stderr,"rtorrent: got requested torrent from peer, wanted", infohash in self.requestedtorrents, `self.requestedtorrents`
+        if infohash not in self.requestedtorrents:
+           return
+
+        usercallback = self.requestedtorrents[infohash]
+        del self.requestedtorrents[infohash]
+        
+        remote_torrent_usercallback_lambda = lambda:usercallback(infohash,metadata,filename)
+        self.session.uch.perform_usercallback(remote_torrent_usercallback_lambda)
diff --git a/instrumentation/next-share/BaseLib/Core/SocialNetwork/SocialNetworkMsgHandler.py b/instrumentation/next-share/BaseLib/Core/SocialNetwork/SocialNetworkMsgHandler.py
new file mode 100644
index 0000000..02f8825
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/SocialNetwork/SocialNetworkMsgHandler.py
@@ -0,0 +1,79 @@
+# Written by Arno Bakker, Jie Yang
+# see LICENSE.txt for license information
+
+
+import sys
+
+from BaseLib.Core.BitTornado.BT1.MessageID import *
+from BaseLib.Core.Overlay.SecureOverlay import OLPROTO_VER_FIFTH
+from BaseLib.Core.SocialNetwork.OverlapMsgHandler import OverlapMsgHandler
+
+DEBUG = False
+
+class SocialNetworkMsgHandler:
+    
+    __single = None
+    
+    def __init__(self):
+        if SocialNetworkMsgHandler.__single:
+            raise RuntimeError, "SocialNetworkMsgHandler is singleton"
+        SocialNetworkMsgHandler.__single = self
+        self.overlap = OverlapMsgHandler()
+
+    def getInstance(*args, **kw):
+        if SocialNetworkMsgHandler.__single is None:
+            SocialNetworkMsgHandler(*args, **kw)
+        return SocialNetworkMsgHandler.__single
+    getInstance = staticmethod(getInstance)
+        
+
+    def register(self,overlay_bridge,launchmany,config):
+        if DEBUG:
+            print >> sys.stderr,"socnet: register"
+        self.overlay_bridge = overlay_bridge
+        self.config = config
+        self.overlap.register(overlay_bridge,launchmany)
+
+    #
+    # Incoming messages
+    # 
+    def handleMessage(self,permid,selversion,message):
+        
+        t = message[0]
+        if t == SOCIAL_OVERLAP:
+            if DEBUG:
+                print >> sys.stderr,"socnet: Got SOCIAL_OVERLAP",len(message)
+            if self.config['superpeer']:
+                if DEBUG:
+                    print >> sys.stderr,"socnet: overlap: Ignoring, we are superpeer"
+                return True
+            else:
+                return self.overlap.recv_overlap(permid,message,selversion)
+
+        else:
+            if DEBUG:
+                print >> sys.stderr,"socnet: UNKNOWN OVERLAY MESSAGE", ord(t)
+            return False
+
+    #
+    # Incoming connections
+    #
+    def handleConnection(self,exc,permid,selversion,locally_initiated):
+        
+        if DEBUG:
+            print >> sys.stderr,"socnet: handleConnection",exc,"v",selversion,"local",locally_initiated
+        if exc is not None:
+            return
+        
+        if selversion < OLPROTO_VER_FIFTH:
+            return True
+
+        if self.config['superpeer']:
+            if DEBUG:
+                print >> sys.stderr,"socnet: overlap: Ignoring connection, we are superpeer"
+            return True
+
+        self.overlap.initiate_overlap(permid,locally_initiated)
+        return True
+
+    
\ No newline at end of file
diff --git a/instrumentation/next-share/BaseLib/Core/SocialNetwork/__init__.py b/instrumentation/next-share/BaseLib/Core/SocialNetwork/__init__.py
new file mode 100644
index 0000000..86ac17b
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/SocialNetwork/__init__.py
@@ -0,0 +1,3 @@
+# Written by Arno Bakker 
+# see LICENSE.txt for license information
+
diff --git a/instrumentation/next-share/BaseLib/Core/Statistics/ChannelCrawler.py b/instrumentation/next-share/BaseLib/Core/Statistics/ChannelCrawler.py
new file mode 100644
index 0000000..4be1183
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Statistics/ChannelCrawler.py
@@ -0,0 +1,112 @@
+# Written by Boudewijn Schoon
+# see LICENSE.txt for license information
+
+import sys
+import cPickle
+from time import strftime
+
+from BaseLib.Core.Overlay.SecureOverlay import OLPROTO_VER_THIRTEENTH
+# OLPROTO_VER_SEVENTH --> Sixth public release, >= 4.5.0, supports CRAWLER_REQUEST and CRAWLER_REPLY messages
+# OLPROTO_VER_EIGHTH  --> Seventh public release, >= 5.0, supporting BuddyCast with clicklog info.
+
+from BaseLib.Core.BitTornado.BT1.MessageID import CRAWLER_CHANNEL_QUERY
+from BaseLib.Core.CacheDB.sqlitecachedb import SQLiteCacheDB
+from BaseLib.Core.Utilities.utilities import show_permid, show_permid_short
+from BaseLib.Core.Statistics.Crawler import Crawler
+
+DEBUG = False
+
+class ChannelCrawler:
+    __single = None
+
+    @classmethod
+    def get_instance(cls, *args, **kargs):
+        if not cls.__single:
+            cls.__single = cls(*args, **kargs)
+        return cls.__single
+
+    def __init__(self):
+        self._sqlite_cache_db = SQLiteCacheDB.getInstance()
+
+        crawler = Crawler.get_instance()
+        if crawler.am_crawler():
+            self._file = open("channelcrawler.txt", "a")
+            self._file.write("".join(("# ", "*" * 80, "\n# ", strftime("%Y/%m/%d %H:%M:%S"), " Crawler started\n")))
+            self._file.flush()
+        else:
+            self._file = None
+
+    def query_initiator(self, permid, selversion, request_callback):
+        """
+        Established a new connection. Send a CRAWLER_CHANNEL_QUERY request.
+        @param permid The Tribler peer permid
+        @param selversion The oberlay protocol version
+        @param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload)
+        """
+        if DEBUG: print >>sys.stderr, "channelcrawler: query_initiator", show_permid_short(permid)
+        sql = []
+        if selversion >= OLPROTO_VER_THIRTEENTH:
+            sql.extend(("SELECT 'channel_files', publisher_id, count(*) FROM ChannelCast group by publisher_id",
+                        "SELECT 'my_votes', mod_id, voter_id, vote, time_stamp FROM VoteCast where voter_id='" + show_permid(permid) + "' order by time_stamp desc limit 100",
+                        "SELECT 'other_votes', mod_id, voter_id, vote, time_stamp FROM VoteCast where voter_id<>'" + show_permid(permid) + "' order by time_stamp desc limit 100"))
+
+        request_callback(CRAWLER_CHANNEL_QUERY, ";".join(sql), callback=self._after_request_callback)
+
+    def _after_request_callback(self, exc, permid):
+        """
+        Called by the Crawler with the result of the request_callback
+        call in the query_initiator method.
+        """
+        if not exc:
+            if DEBUG: print >>sys.stderr, "channelcrawler: request send to", show_permid_short(permid)
+            self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), "REQUEST", show_permid(permid), "\n")))
+            self._file.flush()
+
+    def handle_crawler_request(self, permid, selversion, channel_id, message, reply_callback):
+        """
+        Received a CRAWLER_CHANNEL_QUERY request.
+        @param permid The Crawler permid
+        @param selversion The overlay protocol version
+        @param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair
+        @param message The message payload
+        @param reply_callback Call this function once to send the reply: reply_callback(payload [, error=123])
+        """
+        if DEBUG:
+            print >> sys.stderr, "channelcrawler: handle_crawler_request", show_permid_short(permid), message
+
+        # execute the sql
+        try:
+            cursor = self._sqlite_cache_db.execute_read(message)
+
+        except Exception, e:
+            reply_callback(str(e), error=1)
+        else:
+            if cursor:
+                reply_callback(cPickle.dumps(list(cursor), 2))
+            else:
+                reply_callback("error", error=2)
+
+    def handle_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback):
+        """
+        Received a CRAWLER_CHANNEL_QUERY reply.
+        @param permid The Crawler permid
+        @param selversion The overlay protocol version
+        @param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair
+        @param error The error value. 0 indicates success.
+        @param message The message payload
+        @param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload)
+        """
+        if error:
+            if DEBUG:
+                print >> sys.stderr, "channelcrawler: handle_crawler_reply", error, message
+
+            self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), "  REPLY", show_permid(permid), str(error), message, "\n")))
+            self._file.flush()
+
+        else:
+            if DEBUG:
+                print >> sys.stderr, "channelcrawler: handle_crawler_reply", show_permid_short(permid), cPickle.loads(message)
+
+            self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), "  REPLY", show_permid(permid), str(error), str(cPickle.loads(message)), "\n")))
+            self._file.flush()
+
diff --git a/instrumentation/next-share/BaseLib/Core/Statistics/Crawler.py b/instrumentation/next-share/BaseLib/Core/Statistics/Crawler.py
new file mode 100644
index 0000000..cf3458f
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Statistics/Crawler.py
@@ -0,0 +1,570 @@
+# Written by Boudewijn Schoon
+# see LICENSE.txt for license information
+
+# todo
+# - try to connect first, than start the initiator. now we start the
+#   initiator and we often fail to connect
+
+import sys
+import time
+import random
+from traceback import print_exc,print_stack
+
+from BaseLib.Core.BitTornado.BT1.MessageID import CRAWLER_REQUEST, CRAWLER_REPLY, getMessageName
+from BaseLib.Core.CacheDB.SqliteCacheDBHandler import CrawlerDBHandler
+from BaseLib.Core.Overlay.OverlayThreadingBridge import OverlayThreadingBridge
+from BaseLib.Core.Overlay.SecureOverlay import OLPROTO_VER_SEVENTH
+from BaseLib.Core.Utilities.utilities import show_permid_short
+
+DEBUG = False
+
+# when a message payload exceedes 32KB it is divided into multiple
+# messages
+MAX_PAYLOAD_LENGTH = 32 * 1024
+
+# after 1 hour the channels for any outstanding CRAWLER_REQUEST
+# messages will be closed
+CHANNEL_TIMEOUT = 60 * 60
+
+# the FREQUENCY_FLEXIBILITY tels the client how strict it must adhere
+# to the frequency. the value indicates how many seconds a request
+# will be allowed before the actual frequency deadline
+FREQUENCY_FLEXIBILITY = 5
+
+# Do not attempt to re-initiate communication after more than x
+# connection failures
+MAX_ALLOWED_FAILURES = 26
+
+class Crawler:
+    __singleton = None
+
+    @classmethod
+    def get_instance(cls, *args, **kargs):
+        if not cls.__singleton:
+            cls.__singleton = cls(*args, **kargs)
+        return cls.__singleton
+
+    def __init__(self, session):
+        if self.__singleton:
+            raise RuntimeError, "Crawler is Singleton"
+        self._overlay_bridge = OverlayThreadingBridge.getInstance()
+        self._session = session
+        self._crawler_db = CrawlerDBHandler.getInstance()
+
+        # _message_handlers contains message-id:(request-callback, reply-callback, last-request-timestamp)
+        # the handlers are called when either a CRAWL_REQUEST or CRAWL_REPLY message is received
+        self._message_handlers = {}
+
+        # _crawl_initiators is a list with (initiator-callback,
+        # frequency, accept_frequency) tuples the initiators are called
+        # when a new connection is received
+        self._crawl_initiators = []
+
+        # _initiator_dealines contains [deadline, frequency,
+        # accept_frequency, initiator-callback, permid, selversion,
+        # failure-counter] deadlines register information on when to
+        # call the crawl initiators again for a specific permid
+        self._initiator_deadlines = []
+        
+        # _dialback_deadlines contains message_id:(deadline, permid) pairs
+        # client peers should connect back to -a- crawler indicated by
+        # permid after deadline expired
+        self._dialback_deadlines = {}
+
+        # _channels contains permid:buffer-dict pairs. Where
+        # buffer_dict contains channel-id:(timestamp, buffer,
+        # channel_data) pairs. Where buffer is the payload from
+        # multipart messages that are received so far. Channels are
+        # used to match outstanding replies to given requests
+        self._channels = {}
+
+        # start checking for expired deadlines
+        self._check_deadlines(True)
+
+        # start checking for ancient channels
+        self._check_channels()
+
+    def register_crawl_initiator(self, initiator_callback, frequency=3600, accept_frequency=None):
+        """
+        Register a callback that is called each time a new connection
+        is made and subsequently each FREQUENCY seconds.
+
+        ACCEPT_FREQUENCY defaults to FREQUENCY and indicates the
+        minimum seconds that must expire before a crawler request
+        message is accepted.
+
+        Giving FREQUENCY = 10 and ACCEPT_FREQUENCY = 0 will call
+        INITIATOR_CALLBACK every 10 seconds and will let the receiving
+        peers accept allways.
+
+        Giving FREQUENCY = 10 and ACCEPT_FREQUENCY = 20 will call
+        INITIATOR_CALLBACK every 10 seconds and will cause frequency
+        errors 50% of the time.
+        """
+        if accept_frequency is None:
+            accept_frequency = frequency
+        self._crawl_initiators.append((initiator_callback, frequency, accept_frequency))
+
+    def register_message_handler(self, id_, request_callback, reply_callback):
+        self._message_handlers[id_] = (request_callback, reply_callback, 0)
+
+    def am_crawler(self):
+        """
+        Returns True if this running Tribler is a Crawler
+        """
+        return self._session.get_permid() in self._crawler_db.getCrawlers()
+
+    def _acquire_channel_id(self, permid, channel_data):
+        """
+        Claim a unique one-byte id to match a request to a reply.
+
+        PERMID the peer to communicate with
+        CHANNEL_DATA optional data associated with this channel
+        """
+        if permid in self._channels:
+            channels = self._channels[permid]
+        else:
+            channels = {}
+            self._channels[permid] = channels
+
+        # find a free channel-id randomly
+        channel_id = random.randint(1, 255)
+        attempt = 0
+        while channel_id in channels:
+            attempt += 1
+            if attempt > 64:
+                channel_id = 0
+                break
+            channel_id = random.randint(1, 255)
+
+        if channel_id == 0:
+            # find a free channel-id sequentialy
+            channel_id = 255
+            while channel_id in channels and channel_id != 0:
+                channel_id -= 1
+
+        if channel_id:
+            # create a buffer to receive the reply
+            channels[channel_id] = [time.time() + CHANNEL_TIMEOUT, "", channel_data]
+
+        # print >>sys.stderr, "crawler: _acquire_channel_id:", show_permid_short(permid), len(channels), "channels used"
+
+        # a valid channel-id or 0 when no channel-id is left
+        return channel_id
+
+    def _release_channel_id(self, permid, channel_id):
+        if permid in self._channels:
+            if channel_id in self._channels[permid]:
+                del self._channels[permid][channel_id]
+            if not self._channels[permid]:
+                del self._channels[permid]
+
+    def _post_connection_attempt(self, permid, success):
+        """
+        This method is called after a succesfull or failed connection
+        attempt
+        """
+        if success:
+            # reset all failure counters for this permid
+            for tup in (tup for tup in self._initiator_deadlines if tup[4] == permid):
+                tup[6] = 0
+
+        else:
+            def increase_failure_counter(tup):
+                if tup[4] == permid:
+                    if tup[6] > MAX_ALLOWED_FAILURES:
+                        # remove from self._initiator_deadlines
+                        return False
+                    else:
+                        # increase counter but leave in self._initiator_deadlines
+                        tup[6] += 1
+                        return True
+                else:
+                    return True
+
+            self._initiator_deadlines = filter(increase_failure_counter, self._initiator_deadlines)
+
+    def send_request(self, permid, message_id, payload, frequency=3600, callback=None, channel_data=None):
+        """
+        This method ensures that a connection to PERMID exists before
+        sending the message
+
+        Returns the channel-id.
+
+        MESSAGE_ID is a one character crawler specific ID (defined in MessageID.py).
+        PAYLOAD is message specific sting.
+        FREQUENCY is an integer defining the time, in seconds, until a next message with MESSAGE_ID is accepted by the client-side crawler.
+        CALLBACK is either None or callable. Called with parameters EXC and PERMID. EXC is None for success or an Exception for failure.
+        CHANNEL_DATA can be anything related to this specific request. It is supplied with the handle-reply callback.
+        """
+        # reserve a new channel-id
+        channel_id = self._acquire_channel_id(permid, channel_data)
+
+        def _after_connect(exc, dns, permid, selversion):
+            self._post_connection_attempt(permid, not exc)
+            if exc:
+                # could not connect.
+                if DEBUG: print >>sys.stderr, "crawler: could not connect", dns, show_permid_short(permid), exc
+                self._release_channel_id(permid, channel_id)
+                if callback:
+                    callback(exc, permid)
+            else:
+                self._send_request(permid, message_id, channel_id, payload, frequency=frequency, callback=callback)
+
+#         if DEBUG: print >>sys.stderr, "crawler: connecting (send_request)...", show_permid_short(permid)
+        if channel_id == 0:
+            if DEBUG: print >>sys.stderr, "crawler: send_request: Can not acquire channel-id", show_permid_short(permid)
+        else:
+            self._overlay_bridge.connect(permid, _after_connect)
+        return channel_id
+
+    def _send_request(self, permid, message_id, channel_id, payload, frequency=3600, callback=None):
+        """
+        Send a CRAWLER_REQUEST message to permid. This method assumes
+        that connection exists to the permid.
+
+        @param permid The destination peer
+        @param message_id The message id
+        @param payload The message content
+        @param frequency Destination peer will return a frequency-error when this message_id has been received within the last frequency seconds
+        @param callback Callable function/method is called when request is send with 2 paramaters (exc, permid)
+        @return The message channel-id > 0 on success, and 0 on failure
+        """
+        # Sending a request from a Crawler to a Tribler peer
+        #     SIZE    INDEX
+        #     1 byte: 0      CRAWLER_REQUEST (from BaseLib.Core.BitTornado.BT1.MessageID)
+        #     1 byte: 1      --MESSAGE-SPECIFIC-ID--
+        #     1 byte: 2      Channel id
+        #     2 byte: 3+4    Frequency
+        #     n byte: 5...   Request payload
+        def _after_send_request(exc, permid):
+            if DEBUG:
+                if exc:
+                    print >> sys.stderr, "crawler: could not send request to", show_permid_short(permid), exc
+            if exc:
+                self._release_channel_id(permid, channel_id)
+
+            # call the optional callback supplied with send_request
+            if callback:
+                callback(exc, permid)
+
+        if DEBUG: print >> sys.stderr, "crawler: sending", getMessageName(CRAWLER_REQUEST+message_id), "with", len(payload), "bytes payload to", show_permid_short(permid)
+        self._overlay_bridge.send(permid, "".join((CRAWLER_REQUEST,
+                                                   message_id,
+                                                   chr(channel_id & 0xFF),
+                                                   chr((frequency >> 8) & 0xFF) + chr(frequency & 0xFF),
+                                                   str(payload))), _after_send_request)
+        return channel_id
+
+    def handle_request(self, permid, selversion, message):
+        """
+        Received CRAWLER_REQUEST message from OverlayApps
+        """
+        if selversion >= OLPROTO_VER_SEVENTH and len(message) >= 5:
+
+            message_id = message[1]
+            channel_id = ord(message[2])
+            frequency = ord(message[3]) << 8 | ord(message[4])
+
+            if message_id in self._message_handlers:
+                now = time.time()
+                request_callback, reply_callback, last_request_timestamp = self._message_handlers[message_id]
+
+                # frequency: we will report a requency error when we have
+                # received this request within FREQUENCY seconds
+                if last_request_timestamp + frequency < now + FREQUENCY_FLEXIBILITY:
+
+                    if not permid in self._channels:
+                        self._channels[permid] = {}
+                    self._channels[permid][channel_id] = [time.time() + CHANNEL_TIMEOUT, "", None]
+
+                    # store the new timestamp
+                    self._message_handlers[message_id] = (request_callback, reply_callback, now)
+
+                    def send_reply_helper(payload="", error=0, callback=None):
+                        return self.send_reply(permid, message_id, channel_id, payload, error=error, callback=callback)
+
+                    # 20/10/08. Boudewijn: We will no longer disconnect
+                    # based on the return value from the message handler
+                    try:
+                        request_callback(permid, selversion, channel_id, message[5:], send_reply_helper)
+                    except:
+                        print_exc()
+
+                    # 11/11/08. Boudewijn: Because the client peers may
+                    # not always be connectable, the client peers will
+                    # actively seek to connect to -a- crawler after
+                    # frequency expires. 
+                    self._dialback_deadlines[message_id] = (now + frequency, permid)
+
+                    return True
+
+                else:
+                    # frequency error
+                    self.send_reply(permid, message_id, channel_id, "frequency error", error=254)
+                    return True
+            else:
+                # invalid / unknown message. may be caused by a
+                # crawler sending newly introduced messages
+                self.send_reply(permid, message_id, channel_id, "unknown message", error=253)
+                return True
+        else:
+            # protocol version conflict or invalid message
+            return False
+
+    def send_reply(self, permid, message_id, channel_id, payload, error=0, callback=None):
+        """
+        This method ensures that a connection to PERMID exists before sending the message
+        """
+        def _after_connect(exc, dns, permid, selversion):
+            self._post_connection_attempt(permid, not exc)
+            if exc:
+                # could not connect.
+                if DEBUG: print >>sys.stderr, "crawler: could not connect", dns, show_permid_short(permid), exc
+                if callback:
+                    callback(exc, permid)
+            else:
+                self._send_reply(permid, message_id, channel_id, payload, error=error, callback=callback)
+
+#         if DEBUG: print >>sys.stderr, "crawler: connecting... (send_reply)", show_permid_short(permid)
+        self._overlay_bridge.connect(permid, _after_connect)
+
+    def _send_reply(self, permid, message_id, channel_id, payload, error=0, callback=None):
+        """
+        Send a CRAWLER_REPLY message to permid. This method assumes
+        that connection exists to the permid.
+        
+        @param permid The destination peer
+        @param message_id The message id
+        @param channel_id The channel id. Used to match replies to requests
+        @param payload The message content
+        @param error The error code. (0: no-error, 253: unknown-message, 254: frequency-error, 255: reserved)
+        @param callback Callable function/method is called when request is send with 2 paramaters (exc, permid)
+        @return The message channel-id > 0 on success, and 0 on failure
+        """
+        # Sending a reply from a Tribler peer to a Crawler
+        #     SIZE    INDEX
+        #     1 byte: 0      CRAWLER_REPLY (from BaseLib.Core.BitTornado.BT1.MessageID)
+        #     1 byte: 1      --MESSAGE-SPECIFIC-ID--
+        #     1 byte: 2      Channel id
+        #     1 byte: 3      Parts left
+        #     1 byte: 4      Indicating success (0) or failure (non 0)
+        #     n byte: 5...   Reply payload
+        if len(payload) > MAX_PAYLOAD_LENGTH:
+            remaining_payload = payload[MAX_PAYLOAD_LENGTH:]
+
+            def _after_send_reply(exc, permid):
+                """
+                Called after the overlay attempted to send a reply message
+                """
+                if DEBUG:
+                    print >> sys.stderr, "crawler: _after_send_reply", show_permid_short(permid), exc
+                if not exc:
+                    self.send_reply(permid, message_id, channel_id, remaining_payload, error=error)
+                # call the optional callback supplied with send_request
+                if callback:
+                    callback(exc, permid)
+
+            # 03/06/09 boudewijn: parts_left may be no larger than 255
+            # because we only use one byte to store the 'parts
+            # left'. This does not mean that there can't be more than
+            # 255 parts!
+            parts_left = min(255, int(len(payload) / MAX_PAYLOAD_LENGTH))
+            payload = payload[:MAX_PAYLOAD_LENGTH]
+
+        else:
+            def _after_send_reply(exc, permid):
+                if DEBUG:
+                    if exc:
+                        print >> sys.stderr, "crawler: could not send request", show_permid_short(permid), exc
+                # call the optional callback supplied with send_request
+                if callback:
+                    callback(exc, permid)
+
+            parts_left = 0
+
+            # remove from self._channels if it is still there (could
+            # have been remove during periodic timeout check)
+            if permid in self._channels and channel_id in self._channels[permid]:
+                del self._channels[permid][channel_id]
+                if not self._channels[permid]:
+                    del self._channels[permid]
+
+        if DEBUG: print >> sys.stderr, "crawler: sending", getMessageName(CRAWLER_REPLY+message_id), "with", len(payload), "bytes payload to", show_permid_short(permid)
+        self._overlay_bridge.send(permid, "".join((CRAWLER_REPLY,
+                                                   message_id,
+                                                   chr(channel_id & 0xFF),
+                                                   chr(parts_left & 0xFF),
+                                                   chr(error & 0xFF),
+                                                   str(payload))), _after_send_reply)
+        return channel_id
+
+    def handle_reply(self, permid, selversion, message):
+        """
+        Received CRAWLER_REPLY message from OverlayApps
+        """
+        if selversion >= OLPROTO_VER_SEVENTH and len(message) >= 5 and message[1] in self._message_handlers:
+            
+            message_id = message[1]
+            channel_id = ord(message[2])
+            parts_left = ord(message[3])
+            error = ord(message[4])
+
+            # A request must exist in self._channels, otherwise we did
+            # not request this reply
+            if permid in self._channels and channel_id in self._channels[permid]:
+
+                # add part to buffer
+                self._channels[permid][channel_id][1] += message[5:]
+
+                if parts_left:
+                    # todo: register some event to remove the buffer
+                    # after a time (in case connection is lost before
+                    # all parts are received)
+
+                    if DEBUG: print >> sys.stderr, "crawler: received", getMessageName(CRAWLER_REPLY+message_id), "with", len(message), "bytes payload from", show_permid_short(permid), "with", parts_left, "parts left"
+                    # Can't do anything until all parts have been received
+                    return True
+                else:
+                    timestamp, payload, channel_data = self._channels[permid].pop(channel_id)
+                    if DEBUG:
+                        if error == 253:
+                            # unknown message error (probably because
+                            # the crawler is newer than the peer)
+                            print >> sys.stderr, "crawler: received", getMessageName(CRAWLER_REPLY+message_id), "with", len(message), "bytes payload from", show_permid_short(permid), "indicating an unknown message error"
+                        if error == 254:
+                            # frequency error (we did this request recently)
+                            print >> sys.stderr, "crawler: received", getMessageName(CRAWLER_REPLY+message_id), "with", len(message), "bytes payload from", show_permid_short(permid), "indicating a frequency error"
+                        else:
+                            print >> sys.stderr, "crawler: received", getMessageName(CRAWLER_REPLY+message_id), "with", len(payload), "bytes payload from", show_permid_short(permid)
+                    if not self._channels[permid]:
+                        del self._channels[permid]
+
+                    def send_request_helper(message_id, payload, frequency=3600, callback=None, channel_data=None):
+                        return self.send_request(permid, message_id, payload, frequency=frequency, callback=callback, channel_data=channel_data)
+
+                    # 20/10/08. Boudewijn: We will no longer
+                    # disconnect based on the return value from the
+                    # message handler
+                    try:
+                        self._message_handlers[message_id][1](permid, selversion, channel_id, channel_data, error, payload, send_request_helper)
+                    except:
+                        print_exc()
+                    return True
+            else:
+                # reply from unknown permid or channel
+                if DEBUG: print >> sys.stderr, "crawler: received", getMessageName(CRAWLER_REPLY+message_id), "with", len(payload), "bytes payload from", show_permid_short(permid), "from unknown peer or unused channel"
+                
+        if DEBUG:
+            if len(message) >= 2:
+                message_id = message[1]
+            else:
+                message_id = ""
+            print >> sys.stderr, "crawler: received", getMessageName(CRAWLER_REPLY+message_id), "with", len(message), "bytes from", show_permid_short(permid), "from unknown peer or unused channel"
+        return False
+
+    def handle_connection(self, exc, permid, selversion, locally_initiated):
+        """
+        Called when overlay received a connection. Note that this
+        method is only registered with OverlayApps when running as a
+        crawler (determined by our public key).
+        """
+        if exc:
+            # connection lost
+            if DEBUG:
+                print >>sys.stderr, "crawler: overlay connection lost", show_permid_short(permid), exc
+                print >>sys.stderr, repr(permid)
+
+        elif selversion >= OLPROTO_VER_SEVENTH:
+            # verify that we do not already have deadlines for this permid
+            already_known = False
+            for tup in self._initiator_deadlines:
+                if tup[4] == permid:
+                    already_known = True
+                    break
+
+            if not already_known:
+                if DEBUG:
+                    print >>sys.stderr, "crawler: new overlay connection", show_permid_short(permid)
+                    print >>sys.stderr, repr(permid)
+                for initiator_callback, frequency, accept_frequency in self._crawl_initiators:
+                    self._initiator_deadlines.append([0, frequency, accept_frequency, initiator_callback, permid, selversion, 0])
+
+                self._initiator_deadlines.sort()
+
+                # Start sending crawler requests
+                self._check_deadlines(False)
+        else:
+            if DEBUG:
+                print >>sys.stderr, "crawler: new overlay connection (can not use version %d)" % selversion, show_permid_short(permid)
+                print >>sys.stderr, repr(permid)
+            
+    def _check_deadlines(self, resubmit):
+        """
+        Send requests to permid and re-register to be called again
+        after frequency seconds
+        """
+        now = time.time()
+
+        # crawler side deadlines...
+        if self._initiator_deadlines:
+            for tup in self._initiator_deadlines:
+                deadline, frequency, accept_frequency, initiator_callback, permid, selversion, failure_counter = tup
+                if now > deadline + FREQUENCY_FLEXIBILITY:
+                    def send_request_helper(message_id, payload, frequency=accept_frequency, callback=None, channel_data=None):
+                        return self.send_request(permid, message_id, payload, frequency=frequency, callback=callback, channel_data=channel_data)
+                    # 20/10/08. Boudewijn: We will no longer disconnect
+                    # based on the return value from the message handler
+                    try:
+                        initiator_callback(permid, selversion, send_request_helper)
+                    except Exception:
+                        print_exc()
+
+                    # set new deadline
+                    tup[0] = now + frequency
+                else:
+                    break
+
+            # resort
+            self._initiator_deadlines.sort()
+
+        # client side deadlines...
+        if self._dialback_deadlines:
+
+            def _after_connect(exc, dns, permid, selversion):
+                if DEBUG:
+                    if exc:
+                        print >>sys.stderr, "crawler: dialback to crawler failed", dns, show_permid_short(permid), exc
+                    else:
+                        print >>sys.stderr, "crawler: dialback to crawler established", dns, show_permid_short(permid)
+
+            for message_id, (deadline, permid) in self._dialback_deadlines.items():
+                if now > deadline + FREQUENCY_FLEXIBILITY:
+                    self._overlay_bridge.connect(permid, _after_connect)
+                    del self._dialback_deadlines[message_id]
+
+        if resubmit:
+            self._overlay_bridge.add_task(lambda:self._check_deadlines(True), 5)
+
+    def _check_channels(self):
+        """
+        Periodically removes permids after no connection was
+        established for a long time
+        """
+        now = time.time()
+        to_remove_permids = []
+        for permid in self._channels:
+            to_remove_channel_ids = []
+            for channel_id, (deadline, _, _) in self._channels[permid].iteritems():
+                if now > deadline:
+                    to_remove_channel_ids.append(channel_id)
+            for channel_id in to_remove_channel_ids:
+                del self._channels[permid][channel_id]
+            if not self._channels[permid]:
+                to_remove_permids.append(permid)
+        for permid in to_remove_permids:
+            del self._channels[permid]
+
+        # resubmit
+        self._overlay_bridge.add_task(self._check_channels, 60)
+
diff --git a/instrumentation/next-share/BaseLib/Core/Statistics/DatabaseCrawler.py b/instrumentation/next-share/BaseLib/Core/Statistics/DatabaseCrawler.py
new file mode 100644
index 0000000..decc109
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Statistics/DatabaseCrawler.py
@@ -0,0 +1,125 @@
+# Written by Boudewijn Schoon
+# see LICENSE.txt for license information
+
+import sys
+import cPickle
+from time import strftime
+
+from BaseLib.Core.Overlay.SecureOverlay import OLPROTO_VER_SEVENTH, OLPROTO_VER_EIGHTH, OLPROTO_VER_ELEVENTH
+# OLPROTO_VER_SEVENTH --> Sixth public release, >= 4.5.0, supports CRAWLER_REQUEST and CRAWLER_REPLY messages
+# OLPROTO_VER_EIGHTH  --> Seventh public release, >= 5.0, supporting BuddyCast with clicklog info.
+
+from BaseLib.Core.BitTornado.BT1.MessageID import CRAWLER_DATABASE_QUERY
+from BaseLib.Core.CacheDB.sqlitecachedb import SQLiteCacheDB
+from BaseLib.Core.Utilities.utilities import show_permid, show_permid_short
+from BaseLib.Core.Statistics.Crawler import Crawler
+
+DEBUG = False
+
+class DatabaseCrawler:
+    __single = None
+
+    @classmethod
+    def get_instance(cls, *args, **kargs):
+        if not cls.__single:
+            cls.__single = cls(*args, **kargs)
+        return cls.__single
+
+    def __init__(self):
+        self._sqlite_cache_db = SQLiteCacheDB.getInstance()
+
+        crawler = Crawler.get_instance()
+        if crawler.am_crawler():
+            self._file = open("databasecrawler.txt", "a")
+            self._file.write("".join(("# ", "*" * 80, "\n# ", strftime("%Y/%m/%d %H:%M:%S"), " Crawler started\n")))
+            self._file.flush()
+        else:
+            self._file = None
+
+    def query_initiator(self, permid, selversion, request_callback):
+        """
+        Established a new connection. Send a CRAWLER_DATABASE_QUERY request.
+        @param permid The Tribler peer permid
+        @param selversion The oberlay protocol version
+        @param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload)
+        """
+        if DEBUG: print >>sys.stderr, "databasecrawler: query_initiator", show_permid_short(permid)
+        sql = []
+        if selversion >= OLPROTO_VER_SEVENTH:
+            sql.extend(("SELECT 'peer_count', count(*) FROM Peer",
+                        "SELECT 'torrent_count', count(*) FROM Torrent"))
+
+        if selversion >= OLPROTO_VER_ELEVENTH:
+            sql.extend(("SELECT 'my_subscriptions', count(*) FROM VoteCast where voter_id='" + show_permid(permid) + "' and vote=2",
+                        "SELECT 'my_negative_votes', count(*) FROM VoteCast where voter_id='" + show_permid(permid) + "' and vote=-1",
+                        "SELECT 'my_channel_files', count(*) FROM ChannelCast where publisher_id='" + show_permid(permid) + "'",
+                        "SELECT 'all_subscriptions', count(*) FROM VoteCast where vote=2",
+                        "SELECT 'all_negative_votes', count(*) FROM VoteCast where vote=-1"))
+
+        # if OLPROTO_VER_EIGHTH <= selversion <= 11:
+        #     sql.extend(("SELECT 'moderations_count', count(*) FROM ModerationCast"))
+
+        # if selversion >= OLPROTO_VER_EIGHTH:
+        #     sql.extend(("SELECT 'positive_votes_count', count(*) FROM Moderators where status=1",
+        #                 "SELECT 'negative_votes_count', count(*) FROM Moderators where status=-1"))
+
+        request_callback(CRAWLER_DATABASE_QUERY, ";".join(sql), callback=self._after_request_callback)
+
+    def _after_request_callback(self, exc, permid):
+        """
+        Called by the Crawler with the result of the request_callback
+        call in the query_initiator method.
+        """
+        if not exc:
+            if DEBUG: print >>sys.stderr, "databasecrawler: request send to", show_permid_short(permid)
+            self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), "REQUEST", show_permid(permid), "\n")))
+            self._file.flush()
+
+    def handle_crawler_request(self, permid, selversion, channel_id, message, reply_callback):
+        """
+        Received a CRAWLER_DATABASE_QUERY request.
+        @param permid The Crawler permid
+        @param selversion The overlay protocol version
+        @param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair
+        @param message The message payload
+        @param reply_callback Call this function once to send the reply: reply_callback(payload [, error=123])
+        """
+        if DEBUG:
+            print >> sys.stderr, "databasecrawler: handle_crawler_request", show_permid_short(permid), message
+
+        # execute the sql
+        try:
+            cursor = self._sqlite_cache_db.execute_read(message)
+
+        except Exception, e:
+            reply_callback(str(e), error=1)
+        else:
+            if cursor:
+                reply_callback(cPickle.dumps(list(cursor), 2))
+            else:
+                reply_callback("error", error=2)
+
+    def handle_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback):
+        """
+        Received a CRAWLER_DATABASE_QUERY reply.
+        @param permid The Crawler permid
+        @param selversion The overlay protocol version
+        @param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair
+        @param error The error value. 0 indicates success.
+        @param message The message payload
+        @param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload)
+        """
+        if error:
+            if DEBUG:
+                print >> sys.stderr, "databasecrawler: handle_crawler_reply", error, message
+
+            self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), "  REPLY", show_permid(permid), str(error), message, "\n")))
+            self._file.flush()
+
+        else:
+            if DEBUG:
+                print >> sys.stderr, "databasecrawler: handle_crawler_reply", show_permid_short(permid), cPickle.loads(message)
+
+            self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), "  REPLY", show_permid(permid), str(error), str(cPickle.loads(message)), "\n")))
+            self._file.flush()
+
diff --git a/instrumentation/next-share/BaseLib/Core/Statistics/FriendshipCrawler.py b/instrumentation/next-share/BaseLib/Core/Statistics/FriendshipCrawler.py
new file mode 100644
index 0000000..3e3a3d1
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Statistics/FriendshipCrawler.py
@@ -0,0 +1,136 @@
+# Written by Ali Abbas
+# see LICENSE.txt for license information
+
+import sys
+import time
+from traceback import print_exc
+
+from BaseLib.Core.BitTornado.BT1.MessageID import CRAWLER_FRIENDSHIP_STATS
+from BaseLib.Core.BitTornado.bencode import bencode,bdecode
+from BaseLib.Core.CacheDB.SqliteFriendshipStatsCacheDB import FriendshipStatisticsDBHandler
+from BaseLib.Core.CacheDB.sqlitecachedb import bin2str
+
+DEBUG = False
+
+class FriendshipCrawler:
+    __single = None
+
+    @classmethod
+    def get_instance(cls, *args, **kargs):
+        if not cls.__single:
+            cls.__single = cls(*args, **kargs)
+        return cls.__single
+
+    def __init__(self,session):
+        self.session = session
+        self.friendshipStatistics_db = FriendshipStatisticsDBHandler.getInstance()
+
+    def query_initiator(self, permid, selversion, request_callback):
+        """
+        Established a new connection. Send a CRAWLER_DATABASE_QUERY request.
+        @param permid The Tribler peer permid
+        @param selversion The oberlay protocol version
+        @param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload)
+        """
+        if DEBUG: 
+            print >>sys.stderr, "FriendshipCrawler: friendship_query_initiator"
+
+        get_last_updated_time = self.friendshipStatistics_db.getLastUpdateTimeOfThePeer(permid)
+         
+        msg_dict = {'current time':get_last_updated_time}
+        msg = bencode(msg_dict)
+        return request_callback(CRAWLER_FRIENDSHIP_STATS,msg)
+
+    def handle_crawler_request(self, permid, selversion, channel_id, message, reply_callback):
+        """
+        Received a CRAWLER_FRIENDSHIP_QUERY request.
+        @param permid The Crawler permid
+        @param selversion The overlay protocol version
+        @param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair
+        @param message The message payload
+        @param reply_callback Call this function once to send the reply: reply_callback(payload [, error=123])
+        """
+        if DEBUG:
+            print >> sys.stderr, "FriendshipCrawler: handle_friendship_crawler_database_query_request", message
+
+        try:
+            d = bdecode(message)
+            
+            stats = self.getStaticsFromFriendshipStatisticsTable(self.session.get_permid(),d['current time'])
+            msg_dict = {'current time':d['current time'],'stats':stats}
+            msg = bencode(msg_dict)
+            reply_callback(msg)
+
+        except Exception, e:
+            print_exc()
+            reply_callback(str(e), 1)
+
+        return True
+
+    def handle_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback):
+        """
+        Received a CRAWLER_FRIENDSHIP_STATS request.
+        @param permid The Crawler permid
+        @param selversion The overlay protocol version
+        @param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair
+        @param error The error value. 0 indicates success.
+        @param message The message payload
+        @param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload)
+        """
+
+        if error:
+            if DEBUG:
+                print >> sys.stderr, "friendshipcrawler: handle_crawler_reply"
+                print >> sys.stderr, "friendshipcrawler: error", error, message
+
+        else:
+            try:
+                d = bdecode(message)
+            except Exception:
+                print_exc()
+            else:
+                if DEBUG:
+                    print >> sys.stderr, "friendshipcrawler: handle_crawler_reply"
+                    print >> sys.stderr, "friendshipcrawler: friendship: Got",`d`
+
+                self.saveFriendshipStatistics(permid,d['current time'],d['stats'])
+
+        return True 
+        
+    def getStaticsFromFriendshipStatisticsTable(self, mypermid, last_update_time):
+        ulist = self.friendshipStatistics_db.getAllFriendshipStatistics(mypermid, last_update_time)
+        # Arno, 2010-02-04: Make sure Unicode B64 permids are converted to str, 
+        # bencode can't do that anymore.
+        elist = []
+        for utuple in ulist:
+            etuple = []
+            for uelem in utuple:
+                if isinstance(uelem,unicode):
+                    eelem = uelem.encode("UTF-8")
+                else:
+                    eelem = uelem
+                etuple.append(eelem)
+            elist.append(etuple)
+                
+        return elist
+        
+    
+    def saveFriendshipStatistics(self,permid,currentTime,stats):
+        if stats:
+            # 20/10/08. Boudewijn: A mistake in the code results in
+            # only 7 items in the list instead of 8. We add one here
+            # to get things working.
+            for stat in stats:
+                if len(stat) == 7:
+                    stat.append(0)
+                if len(stat) == 7 or len(stat) == 8:
+                    stat.append(bin2str(permid))
+
+            self.friendshipStatistics_db.saveFriendshipStatisticData(stats)
+    
+    def getLastUpdateTime(self, permid):
+        
+        mypermid = self.session.get_permid()
+        
+        return self.friendshipStatistics_db.getLastUpdateTimeOfThePeer(permid)
+        
diff --git a/instrumentation/next-share/BaseLib/Core/Statistics/Logger.py b/instrumentation/next-share/BaseLib/Core/Statistics/Logger.py
new file mode 100644
index 0000000..198b67c
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Statistics/Logger.py
@@ -0,0 +1,220 @@
+# Written by Jie Yang
+# see LICENSE.txt for license information
+#
+# Log version 3 = BuddyCast message V8
+
+import sys
+import os
+import time
+import socket
+import threading
+from traceback import print_exc
+            
+DEBUG = False
+
+log_separator = ' '
+logger = None
+
+# To be compatible with Logger from http://linux.duke.edu/projects/mini/logger/ 
+# for 2fastbt (revision <=825). 
+def create_logger(file_name):
+    global logger
+
+    logger = Logger(3, file_name) 
+
+
+def get_logger():    
+    global logger
+
+    if logger is None:
+        create_logger("global.log")
+        
+    return logger
+
+def get_today():    # UTC based
+    return time.gmtime(time.time())[:3]
+
+class Logger:
+    """
+    Atrributes (defulat value):
+      threshold (): message will not be logged if its output_level is bigger 
+                     than this threshould
+      file_name (): log file name
+      file_dir ('.'): directory of log file. It can be absolute or relative path.
+      prefix (''): prefix of log file
+      prefix_date (False): if it is True, insert 'YYYYMMDD-' between prefix 
+                     and file_name, e.g., sp-20060302-buddycast.log given 
+                     prefix = 'sp-' and file_name = 'buddycast.log'
+      open_mode ('a+b'): mode for open.
+    """
+    
+    def __init__(self, threshold, file_name, file_dir = '.', prefix = '', 
+                 prefix_date = False, open_mode = 'a+b'):
+        
+        self.threshold = threshold            
+        self.Log = self.log
+        if file_name == '':
+            self.logfile = sys.stderr
+        else:
+            try:
+                if not os.access(file_dir, os.F_OK):
+                    try: 
+                        os.mkdir(file_dir)
+                    except os.error, msg:
+                        raise "logger: mkdir error: " + msg
+                file_path = self.get_file_path(file_dir, prefix,
+                                               prefix_date, file_name)
+                self.logfile = open(file_path, open_mode)
+            except Exception, msg:
+                self.logfile = None
+                print >> sys.stderr, "logger: cannot open log file", \
+                         file_name, file_dir, prefix, prefix_date, msg
+                print_exc() 
+                
+    def __del__(self):
+        self.close()
+        
+    def get_file_path(self, file_dir, prefix, prefix_date, file_name):
+        if prefix_date is True:    # create a new file for each day
+            today = get_today()
+            date = "%04d%02d%02d" % today
+        else:
+            date = ''
+        return os.path.join(file_dir, prefix + date + file_name)
+    
+    def log(self, level, msg, showtime=True):
+        if level <= self.threshold:
+            if self.logfile is None: 
+                return
+            if showtime:
+                time_stamp = "%.01f"%time.time()
+                self.logfile.write(time_stamp + log_separator)
+            if isinstance(msg, str):
+                self.logfile.write(msg)
+            else:
+                self.logfile.write(repr(msg))
+            self.logfile.write('\n')
+            self.logfile.flush()
+            
+    def close(self):
+        if self.logfile is not None:
+            self.logfile.close()
+            
+
+class OverlayLogger:
+    __single = None
+    __lock = threading.RLock()
+    
+    def __init__(self, file_name, file_dir = '.'):
+        if OverlayLogger.__single:
+            raise RuntimeError, "OverlayLogger is singleton2"
+        
+        self.file_name = file_name
+        self.file_dir = file_dir
+        OverlayLogger.__single = self
+        self.Log = self.log
+        self.__call__ = self.log
+
+    def getInstance(*args, **kw):
+        OverlayLogger.__lock.acquire()
+        try:
+            if OverlayLogger.__single is None:
+                OverlayLogger(*args, **kw)
+            return OverlayLogger.__single
+        finally:
+            OverlayLogger.__lock.release()
+    getInstance = staticmethod(getInstance)
+    
+    def log(self, *msgs):
+        """
+        # MSG must be the last one. Permid should be in the rear to be readable
+        BuddyCast log for superpeer format: (V2)    
+          CONN_TRY IP PORT PERMID
+          CONN_ADD IP PORT PERMID SELVERSION
+          CONN_DEL IP PORT PERMID REASON
+          SEND_MSG IP PORT PERMID SELVERSION MSG_ID MSG
+          RECV_MSG IP PORT PERMID SELVERSION MSG_ID MSG
+          
+          #BUCA_CON Permid1, Permid2, ...
+          
+          BUCA_STA xx xx xx ...    # BuddyCast status
+              1  Pr    # nPeer
+              2  Pf    # nPref
+              3  Tr    # nTorrent
+              
+              #4  Cc    # nConntionCandidates (this one was missed before v4.1, and will not be included either in this version)
+              4  Bs    # nBlockSendList
+              5  Br    # nBlockRecvList
+              
+              6  SO    # nConnectionsInSecureOver
+              7  Co    # nConnectionsInBuddyCast
+              
+              8  Ct    # nTasteConnectionList
+              9  Cr   # nRandomConnectionList
+              10  Cu   # nUnconnectableConnectionList
+        """
+        
+        log_msg = ''
+        nmsgs = len(msgs)
+        if nmsgs < 2:
+            print >> sys.stderr, "Error message for log", msgs
+            return
+        
+        else:
+            for i in range(nmsgs):
+                if isinstance(msgs[i], tuple) or isinstance(msgs[i], list):
+                    log_msg += log_separator
+                    for msg in msgs[i]:
+                        try:
+                            log_msg += str(msg)
+                        except:
+                            log_msg += repr(msg)
+                        log_msg += log_separator
+                else:
+                    try:
+                        log_msg += str(msgs[i])
+                    except:
+                        log_msg += repr(msgs[i])
+                    log_msg += log_separator
+                
+        if log_msg:
+            self._write_log(log_msg)
+        
+    def _write_log(self, msg):
+        # one logfile per day. 
+        today = get_today()
+        if not hasattr(self, 'today'):
+            self.logger = self._make_logger(today)
+        elif today != self.today:    # make a new log if a new day comes
+            self.logger.close()
+            self.logger = self._make_logger(today)
+        self.logger.log(3, msg)
+            
+    def _make_logger(self, today):
+        self.today = today
+        hostname = socket.gethostname()
+        logger = Logger(3, self.file_name, self.file_dir, hostname, True)
+        logger.log(3, '# Tribler Overlay Log Version 3', showtime=False)    # mention the log version at the first line
+        logger.log(3, '# BUCA_STA: nRound   nPeer nPref nTorrent   ' + \
+                   'nBlockSendList nBlockRecvList   ' + \
+                   'nConnectionsInSecureOver nConnectionsInBuddyCast  ' + \
+                   'nTasteConnectionList nRandomConnectionList nUnconnectableConnectionList', 
+                   showtime=False)
+        logger.log(3, '# BUCA_STA: Rd  Pr Pf Tr  Bs Br  SO Co  Ct Cr Cu', showtime=False)
+        return logger
+        
+if __name__ == '__main__':
+    create_logger('test.log')
+    get_logger().log(1, 'abc' + ' ' + str(['abc', 1, (2,3)]))
+    get_logger().log(0, [1,'a',{(2,3):'asfadf'}])
+    #get_logger().log(1, open('log').read())
+    
+    ol = OverlayLogger('overlay.log')
+    ol.log('CONN_TRY', '123.34.3.45', 34, 'asdfasdfasdfasdfsadf')
+    ol.log('CONN_ADD', '123.34.3.45', 36, 'asdfasdfasdfasdfsadf', 3)
+    ol.log('CONN_DEL', '123.34.3.45', 38, 'asdfasdfasdfasdfsadf', 'asbc')
+    ol.log('SEND_MSG', '123.34.3.45', 39, 'asdfasdfasdfasdfsadf', 2, 'BC', 'abadsfasdfasf')
+    ol.log('RECV_MSG', '123.34.3.45', 30, 'asdfasdfasdfasdfsadf', 3, 'BC', 'bbbbbbbbbbbbb')
+    ol.log('BUCA_STA', (1,2,3), (4,5,6), (7,8), (9,10,11))
+    ol.log('BUCA_CON', ['asfd','bsdf','wevs','wwrewv'])
+    
\ No newline at end of file
diff --git a/instrumentation/next-share/BaseLib/Core/Statistics/PunctureCrawler.py b/instrumentation/next-share/BaseLib/Core/Statistics/PunctureCrawler.py
new file mode 100644
index 0000000..d776e54
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Statistics/PunctureCrawler.py
@@ -0,0 +1,165 @@
+# Written by Gertjan Halkes
+# see LICENSE.txt for license information
+
+# Crawler and logging module for UDPPuncture testing
+
+from BaseLib.Core.Session import Session
+from BaseLib.Core.Statistics.Crawler import Crawler
+from BaseLib.Core.BitTornado.BT1.MessageID import CRAWLER_PUNCTURE_QUERY
+from BaseLib.Core.Utilities.utilities import show_permid, show_permid_short
+import os
+import time
+import sys
+import zlib
+import thread
+
+DEBUG = False
+
+def get_reporter_instance():
+    return SimpleFileReporter.get_instance()
+
+class SimpleFileReporter:
+    __single = None
+    lock = thread.allocate_lock()
+
+    @classmethod
+    def get_instance(cls, *args, **kw):
+        cls.lock.acquire()
+        try:
+            if not cls.__single:
+                cls.__single = cls()
+        finally:
+            cls.lock.release()
+        return cls.__single
+    
+    def __init__(self):
+        self.file = None
+        self.path = os.path.join(Session.get_instance().get_state_dir(), "udppuncture.log")
+
+    def add_event(self, ignore, msg):
+        SimpleFileReporter.lock.acquire()
+        try:
+            try:
+                if not self.file:
+                    self.file = open(self.path, 'a+b')
+                self.file.write('%.2f %s\n' %(time.time(), msg))
+                self.file.flush()
+            except:
+                if DEBUG:
+                    print >>sys.stderr, 'Error writing puncture log'
+        finally:
+            SimpleFileReporter.lock.release()
+
+class PunctureCrawler:
+    __single = None
+
+    @classmethod
+    def get_instance(cls, *args, **kargs):
+        if not cls.__single:
+            cls.__single = cls(*args, **kargs)
+        return cls.__single
+
+    def __init__(self):
+        crawler = Crawler.get_instance()
+        if crawler.am_crawler():
+            self._file = open("puncturecrawler.txt", "a")
+            self._file.write("# Crawler started at %.2f\n" % time.time())
+            self._file.flush()
+            self._repexlog = None
+        else:
+            self.reporter = get_reporter_instance()
+
+    def query_initiator(self, permid, selversion, request_callback):
+        """
+        Established a new connection. Send a CRAWLER_PUNCTURE_QUERY request.
+        @param permid The Tribler peer permid
+        @param selversion The overlay protocol version
+        @param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload)
+        """
+        request_callback(CRAWLER_PUNCTURE_QUERY, '', callback=self._after_request_callback)
+
+    def _after_request_callback(self, exc, permid):
+        """
+        Called by the Crawler with the result of the request_callback
+        call in the query_initiator method.
+        """
+        if not exc:
+            if DEBUG: print >>sys.stderr, "puncturecrawler: request sent to", show_permid_short(permid)
+            self._file.write("REQUEST %s %.2f\n" % (show_permid(permid), time.time()))
+            self._file.flush()
+
+    def handle_crawler_request(self, permid, selversion, channel_id, message, reply_callback):
+        """
+        Received a CRAWLER_UDPUNCTURE_QUERY request.
+        @param permid The Crawler permid
+        @param selversion The overlay protocol version
+        @param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair
+        @param message The message payload
+        @param reply_callback Call this function once to send the reply: reply_callback(payload [, error=123])
+        """
+        if DEBUG:
+            print >> sys.stderr, "puncturecrawler: handle_crawler_request", show_permid_short(permid), message
+
+        SimpleFileReporter.lock.acquire()
+        try:
+            if not self.reporter.file:
+                try:
+                    self.reporter.file = open(self.reporter.path, 'a+b')
+                except Exception, e:
+                    reply_callback(str(e), error=1)
+                    return
+
+            file = self.reporter.file
+            try:
+                file.seek(0)
+                result = ("%.2f CRAWL\n" % time.time()) + file.read()
+                result = zlib.compress(result)
+                reply_callback(result)
+                file.truncate(0)
+            except Exception, e:
+                reply_callback(str(e), error=1)
+            # Regardless of whether the whole operation succeeds, make sure that we continue writing at end of file
+            try:
+                file.seek(0, os.SEEK_END)
+            except:
+                pass
+        finally:
+            SimpleFileReporter.lock.release()
+
+    def handle_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback):
+        """
+        Received a CRAWLER_UDPUNCTURE_QUERY reply.
+        @param permid The Crawler permid
+        @param selversion The overlay protocol version
+        @param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair
+        @param error The error value. 0 indicates success.
+        @param message The message payload
+        @param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload)
+        """
+        try:
+            if error:
+                if DEBUG:
+                    print >> sys.stderr, "puncturecrawler: handle_crawler_reply", error, message
+
+                self._file.write("ERROR %s %.2f %d %s\n" % (show_permid(permid), time.time(), error, message))
+                self._file.flush()
+
+            else:
+                if DEBUG:
+                    print >> sys.stderr, "puncturecrawler: handle_crawler_reply", show_permid_short(permid)
+
+                # 25/05/10 Boudewijn: We found that, for unknown
+                # reasons, the decompressed(message) contains many
+                # gigabytes worth of \0 characters.  For now we filter
+                # them out until Gert Jan can determine the actual
+                # cause.
+                data = zlib.decompress(message)
+                filtered = filter(lambda char: char != "\0", data)
+
+                self._file.write("REPLY %s %.2f\n" % (show_permid(permid), time.time()))
+                self._file.write("# reply sizes: on-the-wire=%d, decompressed=%d, filtered=%d\n" % (len(message), len(data), len(filtered)))
+                self._file.write(filtered)
+                self._file.flush()
+        except:
+            if DEBUG:
+                print >>sys.stderr, "puncturecrawler: error writing to file"
diff --git a/instrumentation/next-share/BaseLib/Core/Statistics/RepexCrawler.py b/instrumentation/next-share/BaseLib/Core/Statistics/RepexCrawler.py
new file mode 100644
index 0000000..a3cef81
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Statistics/RepexCrawler.py
@@ -0,0 +1,115 @@
+# Based on DatabaseCrawler.py written by Boudewijn Schoon
+# Modified by Raynor Vliegendhart
+# see LICENSE.txt for license information
+
+import sys
+import cPickle
+import base64
+from time import strftime
+
+from BaseLib.Core.BitTornado.BT1.MessageID import CRAWLER_REPEX_QUERY
+from BaseLib.Core.Utilities.utilities import show_permid, show_permid_short
+from BaseLib.Core.Statistics.Crawler import Crawler
+
+from BaseLib.Core.DecentralizedTracking.repex import RePEXLogDB
+
+DEBUG = False
+
+"""
+repexcrawler.txt:
+
+# ******************************************************************************
+# 2009/10/14 10:12:46 Crawler started
+2009/10/14 10:14:03; REQUEST; permid;
+2009/10/14 10:17:42;   REPLY; permid; 0; base64_pickle_peerhistory;
+2009/10/14 10:19:54;   REPLY; permid; 1; exception_msg;
+"""
+
+class RepexCrawler:
+    __single = None
+
+    @classmethod
+    def get_instance(cls, *args, **kargs):
+        if not cls.__single:
+            cls.__single = cls(*args, **kargs)
+        return cls.__single
+
+    def __init__(self,session):
+        crawler = Crawler.get_instance()
+        if crawler.am_crawler():
+            self._file = open("repexcrawler.txt", "a")
+            self._file.write("".join(("# ", "*" * 78, "\n# ", strftime("%Y/%m/%d %H:%M:%S"), " Crawler started\n")))
+            self._file.flush()
+            self._repexlog = None
+        else:
+            self._file = None
+            self._repexlog = RePEXLogDB.getInstance(session)
+
+    def query_initiator(self, permid, selversion, request_callback):
+        """
+        Established a new connection. Send a CRAWLER_REPEX_QUERY request.
+        @param permid The Tribler peer permid
+        @param selversion The overlay protocol version
+        @param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload)
+        """
+        if DEBUG: print >>sys.stderr, "repexcrawler: query_initiator", show_permid_short(permid)
+        
+        request_callback(CRAWLER_REPEX_QUERY, '', callback=self._after_request_callback)
+
+    def _after_request_callback(self, exc, permid):
+        """
+        Called by the Crawler with the result of the request_callback
+        call in the query_initiator method.
+        """
+        if not exc:
+            if DEBUG: print >>sys.stderr, "repexcrawler: request sent to", show_permid_short(permid)
+            self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), "REQUEST", show_permid(permid), "\n")))
+            self._file.flush()
+
+    def handle_crawler_request(self, permid, selversion, channel_id, message, reply_callback):
+        """
+        Received a CRAWLER_REPEX_QUERY request.
+        @param permid The Crawler permid
+        @param selversion The overlay protocol version
+        @param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair
+        @param message The message payload
+        @param reply_callback Call this function once to send the reply: reply_callback(payload [, error=123])
+        """
+        if DEBUG:
+            print >> sys.stderr, "repexcrawler: handle_crawler_request", show_permid_short(permid), message
+
+        # retrieve repex history
+        try:
+            repexhistory = self._repexlog.getHistoryAndCleanup()
+
+        except Exception, e:
+            reply_callback(str(e), error=1)
+        else:
+            reply_callback(cPickle.dumps(repexhistory, 2))
+
+    def handle_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback):
+        """
+        Received a CRAWLER_REPEX_QUERY reply.
+        @param permid The Crawler permid
+        @param selversion The overlay protocol version
+        @param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair
+        @param error The error value. 0 indicates success.
+        @param message The message payload
+        @param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload)
+        """
+        if error:
+            if DEBUG:
+                print >> sys.stderr, "repexcrawler: handle_crawler_reply", error, message
+
+            self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), "  REPLY", show_permid(permid), str(error), message, "\n")))
+            self._file.flush()
+
+        else:
+            if DEBUG:
+                print >> sys.stderr, "repexcrawler: handle_crawler_reply", show_permid_short(permid), cPickle.loads(message)
+            
+            # The message is pickled, which we will just write to file.
+            # To make later parsing easier, we base64 encode it
+            self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), "  REPLY", show_permid(permid), str(error), base64.b64encode(message), "\n")))
+            self._file.flush()
+
diff --git a/instrumentation/next-share/BaseLib/Core/Statistics/SeedingStatsCrawler.py b/instrumentation/next-share/BaseLib/Core/Statistics/SeedingStatsCrawler.py
new file mode 100644
index 0000000..46c091e
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Statistics/SeedingStatsCrawler.py
@@ -0,0 +1,178 @@
+# Written by Boxun Zhang, Boudewijn Schoon
+# see LICENSE.txt for license information
+
+import sys
+import cPickle
+
+from BaseLib.Core.BitTornado.BT1.MessageID import CRAWLER_SEEDINGSTATS_QUERY
+from BaseLib.Core.CacheDB.SqliteSeedingStatsCacheDB import *
+
+DEBUG = False
+
+class SeedingStatsCrawler:
+    __single = None
+
+    @classmethod
+    def get_instance(cls, *args, **kargs):
+        if not cls.__single:
+            cls.__single = cls(*args, **kargs)
+        return cls.__single
+
+    def __init__(self):
+        self._sqlite_cache_db = SQLiteSeedingStatsCacheDB.getInstance()
+    
+    def query_initiator(self, permid, selversion, request_callback):
+        """
+        Established a new connection. Send a CRAWLER_DATABASE_QUERY request.
+        @param permid The Tribler peer permid
+        @param selversion The oberlay protocol version
+        @param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload)
+        """
+        if DEBUG: 
+            print >>sys.stderr, "crawler: SeedingStatsDB_update_settings_initiator"
+        read_query = "SELECT * FROM SeedingStats WHERE crawled = 0"
+        write_query = "UPDATE SeedingStats SET crawled = 1 WHERE crawled = 0"
+        return request_callback(CRAWLER_SEEDINGSTATS_QUERY, cPickle.dumps([("read", read_query), ("write", write_query)], 2))
+    
+    def update_settings_initiator(self, permid, selversion, request_callback):
+        """
+        Established a new connection. Send a CRAWLER_DATABASE_QUERY request.
+        @param permid The Tribler peer permid
+        @param selversion The oberlay protocol version
+        @param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload)
+        """
+        if DEBUG: 
+            print >>sys.stderr, "crawler: SeedingStatsDB_update_settings_initiator"
+        
+        try:
+            sql_update = "UPDATE SeedingStatsSettings SET crawling_interval=%s WHERE crawling_enabled=%s"%(1800, 1)
+        except:
+            print_exc()
+        else:
+            return request_callback(CRAWLER_SEEDINGSTATS_QUERY, cPickle.dumps(sql_update, 2))
+               
+    
+    def handle_crawler_request(self, permid, selversion, channel_id, message, reply_callback):
+        """
+        Received a CRAWLER_DATABASE_QUERY request.
+        @param permid The Crawler permid
+        @param selversion The overlay protocol version
+        @param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair
+        @param message The message payload
+        @param reply_callback Call this function once to send the reply: reply_callback(payload [, error=123])
+
+        MESSAGE contains a cPickled list. Each list element is a
+        tuple. Each tuple consists of a string (either 'read' or
+        'write') and a string (the query)
+        """
+        if DEBUG:
+            print >> sys.stderr, "crawler: handle_crawler_request", len(message)
+
+        results = []
+        try:
+            items = cPickle.loads(message)
+            if DEBUG:
+                print >> sys.stderr, "crawler: handle_crawler_request", items
+
+            for action, query in items:
+                if action == "read":
+                    cursor = self._sqlite_cache_db.execute_read(query)
+                elif action == "write":
+                    cursor = self._sqlite_cache_db.execute_write(query)
+                else:
+                    raise Exception("invalid payload")
+
+                if cursor:
+                    results.append(list(cursor))
+                else:
+                    results.append(None)
+        except Exception, e:
+            if DEBUG:
+                print >> sys.stderr, "crawler: handle_crawler_request", e
+            results.append(str(e))
+            reply_callback(cPickle.dumps(results, 2), 1)
+        else:
+            reply_callback(cPickle.dumps(results, 2))
+
+        return True
+
+
+    def handle_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, reply_callback):
+        """
+        Received a CRAWLER_DATABASE_QUERY request.
+        @param permid The Crawler permid
+        @param selversion The overlay protocol version
+        @param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair
+        @param error The error value. 0 indicates success.
+        @param message The message payload
+        @param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload)
+        """
+        if error:
+            if DEBUG:
+                print >> sys.stderr, "seedingstatscrawler: handle_crawler_reply"
+                print >> sys.stderr, "seedingstatscrawler: error", error
+
+        else:
+            try:
+                results = cPickle.loads(message)
+
+                if DEBUG:
+                    print >> sys.stderr, "seedingstatscrawler: handle_crawler_reply"
+                    print >> sys.stderr, "seedingstatscrawler:", results
+
+                # the first item in the list contains the results from the select query
+                if results[0]:
+                    values = map(tuple, results[0])
+                    self._sqlite_cache_db.insertMany("SeedingStats", values)
+            except Exception, e:
+
+                # 04/11/08 boudewijn: cPickle.loads(...) sometimes
+                # results in EOFError. This may be caused by message
+                # being interpreted as non-binary.
+                f = open("seedingstats-EOFError.data", "ab")
+                f.write("--\n%s\n--\n" % message)
+                f.close()
+
+                print_exc()
+                return False
+
+        return True
+
+    
+    def handle_crawler_update_settings_request(self, permid, selversion, channel_id, message, reply_callback):
+        """
+        Received a CRAWLER_DATABASE_QUERY request.
+        @param permid The Crawler permid
+        @param selversion The overlay protocol version
+        @param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair
+        @param message The message payload
+        @param reply_callback Call this function once to send the reply: reply_callback(payload [, error=123])
+        """
+        if DEBUG:
+            print >> sys.stderr, "crawler: handle_crawler_SeedingStats_request", message
+
+        # execute the sql
+        sql_update = cPickle.loads(message)
+        
+        try:
+            self._sqlite_cache_db.execute_write(sql_update)
+        except Exception, e:
+            reply_callback(str(e), 1)
+        else:
+            reply_callback(cPickle.dumps('Update succeeded.', 2))
+        
+        return True
+
+    def handle_crawler_update_setings_reply(self, permid, selversion, channel_id, message, reply_callback):
+        """
+        Received a CRAWLER_DATABASE_QUERY request.
+        @param permid The Crawler permid
+        @param selversion The overlay protocol version
+        @param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair
+        @param message The message payload
+        @param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload)
+        """
+        if DEBUG:
+            print >> sys.stderr, "olapps: handle_crawler_SeedingStats_reply"
+
+        return True
diff --git a/instrumentation/next-share/BaseLib/Core/Statistics/Status/LivingLabReporter.py b/instrumentation/next-share/BaseLib/Core/Statistics/Status/LivingLabReporter.py
new file mode 100644
index 0000000..50f201d
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Statistics/Status/LivingLabReporter.py
@@ -0,0 +1,220 @@
+# Written by Njaal Borch
+# see LICENSE.txt for license information
+
+#
+# Arno TODO: Merge with Core/Statistics/Status/*
+#
+
+import time
+import sys
+
+import httplib
+
+import XmlPrinter
+import xml.dom.minidom
+
+import Status
+from BaseLib.Core.Utilities.timeouturlopen import find_proxy
+
+STRESSTEST = False
+DEBUG = False
+
+
+class LivingLabPeriodicReporter(Status.PeriodicStatusReporter):
+    """
+    This reporter creates an XML report of the status elements
+    that are registered and sends them using an HTTP Post at
+    the given interval.  Made to work with the P2P-Next lab.
+    """
+    
+    host = "p2pnext-statistics.comp.lancs.ac.uk"
+    #path = "/testpost/"
+    path = "/post/"
+    
+    def __init__(self, name, frequency, id, error_handler=None,
+                 print_post=False):
+        """
+        Periodically report to the P2P-Next living lab status service
+
+        name: The name of this reporter (ignored)
+        frequency: How often (in seconds) to report
+        id: The ID of this device (e.g. permid)
+        error_handler: Optional error handler that will be called if the
+        port fails
+        print_post: Print post to stderr when posting to the lab (largely
+        useful for debugging)
+        
+        """
+        Status.PeriodicStatusReporter.__init__(self,
+                                               name,
+                                               frequency,
+                                               error_handler)
+        self.device_id = id
+        self.print_post = print_post 
+        self.num_reports = 0
+
+    def new_element(self, doc, name, value):
+        """
+        Helper function to save some lines of code
+        """
+
+        element = doc.createElement(name)
+        value = doc.createTextNode(str(value))
+        element.appendChild(value)
+        
+        return element
+        
+    def report(self):
+        """
+        Create the report in XML and send it
+        """
+
+        # Create the report
+        doc = xml.dom.minidom.Document()
+        root = doc.createElement("nextsharedata")
+        doc.appendChild(root)
+        
+        # Create the header
+        header = doc.createElement("header")
+        root.appendChild(header)
+        header.appendChild(self.new_element(doc, "deviceid", self.device_id))
+        header.appendChild(self.new_element(doc, "timestamp",
+                                           long(round(time.time()))))
+        
+        version = "cs_v2a"
+        header.appendChild(self.new_element(doc, "swversion", version))
+        
+
+        elements = self.get_elements()
+        if len(elements) > 0:
+        
+            # Now add the status elements
+            if len(elements) > 0:
+                report = doc.createElement("event")
+                root.appendChild(report)
+
+                report.appendChild(self.new_element(doc, "attribute",
+                                                   "statusreport"))
+                report.appendChild(self.new_element(doc, "timestamp",
+                                                   long(round(time.time()))))
+                for element in elements:
+                    print element.__class__
+                    report.appendChild(self.new_element(doc,
+                                                       element.get_name(),
+                                                       element.get_value()))
+
+        events = self.get_events()
+        if len(events) > 0:
+            for event in events:
+                report = doc.createElement(event.get_type())
+                root.appendChild(report)
+                report.appendChild(self.new_element(doc, "attribute",
+                                                   event.get_name()))
+                if event.__class__ == Status.EventElement:
+                    report.appendChild(self.new_element(doc, "timestamp",
+                                                       event.get_time()))
+                elif event.__class__ == Status.RangeElement:
+                    report.appendChild(self.new_element(doc, "starttimestamp",
+                                                       event.get_start_time()))
+                    
+                    report.appendChild(self.new_element(doc, "endtimestamp",
+                                                       event.get_end_time()))
+                for value in event.get_values():
+                    report.appendChild(self.new_element(doc, "value", value))
+                    
+        if len(elements) == 0 and len(events) == 0:
+            return # Was nothing here for us
+        
+        # all done
+        xml_printer = XmlPrinter.XmlPrinter(root)
+        if self.print_post:
+            print >> sys.stderr, xml_printer.to_pretty_xml()
+        xml_str = xml_printer.to_xml()
+
+        # Now we send this to the service using a HTTP POST
+        self.post(xml_str)
+
+    def post(self, xml_str):
+        """
+        Post a status report to the living lab using multipart/form-data
+        This is a bit on the messy side, but it does work
+        """
+
+        #print >>sys.stderr, xml_str
+        
+        self.num_reports += 1
+        
+        boundary = "------------------ThE_bOuNdArY_iS_hErE_$"
+        headers = {"Host":self.host,
+                   "User-Agent":"NextShare status reporter 2009.4",
+                   "Content-Type":"multipart/form-data; boundary=" + boundary}
+
+        base = ["--" + boundary]
+        base.append('Content-Disposition: form-data; name="NextShareData"; filename="NextShareData"')
+        base.append("Content-Type: text/xml")
+        base.append("")
+        base.append(xml_str)
+        base.append("--" + boundary + "--")
+        base.append("")
+        base.append("")
+        body = "\r\n".join(base)
+
+        # Arno, 2010-03-09: Make proxy aware and use modern httplib classes
+        wanturl = 'http://'+self.host+self.path
+        proxyhost = find_proxy(wanturl)
+        if proxyhost is None:
+            desthost = self.host
+            desturl = self.path
+        else:
+            desthost = proxyhost
+            desturl = wanturl
+
+        h = httplib.HTTPConnection(desthost)
+        h.putrequest("POST", desturl)
+        h.putheader("Host",self.host)
+        h.putheader("User-Agent","NextShare status reporter 2010.3")
+        h.putheader("Content-Type", "multipart/form-data; boundary=" + boundary)
+        h.putheader("Content-Length",str(len(body)))
+        h.endheaders()
+        h.send(body)
+
+        resp = h.getresponse()
+        if DEBUG:
+            # print >>sys.stderr, "LivingLabReporter:\n", xml_str
+            print >>sys.stderr, "LivingLabReporter:", `resp.status`, `resp.reason`, "\n", resp.getheaders(), "\n", resp.read().replace("\\n", "\n")
+
+        if resp.status != 200:
+            if self.error_handler:
+                try:
+                    self.error_handler(resp.status, resp.read())
+                except Exception, e:
+                    pass
+            else:
+                print >> sys.stderr, "Error posting but no error handler:", \
+                      errcode, h.file.read()
+        
+
+if __name__ == "__main__":
+    """
+    Small test routine to check an actual post (unittest checks locally)
+    """
+
+    status = Status.get_status_holder("UnitTest")
+    def test_error_handler(code, message):
+        """
+        Test error-handler
+        """
+        print "Error:", code, message
+        
+    reporter = LivingLabPeriodicReporter("Living lab test reporter",
+                                         1.0, test_error_handler)
+    status.add_reporter(reporter)
+    s = status.create_status_element("TestString", "A test string")
+    s.set_value("Hi from Njaal")
+
+    time.sleep(2)
+
+    print "Stopping reporter"
+    reporter.stop()
+
+    print "Sent %d reports"% reporter.num_reports
diff --git a/instrumentation/next-share/BaseLib/Core/Statistics/Status/Status.py b/instrumentation/next-share/BaseLib/Core/Statistics/Status/Status.py
new file mode 100644
index 0000000..596fcbc
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Statistics/Status/Status.py
@@ -0,0 +1,560 @@
+# Written by Njaal Borch
+# see LICENSE.txt for license information
+
+import threading
+import time
+
+# Factory vars
+global status_holders
+status_holders = {}
+global status_lock
+status_lock = threading.Lock()
+
+def get_status_holder(name):
+    global status_lock
+    global status_holders
+    status_lock.acquire()
+    try:
+        if not name in status_holders:
+            status_holders[name] = StatusHolder(name)
+
+        return status_holders[name]
+    finally:
+        status_lock.release()
+
+class StatusException(Exception):
+    """
+    Parent exception for all status based exceptions
+    """
+    pass
+
+class NoSuchElementException(StatusException):
+    """
+    No such element found
+    """
+    pass
+
+class NoSuchReporterException(StatusException):
+    """
+    Unknown reporter
+    """
+    pass
+
+# Policies
+#ON_CHANGE = 1
+#PERIODIC = 2
+
+class StatusHolder:
+
+    """
+    A class to hold (and report) status information for an application.
+    A status holder can have multiple reporters, that will report status
+    information on change or periodically.
+
+    """
+
+
+    def __init__(self, name):
+        """
+        Do not create new status objects if you don't know what you're doing.
+        Use the getStatusHolder() function to retrieve status objects.
+        """
+        self.name = name
+        self.elements = {}
+        self.reporters = {}
+        self.lock = threading.Lock()
+        self.events = []
+
+    def reset(self):
+        """
+        Reset everything to blanks!
+        """
+        self.elements = {}
+        self.reporters = {}
+        self.events = []
+
+    def get_name(self):
+        """
+        Return the name of this status holder
+        """
+        return self.name
+
+    def get_reporter(self, name):
+        """
+        Get a given reporter from the status holder, using the name of the
+        reporter.
+        """
+        assert name
+        
+        self.lock.acquire()
+        try:
+            if not name in self.reporters:
+                raise Exception("No such reporter '%s'"%name)
+            return self.reporters[name]
+        finally:
+            self.lock.release()
+            
+    def add_reporter(self, reporter):
+        """
+        Add a reporter to this status object.
+        """
+        assert reporter
+        
+        self.lock.acquire()
+        try:
+            if reporter.name in self.reporters:
+                raise Exception("Already have reporter '%s' registered"% \
+                                reporter.name)
+            self.reporters[reporter.name] = reporter
+
+            # The reporter must contact me later
+            reporter.add_status_holder(self)
+            
+            # If we have any other reporters, copy the elements
+            # to the new one
+            for element in self.elements.values():
+                reporter.add_element(element)
+        finally:
+            self.lock.release()
+            
+
+    def _add_element(self, new_element):
+        for reporter in self.reporters.values():
+            reporter.add_element(new_element)
+        
+
+    def create_status_element(self, name, initial_value=None):
+        assert name
+        
+        new_element = StatusElement(name, initial_value)
+
+        self.lock.acquire()
+        try:
+            if name in self.elements:
+                raise Exception("Already have a status element with the given name")
+            self.elements[name] = new_element
+            self._add_element(new_element)
+        finally:
+            self.lock.release()
+
+        return new_element
+            
+    def get_status_element(self, name):
+        """
+        Get a status element from the Status Holder by name
+        """
+        assert name
+        
+        self.lock.acquire()
+        try:
+            if not name in self.elements:
+                raise NoSuchElementException(name)
+            return self.elements[name]
+        finally:
+            self.lock.release()
+        
+    def get_or_create_status_element(self, name, initial_value=None):
+        self.lock.acquire()
+        if not name in self.elements:
+            self.lock.release()
+            return self.create_status_element(name, initial_value)
+        try:
+            return self.elements[name]
+        finally:
+            self.lock.release()
+                                     
+    def remove_status_element(self, element):
+        """
+        Remove a status element
+        """
+        assert element
+        
+        self.lock.acquire()
+        try:
+            if not element.name in self.elements:
+                raise NoSuchElementException(element.name)
+            del self.elements[element.name]
+
+            # Also remove this element to the policy
+            for reporter in self.reporters.values():
+                # TODO: More elegant here
+                try:
+                    reporter.remove_element(element)
+                except:
+                    pass
+
+        finally:
+            self.lock.release()
+            
+    def create_event(self, name, values=[]):
+        return EventElement(name, values)
+
+    def add_event(self, event):
+        self.lock.acquire()
+        try:
+            self.events.append(event)
+            self._add_element(event)
+        finally:
+            self.lock.release()
+
+    def remove_range(self, range):
+        self.remove_event(range)
+        
+    def remove_event(self, event):
+        self.lock.acquire()
+        try:
+            if event in self.events:
+                self.events.remove(event)
+        finally:
+            self.lock.release()
+        
+    def create_and_add_event(self, name, values=[]):
+        self.add_event(self.create_event(name, values))
+
+    def create_range(self, name, values=[]):
+        return RangeElement(name, values)
+
+    def add_range(self, range):
+        self.add_event(range)
+        
+    def create_and_add_range(self, name, values=[]):
+        self.add_range(self.create_range(name, values))
+
+    def get_elements(self):
+        """
+        Reporters will use this to get a copy of all
+        elements that should be reported
+        """
+        self.lock.acquire()
+        try:
+            return self.elements.values()[:]
+        finally:
+            self.lock.release()
+
+    def get_events(self):
+        """
+        Reporters will use this to get a copy of all
+        events that should be reported
+        """
+        self.lock.acquire()
+        try:
+            events = self.events
+            self.events = []
+            return events
+        finally:
+            self.lock.release()
+
+        
+        
+class BaseElement:
+    type = "BaseElement"
+
+    def __init__(self, name):
+        """
+        Create a new element.  DO NOT USE THIS - use
+        create_status_element() using a Status Holder object
+        """
+        assert name
+        self.name = name
+        self.callbacks = []
+        self.lock = threading.Lock()
+
+    def get_type(self):
+        return self.type
+
+    def add_callback(self, callback):
+        """
+        Add a callback that will be executed when this element is changed.
+        The callback function will be passed the status element itself
+        """
+        self.callbacks.append(callback)
+
+    def remove_callback(self, callback):
+        """
+        Remove an already registered callback
+        """
+        if not callback in self.callbacks:
+            raise Exception("Cannot remove unknown callback")
+
+    def get_name(self):
+        return self.name
+
+                           
+    def _updated(self):
+        """
+        When a status element is changed, this method must be called to
+        notify any reporters
+        """
+
+        # TODO: Lock or make a copy?
+        
+        for callback in self.callbacks:
+            try:
+                callback(self)
+            except Exception, e:
+                import sys
+                print >> sys.stderr, "Exception in callback", \
+                      callback,"for parameter",self.name,":",e
+
+        
+class StatusElement(BaseElement):
+    """
+    Class to hold status information
+    """
+    type = "status report"
+
+    def __init__(self, name, initial_value=None):
+        """
+        Create a new element.  DO NOT USE THIS - use
+        create_status_element() using a Status Holder object
+        """
+        BaseElement.__init__(self, name)
+        self.value = initial_value
+
+    def set_value(self, value):
+        """
+        Update the value of this status element
+        """
+        
+        self.value = value
+        self._updated()
+        
+    def get_value(self):
+        return self.value
+
+    def inc(self, value=1):
+        """
+        Will only work for numbers!
+        """
+        self.lock.acquire()
+        try:
+            self.value += value
+            self._updated()
+        except:
+            raise Exception("Can only increment numbers")
+        finally:
+            self.lock.release()
+
+    def dec(self, value=1):
+        """
+        Will only work for numbers!
+        """
+        self.lock.acquire()
+        try:
+            self.value -= value
+            self._updated()
+        except:
+            raise Exception("Can only increment numbers")
+        finally:
+            self.lock.release()
+
+        
+class EventElement(BaseElement):
+    type = "event"
+
+    def __init__(self, name, values=[]):
+        """
+        Create a new element.  DO NOT USE THIS - use
+        create_status_element() using a Status Holder object
+        """
+        self.time = long(time.time())
+        BaseElement.__init__(self, name)
+        self.values = values
+
+    def get_time(self):
+        return self.time
+
+    def add_value(self, value):
+        self.lock.acquire()
+        try:
+            self.values.append(value)
+        finally:
+            self.lock.release()
+
+    def get_values(self):
+        """
+        Return the values as a copy to ensure that there are no
+        synchronization issues
+        """
+        self.lock.acquire()
+        try:
+            return self.values[:]
+        finally:
+            self.lock.release()
+
+class RangeElement(BaseElement):
+    type = "range"
+
+    def __init__(self, name, values=[]):
+        self.start_time = self.end_time = long(time.time())
+        BaseElement.__init__(self, name, "range")
+        self.values = values
+
+    def get_start_time(self):
+        return self.start_time
+
+    def get_end_time(self):
+        return self.end_time
+        
+    def add_value(self, value):
+        self.lock()
+        try:
+            self.end_time = long(time.time())
+            self.values.append(value)
+        finally:
+            self.lock.release()
+            
+    def get_values(self):
+        """
+        Return the values as a copy to ensure that there are no
+        synchronization issues
+        """
+        self.lock()
+        try:
+            return self.values[:]
+        finally:
+            self.lock.release()
+        
+class StatusReporter:
+    """
+    This is the basic status reporter class.  It cannot be used
+    directly, but provides a base for all status reporters.
+    The status reporter is threadsafe
+    """
+    
+    def __init__(self, name):
+        self.name = name
+        self.lock = threading.Lock()
+        self.status_holders = []
+        
+    def add_status_holder(self, holder):
+        if not holder in self.status_holders:
+            self.status_holders.append(holder)
+        
+    def get_elements(self):
+        """
+        Return all elements that should be reported
+        """
+        elements = []
+        for holder in self.status_holders:
+            elements += holder.get_elements()
+        return elements
+
+    def get_events(self):
+        """
+        Return all elements that should be reported
+        """
+        events = []
+        for holder in self.status_holders:
+            events += holder.get_events()
+        return events
+
+
+class OnChangeStatusReporter(StatusReporter):
+    """
+    A basic status reporter which calls 'report(element)' whenever
+    it is changed
+    """
+    elements = []
+    
+    def add_element(self, element):
+        """
+        Add element to this reporter
+        """
+        element.add_callback(self.report)
+        
+    def remove_element(self, element):
+        """
+        Remove an element from this reporter
+        """
+        element.remove_callback(self.report)
+        
+    def report(self, element):
+        """
+        This function must be implemented by and extending class. Does nothing.
+        """
+        pass # To be implemented by the actual reporter
+
+class PeriodicStatusReporter(StatusReporter):
+    """
+    Base class for a periodic status reporter, calling report(self)
+    at given times.  To ensure a nice shutdown, execute stop() when
+    stopping.
+    
+    """
+    
+    def __init__(self, name, frequency, error_handler=None):
+        """
+        Frequency is a float in seconds
+        Error-handler will get an error code and a string as parameters,
+        the meaning will be up to the implemenation of the
+        PeriodicStatusReporter.
+        """
+        
+        StatusReporter.__init__(self, name)
+        self.frequency = frequency
+        self.parameters = []
+        self.error_handler = error_handler
+
+        # Set up the timer
+        self.running = True
+        self.create_timer()
+        
+    def create_timer(self):
+        self.timer = threading.Timer(self.frequency, self.on_time_event)
+        self.timer.setName("PeriodicStatusReporter")
+        self.timer.setDaemon(True)
+        self.timer.start()
+
+    def stop(self, block=False):
+        """
+        Stop this reporter.  If block=True this function will not return
+        until the reporter has actually stopped
+        """
+        self.timer.cancel()
+        
+        self.on_time_event()
+
+        self.running = False
+        self.timer.cancel()
+        self.timer.join()
+        
+    def report(self):
+        """
+        This function must be overloaded, does nothing
+        """
+        raise Exception("Not implemented")
+
+    def add_element(self, element):
+        """
+        Overload if you want your periodic reporter to only
+        report certain elements of a holder. Normally this does
+        nothing, but report fetches all elements
+        """
+        pass
+
+    def on_time_event(self):
+        """
+        Callback function for timers
+        """
+        if self.running:
+            
+            self.create_timer()
+            try:
+                self.report()
+            except Exception, e:
+                if self.error_handler:
+                    try:
+                        self.error_handler(0, str(e))
+                    except:
+                        pass
+                else:
+                    print "Error but no error handler:", e
+                    #import traceback
+                    #traceback.print_stack()
+        
+if __name__ == "__main__":
+    # Some basic testing (full unit tests are in StatusTest.py)
+    
+    print "Run unit tests"
+    raise SystemExit(-1)
diff --git a/instrumentation/next-share/BaseLib/Core/Statistics/Status/XmlPrinter.py b/instrumentation/next-share/BaseLib/Core/Statistics/Status/XmlPrinter.py
new file mode 100644
index 0000000..a39f12e
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Statistics/Status/XmlPrinter.py
@@ -0,0 +1,186 @@
+# Written by Njaal Borch
+# see LICENSE.txt for license information
+
+def to_unicode(string):
+    """
+    Function to change a string (unicode or not) into a unicode string
+    Will try utf-8 first, then latin-1.
+    TODO: Is there a better way?  There HAS to be!!!
+    """
+
+    if string.__class__ != str:
+        return string
+    try:
+        return unicode(string, "utf-8")
+    except:
+        pass
+    print "Warning: Fallback to latin-1 for unicode conversion"
+    return unicode(string, "latin-1")
+
+
+class XmlPrinter:
+
+    """
+    An XML printer that will print XML *with namespaces*
+
+    Why minidom.toxml() does not do so really makes absolutenly no sense
+    
+    """
+
+
+    def __init__(self, doc):
+        """
+        doc should be a xml.dom.minidom document
+        
+        """
+
+        self.root = doc
+        self.namespace_counter=0
+        
+    def to_xml(self, encoding="UTF8"):
+        """
+        Like minidom toxml, just using namespaces too
+        """
+        return self._toxml(self.root, indent='', newl='').encode(encoding, "replace")
+    
+    def to_pretty_xml(self, indent=' ', newl='\n', encoding="UTF8"):
+        """
+        Like minidom toxml, just using namespaces too
+        """
+        return self._toxml(self.root, indent, newl).encode(encoding, "replace")
+    
+
+    def _make_header(self, encoding):
+        
+        return u'\n'%encoding
+
+    def _new_namespace(self, namespace):
+        # Make new namespace
+        ns_short = "ns%d"%self.namespace_counter
+        self.namespace_counter += 1
+        return ns_short
+        
+    def _toxml(self, element, indent=' ', newl='\n', encoding='UTF8', namespaces=None):
+        """
+        Recursive, internal function - do not use directly
+        """
+
+        if not element:
+            return ""
+
+        if not namespaces:
+            namespaces = {}
+        buffer = u""
+        define_ns_list = []
+
+        if element == self.root:
+            # Print the header
+            buffer = self._make_header(encoding)
+
+        if element.nodeType == element.TEXT_NODE:
+            buffer += indent + to_unicode(element.nodeValue) + newl
+            return buffer
+        if element.nodeType == element.ELEMENT_NODE:
+            ns = element.namespaceURI
+            name = to_unicode(element.localName)
+            if name.find(" ") > -1:
+                raise Exception("Refusing spaces in tag names")
+            
+            if namespaces.has_key(ns):
+                ns_short = namespaces[ns]
+                define_ns = False
+            else:
+                if ns not in ["", None]:
+                    ns_short = self._new_namespace(ns)
+                    define_ns_list.append((ns, ns_short))
+                else:
+                    ns_short = None
+                    
+                define_ns = True
+                namespaces[ns] = ns_short
+
+            # Should we define more namespaces?  Will peak into the
+            # children and see if there are any
+            for child in element.childNodes:
+                if child.nodeType != child.ELEMENT_NODE:
+                    continue
+
+                if not namespaces.has_key(child.namespaceURI) and \
+                       child.namespaceURI not in [None, ""]:
+                    # Should define this one too!
+                    new_ns = self._new_namespace(child.namespaceURI)
+                    define_ns_list.append((child.namespaceURI, new_ns))
+                    namespaces[child.namespaceURI] = new_ns
+            buffer += indent
+            
+            # If we have no children, we will write 
+            if not element.hasChildNodes():
+                if ns != None:
+                    if define_ns:
+                        if ns_short:
+                            buffer += '<%s:%s xmlns:%s="%s"/>%s'%\
+                                      (ns_short, name,ns_short,ns,newl)
+                        else:
+                            buffer += '<%s xmlns="%s"/>%s'%(name,ns,newl)
+                    else:
+                        if ns_short:
+                            buffer += '<%s:%s/>%s'%(ns_short, name, newl)
+                        else:
+                            buffer += '<%s/>%s'%(name, newl)
+
+                else:
+                    buffer += '<%s/>%s'%(name, newl)
+
+                # Clean up - namespaces is passed as a reference, and is
+                # as such not cleaned up.  Let it be so to save some speed
+                for (n,short) in define_ns_list:
+                    del namespaces[n]
+                return buffer
+
+            # Have children
+            ns_string = ""
+            if len(define_ns_list) > 0:
+                for (url, short) in define_ns_list:
+                    ns_string += ' xmlns:%s="%s"'%(short, url)
+                        
+            if ns != None:
+                if define_ns:
+                    if ns_short:
+                        # Define all namespaces of next level children too
+                        buffer += '<%s:%s xmlns:%s="%s"%s>%s'%\
+                                  (ns_short, name, ns_short, ns, ns_string, newl)
+                    else:
+                        buffer += '<%s xmlns="%s"%s>%s'%(name,ns,ns_string,newl)
+                else:
+                    if ns_short:
+                        buffer += '<%s:%s%s>%s'%(ns_short, name, ns_string, newl)
+                    else:
+                        buffer += '<%s%s>%s'%(name, ns_string, newl)
+            elif ns_string:
+                buffer += '<%s %s>%s'%(name, ns_string, newl)
+            else:
+                buffer += '<%s>%s'%(name, newl)
+
+            # Recursively process
+            for child in element.childNodes:
+                new_indent = indent
+                if new_indent:
+                    new_indent += "  "
+                buffer += self._toxml(child, new_indent, newl, encoding, namespaces)
+            if ns_short:
+                buffer += "%s%s"%(indent, ns_short, name, newl)
+            else:
+                buffer += "%s%s"%(indent, name, newl)
+
+            for (n, short) in define_ns_list:
+                del namespaces[n]
+            try:
+                return buffer
+            except Exception,e:
+                print "-----------------"
+                print "Exception:",e
+                print "Buffer:",buffer
+                print "-----------------"
+                raise e
+
+        raise Exception("Could not serialize DOM")
diff --git a/instrumentation/next-share/BaseLib/Core/Statistics/Status/__init__.py b/instrumentation/next-share/BaseLib/Core/Statistics/Status/__init__.py
new file mode 100644
index 0000000..c49ce08
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Statistics/Status/__init__.py
@@ -0,0 +1,42 @@
+# Written by Njaal Borch
+# see LICENSE.txt for license information
+
+"""
+Status gathering module with some simple reporting functionality
+
+Usage example:
+
+status = Status.get_status_holder("somename") # Get status object
+reporter = MyReporter("MyReporter")           # Create the reporter
+status.add_reporter(reporter)                 # Add the reporter to the status object
+
+# Create new element
+elem = status.create_status_element("ElementName",
+                                    "Description",
+                                    initial_value=None)
+elem.set_value(somevalue)
+
+# The element will now be reported by the reporter.
+
+A reporter can be created easily like this:
+
+# Print name=value when the element is changed
+class MyOnChangeStatusReporter(Status.OnChangeStatusReporter):
+
+    def report(self, element):
+        print element.name,"=",element.value
+
+
+# Print name=value for all elements when the periodic reporter runs
+class MyPeriodicStatusReporter(Status.PeriodicStatusReporter):
+    def report(self):
+        for elems in self.elements[:]:
+            print element.name,"=",element.value
+
+
+See the StatusTest.py class for more examples
+
+"""
+
+from Status import *
+from LivingLabReporter import LivingLabPeriodicReporter
diff --git a/instrumentation/next-share/BaseLib/Core/Statistics/VideoPlaybackCrawler.py b/instrumentation/next-share/BaseLib/Core/Statistics/VideoPlaybackCrawler.py
new file mode 100644
index 0000000..d1a5117
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Statistics/VideoPlaybackCrawler.py
@@ -0,0 +1,211 @@
+"""
+Crawling the VideoPlayback statistics database
+"""
+
+from time import strftime
+import cPickle
+import sys
+import threading
+import zlib
+
+from BaseLib.Core.BitTornado.BT1.MessageID import CRAWLER_VIDEOPLAYBACK_INFO_QUERY, CRAWLER_VIDEOPLAYBACK_EVENT_QUERY
+from BaseLib.Core.CacheDB.SqliteVideoPlaybackStatsCacheDB import VideoPlaybackDBHandler
+from BaseLib.Core.Overlay.SecureOverlay import OLPROTO_VER_EIGHTH, OLPROTO_VER_TENTH
+from BaseLib.Core.Statistics.Crawler import Crawler
+from BaseLib.Core.Utilities.utilities import show_permid, show_permid_short
+
+DEBUG = False
+
+class VideoPlaybackCrawler:
+    __single = None    # used for multi-threaded singletons pattern
+    lock = threading.Lock()
+
+    @classmethod
+    def get_instance(cls, *args, **kargs):
+        # Singleton pattern with double-checking to ensure that it can only create one object
+        if cls.__single is None:
+            cls.lock.acquire()   
+            try:
+                if cls.__single is None:
+                    cls.__single = cls(*args, **kargs)
+            finally:
+                cls.lock.release()
+        return cls.__single
+    
+    def __init__(self):
+        if VideoPlaybackCrawler.__single is not None:
+            raise RuntimeError, "VideoPlaybackCrawler is singleton"
+
+        crawler = Crawler.get_instance()
+        if crawler.am_crawler():
+            self._file = open("videoplaybackcrawler.txt", "a")
+            self._file.write("".join(("# ", "*" * 80, "\n# ", strftime("%Y/%m/%d %H:%M:%S"), " Crawler started\n")))
+            self._file.flush()
+            self._event_db = None
+
+        else:
+            self._file = None
+            self._event_db = VideoPlaybackDBHandler.get_instance()
+
+    def query_initiator(self, permid, selversion, request_callback):
+        """
+        <>
+        Established a new connection. Send a CRAWLER_VIDEOPLAYBACK_INFO_QUERY request.
+        @param permid The Tribler peer permid
+        @param selversion The oberlay protocol version
+        @param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload)
+        """
+        if selversion >= OLPROTO_VER_TENTH:
+            if DEBUG: print >>sys.stderr, "videoplaybackcrawler: query_initiator", show_permid_short(permid), "version", selversion
+            # Overlay version 10 provided a simplification in the VOD
+            # stats collecting. We now have only one database table:
+            # playback_event that has only 3 columns: key, timestamp,
+            # and event.
+            request_callback(CRAWLER_VIDEOPLAYBACK_EVENT_QUERY, "SELECT key, timestamp, event FROM playback_event; DELETE FROM playback_event;", callback=self._after_event_request_callback)
+            
+        elif selversion >= OLPROTO_VER_EIGHTH:
+            if DEBUG: print >>sys.stderr, "videoplaybackcrawler: query_initiator", show_permid_short(permid), "version", selversion
+            # boudewijn: order the result DESC! From the resulting
+            # list we will not remove the first entries from the
+            # database because this (being the last item added) may
+            # still be actively used.
+            request_callback(CRAWLER_VIDEOPLAYBACK_INFO_QUERY, "SELECT key, timestamp, piece_size, num_pieces, bitrate, nat FROM playback_info ORDER BY timestamp DESC LIMIT 50", callback=self._after_info_request_callback)
+
+        else:
+            if DEBUG: print >>sys.stderr, "videoplaybackcrawler: query_info_initiator", show_permid_short(permid), "unsupported overlay version"
+
+    def _after_info_request_callback(self, exc, permid):
+        """
+        <>
+        Called by the Crawler with the result of the request_callback
+        call in the query_initiator method.
+        """
+        if not exc:
+            if DEBUG: print >>sys.stderr, "videoplaybackcrawler: request send to", show_permid_short(permid)
+            self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), "INFO REQUEST", show_permid(permid), "\n")))
+            self._file.flush()
+
+    def handle_info_crawler_request(self, permid, selversion, channel_id, message, reply_callback):
+        pass
+
+    def handle_info_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback):
+        """
+        <>
+        Received a CRAWLER_VIDEOPLAYBACK_INFO_QUERY reply.
+        @param permid The Crawler permid
+        @param selversion The overlay protocol version
+        @param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair
+        @param error The error value. 0 indicates success.
+        @param message The message payload
+        @param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload)
+        """
+        if error:
+            if DEBUG:
+                print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", error, message
+
+            self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), "   INFO REPLY", show_permid(permid), str(error), message, "\n")))
+            self._file.flush()
+
+        else:
+            if DEBUG:
+                print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", show_permid_short(permid), cPickle.loads(message)
+
+            info = cPickle.loads(message)
+            self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), "   INFO REPLY", show_permid(permid), str(error), str(info), "\n")))
+            self._file.flush()
+
+            i = 0
+            for key, timestamp, piece_size, num_pieces, bitrate, nat in info:
+                i += 1
+                # do not remove the first item. the list is ordered
+                # DESC so the first item is the last that is added to
+                # the database and we can't affored to remove it, as
+                # it may cause exceptions in the running playback.
+                if i == 1:
+                    sql = """
+SELECT timestamp, origin, event FROM playback_event WHERE key = '%s' ORDER BY timestamp ASC LIMIT 50;
+DELETE FROM playback_event WHERE key = '%s';
+""" % (key, key)
+                else:
+                    sql = """
+SELECT timestamp, origin, event FROM playback_event WHERE key = '%s' ORDER BY timestamp ASC LIMIT 50;
+DELETE FROM playback_event WHERE key = '%s';
+DELETE FROM playback_info WHERE key = '%s';
+""" % (key, key, key)
+                    
+                # todo: optimize to not select key for each row
+                request_callback(CRAWLER_VIDEOPLAYBACK_EVENT_QUERY, sql, channel_data=key, callback=self._after_event_request_callback, frequency=0)
+
+    def _after_event_request_callback(self, exc, permid):
+        """
+        <>
+        Called by the Crawler with the result of the request_callback
+        call in the handle_crawler_reply method.
+        """
+        if not exc:
+            if DEBUG: print >>sys.stderr, "videoplaybackcrawler: request send to", show_permid_short(permid)
+            self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), " INFO REQUEST", show_permid(permid), "\n")))
+            self._file.flush()
+
+    def handle_event_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback):
+        """
+        <>
+        Received a CRAWLER_VIDEOPLAYBACK_EVENT_QUERY reply.
+        @param permid The Crawler permid
+        @param selversion The overlay protocol version
+        @param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair
+        @param channel_data Data associated with the request
+        @param error The error value. 0 indicates success.
+        @param message The message payload
+        @param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload)
+        """
+        if error:
+            if DEBUG:
+                print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", error, message
+
+            self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), "  EVENT REPLY", show_permid(permid), str(error), str(channel_data), message, "\n")))
+            self._file.flush()
+
+        elif selversion >= OLPROTO_VER_TENTH:
+            # Overlay version 10 sends the reply pickled and zipped
+            if DEBUG:
+                print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", show_permid_short(permid), len(message), "bytes zipped"
+
+            info = cPickle.loads(zlib.decompress(message))
+            self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), "  EVENT REPLY", show_permid(permid), str(error), str(channel_data), str(info), "\n")))
+            self._file.flush()
+            
+        elif selversion >= OLPROTO_VER_EIGHTH:
+            if DEBUG:
+                print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", show_permid_short(permid), cPickle.loads(message)
+
+            info = cPickle.loads(message)
+            self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), "  EVENT REPLY", show_permid(permid), str(error), str(channel_data), str(info), "\n")))
+            self._file.flush()
+
+    def handle_event_crawler_request(self, permid, selversion, channel_id, message, reply_callback):
+        """
+        <>
+        Received a CRAWLER_VIDEOPLAYBACK_EVENT_QUERY request.
+        @param permid The Crawler permid
+        @param selversion The overlay protocol version
+        @param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair
+        @param message The message payload
+        @param reply_callback Call this function once to send the reply: reply_callback(payload [, error=123])
+        """
+        if DEBUG:
+            print >> sys.stderr, "videoplaybackcrawler: handle_event_crawler_request", show_permid_short(permid), message
+
+        # execute the sql
+        try:
+            cursor = self._event_db._db.execute_read(message)
+
+        except Exception, e:
+            reply_callback(str(e), error=1)
+        else:
+            if cursor:
+                reply_callback(zlib.compress(cPickle.dumps(list(cursor), 2), 9))
+            else:
+                reply_callback("error", error=2)
+
+    
diff --git a/instrumentation/next-share/BaseLib/Core/Statistics/__init__.py b/instrumentation/next-share/BaseLib/Core/Statistics/__init__.py
new file mode 100644
index 0000000..395f8fb
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Statistics/__init__.py
@@ -0,0 +1,2 @@
+# Written by Arno Bakker
+# see LICENSE.txt for license information
diff --git a/instrumentation/next-share/BaseLib/Core/Statistics/crawler.txt b/instrumentation/next-share/BaseLib/Core/Statistics/crawler.txt
new file mode 100644
index 0000000..fec1ba1
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Statistics/crawler.txt
@@ -0,0 +1,31 @@
+#
+#    Anonymous performance data gathering
+#
+# For improvements to our algorithms the scientists behind this
+# software need to have some insight how the P2P network operates.
+# The PermIDs listed in this file have some access to internal P2P
+# statistics when conducting a network crawl.
+# Collected data will never be shared with third parties or used
+# for non-scientific purposes.
+# 
+# Please delete the PermIDs to disable this feature.
+#
+# permid
+# lucia's old crawler
+MFIwEAYHKoZIzj0CAQYFK4EEABoDPgAEAKa2aWZv65UoFv0OR8BbVSnlmTPrYKcwwpGHEhK3AO2PpxiGlv/Y2mTP2kg+VXLaBBmfpdYWPA4eSdpq
+MFIwEAYHKoZIzj0CAQYFK4EEABoDPgAEAe6fNHWhKsReFj8/RIN6rBHWRzT4VkLddvhJZ5jmAQf5c7ZmqkdFQ/F21DKbC8V1Otmf6YO00ufe5D/o
+
+# General crawler PermIDs: DO NOT CHANGE OR ADD WITHOUT CONSULTION!
+MFIwEAYHKoZIzj0CAQYFK4EEABoDPgAEAOydlMAfRpmhT+jKr0gI8EanNLyt+Y/FEFcjTAoFAKCmNGMGrBl22ZICZBi+oPo0p6FpWECrf2oGg2WM
+MFIwEAYHKoZIzj0CAQYFK4EEABoDPgAEAK1cQH+R2B6oOPNgCcgiAruKlWAYZGzryZm6P0B3AMzocJszITiPPIsGujeg0saYZ6+VmzuncOCvVOWY
+MFIwEAYHKoZIzj0CAQYFK4EEABoDPgAEAPKqwAWYmpi3yjhnQTV1kOHU3y8gbNVyFGbAJaQMAAQjDYrSOHJTeKIAaYZFieGU6K8FnmJKlC4qLHxh
+MFIwEAYHKoZIzj0CAQYFK4EEABoDPgAEAMOfw9qZb/9Eqqy+75FWQGOi8vAkt7P32S+EEjVbAN67PY2fTjHNdFlZlhjqotTzJdYc1299OWCV3Nf+
+MFIwEAYHKoZIzj0CAQYFK4EEABoDPgAEAL2CbiMwdGFLFQK93Je7H1a+Gi2QWV8B9n+Fwdq6AdAH04s1unhfTEP6cw1UlAdg4rZEY27GsINGsmD+
+MFIwEAYHKoZIzj0CAQYFK4EEABoDPgAEABZzmfFlN7PryBasdECMITSm8XJEQ4WU2Te99YeqARS2i2aLDxPYhFTOfBuYN4MrFLwpDxmRm7Gvdp2m
+MFIwEAYHKoZIzj0CAQYFK4EEABoDPgAEAbR4tcEbSSikh7oULmXjpl5tYKdKvR3Qn1UH913lAW2GK0k2bF8hO7RIdu971gZpgNUew33kiWE/IREP
+MFIwEAYHKoZIzj0CAQYFK4EEABoDPgAEAXvNe65EBsnBAy/s4dp1kJDa9KXnfTHAOO8OADt+Abm83AAXdeeTwyBboyioaMMlIuUyS/9GwXay2ZLA
+MFIwEAYHKoZIzj0CAQYFK4EEABoDPgAEAHXzQ+9sH0II55c3TfpFz+LZwqNpHCOHYq0iXkmFALZKYSNA3/WvyncKCh9mbpWUtbusf06/HYhHHxUg
+MFIwEAYHKoZIzj0CAQYFK4EEABoDPgAEAFuyErJwV2MBqhjLbjXA0D5PkvY1O9thUbx4QB3CAQOxYlZUtgUP09mc8K+uEuoHzOKdN2h4KoB/G8Ae
+
+
+
diff --git a/instrumentation/next-share/BaseLib/Core/Statistics/tribler_friendship_stats_sdb.sql b/instrumentation/next-share/BaseLib/Core/Statistics/tribler_friendship_stats_sdb.sql
new file mode 100644
index 0000000..f215540
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Statistics/tribler_friendship_stats_sdb.sql
@@ -0,0 +1,38 @@
+-- Tribler Friendship Statistics Database
+
+BEGIN TRANSACTION create_table;
+
+----------------------------------------
+
+CREATE TABLE FriendshipStatistics (
+  source_permid        text NOT NULL,
+  target_permid        text NOT NULL,
+  isForwarder          integer DEFAULT 0,
+  request_time         numeric,
+  response_time        numeric,
+  no_of_attempts       integer DEFAULT 0,
+  no_of_helpers		   integer DEFAULT 0,
+  modified_on		   numeric,
+  crawled_permid       text NOT NULL DEFAULT client
+);
+
+----------------------------------------
+
+CREATE TABLE MyInfo (
+  entry  PRIMARY KEY,
+  value  text
+);
+
+----------------------------------------
+
+COMMIT TRANSACTION create_table;
+
+----------------------------------------
+
+BEGIN TRANSACTION init_values;
+
+-- Version 1: Initial version, published in Tribler 4.5.0
+-- Version 2: Added crawled_permid to FriendshipStatistics table.
+INSERT INTO MyInfo VALUES ('version', 2);
+
+COMMIT TRANSACTION init_values;
\ No newline at end of file
diff --git a/instrumentation/next-share/BaseLib/Core/Statistics/tribler_seedingstats_sdb.sql b/instrumentation/next-share/BaseLib/Core/Statistics/tribler_seedingstats_sdb.sql
new file mode 100644
index 0000000..b2bad53
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Statistics/tribler_seedingstats_sdb.sql
@@ -0,0 +1,41 @@
+-- Tribler Seeding Statistics Database
+
+BEGIN TRANSACTION create_table;
+
+----------------------------------------
+
+CREATE TABLE SeedingStats (
+  timestamp  	real,
+  permID		text,
+  info_hash    	text,
+  seeding_time	real,
+  reputation	real,
+  crawled      	integer
+);
+
+----------------------------------------
+
+CREATE TABLE SeedingStatsSettings (
+  version				integer PRIMARY KEY,
+  crawling_interval  	integer,
+  crawling_enabled		integer
+);
+
+----------------------------------------
+
+CREATE TABLE MyInfo (
+  entry  PRIMARY KEY,
+  value  text
+);
+
+----------------------------------------

+COMMIT TRANSACTION create_table;
+
+----------------------------------------
+
+BEGIN TRANSACTION init_values;
+
+INSERT INTO MyInfo VALUES ('version', 1);
+INSERT INTO SeedingStatsSettings VALUES (1, 1800, 1);
+
+COMMIT TRANSACTION init_values;
diff --git a/instrumentation/next-share/BaseLib/Core/Statistics/tribler_videoplayback_stats.sql b/instrumentation/next-share/BaseLib/Core/Statistics/tribler_videoplayback_stats.sql
new file mode 100644
index 0000000..869b9bf
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Statistics/tribler_videoplayback_stats.sql
@@ -0,0 +1,35 @@
+-- Tribler Video Playback Statistics Database
+
+BEGIN TRANSACTION create_table;
+
+----------------------------------------
+
+CREATE TABLE playback_event (
+  key                   text NOT NULL,
+  timestamp             real NOT NULL,
+  event                 text NOT NULL
+);  
+
+CREATE INDEX playback_event_idx 
+  ON playback_event (key, timestamp);
+
+----------------------------------------
+
+CREATE TABLE MyInfo (
+  entry  PRIMARY KEY,
+  value  text
+);
+
+----------------------------------------
+
+COMMIT TRANSACTION create_table;
+
+----------------------------------------
+
+BEGIN TRANSACTION init_values;
+
+-- Version 1: Initial version, published in Tribler 5.0.0
+-- Version 2: Simplified the database. Now everything is an event. Published in Tribler 5.1.0
+INSERT INTO MyInfo VALUES ('version', 2);
+
+COMMIT TRANSACTION init_values;
diff --git a/instrumentation/next-share/BaseLib/Core/Subtitles/MetadataDomainObjects/Languages.py b/instrumentation/next-share/BaseLib/Core/Subtitles/MetadataDomainObjects/Languages.py
new file mode 100644
index 0000000..1d7a025
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Subtitles/MetadataDomainObjects/Languages.py
@@ -0,0 +1,219 @@
+# Written by Andrea Reale
+# see LICENSE.txt for license information
+
+from __future__ import with_statement
+import csv
+import codecs
+from BaseLib.Core.Utilities.utilities import binaryStringToUint, uintToBinaryString
+
+MAX_SUPPORTED_LANGS = 32
+
+DEFAULT_LANG_CONF_FILE = "res/subs_languages.csv"
+
+
+def _loadLanguages(langFilePath):
+    """
+    Read a list of languages from a csv file
+    
+    Reads a list of language codes and the relative language
+    description from a csv text file. On each line of the file
+    there must be a couple of ISO 693-2 formatted language code
+    'code' and the textual description for the language.
+    e.g. ita, Italian
+    """
+    languages = {}
+    with codecs.open(langFilePath, "r","utf-8") as csvfile:
+        csvreader = csv.reader(csvfile)
+        for row in csvreader:
+            # Must be exactly two entries code, description
+            if len(row) != 2 :
+                raise ValueError("Erroneous format in csv")
+            # Only check if the code is a three character code, not
+            # if it is really a valid ISO 639-2 Cod
+            if len(row[0]) != 3 :
+                raise ValueError("Lang codes must be 3 characters length")
+                
+            languages[row[0]] = row[1]
+    
+    return languages
+
+_languages = {
+    'ara':'Arabic',
+    'ben':'Bengali',
+    'ces':'Czech',
+    'dan':'Danish',
+    'deu':'German',
+    'ell':'Greek',
+    'eng':'English',
+    'fas':'Persian',
+    'fin':'Finnish',
+    'fra':'French',
+    'hin':'Hindi',
+    'hrv':'Croatian',
+    'hun':'Hungarian',
+    'ita':'Italian',
+    'jav':'Javanese',
+    'jpn':'Japanese',
+    'kor':'Korean',
+    'lit':'Latvia',
+    'msa':'Malay',
+    'nld':'Dutch',
+    'pan':'Panjabi',
+    'pol':'Polish',
+    'por':'Portuguese',
+    'ron':'Romanian',
+    'rus':'Russian',
+    'spa':'Spanish',
+    'srp':'Serbian',
+    'swe':'Swedish',
+    'tur':'Turkish',
+    'ukr':'Ukranian',
+    'vie':'Vietnamese',
+    'zho':'Chinese'
+}
+
+
+class Languages(object):
+    '''
+    Performs the translation between supported languages and bitstrings.
+    '''
+            
+    def __init__(self, lang_dict=_languages):
+        '''
+        Constructor
+        '''
+        
+        # Contains paris of the type { lang_code : Long language Name}
+        # its values are read from a file
+        self.supportedLanguages = {}
+        
+        # for each language code defined in supportedLanguages
+        # maps contains the bit string representing that language
+        self.langMappings = {}
+        
+        self.supportedLanguages = lang_dict
+        
+        self._supportedCodes = frozenset(self.supportedLanguages.keys())
+        
+        if len(self.supportedLanguages) > MAX_SUPPORTED_LANGS:
+            raise ValueError("Maximum number of supported languages is %d" %
+                             MAX_SUPPORTED_LANGS)
+            
+        self._initMappings()                
+
+
+    def _initMappings(self):
+        """
+        Assigns bitmasks to languages.
+        
+        Assigns bitmasks to language codes. Language codes are sorted
+        lexicographically and the first bitmask (i.e. 0x1) is given to
+        the first code in this order.
+        """
+        counter = 0
+        sortedKeys = sorted(self.supportedLanguages.keys())
+        for code in sortedKeys:
+            self.langMappings[code] = 1 << counter
+            counter += 1
+        
+        
+            
+    def getMaskLength(self):
+        """
+        Returns the length of the languages bit mask.
+        
+        Returns the necessary length to contain the language bit mask
+        for the languages represented by this instance.
+        It is always a power of two, even if less bits would actually be
+        required
+        """
+        
+        # always returnes the maximum number of supported languages
+        return MAX_SUPPORTED_LANGS
+    
+
+    
+    def maskToLangCodes(self, mask):
+        """
+        Given a int bitmask returns the list of languages it represents.
+        
+        Translates the bitmask passed in as parameters into a list
+        of language codes that represent that bitmask.
+        
+        @param mask: a bitmask representing languages (integer)
+        @return: a list of language codes string
+        @precondition: mask < 2**32 -1
+
+        """
+        assert mask < 2**32 , "Mask mast be a 32 bit value"
+        assert mask >=0 , "Mask must  be positive"
+        codeslist = []
+        
+        for code, cur_mask in self.langMappings.iteritems():
+            if mask & cur_mask != 0 :
+                codeslist.append(code)
+        
+        return sorted(codeslist)
+    
+    
+    
+    def langCodesToMask(self, codes):
+        """
+        Given a list of languages returns the bitmask representing it.
+        
+        Translates a list of language codes in a bitmask representing it.
+        Converse operation of masktoLangCodes.
+        
+        @param codes: a list of language codes. That code must be one of the
+                      keys of self.supportedLanguages.keys()
+        """
+        
+        validCodes = self.supportedLanguages.keys()
+        
+        #mask is the integer value of the bitfield
+        mask = 0
+        for lang in codes:
+            #precondition: every entry in codes is contained in 
+            #self.supportedLanguages.keys
+            if lang not in validCodes:
+                raise ValueError(lang + " is not a supported language code")
+            mask = mask | self.langMappings[lang]
+        
+        return mask
+    
+    
+    def isLangCodeSupported(self, langCode):
+        """
+        Checks whether a given language code is supported.
+        
+        Returns true if the language code is one of the supported languages
+        for subtitles
+        """
+        return langCode in self._supportedCodes
+    
+    def isLangListSupported(self, listOfLangCodes):
+        """
+        Checks whether a list of language codes is fully supported.
+        
+        Returns true only if every entry in the list passed in as parameter
+        is supported as a language for subtitles.
+        """
+        givenCodes = set(listOfLangCodes)
+        return givenCodes & self._supportedCodes == givenCodes
+    
+    
+    
+            
+
+class LanguagesProvider(object):
+    
+    _langInstance = None
+        
+    @staticmethod
+    def getLanguagesInstance():
+        if LanguagesProvider._langInstance is None:
+            #lang_dict = _loadLanguages(DEFAULT_LANG_CONF_FILE)
+            LanguagesProvider._langInstance = Languages(_languages)    
+        return LanguagesProvider._langInstance
+    
+    
diff --git a/instrumentation/next-share/BaseLib/Core/Subtitles/MetadataDomainObjects/MetadataDTO.py b/instrumentation/next-share/BaseLib/Core/Subtitles/MetadataDomainObjects/MetadataDTO.py
new file mode 100644
index 0000000..bce956f
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Subtitles/MetadataDomainObjects/MetadataDTO.py
@@ -0,0 +1,300 @@
+# Written by Andrea Reale
+# see LICENSE.txt for license information
+
+from BaseLib.Core.BitTornado.bencode import bencode, bdecode
+from BaseLib.Core.Subtitles.MetadataDomainObjects.Languages import \
+    LanguagesProvider
+from BaseLib.Core.Subtitles.MetadataDomainObjects.MetadataExceptions import \
+    SerializationException
+from BaseLib.Core.Subtitles.MetadataDomainObjects.SubtitleInfo import SubtitleInfo
+from BaseLib.Core.Overlay.permid import sign_data, verify_data
+from BaseLib.Core.Utilities.utilities import isValidInfohash, isValidPermid, \
+    uintToBinaryString, binaryStringToUint
+from math import floor
+import sys
+import time
+
+DEBUG = False
+
+_languagesUtil = LanguagesProvider.getLanguagesInstance()
+
+class MetadataDTO(object):
+    '''
+    Metdata DataTransferObject
+    '''
+
+
+    def __init__(self, publisher,infohash,timestamp = None,
+                 description=u"", subtitles=None,signature=None):
+        """
+        Create a MetataDTO instance.
+        
+        publisher and infohash are mandatory to be not null
+        @param publisher: the permid  of the owner of the 
+                          channel this instance refers to
+        @param infohash: the infohash of the item in the channel this instance
+                         refers to 
+        @param timestamp: the timestamp of the creation of this metadata
+                          instance. This can be later reset with 
+                          resetTimestamp()
+        @param description: an optional utf-8 string description for the item. 
+                            Defaults to an empty string
+        @param subtitles: a dictionary of type {langCode : SubtitleInfo}
+        @param signature: signature of the packed version of this metadataDTO.
+                          Defaults to None. It can be later signed with sign()
+        """
+        
+        assert publisher is not None
+        assert infohash is not None
+        assert isValidPermid(publisher)
+        assert isValidInfohash(infohash)
+        
+        #stringified permid of the owner of the channel
+        self.channel = publisher
+        
+        #stringified infohash (bin2str) of the torrent
+        self.infohash = infohash
+        if timestamp is not None:
+            timestring = int(floor(timestamp))
+        else:
+            timestring = int(floor(time.time()))
+        
+        #integer timestamp of the creation of this content
+        #(the content, not the MetadataDTO instance)
+        self.timestamp = timestring
+        
+        #utf-8 string description
+        if isinstance(description, str):
+            description = unicode(description, "utf-8")
+            
+        self.description = description
+        
+        if subtitles is None:
+            subtitles = {}
+        self._subtitles = subtitles
+        self.signature = signature
+        
+        
+    def resetTimestamp(self):
+        """
+        Sets the timestamp to the current time.
+        """
+        self.timestamp = int(floor(time.time()))
+    
+    def addSubtitle(self, subtitle):
+        '''
+        Adds a subtitle instance to the metadata dto.
+        
+        subtitle must be an instance of SubtitleInfo, and its language
+        field must be correctly set to an ISO-639-2 language code
+        (see Languages).
+        
+        @param subtitle: a SubtitleInfo instance
+        @precondition: subtitle.lang is not None
+        '''
+        assert isinstance(subtitle, SubtitleInfo)
+        assert subtitle.lang is not None
+        
+        self._subtitles[subtitle.lang] = subtitle
+    
+    def removeSubtitle(self, lang):
+        '''
+        Remove a subtitle instance from the dto.
+        
+        If the subtitles with the given language does not exist, it
+        does nothing.
+        
+        @param lang: a language code for the subtitle to be removed
+        '''
+        if lang in self._subtitles.keys():
+            del self._subtitles[lang]
+        
+    def getSubtitle(self,lang):
+        '''
+        Returns a SubtitleInfo instance for the given language if it exists.
+        
+        @param lang: an ISO-639-2 3 characters language code
+        
+        @rtype: SubtitleInfo.SubtitleInfo
+        @return: a SubtitleInfo instance, or None
+        '''
+        if lang not in self._subtitles.keys():
+            return None
+        else:
+            return self._subtitles[lang]
+    
+    def getAllSubtitles(self):
+        '''
+        Returns a copy of the subtitles for this dto.
+        
+        Notice that modifying this copy does not affect the languages in the
+        metadata dto
+        '''
+        return self._subtitles.copy()
+        
+        
+        
+    def sign(self,keypair):
+        """
+        Signs the packed version of this instance.
+        
+        See _packData to see what packed version means.
+        
+        @param keypair: an ec keypair that will be used to create
+                        the signature
+        """ 
+        bencoding = self._packData()
+        signature = sign_data(bencoding, keypair)
+        self.signature = signature
+    
+    def verifySignature(self):
+        """
+        Verifies the signature field of this instance.
+        
+        The signature is verified agains the packed version of this
+        instance. See _packData
+        
+        """
+        assert self.signature is not None
+        toVerify = self._packData()
+        binaryPermId = self.channel
+        return verify_data(toVerify, binaryPermId, self.signature)
+        
+      
+        
+    def _packData(self):
+        """
+        Creates a bencode binary representation of this metadata instance.
+        
+        This representation is the one that is sent with ChannelCast messages.
+        """
+        if self.description is not None:
+            assert isinstance(self.description, unicode)      
+        if self.description is None:
+            self.description = u""
+        
+        
+        
+        bitmask, checksums = self._getSubtitlesMaskAndChecksums()
+        
+        binaryMask = uintToBinaryString(bitmask)
+        # The signature is taken over the bencoding of
+        # binary representations of (channel,infohash,description,timestamp,bitmask)
+        # that is the same message that is sent with channelcast
+        tosign = (self.channel, 
+                  self.infohash, 
+                  self.description.encode("utf-8"),
+                  self.timestamp,
+                  binaryMask,
+                  checksums )
+    
+        bencoding = bencode(tosign)
+        return bencoding
+    
+    def serialize(self):
+        if self.signature is None:
+            raise SerializationException("The content must be signed")
+        pack = bdecode(self._packData())
+        pack.append(self.signature)
+        
+        return pack
+    
+
+        
+        
+        
+    
+    
+    def _getSubtitlesMaskAndChecksums(self):
+        '''
+        computes bitmask and checksums for subtitles.
+        
+        Computes the bitmask for available subtitles and produces also a tuple
+        containing the checksums for the subtitles that are in the bitmask.
+        The checksums are in the same order as the bits in the bitmask.
+        '''
+        
+        languagesList = []
+        checksumsList = []
+        
+        #cycling by sorted keys
+        sortedKeys = sorted(self._subtitles.keys())
+        
+        for key in sortedKeys:
+            sub = self._subtitles[key]
+            assert sub.lang is not None
+            assert sub.lang == key
+            
+            if sub.checksum is None:
+                if sub.subtitleExists():
+                    sub.computueCheksum()
+                else :
+                    if DEBUG:
+                        print >> sys.stderr, "Warning: Cannot get checksum for " + sub.lang \
+                                +" subtitle. Skipping it."
+                    continue
+            languagesList.append(sub.lang)
+            checksumsList.append(sub.checksum)
+            
+            
+        bitmask = _languagesUtil.langCodesToMask(languagesList)
+        checksums = tuple(checksumsList)
+        
+        return bitmask, checksums
+    
+    def __eq__(self, other):
+        if self is other:
+            return True
+        return self.channel == other.channel and \
+               self.infohash == other.infohash and \
+               self.description == other.description and \
+               self.timestamp == other.timestamp and \
+               self.getAllSubtitles() == other.getAllSubtitles()
+    
+    def __ne__(self, other):
+        return not self.__eq__(other)
+    
+    
+#-- Outside the class
+
+def deserialize(packed):
+    assert packed is not None
+        
+    message = packed
+    if(len(message) != 7):
+        raise SerializationException("Wrong number of fields in metadata")
+        
+    channel = message[0]
+    infohash = message[1]
+    description = message[2].decode("utf-8")
+    timestamp = message[3]
+    binarybitmask = message[4]
+    bitmask = binaryStringToUint(binarybitmask)
+    listOfChecksums = message[5]
+    signature = message[6]
+    subtitles = _createSubtitlesDict(bitmask,listOfChecksums)
+    
+    dto = MetadataDTO(channel, infohash, timestamp, description, subtitles, signature)
+    if not dto.verifySignature():
+        raise SerializationException("Invalid Signature!")
+    return dto
+    
+                                      
+
+def _createSubtitlesDict(bitmask, listOfChecksums):
+    langList = _languagesUtil.maskToLangCodes(bitmask)
+    if len(langList) != len(listOfChecksums):
+        raise SerializationException("Unexpected num of checksums")
+     
+    subtitles = {}
+    for i in range(0, len(langList)):
+        sub = SubtitleInfo(langList[i])
+        sub.checksum = listOfChecksums[i]
+        subtitles[langList[i]] = sub
+    return subtitles
+     
+     
+     
+     
+    
+        
diff --git a/instrumentation/next-share/BaseLib/Core/Subtitles/MetadataDomainObjects/MetadataExceptions.py b/instrumentation/next-share/BaseLib/Core/Subtitles/MetadataDomainObjects/MetadataExceptions.py
new file mode 100644
index 0000000..afc50ff
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Subtitles/MetadataDomainObjects/MetadataExceptions.py
@@ -0,0 +1,72 @@
+# Written by Andrea Reale
+# see LICENSE.txt for license information
+
+
+class RichMetadataException(Exception):
+    '''
+    General exception of the RichMetadata subsystem
+    '''
+
+
+    def __init__(self, value):
+        self.value = value
+        
+    def __str__(self):
+        return repr(self.value)
+
+class SerializationException(RichMetadataException):
+    '''
+    Thrown when some problem occurs when trying to transform a Metadata
+    object into the external representation
+    '''
+
+
+    def __init__(self, value):
+        self.value = value
+        
+    def __str__(self):
+        return repr(self.value)
+
+class SignatureException(RichMetadataException):
+    '''
+    Thrown when some problem occurs concerning metadata signature.
+    '''
+    def __init__(self, value):
+        self.value = value
+        
+    def __str__(self):
+        return repr(self.value)
+
+class MetadataDBException(RichMetadataException):
+    '''
+    Thrown when something  violated Metadata and Subtitles DB constraints.
+    '''
+    def __init__(self, value):
+        self.value = value
+        
+    def __str__(self):
+        return repr(self.value)
+    
+class SubtitleMsgHandlerException(RichMetadataException):
+    """
+    Thrown when a problem is encountered in sending o receiving a subtitle
+    message.
+    """
+    def __init__(self, value):
+        self.value = value
+        
+    def __str__(self):
+        return repr(self.value)
+    
+class DiskManagerException(RichMetadataException):
+    '''
+    Thrown by the Disk Manager when problems dealing with disk reading
+    and writings occur
+    '''
+    def __init__(self, value):
+        self.value = value
+        
+    def __str__(self):
+        return repr(self.value)
+
+    
\ No newline at end of file
diff --git a/instrumentation/next-share/BaseLib/Core/Subtitles/MetadataDomainObjects/SubtitleInfo.py b/instrumentation/next-share/BaseLib/Core/Subtitles/MetadataDomainObjects/SubtitleInfo.py
new file mode 100644
index 0000000..7b2cf36
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Subtitles/MetadataDomainObjects/SubtitleInfo.py
@@ -0,0 +1,226 @@
+# Written by Andrea Reale
+# see LICENSE.txt for license information
+
+from __future__ import with_statement
+from BaseLib.Core.Subtitles.MetadataDomainObjects.Languages import \
+    LanguagesProvider
+import base64
+import codecs
+import hashlib
+import os.path
+import sys
+
+DEBUG = False
+
+class SubtitleInfo(object):
+    '''
+    Represents a subtitles in a given language.
+    
+    It contains three fields, namely lang (an ISO 693-2 code), path that is
+    the path into the filesystem to the subtitles file, and checksum that is
+    a base64 representation of the sha1 checksum for that file.
+    It also manages the computation and verification of a sha1 checksum for 
+    a the subtitle.
+        
+    Notice that the path property can be None. This means that tha actual
+    subtitle hasn't been collected and is not available on the local
+    filesystem, In that case the checksum field will be none as well.
+    
+    Also notice that this object is meant to be used as a DTO. Simply changing
+    property in this object won't by themself affect values contained in the
+    Database
+    
+    SYNCHRONIZATION: This objects act only as copies of the data in the DB.
+    If the instance is nevere passed by between different threads
+    no synchronization is needed. 
+    '''
+
+    
+
+    def __init__(self, lang, path=None, checksum=None):
+        """
+        Create a subtitle instance.
+        
+        @param lang: an ISO 639-2 language code. Notice that not every language
+                     code described by the standard is supported, but only
+                     a small subset. See the Languages module
+        @param path: a file system path to the subtitles file
+        @param checksum: a sha1 checksum of the contents 
+                         of the subitles file
+        """
+        self._languages = LanguagesProvider.getLanguagesInstance()
+        if lang not in self._languages.supportedLanguages.keys():
+            raise ValueError("Language" + lang + " not supported")
+        
+        
+        #ISO 639-2 code. See Languages for supported languages
+        self._lang = lang #final property
+        #A string representing the path in the filesystme for this subtitle
+        self._path = path
+        #sha1 checksum
+        self._checksum = checksum
+        
+            
+    def getLang(self):
+        '''
+        Returns the language of the subtitle as a three characters code
+
+        @rtype: str
+        @return: a three characters ISO 639-2 code
+        '''
+        return self._lang
+
+    lang = property(getLang) # "final" property
+        
+    def setPath(self, path):
+        '''
+        Sets the local path for the subtitle. 
+
+        Calling this method does not change what is stored in the DB. You will
+        have to update that data separately (see L{MetadataDBHandler})
+
+        @type path: str
+        @param path: the local path were the subtitle is stored
+        '''
+        self._path = path
+
+            
+    def getPath(self):
+        '''
+        Get the path on the local host for the subtitle file, if available.
+
+        @rtype: str
+        @return: the local path if the subtitle is locally available. Otherwise
+            None.
+        '''
+        return self._path
+
+    
+    path = property(getPath, setPath)
+            
+    def setChecksum(self, checksum):
+        '''
+        Set the checksum for this subtitle instance. 
+
+        ATTENTION: This method should be never called, but instead a the
+        L{computeChecksum} method should be called instead.
+
+        @type checksum: str
+        @param checksum: a 160bit sha1 checksum of the subtitle
+        '''
+        self._checksum = checksum
+
+            
+    def getChecksum(self):
+        '''
+        Returns the SHA-1 checksum of the subtitle.
+
+        @rtype: str
+        @return: a 20byte string representing the SHA-1 checksum of the
+            subtitle
+        '''
+        return self._checksum
+
+        
+    checksum = property(getChecksum, setChecksum)
+        
+    def subtitleExists(self):
+        """
+        Checks wheter a subtitle exist in its specified path.
+        
+        @return: True if self.path is pointing to a local existing file.
+            Otherwise false
+        """
+
+        if self.path is None:
+            return False
+        return os.path.isfile(self.path)
+
+    
+    def computeChecksum(self):
+        """
+        Computes the checksum of the file containing the subtitles
+        and sets its corresponding property.
+
+        @precondition: self.subtitleExists()
+        @postcondition: self.checksum is not None
+        """
+ 
+        assert self.subtitleExists()
+
+        self.checksum = self._doComputeChecksum()
+ 
+        
+    def _doComputeChecksum(self):
+        """
+        Computes the checksum of the file containing the subtitles
+        
+        @precondition: self.subtitleExists()
+        """
+        try:
+            with codecs.open(self.path, "rb", "utf-8", "replace") as subFile:
+                content = subFile.read()
+      
+            hasher = hashlib.sha1()
+            hasher.update(content.encode('utf-8','replace'))
+            
+            return hasher.digest()
+        
+        except IOError:
+            print >> sys.stderr, "Warning: Unable to open " + self.path + " for reading"
+ 
+        
+    
+    def verifyChecksum(self):
+        """
+        Verifies the checksum of the file containing the subtitles.
+        
+        Computes the checksum of the file pointed by self.path
+        and checks whether it is equal to the one in self.checksum
+        
+        @precondition: self.subtitleExists()
+        @precondition: self.checksum is not None
+
+        @rtype: boolean
+        @return: True if the verification is ok.
+
+        @raises AssertionError: if one of the preconditions does not hold
+        """
+
+        assert self.subtitleExists(), "Cannot compute checksum: subtitle file not found"
+        assert self.checksum is not None, "Cannot verify checksum: no checksum to compare with"
+        
+        computed = self._doComputeChecksum()
+        return computed == self.checksum
+
+    
+    def __str__(self):
+
+        if self.path is not None:
+            path = self.path
+        else:
+            path = "None"
+        return "subtitle: [lang=" + self.lang +"; path=" + path \
+                + "; sha1=" + base64.encodestring(self.checksum).rstrip() + "]"
+
+            
+    def __eq__(self,other):
+        '''
+        Test instances of SubtitleInfo for equality.
+        
+        Two subtitle instances are considered equal if they have the same
+        language and the same file checksum
+        '''
+
+        if self is other:
+            return True
+        return self.lang == other.lang and self.checksum == other.checksum
+                #and self.path == other.path
+
+                
+        
+    def __ne__(self,other):
+        return not self.__eq__(other)
+        
+        
+    
diff --git a/instrumentation/next-share/BaseLib/Core/Subtitles/MetadataDomainObjects/__init__.py b/instrumentation/next-share/BaseLib/Core/Subtitles/MetadataDomainObjects/__init__.py
new file mode 100644
index 0000000..bdef2ba
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Subtitles/MetadataDomainObjects/__init__.py
@@ -0,0 +1,2 @@
+# Written by Andrea Reale
+# see LICENSE.txt for license information
diff --git a/instrumentation/next-share/BaseLib/Core/Subtitles/MetadataDomainObjects/res/subs_languages.csv b/instrumentation/next-share/BaseLib/Core/Subtitles/MetadataDomainObjects/res/subs_languages.csv
new file mode 100644
index 0000000..2261bc6
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Subtitles/MetadataDomainObjects/res/subs_languages.csv
@@ -0,0 +1,32 @@
+ara,Arabic 
+ben,Bengali 
+ces,Czech
+dan,Danish
+deu,German
+ell,Greek
+eng,English
+fas,Persian
+fin,Finnish
+fra,French
+hin,Hindi 
+hrv,Croatian
+hun,Hungarian
+ita,Italian
+jav,Javanese
+jpn,Japanese
+kor,Korean
+lit,Latvia 
+msa,Malay
+nld,Dutch
+pan,Panjabi
+pol,Polish
+por,Portuguese
+ron,Romanian
+rus,Russian
+spa,Spanish
+srp,Serbian
+swe,Swedish
+tur,Turkish
+ukr,Ukranian
+vie,Vietnamese
+zho,Chinese
diff --git a/instrumentation/next-share/BaseLib/Core/Subtitles/PeerHaveManager.py b/instrumentation/next-share/BaseLib/Core/Subtitles/PeerHaveManager.py
new file mode 100644
index 0000000..6bddd0a
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Subtitles/PeerHaveManager.py
@@ -0,0 +1,204 @@
+# Written by Andrea Reale
+# see LICENSE.txt for license information
+
+from __future__ import with_statement
+import time
+from BaseLib.Core.Subtitles.MetadataDomainObjects import Languages
+import threading
+
+
+
+PEERS_RESULT_LIMIT = 5
+HAVE_VALIDITY_TIME = 7*86400 # one week (too big? to small?)
+
+# how often (in seconds) old have messages will be removed from the database
+# -1 means that they will be cleaned up only at Tribler's startup
+CLEANUP_PERIOD = -1
+
+class PeersHaveManager(object):
+    '''
+    Manages the insertion, retrieval and manipulation of 
+    subtitle have messages from other peers.
+    
+    The public interface consists only of the two methods:
+    
+    + getPeersHaving(channel, infohash, bitmask)
+    + newHaveReceived(channel, infohash, peer_id, havemask)
+    
+    See method descriptions for further details
+    '''
+    
+    __single = None
+    _singletonLock = threading.RLock()
+    def __init__(self):
+        
+        with PeersHaveManager._singletonLock:
+            #Singleton pattern not enforced: this makes testing easier
+            PeersHaveManager.__single = self
+            
+        self._haveDb = None
+        self._olBridge = None
+        self._cleanupPeriod = CLEANUP_PERIOD
+        self._haveValidityTime = HAVE_VALIDITY_TIME
+        self._langsUtility = Languages.LanguagesProvider.getLanguagesInstance()
+        self._firstCleanedUp = False
+        
+        self._registered = False
+        
+    @staticmethod
+    def getInstance():
+        with PeersHaveManager._singletonLock:
+            if PeersHaveManager.__single == None:
+                PeersHaveManager()
+        
+        return PeersHaveManager.__single
+        
+    def register(self, haveDb, olBridge):
+        '''
+        Inject dependencies
+        
+        @type haveDb: BaseLib.Core.CacheDB.MetadataDBHandler
+        @type olBridge: OverlayBridge
+
+        '''
+        assert haveDb is not None
+        assert olBridge is not None
+        
+        self._haveDb = haveDb
+        self._olBridge = olBridge
+        
+        self._registered = True
+        
+    def isRegistered(self):
+        return self._registered
+    
+    
+    def getPeersHaving(self, channel, infohash, bitmask, limit=PEERS_RESULT_LIMIT):
+        '''
+        Returns a list of permids of peers having all the subtitles for
+        (channel, infohash) specified in the bitmask
+        
+        Notice that if there exist a peer that has only some of the subtitles
+        specified in the bitmask, that peer will not be included
+        in the returned list.
+        
+        This implementation returns the peers sorted by most recently received
+        have message first.
+        
+        @type channel: str
+        @param channel: binary channel_id
+        
+        @type infohash: str
+        @param infohash: binary infohash
+        
+        @type bitmask: int
+        @param bitmask: a 32 bit bitmask specifieng the desired subtitles languages
+                        for returned peers to have.
+                        
+        @type limit: int
+        @param limit: an upper bound on the size of the returned list. Notice
+                      that anyway the returned list may be smaller then limit
+                      (Default 5)
+                      
+        @rtype: list
+        @return: a list of binary permids of peers that have all the subitles
+                 specified by the bitmask. If there is no suitable entry the returned
+                 list will be empty
+        '''
+        
+        # results are already ordered by timestamp due the current
+        # MetadataDBHandler implementation
+        peersTuples = self._haveDb.getHaveEntries(channel, infohash)
+        peers_length = len(peersTuples)
+        length = peers_length if peers_length < limit else limit
+        
+        results = list()
+        
+        for i in range(length):
+            peer_id, havemask, timestamp = peersTuples[i]
+            if havemask & bitmask == bitmask:
+                results.append(peer_id)
+                
+        if len(results) == 0:
+            #if no results, and if the channel owner was not in the initial
+            #list, consider him always as a valid source
+            results.append(channel)
+                
+        return results
+        
+    
+    def newHaveReceived(self, channel, infohash, peer_id, havemask):
+        '''
+        Notify the PeerHaveManager that a new SUBTITLE HAVE announcement has been
+        received.
+        
+        @type channel: str
+        @param channel: binary channel_id 
+        
+        @type infohash: str
+        @param infohash: binary infohash
+        
+        @type peer_id: str
+        @param channel: binary permid of the peer that sent
+                        this havemask
+                        
+        @type havemask: int
+        @param havemask: integer bitmask representing which combination of subtitles
+                         peer_id has for the given (channel, infohash) pair
+        '''
+        
+        
+        timestamp = int(time.time())
+        self._haveDb.insertOrUpdateHave(channel, infohash, peer_id, havemask, timestamp)
+        
+    
+    def retrieveMyHaveMask(self, channel, infohash):
+        '''
+        Creates the havemask for locally available subtitles for channel,infohash
+        
+        @type channel: str
+        @param channel: a channelid to retrieve the local availability mask for (binary)
+        @type infohash: str
+        @param infohash: the infohash of the torrent to retrieve to local availability mask
+                        for (binary)
+        
+        @rtype: int
+        @return: a bitmask reprsenting wich subtitles languages are locally available
+                 for the given (channel, infohash) pair. If no one is available, or even
+                 if no rich metadata has been ever received for that pair, a zero bitmask
+                 will be returned. (i.e. this method should never thorow an exception if the
+                 passed parametrers are formally valid)
+        '''
+        
+        localSubtitlesDict = self._haveDb.getLocalSubtitles(channel, infohash)
+        
+        havemask = self._langsUtility.langCodesToMask(localSubtitlesDict.keys())
+        
+        return havemask
+    
+    def startupCleanup(self):
+        '''
+        Cleanup old entries in the have database.
+        
+        This method is meant to be called only one time in PeersManager instance lifetime,
+        i.e. at Tribler's startup. Successive calls will have no effect.
+        
+        If CLEANUP_PERIOD is set to a positive value, period cleanups actions will be
+        scheduled.
+        '''
+        if not self._firstCleanedUp:
+            self._firstCleanedUp = True
+            self._schedulePeriodicCleanup()
+        
+    def _schedulePeriodicCleanup(self):
+        
+        minimumAllowedTS = int(time.time()) - self._haveValidityTime
+        self._haveDb.cleanupOldHave(minimumAllowedTS)
+        
+        if self._cleanupPeriod > 0:
+            self._olBridge.add_task(self._schedulePeriodicCleanup, self._cleanupPeriod)
+            
+        
+
+
+        
\ No newline at end of file
diff --git a/instrumentation/next-share/BaseLib/Core/Subtitles/RichMetadataInterceptor.py b/instrumentation/next-share/BaseLib/Core/Subtitles/RichMetadataInterceptor.py
new file mode 100644
index 0000000..a57d157
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Subtitles/RichMetadataInterceptor.py
@@ -0,0 +1,304 @@
+# Written by Andrea Reale
+# see LICENSE.txt for license information
+
+import sys
+from BaseLib.Core.Subtitles.MetadataDomainObjects.MetadataDTO import deserialize
+from BaseLib.Core.Subtitles.MetadataDomainObjects.MetadataExceptions import SerializationException,\
+    RichMetadataException
+from BaseLib.Core.Utilities.utilities import isValidPermid, bin2str,\
+    show_permid_short, uintToBinaryString, binaryStringToUint
+from copy import copy
+from BaseLib.Core.simpledefs import NTFY_RICH_METADATA, NTFY_UPDATE, NTFY_INSERT
+
+
+DEBUG = False
+
+
+class RichMetadataInterceptor(object):
+    
+  
+    
+    def __init__(self, metadataDbHandler, voteCastDBHandler, myPermId,
+                 subSupport=None, peerHaveManager = None, notifier = None):
+        '''
+        Builds an instance of RichMetadataInterceptor.
+        
+        @param metadataDbHandler: an registered instance of
+            L{MetadataDBHandler}
+        @param voteCastDBHandler: a registered instance of VoteCastDBHandler
+        @param myPermId: the PermId of the client.
+        @param subSupport: a registered instance of L{SubtitlesSupport}
+        @param peerHaveManager: an instance of L{PeerHaveManager}
+        @param notifier: an instance of Notifier
+        '''
+#        assert isinstance(metadataDbHandler, MetadataDBHandler), \
+#            "Invalid RichMetadata DB Handler"
+#        assert isinstance(voteCastDBHandler, VoteCastDBHandler), \
+#            "Invalid Votecast DB Handler"
+        #hack to make a fast test DELETE THIS CONDITION
+#        if subSupp != None:
+#            assert isinstance(subSupp, SubtitlesSupport)
+        assert isValidPermid(myPermId),  "Invalid Permid"
+            
+        self.rmdDb = metadataDbHandler
+        self.votecastDB = voteCastDBHandler
+        self.my_permid = myPermId
+        self.subSupport = subSupport
+        self.peerHaveManager = peerHaveManager
+        self.notifier = notifier
+
+    
+    def _splitChannelcastAndRichMetadataContents(self,enrichedChannelcastMessage):
+        '''
+        Takes a "enriched" channelcast message (protocol v.14 - the one with
+        the 'rich_metadata' field inside), and extracts the rich metadata info
+        from it
+        
+        @param enrichedChannelcastMessage: a channelcast message from protocol 
+                                           version 14
+                                           
+        @return: a list tuples like (MetadataDTO, haveMask) instances extracted from the message. or
+                 an empty list if nothing. Along with it there is a list
+                 of the size of each entry in the message that is used to 
+                 collect stats. if the announceStatsLog is disable this list
+                 will always be empty
+        '''
+        if not isinstance(enrichedChannelcastMessage, dict):
+            if DEBUG:
+                print >> sys.stderr, "Invalid channelcast message received"
+            return None
+        
+        rmdData = list()
+        
+        sizeList = list()
+        for signature in iter(enrichedChannelcastMessage):
+            msg = enrichedChannelcastMessage[signature]
+            
+            if 'rich_metadata' in msg.keys():
+                metadataEntry = msg['rich_metadata']
+                if metadataEntry is None \
+                    or not validMetadataEntry(metadataEntry):
+                    continue
+                else:
+                    channel_id = msg['publisher_id']
+                    infohash = msg['infohash']
+
+                    # rebuilding the serialized MetadataDTO structure
+                    # that was broken in self.addRichMetadataContent
+                    binary_havemask = metadataEntry.pop(-1)
+                    havemask = binaryStringToUint(binary_havemask)
+                    
+                    metadataEntry.insert(0,infohash)
+                    metadataEntry.insert(0,channel_id)
+                    try:
+                        curMetadataDTO = deserialize(metadataEntry)
+                    except SerializationException,e:
+                        if DEBUG:
+                            print >> sys.stderr, "Invalid metadata message content: %s" % e
+                        continue
+                    
+                    rmdData.append((curMetadataDTO,havemask))
+        
+        return rmdData, sizeList
+    
+    def handleRMetadata(self, sender_permid, channelCastMessage, fromQuery = False):
+        '''
+        Handles the reception of rich metadata.
+        
+        Called when an "erniched" channelCastMessage (v14) is received.
+        @param sender_permid: the PermId of the peer who sent the message
+        @param channelCastMessage: the received message
+        @return: None
+        '''
+        metadataDTOs, sizeList = \
+          self._splitChannelcastAndRichMetadataContents(channelCastMessage)
+          
+        if DEBUG:
+            print >> sys.stderr, "Handling rich metadata from %s..." % show_permid_short(sender_permid)
+        i=0
+        for md_and_have in metadataDTOs:
+            md = md_and_have[0]
+            havemask = md_and_have[1]
+            
+            vote = self.votecastDB.getVote(bin2str(md.channel), 
+                                       bin2str(self.my_permid))
+            
+            # the next if may seem useless, but since sizeList is defined only when
+            # logging is enabled for debug, I get an error without this conditional statement
+            # because the argument for the debug() call getsEvaluated before the logging
+            # system understands that debug is disabled
+            #if announceStatsLog.isEnabledFor(logging.INFO):
+            if DEBUG:
+                id = "RQ" if fromQuery else "R"
+                print >> sys.stderr, "%c, %s, %s, %s, %d, %d" % \
+                                       (id, md.channel, md.infohash, \
+                                        show_permid_short(sender_permid), md.timestamp,
+                                        sizeList[i])
+                #format "R|S (R: received - S: sent), channel, infohash, sender|destination,metadataCreationTimestamp"
+                # 30-06-2010: "RQ" as received from query
+                i += 1
+        
+            # check if the record belongs to a channel 
+            # who we have "reported spam" (negative vote)
+            if  vote == -1:
+                # if so, ignore the incoming record
+                continue
+            
+            isUpdate =self.rmdDb.insertMetadata(md)
+            
+            self.peerHaveManager.newHaveReceived(md.channel,md.infohash,sender_permid,havemask)
+            
+            if isUpdate is not None:
+                #retrieve the metadataDTO from the database in the case it is an update
+                md = self.rmdDb.getMetadata(md.channel,md.infohash)
+                self._notifyRichMetadata(md, isUpdate)
+            
+            # if I am a subscriber send immediately a GET_SUBS to the 
+            # sender
+            if vote == 2:
+                if DEBUG:
+                    print >> sys.stderr, "Subscribed to channel %s, trying to retrieve" \
+                         "all subtitle contents" % (show_permid_short(md.channel),)
+                
+                self._getAllSubtitles(md)
+
+    def _computeSize(self,msg):
+        import BaseLib.Core.BitTornado.bencode as bencode
+        bencoded = bencode.bencode(msg)
+        return len(bencoded)
+    
+    
+    def _notifyRichMetadata(self, metadataDTO, isUpdate):
+        if self.notifier is not None:
+            eventType = NTFY_UPDATE if isUpdate else NTFY_INSERT
+            self.notifier.notify(NTFY_RICH_METADATA, eventType, (metadataDTO.channel, metadataDTO.infohash))
+            
+    
+    def _getAllSubtitles(self, md):
+        
+        subtitles = md.getAllSubtitles()
+        
+        try:
+            self.subSupport.retrieveMultipleSubtitleContents(md.channel,md.infohash,
+                                                             subtitles.values())
+        except RichMetadataException,e:
+            print >> sys.stderr, "Warning: Retrievement of all subtitles failed: " + str(e)
+        
+    
+    def addRichMetadataContent(self,channelCastMessage, destPermid = None, fromQuery = False):
+        '''
+        Takes plain channelcast message (from OLProto v.13) and adds to it
+        a 'rich_metadata' field.
+        
+        @param channelCastMessage: the old channelcast message in the format of
+                                   protocol v13
+        @param destPermid: the destination of the message. If not None it is used
+                            for logging purposes only. If None, nothing bad happens.
+        @return: the "enriched" channelcast message
+        '''
+        if not len(channelCastMessage) > 0:
+            if DEBUG:
+                print >> sys.stderr, "no entries to enrich with rmd"
+            return channelCastMessage
+        
+        if DEBUG:
+            if fromQuery: 
+                print >> sys.stderr, "Intercepted a channelcast message as answer to a query"
+            else:
+                print >> sys.stderr, "Intercepted a channelcast message as normal channelcast"
+        #otherwise I'm modifying the old one (even if there's nothing bad
+        #it's not good for the caller to see its parameters changed :)
+        newMessage = dict()
+            
+        # a channelcast message is made up of a dictionary of entries
+        # keyed the signature. Every value in the dictionary is itself
+        # a dictionary with the item informatino
+        for key in iter(channelCastMessage):
+            entryContent = copy(channelCastMessage[key])
+            newMessage[key] = entryContent
+            
+            channel_id = entryContent['publisher_id']
+            infohash = entryContent['infohash']
+            #not clean but the fastest way :(
+            # TODO: make something more elegant
+            metadataDTO = self.rmdDb.getMetadata(channel_id, infohash)
+            if metadataDTO is not None:
+                try:
+                    if DEBUG:
+                        print >> sys.stderr, "Enriching a channelcast message with subtitle contents"
+                    metadataPack = metadataDTO.serialize()
+                    
+                    # I can remove from the metadata pack the infohash, and channelId
+                    # since they are already in channelcast and they would be redundant
+                    metadataPack.pop(0)
+                    metadataPack.pop(0)
+                    
+                    #adding the haveMask at the end of the metadata pack
+                    havemask = self.peerHaveManager.retrieveMyHaveMask(channel_id, infohash)
+                    binary_havemask = uintToBinaryString(havemask)
+                    metadataPack.append(binary_havemask)
+                    
+                    
+                    entryContent['rich_metadata'] = metadataPack
+                    
+                    if DEBUG:
+                        size = self._computeSize(metadataPack)
+                        # if available records also the destination of the message
+                        dest = "NA" if destPermid is None else show_permid_short(destPermid)
+                    
+                        id = "SQ" if fromQuery else "S"
+                        # format (S (for sent) | SQ (for sent as response to a query), channel, infohash, destination, timestampe, size)
+                        print >> sys.stderr, "%c, %s, %s, %s, %d, %d" % \
+                            (id, bin2str(metadataDTO.channel), \
+                            bin2str(metadataDTO.infohash), \
+                             dest, metadataDTO.timestamp, size)
+                except Exception,e:
+                    print >> sys.stderr, "Warning: Error serializing metadata: %s", str(e)
+                    return channelCastMessage
+            else:
+                # better to put the field to None, or to avoid adding the
+                # metadata field at all?
+                ##entryContent['rich_metadata'] = None
+                pass
+            
+            
+        
+            
+        return newMessage
+    
+def validMetadataEntry(entry):
+    if entry is None or len(entry) != 6:
+        if DEBUG:
+            print >> sys.stderr, "An invalid metadata entry was found in channelcast message"
+        return False
+    
+    if not isinstance(entry[1], int) or entry[1] <= 0:
+        if DEBUG:
+            print >> sys.stderr, "Invalid rich metadata: invalid timestamp"
+        return False
+   
+    if not isinstance(entry[2], basestring) or not len(entry[2]) == 4: #32 bit subtitles mask
+        if DEBUG:
+            print >> sys.stderr, "Invalid rich metadata: subtitles mask"
+        return False
+    
+    if not isinstance(entry[3], list):
+        if DEBUG:
+            print >> sys.stderr, "Invalid rich metadata: subtitles' checsums"
+        return False
+    else:
+        for checksum in entry[3]:
+            if not isinstance(entry[2], basestring) or not len(checksum) == 20:
+                if DEBUG:
+                    print >> sys.stderr, "Invalid rich metadata: subtitles' checsums"
+                return False
+
+    
+    if not isinstance(entry[2], basestring) or not len(entry[5]) == 4: #32 bit have mask
+        if DEBUG:
+            print >> sys.stderr, "Invalid rich metadata: have mask"
+        return False
+    
+    return True
+    
+    
diff --git a/instrumentation/next-share/BaseLib/Core/Subtitles/SubtitleHandler/DiskManager.py b/instrumentation/next-share/BaseLib/Core/Subtitles/SubtitleHandler/DiskManager.py
new file mode 100644
index 0000000..9e1c326
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Subtitles/SubtitleHandler/DiskManager.py
@@ -0,0 +1,363 @@
+# Written by Andrea Reale
+# see LICENSE.txt for license information
+
+from __future__ import with_statement
+from BaseLib.Core.Subtitles.MetadataDomainObjects.MetadataExceptions import \
+    DiskManagerException
+from BaseLib.Core.osutils import getfreespace
+from random import random
+from traceback import print_exc
+import codecs
+import os
+import sys
+
+DISK_FULL_REJECT_WRITES = 0x0
+DISK_FULL_DELETE_SOME = 0x1
+
+DELETE_RANDOM = 0x02
+DELETE_OLDEST_FIRST = 0x0
+DELETE_NEWEST_FIRST = 0x4
+
+MINIMUM_FILE_SIZE = 4 #KBs. most common.
+
+
+DEFAULT_CONFIG = { "maxDiskUsage" :-1 , #infinity
+                   "diskPolicy" : DISK_FULL_REJECT_WRITES,
+                   "encoding" : "utf-8"}
+
+DEBUG = False
+
+class DiskManager(object):
+    """
+    Manages disk policies.
+    
+    Used for subtitle disk space handling it could be adapted
+    for any space management. The current implementation is
+    NOT THREAD-SAFE
+    
+    The disk manager is a central resource manager for disk space. 
+    A client object who wants to use the disk manager has to register to
+    it using the registerDir method. After that that client will be
+    associated to a directory where the disk manager will try to store
+    files for it.
+    
+    A DiskManager has a _minFreeSpace attribute that determines how much
+    space has to be always left free on the disk. It will perform no writes
+    if that write will make free space go under the _minFreeSpace threshold.
+    
+    When a client registers it must provided some configuration parameters.
+    This parameters comprehend maxDiskUsage that is the maximum disk quota
+    that can be used by that client, diskPolicy that is a bitmask specifying
+    the actions to do when the disk quota has been reached and a write
+    operation is asked, adn encoding that is the default encoding for every
+    file read or write.
+    
+    THIS CLASS IS NOT THREAD-SAFE!!!!
+    """
+    
+    def __init__(self, minFreeSpace=0, baseDir="."):
+        """
+        Create a new instance of DiskManager.
+        
+        @type minFreeSpace: int
+        @param minFreeSpace: the minimum amount of free space in KBs that
+            needs to be always available on disk after any
+            write operation
+        @type baseDir: str
+        @param baseDir: a path. It will be used by the manager to determine
+            which disk he has to use to calculate free space and
+            so.
+        """
+        assert os.path.isdir(baseDir)
+        self._minFreeSpace = minFreeSpace
+        self._registeredDirs = dict()
+        self._baseDir = baseDir
+        
+    def registerDir(self, directory, config=None):
+        """
+        Register a client object to use the services of the disk manager.
+        
+        When a client object wants to use a DiskManager instance it has to
+        provide a directory path, under which to store its files. This path
+        should corrispond to the same disk as the diskmanager _baseDir 
+        attribute. All subsequente write and read operations performed by the
+        disk manager will refer to files in the provided directory.
+        
+        @param directory: a directory path for which to register. This
+                          directory will be used as the base path for all
+                          subsequent file read and writes by the client that
+                          registered for it.
+        @param config: a dictionary containing configurations parameters
+            for the registrations. The keys of that dictionary
+            are:
+                - 'maxDiskUsage': maximum disk page that the client is
+                  allowed to use (in KBs) [-1 for infinity]
+                - 'diskPolicy': a 3 bit bitmask that is a combination of
+                  one of (DISK_FULL_REJECT_WRITES, 
+                  DISK_FULL_DELETE_SOME) and one of
+                  (DELETE_RANDOM, DELETE_OLDEST,
+                  ELETE_NEWEST)
+                - 'fileEncoding": encoding that will be used to read and
+                  write every file under the registered
+                  Dir
+                                    
+        """
+        assert directory is not None
+        assert os.path.isdir(directory), "%s is not a dir" % directory
+        
+        if config is None:
+            config = DEFAULT_CONFIG
+            
+        if "maxDiskUsage" not in config.keys() \
+            or "diskPolicy" not in config.keys() \
+            or "encoding" not in config.keys():
+            if DEBUG:
+                print >> sys.stderr, "Invalid config. Using default"
+            config = DEFAULT_CONFIG
+            
+        dedicatedDiskManager = BaseSingleDirDiskManager(directory, config, self)
+        self._registeredDirs[directory] = dedicatedDiskManager
+    
+    #free space in KBs
+    def getAvailableSpace(self):
+        space = max(0, self._get_free_space() - self._minFreeSpace)
+        return space
+        
+    
+    def _get_free_space(self):
+        """
+        Retrieves current free disk space.
+        """
+        try:
+            freespace = getfreespace(self._baseDir) / 1024.0
+            return freespace 
+        except:
+            print >> sys.stderr, "cannot get free space of", self._baseDir
+            print_exc()
+            return 0
+        
+    def writeContent(self, directory, filename, content):
+        """
+        Write a string into a file.
+        
+        @return: The path of the written file, if everythin is right
+        
+        @precondition: directory is registered
+        @precondition: content is a string
+        @postcondition: minimum free space, and maximum disk usage constraints
+                        are ok
+        """
+        if directory not in self._registeredDirs.keys():
+            msg = "Directory %s not registered" % directory
+            if DEBUG:
+                print >> sys.stderr, msg
+            raise DiskManagerException(msg)
+        
+        return self._registeredDirs[directory].writeContent(filename, content)
+    
+    def readContent(self, directory, filename):
+        """
+        Read the contents of a file.
+        
+        @return: a string containing the contents of the file
+        """
+        if directory not in self._registeredDirs.keys():
+            msg = "Directory %s not registered" % directory
+            if DEBUG:
+                print >> sys.stderr, msg
+                
+            raise DiskManagerException(msg)
+        
+        return self._registeredDirs[directory].readContent(filename)
+
+    def deleteContent(self, directory, filename):
+        if directory not in self._registeredDirs.keys():
+            msg = "Directory %s not registered" % directory
+            if DEBUG:
+                print >> sys.stderr, msg
+            raise DiskManagerException(msg)
+        
+        return self._registeredDirs[directory].deleteContent(filename)
+    
+    def tryReserveSpace(self, directory, amount):
+        """
+        Check if there a given amount of available space. (in KBs)
+        
+        If there is, it does nothing :)
+        (aslo if there isn't) 
+        """
+        if directory not in self._registeredDirs.keys():
+            msg = "Directory %s not registered" % directory
+            if DEBUG:
+                print >> sys.stderr, msg
+            raise DiskManagerException(msg)
+        
+        return self._registeredDirs[directory].tryReserveSpace(amount)
+    
+    def isFilenOnDisk(self, directory, filename):
+        if directory not in self._registeredDirs.keys():
+            msg = "Directory %s not registered" % directory
+            if DEBUG:
+                print >> sys.stderr, msg
+            raise DiskManagerException(msg)
+        
+        return self._registeredDirs[directory].isFileOnDisk(filename)
+        
+        
+    
+class BaseSingleDirDiskManager(object):
+    
+    def __init__(self, workingDir, config, dm):
+        self.workingDir = workingDir
+        self.fileEncoding = config["encoding"]
+        #select the last bit only
+        self.diskFullPolicy = config["diskPolicy"] & 0x1
+        #select the second and third bit from the right
+        self.deletePolicy = config["diskPolicy"] & 0x6
+        self.maxDiskUsage = config["maxDiskUsage"]
+        if self.maxDiskUsage < 0: #infinte
+            self.maxDiskUsage = (2 ** 80) #quite infinite
+        self.dm = dm
+        self.dirUsage = 0
+        self._updateDirectoryUsage()
+    
+    def writeContent(self, filename, content):
+        # assuming that a file system block is 4 KB
+        # and therefore every file has a size that is a multiple of 4 kbs
+        # if the assumption is violated nothing bad happens :)
+        approxSize = max(MINIMUM_FILE_SIZE, (len(content) / 1024.0))
+        sizeInKb = approxSize + (approxSize % MINIMUM_FILE_SIZE)
+        if self.tryReserveSpace(sizeInKb):
+            return self._doWrite(filename, content)
+        else:
+            if self.diskFullPolicy == DISK_FULL_REJECT_WRITES:
+                raise DiskManagerException("Not enough space to write content. Rejecting")
+            elif self.diskFullPolicy == DISK_FULL_DELETE_SOME:
+                if self.makeFreeSpace(sizeInKb):
+                    return self._doWrite(filename, content)
+                else:
+                    raise DiskManagerException("Unable to get enough space to write content.")
+            
+            
+    def readContent(self, filename):
+        path = os.path.join(self.workingDir, filename)
+        if not os.path.isfile(path):
+            raise IOError("Unable to read from %s" % path)
+        with codecs.open(path, "rb", self.fileEncoding,"replace") as xfile:
+            content = xfile.read()
+        
+        return content
+    
+    def deleteContent(self, filename):
+        if DEBUG:
+            print >> sys.stderr, "Deleting " + filename
+        path = os.path.join(self.workingDir, filename)
+        if not os.path.isfile(path):
+            if DEBUG:
+                print >> sys.stderr, "Noting to delete at %s" % path
+            return False
+        try:
+            os.remove(path)
+            self._updateDirectoryUsage()
+            return True
+        except OSError,e:
+            print >> sys.stderr, "Warning: Error removing %s: %s" % (path, e)
+            return False
+    
+    def makeFreeSpace(self, amount):
+        if DEBUG:
+            print >> sys.stderr, "Trying to retrieve %d KB of free space for %s" % (amount, self.workingDir)
+        if amount >= self.maxDiskUsage:
+            return False
+        if amount >= (self.dm.getAvailableSpace() + self._currentDiskUsage()):
+            return False
+        
+        maxTries = 100
+        tries = 0
+        while self._actualAvailableSpace() <= amount:
+            if tries >= maxTries:
+                print >> sys.stderr, "Unable to make up necessary free space for %s" % \
+                         self.workingDir
+                return False
+            toDelete = self._selectOneToDelete()
+            if toDelete is None:
+                return False
+            self.deleteContent(toDelete)
+            tries = +1
+            
+            
+        return True
+    
+    def isFileOnDisk(self, filename):
+        path = os.path.join(self.workingDir, filename)
+        if os.path.isfile(path):
+            return path
+        else:
+            return None
+    
+    def _doWrite(self, filename, content):
+        
+        path = os.path.join(self.workingDir, filename)
+        if os.path.exists(path):
+            if DEBUG:
+                print >> sys.stderr, "File %s exists. Overwriting it."
+            os.remove(path)
+        try:
+            if not isinstance(content,unicode):
+                content = content.decode(self.fileEncoding,'replace')
+            with codecs.open(path, "wb", self.fileEncoding,'replace') as toWrite:
+                toWrite.write(content)
+        except Exception,e:
+            #cleaning up stuff
+            if os.path.exists(path):
+                os.remove(path)
+            raise e
+        
+        self._updateDirectoryUsage()
+        return path
+    
+    def _selectOneToDelete(self):
+        pathlist = map(lambda x : os.path.join(self.workingDir, x),
+                            os.listdir(self.workingDir))
+        candidateList = [xfile for xfile in pathlist
+                         if os.path.isfile(os.path.join(self.workingDir, xfile))]
+        
+        if not len(candidateList) > 0:
+            return None
+            
+        if self.deletePolicy == DELETE_RANDOM:
+            return random.choice(candidateList)
+        else:
+            sortedByLastChange = sorted(candidateList, key=os.path.getmtime)
+            if self.deletePolicy == DELETE_NEWEST_FIRST:
+                return sortedByLastChange[-1]
+            elif self.deletePolicy == DELETE_OLDEST_FIRST:
+                return sortedByLastChange[0]
+
+
+    def tryReserveSpace(self, amount):
+        if amount >= self._actualAvailableSpace():
+            return False
+        else:
+            return True
+    
+    def _currentDiskUsage(self):
+        return self.dirUsage
+    
+    def _updateDirectoryUsage(self):
+        listOfFiles = os.listdir(self.workingDir)
+        listofPaths = \
+            map(lambda name: os.path.join(self.workingDir, name), listOfFiles)
+        
+        #does not count subdirectories
+        dirSize = sum([os.path.getsize(fpath) for fpath in listofPaths])
+        
+        self.dirUsage = dirSize / 1024.0 #Kilobytes
+    
+    def _actualAvailableSpace(self):
+        space = min(self.dm.getAvailableSpace(),
+                   self.maxDiskUsage - self._currentDiskUsage())
+        return space
+    
+        
+            
diff --git a/instrumentation/next-share/BaseLib/Core/Subtitles/SubtitleHandler/SimpleTokenBucket.py b/instrumentation/next-share/BaseLib/Core/Subtitles/SubtitleHandler/SimpleTokenBucket.py
new file mode 100644
index 0000000..aad7090
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Subtitles/SubtitleHandler/SimpleTokenBucket.py
@@ -0,0 +1,63 @@
+# Written by Andrea Reale
+# see LICENSE.txt for license information
+
+
+from time import time
+
+class SimpleTokenBucket(object):
+    """
+    A simple implementation of a token bucket, to
+    control the rate of subtitles being uploaded.
+    
+    1 token corresponds to 1 KB
+    
+    Not threadsafe!!
+    """
+    
+    def __init__(self, fill_rate, capacity = -1):
+        """
+        Creates a token bucket initialy having 0 tokens,
+        with the given fill_rate.
+        
+        @param fill_rate: number of tokens refilled per second.
+                          a token corrisponds to 1KB
+        @param capacity: maximum number of tokens in the bucket.
+        """
+        
+        #infinite bucket! (well, really big at least)
+        if capacity == -1:
+            capacity = 2**30 # 1 TeraByte capacity
+        self.capacity = float(capacity)
+        
+        self._tokens = float(0)
+        
+        self.fill_rate = float(fill_rate)
+        self.timestamp = time()
+
+    def consume(self, tokens):
+        """Consume tokens from the bucket. Returns True if there were
+        sufficient tokens otherwise False."""
+        if tokens <= self.tokens:
+            self._tokens -= tokens
+        else:
+            return False
+        return True
+    
+    def _consume_all(self):
+        """
+        Consumes every token in the bucket
+        """
+        self._tokens = float(0)
+
+    @property
+    def tokens(self):
+        if self._tokens < self.capacity:
+            now = time()
+            delta = self.fill_rate * (now - self.timestamp)
+            self._tokens = min(self.capacity, self._tokens + delta)
+            self.timestamp = now
+        return self._tokens
+    
+    @property
+    def upload_rate(self):
+        return self.fill_rate
\ No newline at end of file
diff --git a/instrumentation/next-share/BaseLib/Core/Subtitles/SubtitleHandler/SubsMessageHandler.py b/instrumentation/next-share/BaseLib/Core/Subtitles/SubtitleHandler/SubsMessageHandler.py
new file mode 100644
index 0000000..4013e06
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Subtitles/SubtitleHandler/SubsMessageHandler.py
@@ -0,0 +1,880 @@
+# Written by Andrea Reale
+# see LICENSE.txt for license information
+
+
+
+from BaseLib.Core.BitTornado.BT1.MessageID import SUBS, GET_SUBS
+from BaseLib.Core.BitTornado.bencode import bencode, bdecode
+from BaseLib.Core.Subtitles.MetadataDomainObjects.Languages import \
+    LanguagesProvider
+from BaseLib.Core.Subtitles.MetadataDomainObjects.MetadataExceptions import \
+    SubtitleMsgHandlerException
+from BaseLib.Core.Overlay.SecureOverlay import OLPROTO_VER_FOURTEENTH
+from BaseLib.Core.Utilities import utilities
+from BaseLib.Core.Utilities.utilities import show_permid_short, validInfohash, \
+    validPermid, bin2str, uintToBinaryString, binaryStringToUint
+from time import time
+from traceback import print_exc
+import sys
+import threading
+    
+SUBS_LOG_PREFIX = "subtitles: "
+
+REQUEST_VALIDITY_TIME = 10 * 60 #10 minutes
+CLEANUP_PERIOD = 5 * 60#5 minutes
+
+DEBUG = False
+    
+class SubsMessageHandler(object):
+    
+    def __init__(self, overlayBridge, tokenBucket, maxSubsSize):
+        self._languagesUtility = LanguagesProvider.getLanguagesInstance()
+        self._overlay_bridge = overlayBridge
+        
+        # handleMessage() is called by the OLThread
+        # registerListener is called by the OLThread
+        # no synchronization should be needed for this list :)
+        self._listenersList = list()
+        
+        self._tokenBucket = tokenBucket
+        
+        #controls the interval the uploadQueue gets checked
+        self._nextUploadTime = 0
+        
+        
+        #dictionary of type { "".join(channel_id,infohash) : _RequestedSubtitlesEntry}
+        #bits get cleaned when subtitles are recevied
+        #when the bitmask is 000 the netry is removed from the dictionary
+        #also entries older then REQUEST_VALIDITY_TIME get dropped
+        
+
+        
+        self.requestedSubtitles = {}
+        self._requestsLock = threading.RLock()
+        
+        self._nextCleanUpTime = int(time()) + CLEANUP_PERIOD
+
+        #subtitles to send get queueued in this queue
+        #each subtitle message to send is a dictionary whose keys are:
+        #permid: destionation of the message
+        #channel_id: identifier of the channel from wich the subtitles to
+        #            upload are
+        #infohash: identifier of the torrent for which the subtitles to 
+        #          upload are
+        #subtitles: a dictionary of the form {langCode : path} for the
+        #           subtitles to send
+        #selversion: 
+    
+        self._uploadQueue = []
+        
+        self._requestValidityTime = REQUEST_VALIDITY_TIME
+        
+        self._maxSubSize = maxSubsSize
+        
+        
+    def setTokenBucket(self, tokenBucket):
+        assert tokenBucket is not None
+        self._tokenBucket = tokenBucket
+    def getTokenBucket(self):
+        return self._tokenBucket
+    
+    tokenBucket = property(getTokenBucket,setTokenBucket)
+            
+    
+    def _getRequestedSubtitlesKey(self, channel_id, infohash):
+        #requested subtitle is a dictionary whose keys are the
+        #concatenation of (channel_id,infohash)
+
+        return "".join((channel_id, infohash))
+    
+    
+    def sendSubtitleRequest(self, dest_permid, requestDetails, 
+                            msgSentCallback = None, usrCallback = None, selversion=-1):
+        """
+        Create and send a subtitle request to dest_permid.
+        
+        Creates, encodes and sends (through the OLBridge) an GET_SUBS request
+        to the given dest_permid. Notice that even when this method return
+        succesfully the message by have been still not sent. 
+        
+        @param dest_permid: the permid of the peer where the message should be 
+                            sent. Binary.
+        @param requestDetails: a dictionary containing the details of the request
+                               to be sent:
+                               a 'channel_id' entry which is the binary channel
+                               identifier (permid) of the desired subtitles
+                               a 'infohash' entry which is the binary infohash
+                               of the torrent the requested subtitles refer to
+                               a 'languages' entry which is a list of 3-characters
+                               codes identifying the need subtitles
+        @type msgSentCallback: function
+        @param msgSentCallback: a function that will be called when the message has been
+                          sent. It must have 5 parameters: exc (bounded to a possible
+                          exception), dest_permid, channel_id, infohash, bitmask)
+        @type usrCallback: function
+        @param usrCallback: a function that will be called whenever some of the requested
+                      subtitles are retrieved. Only one parameter: ie a list that will
+                      be bound to the received language codes
+                          
+        @raise SubtitleMsgHandlerException: if something fails before attempting
+                                            to send the message. 
+        """
+
+        
+        channel_id = requestDetails['channel_id']
+        infohash = requestDetails['infohash']
+        languages = requestDetails['languages']
+        
+        bitmask = self._languagesUtility.langCodesToMask(languages)
+        if bitmask != 0:
+            try:
+                # Optimization: don't connect if we're connected, although it won't 
+                # do any harm.
+                if selversion == -1: # not currently connected
+                    self._overlay_bridge.connect(dest_permid,
+                                                lambda e, d, p, s:
+                                                self._get_subs_connect_callback(e, d, p, s, channel_id,
+                                                                                infohash, bitmask,
+                                                                                msgSentCallback, usrCallback))
+                else:
+                    self._get_subs_connect_callback(None, None, dest_permid,
+                                                   selversion, channel_id, infohash,
+                                                   bitmask, msgSentCallback, usrCallback)
+                
+            except Exception,e:
+                if DEBUG:
+                    print >> sys.stderr, SUBS_LOG_PREFIX + "Unable to send: %s" % str(e)
+                raise SubtitleMsgHandlerException(e)
+        else:
+            raise SubtitleMsgHandlerException("Empty request, nothing to send")
+        
+        
+    def sendSubtitleResponse(self, destination, response_params, selversion = -1):
+        """
+        Send a subtitle response message to destination permid.
+        
+        @param destination: the permid of the destionation of the message
+        @param response_params: a tuple containing channel_id,infohash, and a 
+                                dictionary of contents, in that order
+        @type selversion: int
+        @param selversion: the protocol version of the destination (default -1)
+        """
+        
+        channel_id, infohash, contentsList = response_params
+        
+                
+        task = {
+                'permid' : destination,
+                'channel_id' : channel_id,
+                'infohash' : infohash,
+                'subtitles' : contentsList,
+                'selversion' : selversion
+                }
+        
+
+        self._uploadQueue.append(task)
+        
+        if int(time()) >= self._nextUploadTime:
+            self._checkingUploadQueue()
+            
+        return True
+        
+    
+    def handleMessage(self, permid, selversion, message):
+        """
+        Must return True or False (for what I understood a return value of
+        false closes the connection with permid, but I'm still not sure)
+        """
+        t = message[0]
+        
+        if t == GET_SUBS:   # the other peer requests a torrent
+            if DEBUG:
+                print >> sys.stderr, SUBS_LOG_PREFIX + "Got GET_SUBS len: %s from %s" % \
+                      (len(message), show_permid_short(permid))
+            return self._handleGETSUBS(permid, message, selversion)
+        elif t == SUBS:     # the other peer sends me a torrent
+            if DEBUG:
+                print >> sys.stderr, SUBS_LOG_PREFIX + "Got SUBS len: %s from %s" %\
+                     (len(message), show_permid_short(permid))
+
+            return self._handleSUBS(permid, message, selversion)
+        else:
+            if DEBUG:
+                print >> sys.stderr, SUBS_LOG_PREFIX + "Unknown Overlay Message %d" % ord(t)
+            return False
+    
+    
+    def _handleGETSUBS(self,permid, message, selversion):
+        
+        if selversion < OLPROTO_VER_FOURTEENTH:
+            if DEBUG:
+                print >> sys.stderr, "The peer that sent the GET_SUBS request has an old" \
+                     "protcol version: this is strange. Dropping the msg"
+            return False
+        decoded = self._decodeGETSUBSMessage(message)
+        
+        if decoded is None:
+            if DEBUG:
+                print >> sys.stderr, "Error decoding a GET_SUBS message from %s" %\
+                      utilities.show_permid_short(permid)
+            return False
+    
+        if DEBUG:
+            channel_id, infohash, languages = decoded
+            bitmask = self._languagesUtility.langCodesToMask(languages)
+            print >> sys.stderr, "%s, %s, %s, %s, %d, %d" % ("RG", show_permid_short(permid), 
+                                                     show_permid_short(channel_id),
+                                                     bin2str(infohash), bitmask, len(message))
+        
+        # no synch on _listenersList since both this method
+        # and the registerListener method are called by
+        # the OLThread
+        for listener in self._listenersList:
+            listener.receivedSubsRequest(permid, decoded, selversion)
+        
+        return True
+    
+    
+    
+    def _handleSUBS(self, permid, message, selversion):
+        if selversion < OLPROTO_VER_FOURTEENTH:
+            if DEBUG:
+                print >> sys.stderr, "The peer that sent the SUBS request has an old" \
+                     "protcol version: this is strange. Dropping the msg"
+            return False
+        
+        decoded = self._decodeSUBSMessage(message)
+        
+        if decoded is None:
+            if DEBUG:
+                print >> sys.stderr, "Error decoding a SUBS message from %s" %\
+                      utilities.show_permid_short(permid)
+            return False
+        
+        
+        channel_id, infohash, bitmask,contents = decoded
+        #if no subtitle was requested drop the whole message
+        
+        if DEBUG:
+            print >> sys.stderr, "%s, %s, %s, %s, %d, %d" % ("RS", show_permid_short(permid), 
+                                                     show_permid_short(channel_id),
+                                                     bin2str(infohash), bitmask, len(message))
+        
+        
+
+        requestedSubs = self._checkRequestedSubtitles(channel_id,infohash,bitmask) 
+        if requestedSubs == 0:
+            if DEBUG:
+                print >> sys.stderr, SUBS_LOG_PREFIX + "Received a SUBS message that was not"\
+                      " requested. Dropping"
+            return False
+        
+        requestedSubsCodes = self._languagesUtility.maskToLangCodes(requestedSubs)
+        #drop from the contents subtitles that where not requested
+        
+        
+        for lang in contents.keys():
+            if lang not in requestedSubsCodes:
+                del contents[lang]
+        
+        #remove the received subtitles from the requested 
+        callbacks = \
+            self._removeFromRequestedSubtitles(channel_id, infohash, bitmask)
+
+        
+        
+        #the receiver does not need the bitmask
+        tuple = channel_id, infohash, contents
+        
+        # no synch on _listenersList since both this method
+        # and the registerListener method are called by
+        # the OLThread
+        for listener in self._listenersList:
+            listener.receivedSubsResponse(permid, tuple, callbacks, selversion)
+        
+    
+        return True
+    
+    def registerListener(self, listenerObject):
+        '''
+        Register an object to be notifed about the reception of subtitles
+        related messages.
+
+        Currently the messages that are notified are:
+            - GET_SUBS
+            - SUBS
+
+        The appropriete method on listenerObject will be called by the
+        OverlayThread upon reception of a message
+
+        @param listenerObject: an object having two methods with the following
+            signature:
+                1. receivedSubsRequest(permid, decoded, selversion)
+                2. receivedSubsResponse(permid, decoded, callbacks, selversion)
+            Following is the explanation of the paramets:
+                - permid: is the PermId of the peer of send the request
+                  (response)
+                - decoded is a tuple containing the decoded attributes of the
+                  GET_SUBS message
+                - selversion is the protocol version of the peer who sent the
+                  request (response)
+                - callbacks is a list of pairs. Each pair is like::
+                    (mask, function)
+                  mask is a bitmask, and function is the function that should
+                  be called upon receival of subtitles for that mask.
+
+        '''
+        #Only called by OLThread
+        self._listenersList.append(listenerObject)
+        
+
+    
+    
+    def _get_subs_connect_callback(self, exception, dns, permid, selversion,
+                              channel_id, infohash, bitmask, msgSentCallback, usrCallback):
+        """
+        Called by the Overlay Thread when a connection with permid is established.
+        
+        Performs the actual action of sending a GET_SUBS request to the peer
+        identified by permid. It is called by the OLThread when a connection
+        with that peer is established.
+    
+        """
+        
+        if exception is not None:
+            if DEBUG:
+                print >> sys.stderr, SUBS_LOG_PREFIX + \
+                      "GET_SUBS not sent. Unable to connect to " + \
+                      utilities.show_permid_short(permid)
+        else:
+            
+                    
+            if (selversion > 0 and selversion < OLPROTO_VER_FOURTEENTH):
+                msg = "GET_SUBS not send, the other peers had an old protocol version: %d" %\
+                    selversion
+                if DEBUG:
+                    print >> sys.stderr, msg
+                raise SubtitleMsgHandlerException(msg)
+            
+            if DEBUG:
+                print >> sys.stderr, SUBS_LOG_PREFIX + "sending GET_SUBS to " + \
+                      utilities.show_permid_short(permid)
+            try :
+                message = self._createGETSUBSMessage(channel_id, infohash,
+                                                    bitmask)
+                
+                
+                if DEBUG:
+                    # Format:
+                    # SS|SG, destination, channel, infohash, bitmask, size
+                    print >> sys.stderr, "%s, %s, %s, %s, %d, %d" % ("SG",show_permid_short(permid), 
+                                                             show_permid_short(channel_id),
+                                                             bin2str(infohash),bitmask,len(message))
+                
+                self._overlay_bridge.send(permid, message,
+                                          lambda exc, permid: \
+                                            self._sent_callback(exc,permid,
+                                                            channel_id,
+                                                            infohash,
+                                                            bitmask,
+                                                            msgSentCallback,
+                                                            usrCallback))
+        
+            except Exception,e:
+                print_exc()
+                msg = "GET_SUBS not sent: %s" % str(e)
+                raise SubtitleMsgHandlerException(e)
+            
+    def _sent_callback(self,exc,permid,channel_id,infohash,bitmask, msgSentCallback, usrCallback):
+        """
+        Called by the OverlayThread after a GET_SUBS request has been sent.
+        """
+        if exc is not None:
+            if DEBUG:
+                print >> sys.stderr, SUBS_LOG_PREFIX + "Unable to send GET_SUBS to: " + \
+                      utilities.show_permid_short(permid) + ": " + exc
+        else:
+            if DEBUG:
+                print >> sys.stderr, SUBS_LOG_PREFIX + "GET_SUBS sent to %s" % \
+                       (utilities.show_permid_short(permid))
+            self._addToRequestedSubtitles(channel_id, infohash, bitmask, usrCallback)
+            if msgSentCallback is not None:
+                msgSentCallback(exc,permid,channel_id,infohash,bitmask)
+        
+                
+    def _createGETSUBSMessage(self, channel_id, infohash, bitmask):
+        """
+        Bencodes a GET_SUBS message and adds the appropriate header.
+        """
+        
+        binaryBitmask = uintToBinaryString(bitmask)
+        body = bencode((channel_id, infohash, binaryBitmask))
+        head = GET_SUBS
+        return head + body
+    
+    
+            
+    def _decodeGETSUBSMessage(self, message):
+        """
+        From a bencoded GET_SUBS messages, returns its decoded contents.
+        
+        Decodes and checks for validity a bencoded GET_SUBS messages.
+        If the message is succesfully decoded returns the tuple
+        (channel_id,infohash,languages).
+        
+        channel_id is the binary identifier of the chanel that published
+        the requested subtitles.
+        infohash is the binary identifier of the torrent wich the subtitle
+        refers to
+        languages is a list of 3 characters language codes, for the languages
+        of the requested subtitles
+        
+        @return: (channel_id,infohash,languages) or None if something is wrong
+        """
+        assert message[0] == GET_SUBS, SUBS_LOG_PREFIX + \
+            "Invalid GET_SUBS Message header: %s" % message[0]
+        
+        try:
+            values = bdecode(message[1:])
+        except:
+            if DEBUG:
+                print >> sys.stderr, SUBS_LOG_PREFIX + "Error bdecoding message"
+            return None
+        
+        if len(values) != 3:
+            if DEBUG:
+                print >> sys.stderr, SUBS_LOG_PREFIX + "Invalid number of fields in GET_SUBS"
+            return None
+        channel_id, infohash, bitmask = values[0], values[1], values[2]
+        if not validPermid(channel_id):
+            if DEBUG:
+                print >> sys.stderr, SUBS_LOG_PREFIX + "Invalid channel_id in GET_SUBS"
+            return None
+        elif not validInfohash(infohash):
+            if DEBUG:
+                print >> sys.stderr, SUBS_LOG_PREFIX + "Invalid infohash in GET_SUBS"
+            return None
+        elif not isinstance(bitmask, str) or not len(bitmask)==4:
+            if DEBUG:
+                print >> sys.stderr, SUBS_LOG_PREFIX + "Invalid bitmask in GET_SUBS"
+            return None
+        
+        try:
+            bitmask = binaryStringToUint(bitmask)
+            languages = self._languagesUtility.maskToLangCodes(bitmask)
+        except:
+            if DEBUG:
+                print >> sys.stderr, SUBS_LOG_PREFIX + "Invalid bitmask in GET_SUBS"
+            return None
+        
+        return channel_id, infohash, languages
+    
+    
+    def _decodeSUBSMessage(self, message):
+        """
+        From a bencoded SUBS message, returns its decoded contents.
+        
+        Decodes and checks for validity a bencoded SUBS message.
+        If the message is succesfully decoded returns the tuple
+        (channel_id, infohash, bitmask, contentsDictionary )
+        
+        channel_id is the binary identifier of the chanel that published
+        the requested subtitles.
+        infohash is the binary identifier of the torrent wich the subtitle
+        refers to
+        contentsDictionary is a dictionary having each entry like 
+        {langCode : subtitleContents}.
+        
+        @return: the above described tuple, or None if something is wrong
+        """
+        assert message[0] == SUBS, SUBS_LOG_PREFIX + \
+            "Invalid SUBS Message header: %s" % message[0]
+            
+        try:
+            values = bdecode(message[1:])
+        except:
+            if DEBUG:
+                print >> sys.stderr, SUBS_LOG_PREFIX + "Error bdecoding SUBS message"
+            return None
+        
+        if len(values) != 4:
+            if DEBUG:
+                print >> sys.stderr, SUBS_LOG_PREFIX + "Invalid number of fields in SUBS"
+            return None
+        channel_id, infohash, bitmask, contents = values[0], values[1], \
+            values[2], values[3]
+        
+        if not validPermid(channel_id):
+            if DEBUG:
+                print >> sys.stderr, SUBS_LOG_PREFIX + "Invalid channel_id in SUBS"
+            return None
+        elif not validInfohash(infohash):
+            if DEBUG:
+                print >> sys.stderr, SUBS_LOG_PREFIX + "Invalid infohash in SUBS"
+            return None
+        elif not isinstance(bitmask, str) or not len(bitmask) == 4:
+            if DEBUG:
+                print >> sys.stderr, SUBS_LOG_PREFIX + "Invalid bitmask in SUBS"
+            return None
+        
+        try:
+            bitmask = binaryStringToUint(bitmask)
+            languages = self._languagesUtility.maskToLangCodes(bitmask)
+        except:
+            if DEBUG:
+                print >> sys.stderr, SUBS_LOG_PREFIX + "Invalid bitmask in SUBS"
+            return None
+        
+        if not isinstance(contents, list):
+            if DEBUG:
+                print >> sys.stderr, SUBS_LOG_PREFIX + "Invalid contents in SUBS"
+            return None
+        if len(languages) != len(contents):
+            if DEBUG:
+                print >> sys.stderr, SUBS_LOG_PREFIX + "Bitmask and contents do not match in"\
+                      " SUBS"
+            return None
+        
+        numOfContents = len(languages)
+        if numOfContents == 0:
+            if DEBUG:
+                print >> sys.stderr, SUBS_LOG_PREFIX + "Empty message. Discarding."
+            return None
+        
+        
+        contentsDictionary = dict()
+        for i in range(numOfContents):
+            lang = languages[i]
+            subtitle = contents[i]
+            if not isinstance(subtitle,unicode):
+                try:
+                    subtitle = unicode(subtitle)
+                except:
+                    return None
+            if len(subtitle) <= self._maxSubSize:
+                contentsDictionary[lang] = subtitle
+            else:
+                #drop that subtitle
+                continue
+            
+        bitmask = self._languagesUtility.langCodesToMask(contentsDictionary.keys())
+            
+        
+        return channel_id, infohash, bitmask, contentsDictionary
+    
+
+    def _checkingUploadQueue(self):
+        """
+        Uses a token bucket to control the subtitles upload rate.
+        
+        Every time this method is called, it will check if there are enough
+        tokens in the bucket to send out a SUBS message.
+        Currently fragmentation is not implemented: all the reuquested subtitles
+        are sent in a single SUBS messages if there are enough tokens:
+        too big responses are simply discarded.
+        
+        The method tries to consume all the available tokens of the token
+        bucket until there are no more messages to send. If there are no
+        sufficiente tokens to send a message, another call to this method
+        is scheduled in a point in time sufficiently distant.
+        """
+        
+        if DEBUG:
+            print >> sys.stderr, SUBS_LOG_PREFIX + "Checking the upload queue..."
+        
+        if not self._tokenBucket.upload_rate > 0:
+            return 
+        
+        if not len(self._uploadQueue) > 0:
+            if DEBUG:
+                print >> sys.stderr, SUBS_LOG_PREFIX + "Upload queue is empty."
+            
+        while len(self._uploadQueue) > 0 :
+            responseData = self._uploadQueue[0]
+            encodedMsg = self._createSingleResponseMessage(responseData)
+            
+            if encodedMsg is None:
+                if DEBUG:
+                    print >> sys.stderr, SUBS_LOG_PREFIX + "Nothing to send"
+                del self._uploadQueue[0]
+                continue #check other messages in the queue
+            
+            msgSize = len(encodedMsg) / 1024.0 #in kilobytes
+            
+            if msgSize > self._tokenBucket.capacity: 
+                #message is too big, discarding
+                print >> sys.stderr, "Warning:" + SUBS_LOG_PREFIX + "SUBS message too big. Discarded!"
+                del self._uploadQueue[0]
+                continue #check other messages in the queue
+            
+            #check if there are sufficiente tokens
+            if self._tokenBucket.consume(msgSize):
+                
+                if DEBUG:
+                    # Format:
+                    # S|G, destination, channel, infohash, bitmask, size
+                    keys = responseData['subtitles'].keys()
+                    bitmask = self._languagesUtility.langCodesToMask(keys)
+                    print >> sys.stderr, "%s, %s, %s, %s, %d, %d" % ("SS",show_permid_short(responseData['permid']),
+                                                             show_permid_short(responseData['channel_id']),
+                                                             bin2str(responseData['infohash']),bitmask,int(msgSize*1024))
+                    
+                self._doSendSubtitles(responseData['permid'], encodedMsg, responseData['selversion'])
+                del self._uploadQueue[0]
+            else: 
+                #tokens are insufficient wait the necessary time and check again
+                neededCapacity = max(0, msgSize - self._tokenBucket.tokens)
+                delay = (neededCapacity / self._tokenBucket.upload_rate)
+                self._nextUploadTime = time() + delay
+                self.overlay_bridge.add_task(self._checkingUploadQueue, delay)
+                return
+        
+        #The cycle breaks only if the queue is empty
+    
+    
+    def _createSingleResponseMessage(self, responseData):
+        """
+        Create a bencoded SUBS message to send in response to a GET_SUBS
+        
+        The format of the sent message is a not encoded SUBS character and then
+        the bencoded form of
+        (channel_id,infohash,bitmask,[listOfSubtitleContents])
+        the list of subtitle contents is ordered as the bitmask
+        
+        """
+        
+        orderedKeys = sorted(responseData['subtitles'].keys())
+
+        payload = list()
+        #read subtitle contents
+        for lang in orderedKeys:
+            
+            fileContent = responseData['subtitles'][lang]
+                
+            if fileContent is not None and len(fileContent) <= self._maxSubSize:
+                payload.append(fileContent)
+            else:
+                print >> sys.stderr, "Warning: Subtitle in % for ch: %s, infohash:%s dropped. Bigger then %d" % \
+                            (lang, responseData['channel_id'], responseData['infohash'], 
+                             self._maxSubSize)
+
+                
+        
+        if not len(payload) > 0:
+            if DEBUG:
+                print >> sys.stderr, SUBS_LOG_PREFIX + "No payload to send in SUBS"
+            return None
+        
+        bitmask = \
+            self._languagesUtility.langCodesToMask(orderedKeys)
+        
+        binaryBitmask = uintToBinaryString(bitmask, length=4)
+        header = (responseData['channel_id'], responseData['infohash'], binaryBitmask)
+        
+        message = bencode((
+                           header[0],
+                           header[1],
+                           header[2],
+                           payload
+                           ))
+        
+        return SUBS + message
+    
+    
+    def _doSendSubtitles(self, permid, msg, selversion):
+        """
+        Do sends the SUBS message through the overlay bridge.
+        """
+        if DEBUG:
+            print >> sys.stderr, SUBS_LOG_PREFIX + "Sending SUBS message to %s..." % \
+                  show_permid_short(permid)
+    
+        # Optimization: we know we're currently connected
+        #DOUBLE CHECK THIS. I just assuemed it was true 
+        # since it is true for MetadataHandler
+        self._overlay_bridge.send(permid, msg, self._subs_send_callback)
+    
+    def _subs_send_callback(self, exc, permid):
+        '''
+        Called by the OLThread when a SUBS message is succesfully sent
+        '''
+        if exc is not None:
+            print >> sys.stderr, "Warning: Sending of SUBS message to %s failed: %s" % \
+                (show_permid_short(permid), str(exc))
+        else:
+            if DEBUG:
+                print >> sys.stderr, "SUBS message succesfully sent to %s" % show_permid_short(permid)
+        
+        
+    def _addToRequestedSubtitles(self, channel_id, infohash, bitmask, callback=None):
+        """
+        Add (channel_id, infohash, bitmask) to the history of requested subs.
+        
+        Call this method after a request for subtitles for a torrent
+        identified by infohash in channel channel_id, has been sent for the 
+        languages identified by the bitmask.
+        """
+        
+        assert 0 <= bitmask < 2**32, "bitmask must be a 32  bit integer"
+
+        if(int(time()) >= self._nextCleanUpTime):
+            self._cleanUpRequestedSubtitles() #cleanup old unanswered requests
+        
+        key = self._getRequestedSubtitlesKey(channel_id, infohash)
+        if key in self.requestedSubtitles.keys():
+            rsEntry = self.requestedSubtitles[key]
+            rsEntry.newRequest(bitmask)
+        else :
+            rsEntry = _RequestedSubtitlesEntry()
+            rsEntry.newRequest(bitmask, callback)
+            self.requestedSubtitles[key] = rsEntry
+
+
+    
+    def _cleanUpRequestedSubtitles(self):
+        """
+        Cleans up unanswered requests.
+        
+        A request is considered unanswered when it was last updated more then
+        REQUESTE_VALIDITY_TIME seconds ago.
+        If a response arrives after a request gets deleted, it will be dropped.
+        """
+ 
+        keys = self.requestedSubtitles.keys()
+        now = int(time())
+        for key in keys:
+            rsEntry = self.requestedSubtitles[key]
+            somethingDeleted = rsEntry.cleanUpRequests(self._requestValidityTime)
+            if somethingDeleted:
+                if DEBUG:
+                    print >> sys.stderr, "Deleting subtitle request for key %s: expired.", key
+            
+            #no more requests for the (channel,infohash, pair)
+            if rsEntry.cumulativeBitmask == 0:
+                del self.requestedSubtitles[key]
+                
+        self._nextCleanUpTime = now + CLEANUP_PERIOD
+
+        
+        
+    
+    
+    def _removeFromRequestedSubtitles(self, channel_id, infohash, bitmask):
+        """
+        Remove (channel_id,infohash,bitmask) from the history of requested subs.
+        
+        Call this method after a request for subtitles for a torrent
+        identified by infohash in channel channel_id, has been recevied for the 
+        languages identified by the bitmask.
+        """
+
+        key = self._getRequestedSubtitlesKey(channel_id, infohash)
+        if key not in self.requestedSubtitles.keys():
+            if DEBUG:
+                print >> sys.stderr, SUBS_LOG_PREFIX + "asked to remove a subtitle that" + \
+                        "was never requested from the requestedList"
+            return None
+        else:
+            rsEntry = self.requestedSubtitles[key]
+            callbacks = rsEntry.removeFromRequested(bitmask)
+            
+            if rsEntry.cumulativeBitmask == 0:
+                del self.requestedSubtitles[key]
+            return callbacks
+            
+    def _checkRequestedSubtitles(self, channel_id, infohash, bitmask):
+        """
+        Given a bitmask returns a list of language from the ones in the bitmask
+        that have been actually requested
+        """
+
+        key = self._getRequestedSubtitlesKey(channel_id, infohash)
+        if key not in self.requestedSubtitles.keys():
+            if DEBUG:
+                print >> sys.stderr, SUBS_LOG_PREFIX + "asked to remove a subtitle that" + \
+                        "was never requested from the requested List"
+            return 0
+        else:
+            rsEntry = self.requestedSubtitles[key]
+            reqBitmask = rsEntry.cumulativeBitmask & bitmask
+            return reqBitmask
+
+            
+            
+class _RequestedSubtitlesEntry():
+    '''
+    Convenience class to represent entries in the requestedSubtitles map
+    from the SubtitleHandler.
+    For each (channel, infohash tuple it keeps a cumulative bitmask
+    of all the requested subtitles, and a list of the single different
+    requests. Each single request bears a timestamp that is used
+    to cleanup outdated requests
+    '''
+    
+    def __init__(self):
+        self.requestsList = list()
+        self.cumulativeBitmask = 0
+        
+    def newRequest(self, req_bitmask, callback = None):
+        assert 0 <= req_bitmask < 2**32
+        
+        self.requestsList.append([req_bitmask,callback,int(time())])
+        self.cumulativeBitmask = int(self.cumulativeBitmask | req_bitmask)
+
+            
+    
+    def removeFromRequested(self, rem_bitmask):
+
+        callbacks = list()
+        self.cumulativeBitmask = self.cumulativeBitmask & (~rem_bitmask)
+        
+        length = len(self.requestsList)
+        i=0
+        while i < length:
+            entry = self.requestsList[i]
+            receivedLangs = entry[0] & rem_bitmask
+            #if something was received for the request
+            if receivedLangs != 0:
+                callbacks.append((entry[1],receivedLangs))
+                updatedBitmask = entry[0] & (~receivedLangs)
+                # no more subtitles to receive for 
+                # thath request
+                if updatedBitmask == 0:
+                    del self.requestsList[i]
+                    i -=1
+                    length -=1
+                else:
+                    entry[0] = updatedBitmask
+            i += 1
+             
+        return callbacks
+    
+
+    
+    
+    
+    def cleanUpRequests(self, validityDelta):
+
+        somethingDeleted = False
+        now = int(time())
+        
+        length = len(self.requestsList)
+        i=0
+        while i < length:
+            entry = self.requestsList[i]
+            requestTime = entry[2]
+            #if the request is outdated
+            if requestTime + validityDelta < now :
+                #remove the entry
+                self.cumulativeBitmask = self.cumulativeBitmask & \
+                    (~entry[0])
+                del self.requestsList[i]
+                i -= 1
+                length -= 1
+                somethingDeleted = True
+            
+            i += 1
+        
+        return somethingDeleted
+    
+
diff --git a/instrumentation/next-share/BaseLib/Core/Subtitles/SubtitleHandler/__init__.py b/instrumentation/next-share/BaseLib/Core/Subtitles/SubtitleHandler/__init__.py
new file mode 100644
index 0000000..284cc10
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Subtitles/SubtitleHandler/__init__.py
@@ -0,0 +1,2 @@
+# Written by Andrea Reale
+# see LICENSE.txt for license information
\ No newline at end of file
diff --git a/instrumentation/next-share/BaseLib/Core/Subtitles/SubtitlesHandler.py b/instrumentation/next-share/BaseLib/Core/Subtitles/SubtitlesHandler.py
new file mode 100644
index 0000000..0baf419
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Subtitles/SubtitlesHandler.py
@@ -0,0 +1,611 @@
+# Written by Andrea Reale
+# see LICENSE.txt for license information
+
+from __future__ import with_statement
+from BaseLib.Core.Subtitles.MetadataDomainObjects.Languages import \
+    LanguagesProvider
+from BaseLib.Core.Subtitles.MetadataDomainObjects.MetadataExceptions import \
+    MetadataDBException, RichMetadataException, DiskManagerException
+from BaseLib.Core.CacheDB.Notifier import Notifier
+from BaseLib.Core.Subtitles.SubtitleHandler.DiskManager import DiskManager, \
+    DISK_FULL_DELETE_SOME, DELETE_OLDEST_FIRST
+from BaseLib.Core.Subtitles.SubtitleHandler.SimpleTokenBucket import \
+    SimpleTokenBucket
+from BaseLib.Core.Subtitles.SubtitleHandler.SubsMessageHandler import \
+    SubsMessageHandler
+from BaseLib.Core.Utilities import utilities
+from BaseLib.Core.Utilities.Crypto import sha
+from BaseLib.Core.Utilities.utilities import bin2str, show_permid_short
+from BaseLib.Core.simpledefs import NTFY_ACT_DISK_FULL, NTFY_SUBTITLE_CONTENTS, \
+    NTFY_UPDATE
+import os
+import sys
+
+
+
+
+SUBS_EXTENSION = ".srt"
+SUBS_LOG_PREFIX = "subtitles: "
+
+DEFAULT_MIN_FREE_SPACE = 0 #no treshold
+
+MAX_SUBTITLE_SIZE = 1 * 1024 * 1024    # 1MB subtitles. too big?
+MAX_SUBS_MESSAGE_SIZE = int(2 * MAX_SUBTITLE_SIZE / 1024) #in KBs
+
+
+MAX_SUBTITLE_DISK_USAGE = 200 * (2 ** 10) #200 MBs
+
+DEBUG = False
+
+class SubtitlesHandler(object):
+    
+    
+    __single = None
+    
+    def __init__(self):
+        # notice that singleton pattern is not enforced.
+        # This is better, since this way the code is more easy
+        # to test.
+        
+        SubtitlesHandler.__single = self
+        self.avg_subtitle_size = 100 # 100 KB, experimental avg
+        self.languagesUtility = LanguagesProvider.getLanguagesInstance()
+        
+
+        #instance of MetadataDBHandler
+        self.subtitlesDb = None
+        self.registered = False
+        self.subs_dir = None
+        
+        
+        
+        #other useful attributes are injected by the register method
+        
+
+    @staticmethod
+    def getInstance(*args, **kw):
+        if SubtitlesHandler.__single is None:
+            SubtitlesHandler(*args, **kw)
+        return SubtitlesHandler.__single
+    
+    def register(self, overlay_bridge, metadataDBHandler, session):
+        """
+        Injects the required dependencies on the instance.
+        
+        @param overlay_bridge: a reference to a working instance
+                               of OverlayTrheadingBridge
+        @param metadataDBHandler: a reference to the current instance of
+                           L{MetadataDBHandler}
+        @param session: a reference to the running session
+        """
+        self.overlay_bridge = overlay_bridge
+        self.subtitlesDb = metadataDBHandler
+        self.config_dir = os.path.abspath(session.get_state_dir())
+        subs_path = os.path.join(self.config_dir, session.get_subtitles_collecting_dir())
+        self.subs_dir = os.path.abspath(subs_path)
+        
+        self.min_free_space = DEFAULT_MIN_FREE_SPACE
+        self._upload_rate = session.get_subtitles_upload_rate()
+        self.max_subs_message_size = MAX_SUBS_MESSAGE_SIZE
+        self._session = session
+       
+        #the upload rate is controlled by a token bucket.
+        #a token corresponds to 1 KB.
+        #The max burst size corresponds to 2 subtitles of the maximum size (2 MBs)
+        tokenBucket = SimpleTokenBucket(self._upload_rate,
+                                              self.max_subs_message_size)
+       
+        self._subsMsgHndlr = SubsMessageHandler(self.overlay_bridge, tokenBucket, 
+                                                MAX_SUBTITLE_SIZE)
+        self._subsMsgHndlr.registerListener(self)
+        
+    
+        #assure that the directory exists
+        
+        if os.path.isdir(self.config_dir) :
+            if not os.path.isdir(self.subs_dir):
+                try:
+                    os.mkdir(self.subs_dir)
+                except:
+                    msg = u"Cannot create collecting dir %s " % self.subs_dir
+                    print >> sys.stderr, "Error: %s" % msg
+                    raise IOError(msg)
+        else:
+            msg = u"Configuration dir %s does not exists" % self.subs_dir
+            print >> sys.stderr, "Error: %s" % msg
+            raise IOError(msg)
+        
+        
+        
+           
+        diskManager = DiskManager(self.min_free_space, self.config_dir)
+        self.diskManager = diskManager
+        
+        
+        dmConfig = {"maxDiskUsage" : MAX_SUBTITLE_DISK_USAGE,
+                    "diskPolicy" : DISK_FULL_DELETE_SOME | DELETE_OLDEST_FIRST,
+                     "encoding" : "utf-8"}
+        self.diskManager.registerDir(self.subs_dir, dmConfig)
+        
+        freeSpace = self.diskManager.getAvailableSpace()
+        if DEBUG:
+            print >> sys.stderr, SUBS_LOG_PREFIX + "Avaialble %d MB for subtitle collecting" % (freeSpace / (2 ** 20))
+        
+        #event notifier
+        self._notifier = Notifier.getInstance()
+        
+        self.registered = True
+    
+    
+    
+    
+    
+    def sendSubtitleRequest(self, permid, channel_id, infohash, languages,
+                            callback=None, selversion= -1):
+        """
+        Send a request for subtitle files. Only called by the OLThread
+        
+        Send a GET_SUBS request to the peer indentified by permid.
+        The request asks for several subtitles file, for a given channel_id
+        and torrent infohash. The subtitles file to request are specified
+        by the languages parameter that is a list of 3 characters language
+        codes.
+        
+        The contents of a GET_SUBS request are:
+            - channel_id: the identifier of the channel for which the subtitles
+              were added. (a permid). Binary.
+            - infohash: the infohash of the torrent, the subtitles refer to.
+              Binary.
+            - bitmask:  a 32 bit bitmask (an integer) which specifies the 
+              languages requested
+                      
+        
+        
+        @param permid: the destination of the request (binary)
+        @param channel_id: the identifier of the channel for which the subtitle
+                           was added (binary)
+        @param infohash: the infohash of a torrent the subtitles refers to (binary).
+        @param languages: a list of 3-characters language codes. It must be
+                          on of the supported language codes (see Languages)
+        @param callback: a function that will be called WHENEVER some of the
+                         requested subtitles are received. It must have exactly
+                         one parameter that will be bound to a list of 
+                         the languages that were received
+        @param selversion: the protocol version of the peer whe are sending 
+                            the request to
+        
+        @raise SubtitleMsgHandlerException: if the message failed its attempt to be sent.
+                                      Notice that also if the method returns without
+                                      raising any exception it doesn't mean
+                                      that the message has been sent.
+        """
+        
+        assert utilities.isValidInfohash(infohash), \
+            SUBS_LOG_PREFIX + "Invalid infohash %s" % infohash
+        assert utilities.isValidPermid(permid), \
+            SUBS_LOG_PREFIX + "Invlaid destination permid %s" % permid
+        
+        assert self.languagesUtility.isLangListSupported(languages), \
+             SUBS_LOG_PREFIX + "Some of the languages where not supported"
+            
+
+        
+        if DEBUG:
+            print >> sys.stderr, SUBS_LOG_PREFIX + "preparing to send GET_SUBS to " + \
+                  utilities.show_permid_short(permid)
+        
+
+        
+# Better to leave up to the caller the responsibility to check
+# if the subtitle is already available and as correct checsum and so on..
+#        onDisk = []
+#        for langCode in languages:
+#            
+#            filename = self.diskManager.isFilenOnDisk(self.subs_dir,
+#                    getSubtitleFileRelativeName(channel_id, infohash, langCode))
+#            
+#            
+#            # should I skip this part and just send the request anyway?
+#            # (thus leaving to the caller the responsibility to avoid useless
+#            # requests)
+#            if filename:
+#                log.debug(SUBS_LOG_PREFIX + langCode + 
+#                          " subtitle already on disk. Skipping it"\
+#                          " in the request")
+#                onDisk.append(langCode)
+#                self._notify_sub_is_in(channel_id, infohash, langCode, filename)
+#        
+#        for deleteme in onDisk:
+#            languages.remove(deleteme)
+            
+        if len(languages) == 0:
+            if DEBUG:
+                print >> sys.stderr, SUBS_LOG_PREFIX + " no subtitles to request."
+            return
+            
+        
+        if not self.diskManager.tryReserveSpace(self.subs_dir, len(languages) * self.avg_subtitle_size):
+            self._warn_disk_full()
+            return False
+        
+        requestDetails = dict()
+        requestDetails['channel_id'] = channel_id
+        requestDetails['infohash'] = infohash
+        requestDetails['languages'] = languages
+        
+        
+        self._subsMsgHndlr.sendSubtitleRequest(permid, requestDetails,
+                                                lambda e,d,c,i,b : \
+                                                    self._subsRequestSent(e,d,c,i,b),
+                                                    callback,
+                                                    selversion)
+            
+        
+    
+    def _subsRequestSent(self,exception,dest, channel_id, infohash, bitmask ):
+        '''
+        Gets called when a subtitle request has been succesfully sent.
+        '''
+        pass
+
+    def receivedSubsRequest(self, permid, request, selversion):
+        """
+        Reads a received GET_SUBS message and possibly sends a response.
+        
+        @param permid: the permid of the sender of the GET_SUBS message
+        @param request: a tuple made of channel_id, infohash, language code
+        @param selversion: the protocol version of the requesting peer
+        
+        @return: False if the message had something wrong. (a return value
+                 of False makes the caller close the connection).
+                 Otherwise True
+        """
+        
+        assert self.registered, SUBS_LOG_PREFIX + "Handler not yet registered"
+
+        
+        channel_id, infohash, languages = request #happily unpacking
+        
+        
+        #diction {lang : Subtitle}
+        allSubtitles = self.subtitlesDb.getAllSubtitles(channel_id,
+                                                        infohash)
+        
+        
+        contentsList = {} #{langCode : path}
+        #for each requested language check if the corresponding subtitle
+        #is available
+        for lang in sorted(languages):
+            if lang in allSubtitles.keys():
+                if allSubtitles[lang].subtitleExists():
+                    content = self._readSubContent(allSubtitles[lang].path)
+                    if content is not None:
+                        contentsList[lang] = content 
+                    
+                else:
+                    if DEBUG:
+                        print >> sys.stderr, SUBS_LOG_PREFIX + "File not available for " + \
+                              "channel %s, infohash %s, lang %s" % \
+                              (show_permid_short(channel_id), bin2str(infohash),
+                              lang)
+                    self.subtitlesDb.updateSubtitlePath(channel_id,infohash,lang,None)
+            else:
+                if DEBUG:
+                    print >> sys.stderr, SUBS_LOG_PREFIX + "Subtitle not available for " + \
+                          "channel %s, infohash %s, lang %s" % \
+                          (show_permid_short(channel_id), bin2str(infohash),
+                          lang)
+        
+        if len(contentsList) == 0: #pathlist is empty
+            if DEBUG:
+                print >> sys.stderr, SUBS_LOG_PREFIX + "None of the requested subtitles " + \
+                      " was available. No answer will be sent to %s" % \
+                      show_permid_short(permid)
+            return True
+        
+        
+        
+        return self._subsMsgHndlr.sendSubtitleResponse(permid, 
+                                                (channel_id,infohash,contentsList), 
+                                                selversion)
+
+        
+        
+    
+    def _readSubContent(self,path):
+
+        try:
+            relativeName = os.path.relpath(path, self.subs_dir)
+            fileContent = self.diskManager.readContent(self.subs_dir,
+                                                       relativeName)
+        except IOError,e:
+            if DEBUG:
+                print >> sys.stderr, SUBS_LOG_PREFIX + "Error reading from subs file %s: %s" % \
+                 (relativeName, e)
+            fileContent = None
+            
+        if fileContent is not None and len(fileContent) <= MAX_SUBTITLE_SIZE:
+            return fileContent
+        else:
+            print >> sys.stderr, "Warning: Subtitle %s dropped. Bigger then %d" % \
+                (relativeName, MAX_SUBTITLE_SIZE)
+            return None
+                
+
+            
+        
+
+    
+   
+        
+    def _subs_send_callback(self, exception, permid):
+        """
+        Called by the overlay thread when the send action is completed
+        """
+        if exception is not None:
+            if DEBUG:
+                print >> sys.stderr, SUBS_LOG_PREFIX + "Failed to send metadata to %s: %s" % \
+                      (show_permid_short(permid), str(exception))
+        
+    
+    def receivedSubsResponse(self, permid, msg, callbacks, selversion):
+        """
+        Handles the reception of a SUBS message.
+        
+        Checks against integrity of the contents received in a SUBS message.
+        If the message containes one or more subtitles that were not requested
+        they are dropped.
+        If the message is bigger in size then MAX_SUBS_MSG_SIZE it is dropped.
+        If one subtitle is bigger in size then MAX_SUBTITLE_SIZE it is dropped.
+        Otherwise the message is decoded, the subtitles saved to disk, and 
+        their path added to database.
+        
+        @param permid: the permid of the sender
+        @param msg: a triple of channel_id, infohash, and the contentsDictionary
+        @param callbacks: a list of pairs. The first element is a function to call,
+            the second a bitmask that help building back the parameters
+            of the function
+        @param selversion: the protocol version number of the other peer
+        
+        
+        @return: False if the message is dropped becuase malformed. 
+        """
+        assert self.registered == True, SUBS_LOG_PREFIX + "Subtitles Handler"\
+            " is not registered"
+        
+    
+        channel_id, infohash, contentsDictionary = \
+            msg
+        
+        
+        metadataDTO = self.subtitlesDb.getMetadata(channel_id, infohash)
+        
+        assert metadataDTO is not None, SUBS_LOG_PREFIX + "Inconsistent " \
+            "subtitles DB: a requested subtitle was not available in the db"
+        
+        filepaths = dict()
+        somethingToWrite = False
+        
+        for lang, subtitleContent in contentsDictionary.iteritems():
+            try:
+                filename = self._saveSubOnDisk(channel_id, infohash, lang,
+                                               subtitleContent)
+                filepaths[lang] = filename
+            except IOError,e:
+                if DEBUG:
+                    print >> sys.stderr, SUBS_LOG_PREFIX + "Unable to save subtitle for "\
+                          "channel %s and infohash %s to file: %s" % \
+                          (show_permid_short(channel_id), str(infohash), e)
+                continue
+            except Exception,e:
+                if DEBUG:
+                    print >> sys.stderr, "Unexpected error copying subtitle On Disk: " + str(e)
+                raise e
+            
+            subToUpdate = metadataDTO.getSubtitle(lang)
+            if subToUpdate is None:
+                print >> sys.stderr, "Warning:" + SUBS_LOG_PREFIX + "Subtitles database inconsistency."
+                #is it ok to throw a runtime error or should I gracefully fail?
+                raise MetadataDBException("Subtitles database inconsistency!")
+            
+            subToUpdate.path = filename
+            if not subToUpdate.verifyChecksum():
+                if DEBUG:
+                    print >> sys.stderr, "Received a subtitle having invalid checsum from %s" % \
+                         show_permid_short(permid)
+                subToUpdate.path = None
+                
+                relativeName = os.path.relpath(filename, self.subs_dir)
+                self.diskManager.deleteContent(self.subs_dir, relativeName)
+                continue
+            self.subtitlesDb.updateSubtitlePath(channel_id, infohash, subToUpdate.lang, filename, False)
+            somethingToWrite = True
+        
+        if somethingToWrite:
+            self.subtitlesDb.commit()
+        
+        if DEBUG:    
+            print >> sys.stderr, "Subtitle written on disk and informations on database."
+        
+        self._scheduleUserCallbacks(callbacks)
+        
+        return True
+            
+                
+    def _scheduleUserCallbacks(self, callbacks):
+        
+        # callbacks is a list of tuples such as
+        # (callback_func, bitmask)
+        for entry in callbacks:
+            callback = entry[0]
+            if callback is None:
+                pass
+            else:
+                listOfLanguages = self.languagesUtility.maskToLangCodes(entry[1])
+                def callBack():
+                    to_call = callback
+                    return to_call(listOfLanguages)
+                # Commented because had a problem related to the
+                # scope of the closure
+                #toCall = lambda : callback(listOfLanguages)
+                self.overlay_bridge.add_task(callBack)
+
+        
+        
+        
+        
+    
+    def _saveSubOnDisk(self, channel_id, infohash, lang, subtitleContent):
+        assert self.registered == True, SUBS_LOG_PREFIX + "Subtitles Handler"\
+            " is not registered"
+            
+        filename = getSubtitleFileRelativeName(channel_id, infohash, lang)
+        
+        path = self.diskManager.writeContent(self.subs_dir,
+                                             filename, subtitleContent)
+            
+        return path
+
+    
+    
+    def _notify_sub_is_in(self, channel_id, infohash, langCode, filename):
+        """
+        Notify that a subtitle file is available.
+        
+        Notifies any interested receiver that a subtitle for 
+        (channel_id, infohash, langCode) is available in the file
+        located at path filename.
+        
+        Currently it just prints a cool debug message.
+        """
+        if DEBUG:
+            print >> sys.stderr, SUBS_LOG_PREFIX + "Subtitle is in at" + filename
+        
+        if self._notifier is not None:
+            self.notifier.notify(NTFY_SUBTITLE_CONTENTS, NTFY_UPDATE,
+                                 (channel_id, infohash), langCode, filename)
+    
+    
+    def _warn_disk_full(self):
+        """
+        Notifies the LaunchMany instance that the disk is full.
+        """
+        print >> sys.stderr, "Warning: " + SUBS_LOG_PREFIX + "GET_SUBS: Disk full!"
+        drive, rdir = os.path.splitdrive(os.path.abspath(self.subs_dir))
+        if not drive:
+            drive = rdir
+        self.launchmany.set_activity(NTFY_ACT_DISK_FULL, drive)
+        
+
+      
+    def setUploadRate(self, uploadRate):
+        """
+        Sets the subtitles uploading rate, expressed in KB/s
+        """
+        assert self.registered
+        
+        self._upload_rate = float(uploadRate)
+        self._subsMsgHndlr._tokenBucket.fill_rate = float(uploadRate)
+
+    def getUploadRate(self):
+        """
+        Returns the current setting for the subtitles upload rate, in KB/s
+        """
+        return self._upload_rate
+    
+    def delUploadRate(self):
+        """
+        No, you can't delete the upload_rate property
+        """
+        raise RuntimeError("Operation not supported")
+    
+    upload_rate = property(getUploadRate, setUploadRate, delUploadRate,
+                           "Controls the subtitles uploading rate. Expressed in KB/s")
+    
+    
+    def copyToSubtitlesFolder(self,pathToMove, channel_id, infohash, langCode):
+        """
+        Given the path to an srt, moves it to the subtitle folder, also
+        changing the name to the correct one
+        
+        @return: the complete path of the file if the file was succesfully copied,
+        
+        @raise RichMetadataException: if the subtitle cannot be copied.
+        
+        """
+        
+        if not os.path.isfile(pathToMove):
+            raise RichMetadataException("File not found.")
+        
+        if os.path.getsize(pathToMove) >= MAX_SUBTITLE_SIZE :
+            raise RichMetadataException("Subtitle bigger then %d KBs" % (MAX_SUBTITLE_SIZE/1024))
+        
+        # Not really strong check: anyone can change the extension of a file :)
+        if not pathToMove.endswith(SUBS_EXTENSION):
+            raise RichMetadataException("Only .srt subtitles are supported")
+        
+        filename = getSubtitleFileRelativeName(channel_id, infohash, langCode)
+
+        
+        if self.diskManager.isFilenOnDisk(self.subs_dir, filename):
+            if DEBUG:
+                print >> sys.stderr, "Overwriting previous subtitle %s" % filename
+            try:
+                deleted = self.diskManager.deleteContent(self.subs_dir, filename)
+            except DiskManagerException,e:
+                if DEBUG:
+                    print >> sys.stderr, "Unable to remove subtitle %s" % filename
+                raise RichMetadataException("Unable to remove subtile %s to overwrite: %s"\
+                                            % (filename, str(e)))
+            
+            if not deleted:
+                if DEBUG: 
+                    print >> sys.stderr, "Unable to remove subtitle %s" % filename
+                raise RichMetadataException("Old subtitle %s is write protected"% filename)
+        
+        
+        with open(pathToMove,"rb") as toCopy:
+            encoding = toCopy.encoding
+            content = toCopy.read()
+            
+        if encoding is not None:
+            #convert the contents from their original encoding 
+            # to unicode, and replace possible unknown characters
+            #with U+FFFD
+            content = unicode(content, encoding, 'relplace')
+        else:
+            #convert using the system default encoding
+            content = unicode(content, errors="replace")
+        
+        return self.diskManager.writeContent(self.subs_dir, filename, content)
+
+
+    def getMessageHandler(self):
+        return self._subsMsgHndlr.handleMessage                
+        
+
+        
+    
+    
+        
+def getSubtitleFileRelativeName(channel_id, infohash, langCode):
+    #subtitles filenames are build from the sha1 hash
+    #of the triple (channel_id, infohash, langCode)
+    
+    # channel_id and infohash are binary versions
+    
+    assert utilities.validPermid(channel_id), \
+        "Invalid channel_id %s" % utilities.show_permid_short(channel_id)
+    assert utilities.validInfohash(infohash), \
+        "Invalid infohash %s" % bin2str(infohash)
+    assert LanguagesProvider.getLanguagesInstance().isLangCodeSupported(langCode), \
+        "Unsupported language code %s" % langCode
+        
+    hasher = sha()
+    for data in (channel_id, infohash, langCode):
+        hasher.update(data)
+    subtitleName = hasher.hexdigest() + SUBS_EXTENSION
+        
+    return subtitleName
+
+
diff --git a/instrumentation/next-share/BaseLib/Core/Subtitles/SubtitlesSupport.py b/instrumentation/next-share/BaseLib/Core/Subtitles/SubtitlesSupport.py
new file mode 100644
index 0000000..7255f4e
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Subtitles/SubtitlesSupport.py
@@ -0,0 +1,458 @@
+# Written by Andrea Reale
+# see LICENSE.txt for license information
+
+
+from BaseLib.Core.Subtitles.MetadataDomainObjects.Languages import \
+    LanguagesProvider
+from BaseLib.Core.Subtitles.MetadataDomainObjects.MetadataDTO import MetadataDTO
+from BaseLib.Core.Subtitles.MetadataDomainObjects.MetadataExceptions import \
+    RichMetadataException
+from BaseLib.Core.Subtitles.MetadataDomainObjects.SubtitleInfo import SubtitleInfo
+from BaseLib.Core.Utilities import utilities
+from BaseLib.Core.Utilities.utilities import isValidPermid, bin2str
+import sys
+import threading
+
+DEBUG = False
+
+
+class SubtitlesSupport(object):
+    '''
+    Subtitle dissemination system facade.
+    
+    Acts as the only faced between the subtitle dissemination system and 
+    the GUI (or whoever needs to subtitles).
+    
+    Provides methods to query the subtitles database. Allows publishers to
+    add their own subtitles, and if necessary permits to retrieve the subtitle
+    remotely if not available.
+    '''
+
+    __single = None
+    _singletonLock = threading.RLock()
+    
+    def __init__(self):
+        
+        #singleton pattern not really enforced if someone just calls 
+        # the normal constructor. But this way I can test the instance easier
+        try:
+            SubtitlesSupport._singletonLock.acquire()
+            SubtitlesSupport.__single = self
+        finally:
+            SubtitlesSupport._singletonLock.release()
+            
+        self.richMetadata_db = None
+        self.subtitlesHandler = None
+        self.channelcast_db = None
+        self.langUtility = LanguagesProvider.getLanguagesInstance()
+        self._registered = False
+    
+    @staticmethod
+    def getInstance(*args, **kw):
+        try:
+            SubtitlesSupport._singletonLock.acquire()
+            if SubtitlesSupport.__single == None:
+                SubtitlesSupport(*args, **kw)
+        finally:
+            SubtitlesSupport._singletonLock.release()
+        
+        return SubtitlesSupport.__single
+        
+    def _register(self, richMetadataDBHandler, subtitlesHandler,
+                 channelcast_db, my_permid, my_keypair, peersHaveManger,
+                 ol_bridge):
+        assert richMetadataDBHandler is not None
+        assert subtitlesHandler is not None
+        assert channelcast_db is not None
+        assert peersHaveManger is not None
+        assert ol_bridge is not None
+        assert isValidPermid(my_permid)
+        
+        self.richMetadata_db = richMetadataDBHandler
+        self.subtitlesHandler = subtitlesHandler
+        self.channelcast_db = channelcast_db
+        self.my_permid = my_permid
+        self.my_keypair = my_keypair
+        self._peersHaveManager = peersHaveManger
+        #used to decouple calls to SubtitleHandler
+        self._ol_bridge = ol_bridge
+        self._registered = True
+        
+    
+    def getSubtileInfosForInfohash(self, infohash):
+        '''
+        Retrieve available information about subtitles for the given infohash.
+        
+        Given the infohash of a .torrent, retrieves every
+        information about subtitles published for that .torrent that is
+        currently available in the DB. 
+        
+        @param infohash: a .torrent infohash (binary)
+        @return: a dictionary. The dictionary looks like this::
+                { 
+                  channel_id1 : {langCode : L{SubtitleInfo}, ...} ,
+                  channel_id2 : {langCode : L{SubtitleInfo}, ... },
+                  ...
+                } 
+            Each entry in the dictionary has the following semantics:
+                - channel_id is the permid identifiying the channel (binary).
+                - langCode is an ISO 693-2 three characters language code
+        '''
+        assert utilities.isValidInfohash(infohash)
+        assert self._registered, "Instance is not registered"
+        
+        returnDictionary = dict()
+        
+        #a metadataDTO corrisponds to all metadata for a pair channel, infohash
+        metadataDTOs = self.richMetadata_db.getAllMetadataForInfohash(infohash)
+        
+        for metadataDTO in metadataDTOs:
+            channel = metadataDTO.channel
+            subtitles = metadataDTO.getAllSubtitles()
+            if len(subtitles) > 0 :
+                returnDictionary[channel] = subtitles
+        
+        return returnDictionary
+        
+    
+    
+    def getSubtitleInfos(self, channel, infohash):
+        '''
+        Retrieve subtitles information for the given channel-infohash pair.
+        
+        Searches in the local database for information about subtitles that
+        are currently availabe.
+        
+        @param channel: the channel_id (perm_id) of a channel (binary)
+        @param infohash: a .torrent infohash (binary)
+        @return: a dictionary of SubtitleInfo instances. The keys are the 
+                language codes of the subtitles
+        '''
+        assert self._registered, "Instance is not registered"
+        metadataDTO = self.richMetadata_db.getMetadata(channel,infohash)
+        if metadataDTO is None:
+            #no results
+            return {}
+        else:
+            return metadataDTO.getAllSubtitles()
+        
+    
+    def publishSubtitle(self, infohash, lang, pathToSrtSubtitle):
+        '''
+        Allows an user to publish an srt subtitle file in his channel.
+        
+        Called by a channel owner this method inserts a new subtitle for
+        a torrent published in his channel. 
+        The method assumes that the torrent identified by the infohash
+        parameter is already in the channel, and that the parameter 
+        pathToSrtSubtitle points to an existing srt file on the local
+        filesystem.
+        If a subtitle for the same language was already associated to the 
+        specified infohash and channel, it will be overwritten.
+        After calling this method the newly inserted subtitle will be 
+        disseminated via Channelcast.
+        
+        @param infohash: the infohash of the torrent to associate the subtitle
+                         with, binary
+        @param lang: a 3 characters code for the language of the subtitle as
+                     specified in ISO 639-2. Currently just 32 language codes
+                     will be supported.
+        @param pathToSrtSubtitle: a path in the local filesystem to a subtitle
+                                  in srt format.
+        
+        @raise RichMetadataException: if something "general" goes wrong while
+                                      adding new metadata
+        @raise IOError: if disk related problems occur
+        '''
+        assert utilities.isValidInfohash(infohash), "Invalid Infohash"
+        assert lang is not None and self.langUtility.isLangCodeSupported(lang)
+        assert self._registered, "Instance is not registered"
+  
+        channelid = bin2str(self.my_permid)
+        base64infohash = bin2str(infohash)
+        # consisnstency check: I want to assure that this method is called
+        # for an item that is actually in my channel
+        consinstent = self.channelcast_db.isItemInChannel(channelid,base64infohash)
+        
+        if not consinstent:
+            msg = "Infohash %s not found in my channel. Rejecting subtitle" \
+                    % base64infohash
+            if DEBUG:
+                print >> sys.stderr, msg
+            raise RichMetadataException(msg)
+        
+        try:
+        
+            filepath = \
+                self.subtitlesHandler.copyToSubtitlesFolder(pathToSrtSubtitle,
+                                                            self.my_permid,infohash,
+                                                            lang)   
+        except Exception,e:
+            if DEBUG:
+                print >> sys.stderr, "Failed to read and copy subtitle to appropriate folder: %s" % str(e)
+
+
+        
+        # retrieve existing metadata from my channel, infoahash
+        metadataDTO = self.richMetadata_db.getMetadata(self.my_permid, infohash)
+        # can be none if no metadata was available
+        if metadataDTO is None:
+            metadataDTO = MetadataDTO(self.my_permid, infohash)
+        else:
+            #update the timestamp
+            metadataDTO.resetTimestamp()
+        
+        newSubtitle = SubtitleInfo(lang, filepath)
+        
+        # this check should be redundant, since i should be sure that subtitle
+        # exists at this point
+        if newSubtitle.subtitleExists():
+            newSubtitle.computeChecksum()
+        else:
+            msg = "Inconsistency found. The subtitle was"\
+                                        "not published"
+            if DEBUG:
+                print >> sys.stderr, msg
+            raise RichMetadataException(msg)
+        
+        metadataDTO.addSubtitle(newSubtitle)
+        metadataDTO.sign(self.my_keypair)
+        
+        #channelid is my permid. I received the metadata from myself
+        self.richMetadata_db.insertMetadata(metadataDTO)
+        
+
+
+    def retrieveSubtitleContent(self, channel, infohash, subtitleInfo, callback = None):
+        '''
+        Retrieves the actual subtitle file from a remote peer.
+        
+        If not already locally available this function tries to retrieve the
+        actual subtitle content from remote peers. The parameter subtitleInfo
+        describes the subtitle to retrieve the content for.
+        
+        A callback can be provided. It will be called by the
+        OLThread once the actual subtitle is available, or never
+        in case of failure. 
+        The callback function should have exactly one parameter that will
+        be bound to a new SubtitleInfo instance, with the path field updated
+        to the path where the downloaded subtitle resides.
+        
+        Usually this method should be called when the value of 
+        subtitleInfo.path is None, meaning that the subtitle of the content
+        is not available locally. If subtitleInfo.path is not None, tha path
+        will be checked for validity and in case it is not valid the method
+        will try to fetch a new subtitle. If it points to a valid subtitle
+        with the correct checksum, nothing will be done and the user callback
+        will be immediately scheduled.
+        
+        The current implementation queries for subtitle up to 5 peers
+        ithat manifested the availability for that subtitle through channelcast.
+        The requests are sent in parallel but only the first response is 
+        considered.
+        
+        @param channel: the channel where the subtitle was published. (binary channel_id)
+        
+        @param infohash: the infohash of the item we want to retrieve the
+                         subtitle for. (binary)
+        
+        @param subtitleInfo: an intance of SubtitleInfo describing the
+                             subtitle to be downloaded
+                             
+        @param callback: a function that will be called when the subtitle is 
+                         succesfully retrieved. See the description for
+                         further details. If None nothing will be called.
+        '''
+        assert self._registered, "Instance is not registered"
+        assert subtitleInfo.checksum is not None , "Cannot retrieve a subtitle"\
+            "whose checksum is not known"
+        
+        if subtitleInfo.subtitleExists():
+            if subtitleInfo.verifyChecksum():
+                #subtitle is available call the callback
+                callback(subtitleInfo)
+                return
+            else:
+                #delete the existing subtitle and ask for a new
+                #one
+                if DEBUG:
+                    print >> sys.stderr, "Subtitle is locally available but has invalid" \
+                          "checksum. Issuing another download"
+                subtitleInfo.path = None
+        
+
+        languages = [subtitleInfo.lang]
+        
+        def call_me_when_subtitle_arrives(listOfLanguages):
+            if callback is not None:
+                #since this was a request for a single subtitle
+                assert len(listOfLanguages) == 1
+                
+                #retrieve the updated info from the db
+                sub = self.richMetadata_db.getSubtitle(channel,infohash,
+                                                       listOfLanguages[0])
+                
+                #call the user callback
+                
+                callback(sub)
+            
+            
+        self._queryPeersForSubtitles(channel, infohash, languages,
+                                             call_me_when_subtitle_arrives)
+
+        
+    
+    def retrieveMultipleSubtitleContents(self, channel, infohash, listOfSubInfos, callback=None):
+        '''
+        Query remote peers of severela subtitles given the infohash
+        of the torrent they refer to, and the channel_id of the channel
+        they where published in.
+        
+        @param channel: channel_id (permid) of the channel where the subtitles where published
+                        (binary)
+        @param infohash: infohash of the torrent the subtitles are associated to (binary)
+        @param listOfSubInfos: a list of SubtitleInfo instances, specifing the subtitles to
+                               retrieve
+                               
+        @param callback: a callback function that will be called whenever any of the requested
+                        subtitles are retrieved. The function may be called multiple times
+                        if different requested subtitles arrive at different times, but it is
+                        guaranteed that it will be called at most once for each different
+                        subtitle arrived.
+                        The function MUST have one parameter, that will be bound to a list
+                        of updated SubtitleInfo s, reflecting the subtitles that have been
+                        received
+        
+        @rtype: None
+        @return:  always None
+        '''
+        assert self._registered, "Instance is not registered"
+        
+        languages = []
+        locallyAvailableSubs = []
+        for subtitleInfo in listOfSubInfos:
+            if subtitleInfo.checksum is None:
+                if DEBUG:
+                    print >> sys.stderr, "No checksum for subtitle %s. Skipping it in the request"\
+                        % subtitleInfo
+                continue
+            
+            if subtitleInfo.subtitleExists():
+                if subtitleInfo.verifyChecksum():
+                    #subtitle is available call the callback
+                    locallyAvailableSubs.append(subtitleInfo)
+                    continue
+                else:
+                    #delete the existing subtitle and ask for a new
+                    #one
+                    if DEBUG:
+                        print >> sys.stderr, "Subtitle is locally available but has invalid" \
+                              "checksum. Issuing another download"
+                    subtitleInfo.path = None
+                    
+            languages.append(subtitleInfo.lang)
+        
+            
+        if len(locallyAvailableSubs) > 0 and callback is not None:
+            callback(locallyAvailableSubs)
+        
+        def call_me_when_subtitles_arrive(listOfLanguages):
+            if callback is not None:
+                assert len(listOfLanguages) > 0 
+                
+                subInfos = list()
+                
+                #better to perform a single read from the db
+                allSubtitles = self.richMetadata_db.getAllSubtitles(channel,infohash)
+                for lang in listOfLanguages:
+                    subInfos.append(allSubtitles[lang])
+    
+                callback(subInfos)
+        
+        if len(languages) > 0:
+            self._queryPeersForSubtitles(channel, infohash, languages,
+                                                  call_me_when_subtitles_arrive)
+            
+        
+       
+    def _queryPeersForSubtitles(self, channel, infohash, languages, callback):
+        '''
+        Queries remote peers for subtitle contents  specified by 'infohash' 
+        published in a channel identified by 'channel' in the languages specified
+        by the languages list. 
+        Once any of theses subtitles arrive callback is called.
+        NOTE: calls send() on the OverlayThreadingBridge 
+        
+        @param channel: the channel_id of the channel were the subtitles to retrieve
+                        were published (binary string)
+        @param infohash: the infohash of a torrent to whom the subtitles to retrieve
+                        refer (binary string)
+        @param languages: a list of language codes (see Languages.py) for the subtitles
+                          contents to retrieve
+        @param callback: a callback function that will be called when some (or all) of the
+                         requested subtitles are received. The provided function must
+                         accept one parameter, that will be bound to a list of language codes
+                         corresponding to the languages of the subtitles that were received.
+                         Notice that if subtitles for different languages are received at multiple
+                         times, the callback my be called multiple times. Notice also
+                         that the callback will be called at most once for each of the requested
+                         languages.
+        '''
+        
+        def task():
+            bitmask  = self.langUtility.langCodesToMask(languages)
+            
+            if not bitmask > 0:
+                if DEBUG:
+                    print >> sys.stderr, "Will not send a request for 0 subtitles"
+                return
+                
+            peers_to_query = self._peersHaveManager.getPeersHaving(channel, infohash, bitmask)
+            
+            assert len(peers_to_query) > 0, "Consistency error: there should always be some result"
+        
+            
+            #ask up to 5 peers for the same subtitle. The callback will be called only at the
+            # first received response (the others should be dropped)
+            for peer in peers_to_query:
+                self.subtitlesHandler.sendSubtitleRequest(peer, channel, infohash,
+                                                                   languages, callback)
+        
+        self._ol_bridge.add_task(task)
+        
+            
+            
+    
+        
+        
+        
+            
+    def runDBConsinstencyRoutine(self):
+        '''
+        Clean the database from incorrect data.
+        
+        Checks the databases for the paths of subtitles presumably locally available.
+        If those subtitles are not really available at the given path, updates
+        the database in a consistent way.
+        '''
+        result = self.richMetadata_db.getAllLocalSubtitles()
+        
+        for channel in result:
+            for infohash in result[channel]:
+                for subInfo in result[channel][infohash]:
+                    if not subInfo.subtitleExists():
+                        #If a subtitle published by me was removed delete the whole entry
+                        if channel == self.my_permid:
+                            metadataDTO = self.richMetadata_db.getMetadata(channel,infohash)
+                            metadataDTO.removeSubtitle(subInfo.lang)
+                            metadataDTO.sign(self.my_keypair)
+                            self.richMetadata_db.insertMetadata(metadataDTO)
+                        #otherwise just set the path to none
+                        else:
+                            self.richMetadata_db.updateSubtitlePath(channel, infohash, subInfo.lang,None)
+            
+        
+        
+                
+        
+        
diff --git a/instrumentation/next-share/BaseLib/Core/Subtitles/__init__.py b/instrumentation/next-share/BaseLib/Core/Subtitles/__init__.py
new file mode 100644
index 0000000..f99ac72
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Subtitles/__init__.py
@@ -0,0 +1,3 @@
+# Written by Andrea Reale
+# see LICENSE.txt for license information
+
diff --git a/instrumentation/next-share/BaseLib/Core/TorrentDef.py b/instrumentation/next-share/BaseLib/Core/TorrentDef.py
new file mode 100644
index 0000000..dea40a1
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/TorrentDef.py
@@ -0,0 +1,1110 @@
+# Written by Arno Bakker 
+# see LICENSE.txt for license information
+""" Definition of a torrent, that is, a collection of files or a live stream. """
+import sys
+import os
+import copy
+import math
+from traceback import print_exc,print_stack
+from types import StringType,ListType,IntType,LongType
+
+import BaseLib
+from BaseLib.Core.simpledefs import *
+from BaseLib.Core.defaults import *
+from BaseLib.Core.exceptions import *
+from BaseLib.Core.Base import *
+from BaseLib.Core.BitTornado.bencode import bencode,bdecode
+import BaseLib.Core.APIImplementation.maketorrent as maketorrent
+import BaseLib.Core.APIImplementation.makeurl as makeurl
+from BaseLib.Core.APIImplementation.miscutils import *
+
+from BaseLib.Core.Utilities.utilities import validTorrentFile,isValidURL
+from BaseLib.Core.Utilities.unicode import dunno2unicode
+from BaseLib.Core.Utilities.timeouturlopen import urlOpenTimeout
+from BaseLib.Core.osutils import *
+from BaseLib.Core.Utilities.Crypto import sha
+
+from BaseLib.Core.ClosedSwarm import ClosedSwarm
+from BaseLib.Core.DecentralizedTracking.MagnetLink.MagnetLink import MagnetLink
+
+class TorrentDef(Serializable,Copyable):
+    """
+    Definition of a torrent, that is, all params required for a torrent file,
+    plus optional params such as thumbnail, playtime, etc.
+    
+    Note: to add fields to the torrent definition which are not supported
+    by its API, first create the torrent def, finalize it, then add the
+    fields to the metainfo, and create a new torrent def from that
+    upgraded metainfo using TorrentDef.load_from_dict()
+    
+    This class can also be used to create P2P URLs, by calling set_url_compat()
+    before finalizing. In that case only name, piece length, tracker, bitrate 
+    and source-authentication parameters (for live) are configurable. 
+
+    cf. libtorrent torrent_info
+    """
+    def __init__(self,input=None,metainfo=None,infohash=None):
+        """ Normal constructor for TorrentDef (The input, metainfo and infohash
+        parameters are used internally to make this a copy constructor) """
+        assert infohash is None or isinstance(infohash, str), "INFOHASH has invalid type: %s" % type(infohash)
+        assert infohash is None or len(infohash) == INFOHASH_LENGTH, "INFOHASH has invalid length: %d" % len(infohash)
+        self.readonly = False
+        if input is not None: # copy constructor
+            self.input = input
+            # self.metainfo_valid set in copy() 
+            self.metainfo = metainfo
+            self.infohash = infohash
+            return
+        
+        self.input = {} # fields added by user, waiting to be turned into torrent file
+        # Define the built-in default here
+        self.input.update(tdefdefaults)
+        try:
+            self.input['encoding'] = sys.getfilesystemencoding()
+        except:
+            self.input['encoding'] = sys.getdefaultencoding()
+
+        self.input['files'] = []
+
+        self.metainfo_valid = False
+        self.metainfo = None # copy of loaded or last saved torrent dict
+        self.infohash = None # only valid if metainfo_valid
+        
+        
+        # We cannot set a built-in default for a tracker here, as it depends on
+        # a Session. Alternatively, the tracker will be set to the internal
+        # tracker by default when Session::start_download() is called, if the
+        # 'announce' field is the empty string.
+
+    #
+    # Class methods for creating a TorrentDef from a .torrent file
+    #
+    def load(filename):
+        """
+        Load a BT .torrent or Tribler .tribe file from disk and convert
+        it into a finalized TorrentDef.
+        
+        @param filename  An absolute Unicode filename
+        @return TorrentDef
+        """
+        # Class method, no locking required
+        f = open(filename,"rb")
+        return TorrentDef._read(f)
+    load = staticmethod(load)
+        
+    def _read(stream):
+        """ Internal class method that reads a torrent file from stream,
+        checks it for correctness and sets self.input and self.metainfo
+        accordingly. """
+        bdata = stream.read()
+        stream.close()
+        data = bdecode(bdata)
+        #print >>sys.stderr,data
+        return TorrentDef._create(data)
+    _read = staticmethod(_read)
+        
+    def _create(metainfo): # TODO: replace with constructor
+        # raises ValueErrors if not good
+        validTorrentFile(metainfo) 
+        
+        t = TorrentDef()
+        t.metainfo = metainfo
+        t.metainfo_valid = True
+        # copy stuff into self.input
+        maketorrent.copy_metainfo_to_input(t.metainfo,t.input)
+
+        # For testing EXISTING LIVE, or EXISTING MERKLE: DISABLE, i.e. keep true infohash
+        if t.get_url_compat():
+            t.infohash = makeurl.metainfo2swarmid(t.metainfo)
+        else:
+            # Two places where infohash calculated, here and in maketorrent.py
+            # Elsewhere: must use TorrentDef.get_infohash() to allow P2PURLs.
+            t.infohash = sha(bencode(metainfo['info'])).digest()
+
+        assert isinstance(t.infohash, str), "INFOHASH has invalid type: %s" % type(t.infohash)
+        assert len(t.infohash) == INFOHASH_LENGTH, "INFOHASH has invalid length: %d" % len(t.infohash)
+        
+        #print >>sys.stderr,"INFOHASH",`t.infohash`
+
+        return t
+    
+    _create = staticmethod(_create)
+
+    @staticmethod
+    def retrieve_from_magnet(url, callback):
+        """
+        If the URL conforms to a magnet link, the .torrent info is
+        downloaded and converted into a TorrentDef.  The resulting
+        TorrentDef is provided through CALLBACK.
+
+        Returns True when attempting to obtain the TorrentDef, in this
+        case CALLBACK will always be called.  Otherwise False is
+        returned, in this case CALLBACK will not be called.
+        
+        The thread making the callback should be used very briefly. 
+        """
+        assert isinstance(url, str), "URL has invalid type: %s" % type(url)
+        assert callable(callback), "CALLBACK must be callable"
+        def metainfo_retrieved(metadata):
+            tdef = TorrentDef.load_from_dict(metadata)
+            callback(tdef)
+
+        try:
+            magnet_link = MagnetLink(url, metainfo_retrieved)
+            return magnet_link.retrieve()
+        except:
+            # malformed url
+            return False
+
+    def load_from_url(url):
+        """
+        If the URL starts with 'http:' load a BT .torrent or Tribler .tstream
+        file from the URL and convert it into a TorrentDef. If the URL starts
+        with our URL scheme, we convert the URL to a URL-compatible TorrentDef.
+        
+        @param url URL
+        @return TorrentDef.
+        """
+        # Class method, no locking required
+        if url.startswith(P2PURL_SCHEME):
+            (metainfo,swarmid) = makeurl.p2purl2metainfo(url)
+
+            # Metainfo created from URL, so create URL compatible TorrentDef.
+            metainfo['info']['url-compat'] = 1
+
+            # For testing EXISTING LIVE: ENABLE, for old EXISTING MERKLE: DISABLE 
+            #metainfo['info']['name.utf-8'] = metainfo['info']['name'] 
+
+            t = TorrentDef._create(metainfo)
+            
+            return t
+        else:
+            f = urlOpenTimeout(url)
+            return TorrentDef._read(f)
+    load_from_url = staticmethod(load_from_url)
+
+
+    def load_from_dict(metainfo):
+        """
+        Load a BT .torrent or Tribler .tribe file from the metainfo dictionary
+        it into a TorrentDef
+        
+        @param metainfo A dictionary following the BT torrent file spec.
+        @return TorrentDef.
+        """
+        # Class method, no locking required
+        return TorrentDef._create(metainfo)
+    load_from_dict = staticmethod(load_from_dict)
+
+    
+    #
+    # Convenience instance methods for publishing new content
+    #
+    def add_content(self,inpath,outpath=None,playtime=None):
+        """
+        Add a file or directory to this torrent definition. When adding a
+        directory, all files in that directory will be added to the torrent.
+        
+        One can add multiple files and directories to a torrent definition.
+        In that case the "outpath" parameter must be used to indicate how
+        the files/dirs should be named in the torrent. The outpaths used must
+        start with a common prefix which will become the "name" field of the
+        torrent. 
+
+        To seed the torrent via the core (as opposed to e.g. HTTP) you will
+        need to start the download with the dest_dir set to the top-level
+        directory containing the files and directories to seed. For example,
+        a file "c:\Videos\file.avi" is seeded as follows:
+        
+            tdef = TorrentDef()
+            tdef.add_content("c:\Videos\file.avi",playtime="1:59:20")
+            tdef.set_tracker(s.get_internal_tracker_url())
+            tdef.finalize()
+            dscfg = DownloadStartupConfig()
+            dscfg.set_dest_dir("c:\Video")
+            s.start_download(tdef,dscfg)
+        

+        @param inpath Absolute name of file or directory on local filesystem, 
+        as Unicode string.
+        @param outpath (optional) Name of the content to use in the torrent def
+        as Unicode string.
+        @param playtime (optional) String representing the duration of the 
+        multimedia file when played, in [hh:]mm:ss format. 
+        """
+        if self.readonly:
+            raise OperationNotPossibleAtRuntimeException()
+        
+        s = os.stat(inpath)
+        d = {'inpath':inpath,'outpath':outpath,'playtime':playtime,'length':s.st_size}
+        self.input['files'].append(d)
+        
+        self.metainfo_valid = False
+
+
+    def remove_content(self,inpath):
+        """ Remove a file or directory from this torrent definition 
+
+        @param inpath Absolute name of file or directory on local filesystem, 
+        as Unicode string.
+        """
+        if self.readonly:
+            raise OperationNotPossibleAtRuntimeException()
+        
+        for d in self.input['files']:
+            if d['inpath'] == inpath:
+                self.input['files'].remove(d)
+                break
+
+    def create_live(self,name,bitrate,playtime="1:00:00",authconfig=None):
+        """ Create a live streaming multimedia torrent with a specific bitrate.
+        
+        The authconfig is a subclass LiveSourceAuthConfig with the key 
+        information required to allow authentication of packets from the source,
+        or None. In the latter case there is no source authentication. The other
+        two legal values are:
+        
+        * An instance of ECDSALiveSourceAuthConfig. 
+        * An Instance of RSALiveSourceAuthConfig.
+        

+        When using the ECDSA method, a sequence number, real-time timestamp and 
+        an ECDSA signature of 64 bytes is put in each piece. As a result, the 
+        content in each packet is get_piece_length()-81, so that this into 
+        account when selecting the bitrate.
+        
+        When using the RSA method, a sequence number, real-time timestamp and
+        a RSA signature of keysize/8 bytes is put in each piece.
+        
+        The info from the authconfig is stored in the 'info' part of the 
+        torrent file when finalized, so changing the authentication info changes
+        the identity (infohash) of the torrent.
+         
+        @param name The name of the stream.
+        @param bitrate The desired bitrate in bytes per second.
+        @param playtime The virtual playtime of the stream as a string in 
+        [hh:]mm:ss format.
+        @param authconfig Parameters for the authentication of the source
+        """
+        self.input['bps'] = bitrate
+        self.input['playtime'] = playtime # size of virtual content 
+
+        # For source auth
+        authparams = {}
+        if authconfig is None:
+            authparams['authmethod'] = LIVE_AUTHMETHOD_NONE
+        else: 
+            authparams['authmethod'] = authconfig.get_method()
+            authparams['pubkey'] = authconfig.get_pubkey()
+
+        self.input['live'] = authparams 
+
+        d = {'inpath':name,'outpath':None,'playtime':None,'length':None}
+        self.input['files'].append(d)
+
+    #
+    # Torrent attributes
+    #
+    def set_encoding(self,enc):
+        """ Set the character encoding for e.g. the 'name' field """
+        self.input['encoding'] = enc
+        self.metainfo_valid = False
+        
+    def get_encoding(self):
+        return self.input['encoding']
+
+    def set_thumbnail(self,thumbfilename):
+        """
+        Reads image from file and turns it into a torrent thumbnail
+        The file should contain an image in JPEG format, preferably 171x96.
+        
+        @param thumbfilename Absolute name of image file, as Unicode string.
+        """
+        if self.readonly:
+            raise OperationNotPossibleAtRuntimeException()
+        
+        f = open(thumbfilename,"rb")
+        data = f.read()
+        f.close()
+        self.input['thumb'] = data 
+        self.metainfo_valid = False
+
+
+    def get_thumbnail(self):
+        """ Returns (MIME type,thumbnail data) if present or (None,None)
+        @return A tuple. """
+        if 'thumb' not in self.input or self.input['thumb'] is None:
+            return (None,None)
+        else:
+            thumb = self.input['thumb'] # buffer/string immutable
+            return ('image/jpeg',thumb)
+
+    def set_tracker(self,url):
+        """ Sets the tracker (i.e. the torrent file's 'announce' field). 
+        @param url The announce URL.
+        """
+        if self.readonly:
+            raise OperationNotPossibleAtRuntimeException()
+
+        if not isValidURL(url):
+            raise ValueError("Invalid URL")
+        
+        if url.endswith('/'):
+            # Some tracker code can't deal with / at end
+            url = url[:-1]
+        self.input['announce'] = url 
+        self.metainfo_valid = False
+
+    def get_tracker(self):
+        """ Returns the announce URL.
+        @return URL """
+        return self.input['announce']
+
+    def set_tracker_hierarchy(self,hier):
+        """ Set hierarchy of trackers (announce-list) following the spec
+        at http://www.bittornado.com/docs/multitracker-spec.txt
+        @param hier A hierarchy of trackers as a list of lists.
+        """
+        if self.readonly:
+            raise OperationNotPossibleAtRuntimeException()
+
+        # TODO: check input, in particular remove / at end
+        newhier = []
+        if type(hier) != ListType:
+            raise ValueError("hierarchy is not a list")
+        for tier in hier:
+            if type(tier) != ListType:
+                raise ValueError("tier is not a list")
+            newtier = []
+            for url in tier:
+                if not isValidURL(url):
+                    raise ValueError("Invalid URL: "+`url`)
+                
+                if url.endswith('/'):
+                    # Some tracker code can't deal with / at end
+                    url = url[:-1]
+                newtier.append(url)
+            newhier.append(newtier)
+
+        self.input['announce-list'] = newhier
+        self.metainfo_valid = False
+
+    def get_tracker_hierarchy(self):
+        """ Returns the hierarchy of trackers.
+        @return A list of lists. """
+        return self.input['announce-list']
+
+    def set_dht_nodes(self,nodes):
+        """ Sets the DHT nodes required by the mainline DHT support,
+        See http://www.bittorrent.org/beps/bep_0005.html
+        @param nodes A list of [hostname,port] lists.
+        """
+        if self.readonly:
+            raise OperationNotPossibleAtRuntimeException()
+
+        # Check input
+        if type(nodes) != ListType:
+            raise ValueError("nodes not a list")
+        else:
+            for node in nodes:
+                if type(node) != ListType and len(node) != 2:
+                    raise ValueError("node in nodes not a 2-item list: "+`node`)
+                if type(node[0]) != StringType:
+                    raise ValueError("host in node is not string:"+`node`)
+                if type(node[1]) != IntType:
+                    raise ValueError("port in node is not int:"+`node`)
+                
+        self.input['nodes'] = nodes
+        self.metainfo_valid = False 
+
+    def get_dht_nodes(self):
+        """ Returns the DHT nodes set. 
+        @return A list of [hostname,port] lists. """
+        return self.input['nodes']
+        
+    def set_comment(self,value):
+        """ Set comment field.
+        @param value A Unicode string.
+         """
+        if self.readonly:
+            raise OperationNotPossibleAtRuntimeException()
+
+        self.input['comment'] = value
+        self.metainfo_valid = False
+
+    def get_comment(self):
+        """ Returns the comment field of the def.
+        @return A Unicode string. """
+        return self.input['comment']
+
+    def get_comment_as_unicode(self):
+        """ Returns the comment field of the def as a unicode string.
+        @return A Unicode string. """
+        return dunno2unicode(self.input['comment'])
+
+    def set_created_by(self,value):
+        """ Set 'created by' field.
+        @param value A Unicode string.
+        """
+        if self.readonly:
+            raise OperationNotPossibleAtRuntimeException()
+
+        self.input['created by'] = value
+        self.metainfo_valid = False
+
+    def get_created_by(self):
+        """ Returns the 'created by' field.
+        @return Unicode string. """
+        return self.input['created by']
+
+    def set_urllist(self,value):
+        """ Set list of HTTP seeds following the BEP 19 spec (GetRight style):
+        http://www.bittorrent.org/beps/bep_0019.html
+        @param value A list of URLs.
+        """
+        if self.readonly:
+            raise OperationNotPossibleAtRuntimeException()
+
+        for url in value:
+            if not isValidURL(url):
+                raise ValueError("Invalid URL: "+`url`)
+
+        self.input['url-list'] = value
+        self.metainfo_valid = False
+
+    def get_urllist(self):
+        """ Returns the list of HTTP seeds.
+        @return A list of URLs. """
+        return self.input['url-list']
+
+    def set_httpseeds(self,value):
+        """ Set list of HTTP seeds following the BEP 17 spec (John Hoffman style):
+        http://www.bittorrent.org/beps/bep_0017.html
+        @param value A list of URLs.
+        """
+        if self.readonly:
+            raise OperationNotPossibleAtRuntimeException()
+
+        for url in value:
+            if not isValidURL(url):
+                raise ValueError("Invalid URL: "+`url`)
+
+        self.input['httpseeds'] = value
+        self.metainfo_valid = False
+
+    def get_httpseeds(self):
+        """ Returns the list of HTTP seeds.
+        @return A list of URLs. """
+        return self.input['httpseeds']
+
+    def set_piece_length(self,value):
+        """ Set the size of the pieces in which the content is traded. 
+        The piece size must be a multiple of the chunk size, the unit in which
+        it is transmitted, which is 16K by default (see 
+        DownloadConfig.set_download_slice_size()). The default is automatic 
+        (value 0).
+        @param value A number of bytes as per the text.
+        """
+        if self.readonly:
+            raise OperationNotPossibleAtRuntimeException()
+
+        if not (type(value) == IntType or type(value) == LongType):
+            raise ValueError("Piece length not an int/long")
+
+        self.input['piece length'] = value
+        self.metainfo_valid = False
+
+    def get_piece_length(self):
+        """ Returns the piece size.
+        @return A number of bytes. """
+        return self.input['piece length']
+
+    #
+    # ClosedSwarm fields
+    #
+    def set_cs_keys(self, keys):
+        """ Keys is a list of DER encoded keys
+        """
+        self.input['cs_keys'] = ",".join(keys)
+
+    def get_cs_keys_as_ders(self):
+        """Returns a list of DER encoded keys
+        @return A list of DER encoded keys or [] if not a CS
+        """
+        if 'cs_keys' in self.input and len(self.input['cs_keys']) > 0:
+            return self.input['cs_keys'].split(",")
+        return []
+            
+    def get_cs_keys(self):
+        """ Get the Closed swarm keys for this torrent.
+        @return A list of key objects ready to be used or [] if not a CS
+        """
+        if 'cs_keys' in self.input:
+            keys = self.input['cs_keys'].split(",")
+            
+            cs_keys = []
+            for key in keys:
+                k = ClosedSwarm.pubkey_from_der(key)
+                cs_keys.append(k)
+            return cs_keys
+        return []
+
+    def set_add_md5hash(self,value):
+        """ Whether to add an end-to-end MD5 checksum to the def.
+        @param value Boolean.
+        """
+        if self.readonly:
+            raise OperationNotPossibleAtRuntimeException()
+
+        self.input['makehash_md5'] = value
+        self.metainfo_valid = False
+
+    def get_add_md5hash(self):
+        """ Returns whether to add an MD5 checksum. """
+        return self.input['makehash_md5']
+
+    def set_add_crc32(self,value):
+        """ Whether to add an end-to-end CRC32 checksum to the def.
+        @param value Boolean.
+        """
+        if self.readonly:
+            raise OperationNotPossibleAtRuntimeException()
+
+        self.input['makehash_crc32'] = value
+        self.metainfo_valid = False
+
+    def get_add_crc32(self):
+        """ Returns whether to add an end-to-end CRC32 checksum to the def.
+        @return Boolean. """
+        return self.input['makehash_crc32']
+
+    def set_add_sha1hash(self,value):
+        """ Whether to add end-to-end SHA1 checksum to the def.
+        @param value Boolean.
+        """
+        if self.readonly:
+            raise OperationNotPossibleAtRuntimeException()
+
+        self.input['makehash_sha1'] = value
+        self.metainfo_valid = False
+
+    def get_add_sha1hash(self):
+        """ Returns whether to add an end-to-end SHA1 checksum to the def. 
+        @return Boolean."""
+        return self.input['makehash_sha1']
+    
+    def set_create_merkle_torrent(self,value):
+        """ Create a Merkle torrent instead of a regular BT torrent. A Merkle
+        torrent uses a hash tree for checking the integrity of the content
+        received. As such it creates much smaller torrent files than the
+        regular method. Tribler-specific feature."""
+        if self.readonly:
+            raise OperationNotPossibleAtRuntimeException()
+
+        self.input['createmerkletorrent'] = value
+        self.metainfo_valid = False
+
+    def get_create_merkle_torrent(self):
+        """ Returns whether to create a Merkle torrent.
+        @return Boolean. """
+        return self.input['createmerkletorrent']
+
+    def set_signature_keypair_filename(self,value):
+        """ Set absolute filename of keypair to be used for signature.
+        When set, a signature will be added.
+        @param value A filename containing an Elliptic Curve keypair.
+        """
+        if self.readonly:
+            raise OperationNotPossibleAtRuntimeException()
+
+        self.input['torrentsigkeypairfilename'] = value
+        self.metainfo_valid = False
+
+    def get_signature_keypair_filename(self):
+        """ Returns the filename containing the signing keypair or None.
+        @return Unicode String or None. """
+        return self.input['torrentsigkeypairfilename']
+
+    def get_live(self):
+        """ Returns whether this definition is for a live torrent.
+        @return Boolean. """
+        return bool('live' in self.input and self.input['live'])
+
+
+    def get_live_authmethod(self):
+        """ Returns the method for authenticating the source.
+        
+        LIVE_AUTHMETHOD_ECDSA
+        

+        @return String
+        """
+        return 'live' in self.input and self.input['live']['authmethod']
+
+    def get_live_pubkey(self):
+        """ Returns the public key used for authenticating packets from
+        the source.
+        @return A public key in DER.
+        """
+        if 'live' in self.input and 'pubkey' in self.input['live']:
+            return self.input['live']['pubkey']
+        else:
+            return None
+
+
+    def set_url_compat(self,value):
+        """ Set the URL compatible value for this definition. Only possible
+        for Merkle torrents and live torrents.
+        @param value Integer."""
+        
+        self.input['url-compat'] = value
+
+    def get_url_compat(self):
+        """ Returns whether this definition is URL compatible.
+        @return Boolean. """
+        return 'url-compat' in self.input and self.input['url-compat']
+        
+    #
+    # For P2P-transported Ogg streams
+    #
+    def set_live_ogg_headers(self,value):
+        if self.get_url_compat():
+            raise ValueError("Cannot use P2PURLs for Ogg streams")
+        self.input['ogg-headers'] = value
+
+
+    def get_live_ogg_headers(self):
+        if 'ogg-headers' in self.input:
+            return self.input['ogg-headers']
+        else:
+            return None
+        
+    def set_metadata(self,value):
+        """ Set the P2P-Next metadata 
+        @param value binary string """
+        
+        self.input['ns-metadata'] = value
+
+    def get_metadata(self):
+        """ Returns the stored P2P-Next metadata or None.
+        @return binary string. """
+        if 'ns-metadata' in self.input:
+            return self.input['ns-metadata']
+        else:
+            return None
+
+    def set_initial_peers(self,value):
+        """ Set the initial peers to connect to. 
+        @param value List of (IP,port) tuples """
+        self.input['initial peers'] = value
+
+    def get_initial_peers(self):
+        """ Returns the list of initial peers.
+        @return List of (IP,port) tuples. """
+        if 'initial peers' in self.input:
+            return self.input['initial peers']
+        else:
+            return []
+
+
+    def finalize(self,userabortflag=None,userprogresscallback=None):
+        """ Create BT torrent file by reading the files added with
+        add_content() and calculate the torrent file's infohash. 
+        
+        Creating the torrent file can take a long time and will be carried out
+        by the calling thread. The process can be made interruptable by passing 
+        a threading.Event() object via the userabortflag and setting it when 
+        the process should be aborted. The also optional userprogresscallback 
+        will be called by the calling thread periodically, with a progress 
+        percentage as argument.
+        
+        The userprogresscallback function will be called by the calling thread. 
+        
+        @param userabortflag threading.Event() object
+        @param userprogresscallback Function accepting a fraction as first
+        argument. 
+        """
+        if self.readonly:
+            raise OperationNotPossibleAtRuntimeException()
+        
+        if self.metainfo_valid:
+            return
+
+        if 'live' in self.input:
+            # Make sure the duration is an integral number of pieces, for
+            # security (live source auth).
+            secs = parse_playtime_to_secs(self.input['playtime'])
+            pl = float(self.get_piece_length())
+            length = float(self.input['bps']*secs)
+
+            if DEBUG:
+                print >>sys.stderr,"TorrentDef: finalize: length",length,"piecelen",pl
+            diff = length % pl
+            add = (pl - diff) % pl
+            newlen = int(length + add)
+
+                
+            #print >>sys.stderr,"CHECK INFO LENGTH",secs,newlen
+
+            d = self.input['files'][0]
+            d['length'] = newlen
+
+
+        # Note: reading of all files and calc of hashes is done by calling 
+        # thread.
+        
+        (infohash,metainfo) = maketorrent.make_torrent_file(self.input,userabortflag=userabortflag,userprogresscallback=userprogresscallback)
+        if infohash is not None:
+            
+            if self.get_url_compat():
+                url = makeurl.metainfo2p2purl(metainfo)
+                # Make sure metainfo is preserved, in particular, the url-compat field.
+                swarmid = makeurl.metainfo2swarmid(metainfo) 
+                self.infohash = swarmid
+            else:
+                self.infohash = infohash
+            self.metainfo = metainfo
+            
+            self.input['name'] = metainfo['info']['name']
+            # May have been 0, meaning auto.
+            self.input['piece length'] = metainfo['info']['piece length']
+            self.metainfo_valid = True
+
+        assert self.infohash is None or isinstance(self.infohash, str), "INFOHASH has invalid type: %s" % type(self.infohash)
+        assert self.infohash is None or len(self.infohash) == INFOHASH_LENGTH, "INFOHASH has invalid length: %d" % len(self.infohash)
+
+    def is_finalized(self):
+        """ Returns whether the TorrentDef is finalized or not.
+        @return Boolean. """
+        return self.metainfo_valid
+
+    #
+    # Operations on finalized TorrentDefs
+    #
+    def get_infohash(self):
+        """ Returns the infohash of the torrent, for non-URL compatible
+        torrents. Otherwise it returns the swarm identifier (either the root hash
+        (Merkle torrents) or hash of the live-source authentication key.
+        @return A string of length 20. """
+        if self.metainfo_valid:
+            return self.infohash
+        else:
+            raise TorrentDefNotFinalizedException()
+
+    def get_metainfo(self):
+        """ Returns the torrent definition as a dictionary that follows the BT
+        spec for torrent files. 
+        @return dict
+        """
+        if self.metainfo_valid:
+            return self.metainfo
+        else:
+            raise TorrentDefNotFinalizedException()
+
+    def get_name(self):
+        """ Returns the info['name'] field as raw string of bytes.
+        @return String """
+        if self.metainfo_valid:
+            return self.input['name'] # string immutable
+        else:
+            raise TorrentDefNotFinalizedException()
+
+    def set_name(self,name):
+        """ Set the name of this torrent
+        @param name name of torrent as String
+        """
+        if self.readonly:
+            raise OperationNotPossibleAtRuntimeException()
+
+        self.input['name'] = name
+        self.metainfo_valid = False
+
+        
+    def get_name_as_unicode(self):
+        """ Returns the info['name'] field as Unicode string.
+        @return Unicode string. """
+        if not self.metainfo_valid:
+            raise TorrentDefNotFinalizedException()
+
+        if "name.utf-8" in self.metainfo["info"]:
+            # There is an utf-8 encoded name.  We assume that it is
+            # correctly encoded and use it normally
+            try:
+                return unicode(self.metainfo["info"]["name.utf-8"], "UTF-8")
+            except UnicodeError:
+                pass
+
+        if "name" in self.metainfo["info"]:
+            # Try to use the 'encoding' field.  If it exists, it
+            # should contain something like 'utf-8'
+            if "encoding" in self.metainfo:
+                try:
+                    return unicode(self.metainfo["info"]["name"], self.metainfo["encoding"])
+                except UnicodeError:
+                    pass
+                except LookupError:
+                    # Some encodings are not supported by python.  For
+                    # instance, the MBCS codec which is used by
+                    # Windows is not supported (Jan 2010)
+                    pass
+
+            # Try to convert the names in path to unicode, without
+            # specifying the encoding
+            try:
+                return unicode(self.metainfo["info"]["name"])
+            except UnicodeError:
+                pass
+
+            # Try to convert the names in path to unicode, assuming
+            # that it was encoded as utf-8
+            try:
+                return unicode(self.metainfo["info"]["name"], "UTF-8")
+            except UnicodeError:
+                pass
+
+            # Convert the names in path to unicode by replacing out
+            # all characters that may -even remotely- cause problems
+            # with the '?' character
+            try:
+                def filter_characters(name):
+                    def filter_character(char):
+                        if 0 < ord(char) < 128:
+                            return char
+                        else:
+                            if DEBUG: print >> sys.stderr, "Bad character filter", ord(char), "isalnum?", char.isalnum()
+                            return u"?"
+                    return u"".join([filter_character(char) for char in name])
+                return unicode(filter_characters(self.metainfo["info"]["name"]))
+            except UnicodeError:
+                pass
+
+        # We failed.  Returning an empty string
+        return u""
+
+    def verify_torrent_signature(self):
+        """ Verify the signature on the finalized torrent definition. Returns
+        whether the signature was valid.
+        @return Boolean.
+        """
+        if self.metainfo_valid:
+            return BaseLib.Core.Overlay.permid.verify_torrent_signature(self.metainfo)
+        else:
+            raise TorrentDefNotFinalizedException()
+
+
+    def save(self,filename):
+        """
+        Finalizes the torrent def and writes a torrent file i.e., bencoded dict 
+        following BT spec) to the specified filename. Note this may take a
+        long time when the torrent def is not yet finalized.
+        
+        @param filename An absolute Unicode path name.
+        """
+        if not self.readonly:
+            self.finalize()
+
+        bdata = bencode(self.metainfo)
+        f = open(filename,"wb")
+        f.write(bdata)
+        f.close()
+
+
+    def get_bitrate(self,file=None):
+        """ Returns the bitrate of the specified file. If no file is specified, 
+        we assume this is a single-file torrent.
+        
+        @param file (Optional) the file in the torrent to retrieve the bitrate of.
+        @return The bitrate in bytes per second or None.
+        """ 
+        if not self.metainfo_valid:
+            raise NotYetImplementedException() # must save first
+
+        return maketorrent.get_bitrate_from_metainfo(file,self.metainfo)
+
+    def get_files_with_length(self,exts=None):
+        """ The list of files in the finalized torrent def.
+        @param exts (Optional) list of filename extensions (without leading .)
+        to search for.
+        @return A list of filenames.
+        """
+        return maketorrent.get_files(self.metainfo,exts)
+
+    def get_files(self,exts=None):
+        """ The list of files in the finalized torrent def.
+        @param exts (Optional) list of filename extensions (without leading .)
+        to search for.
+        @return A list of filenames.
+        """
+        return [filename for filename, _ in maketorrent.get_files(self.metainfo, exts)]
+
+    def _get_all_files_as_unicode_with_length(self):
+        """ Get a generator for files in the torrent def. No filtering
+        is possible and all tricks are allowed to obtain a unicode
+        list of filenames.
+        @return A unicode filename generator.
+        """
+        assert self.metainfo_valid, "TorrentDef is not finalized"
+        if "files" in self.metainfo["info"]:
+            # Multi-file torrent
+            join = os.path.join
+            files = self.metainfo["info"]["files"]
+
+            for file_dict in files:
+                if "path.utf-8" in file_dict:
+                    # This file has an utf-8 encoded list of elements.
+                    # We assume that it is correctly encoded and use
+                    # it normally
+                    try:
+                        yield join(*[unicode(element, "UTF-8") for element in file_dict["path.utf-8"]]), file_dict["length"]
+                    except UnicodeError:
+                        pass
+
+                if "path" in file_dict:
+                    # Try to use the 'encoding' field.  If it exists,
+                    # it should contain something like 'utf-8'
+                    if "encoding" in self.metainfo:
+                        encoding = self.metainfo["encoding"]
+                        try:
+                            yield join(*[unicode(element, encoding) for element in file_dict["path"]]), file_dict["length"]
+                        except UnicodeError:
+                            pass
+                        except LookupError:
+                            # Some encodings are not supported by
+                            # python.  For instance, the MBCS codec
+                            # which is used by Windows is not
+                            # supported (Jan 2010)
+                            pass
+
+                    # Try to convert the names in path to unicode,
+                    # without specifying the encoding
+                    try:
+                        yield join(*[unicode(element) for element in file_dict["path"]]), file_dict["length"]
+                    except UnicodeError:
+                        pass
+
+                    # Try to convert the names in path to unicode,
+                    # assuming that it was encoded as utf-8
+                    try:
+                        yield join(*[unicode(element, "UTF-8") for element in file_dict["path"]]), file_dict["length"]
+                    except UnicodeError:
+                        pass
+
+                    # Convert the names in path to unicode by
+                    # replacing out all characters that may -even
+                    # remotely- cause problems with the '?' character
+                    try:
+                        def filter_characters(name):
+                            def filter_character(char):
+                                if 0 < ord(char) < 128:
+                                    return char
+                                else:
+                                    if DEBUG: print >> sys.stderr, "Bad character filter", ord(char), "isalnum?", char.isalnum()
+                                    return u"?"
+                            return u"".join([filter_character(char) for char in name])
+                        yield join(*[unicode(filter_characters(element)) for element in file_dict["path"]]), file_dict["length"]
+                    except UnicodeError:
+                        pass
+
+        else:
+            # Single-file torrent
+            yield self.get_name_as_unicode(), self.metainfo["info"]["length"]
+        
+    def get_files_as_unicode_with_length(self,exts=None):
+        """ The list of files in the finalized torrent def.
+        @param exts (Optional) list of filename extensions (without leading .)
+        to search for.
+        @return A list of filenames.
+        """
+        if not self.metainfo_valid:
+            raise NotYetImplementedException() # must save first
+
+        videofiles = []
+        for filename, length in self._get_all_files_as_unicode_with_length():
+            prefix, ext = os.path.splitext(filename)
+            if ext != "" and ext[0] == ".":
+                ext = ext[1:]
+            if exts is None or ext.lower() in exts:
+                videofiles.append((filename, length))
+        return videofiles
+
+    def get_files_as_unicode(self,exts=None):
+        return [filename for filename, _ in self.get_files_as_unicode_with_length(exts)]
+
+    def get_length(self,selectedfiles=None):
+        """ Returns the total size of the content in the torrent. If the
+        optional selectedfiles argument is specified, the method returns
+        the total size of only those files.
+        @return A length (long)
+        """
+        if not self.metainfo_valid:
+            raise NotYetImplementedException() # must save first
+        
+        (length,filepieceranges) = maketorrent.get_length_filepieceranges_from_metainfo(self.metainfo,selectedfiles)
+        return length
+
+    def get_creation_date(self,default=0):
+        if not self.metainfo_valid:
+            raise NotYetImplementedException() # must save first
+
+        return self.metainfo.get("creation date", default)
+
+    def is_multifile_torrent(self):
+        """ Returns whether this TorrentDef is a multi-file torrent.
+        @return Boolean
+        """
+        if not self.metainfo_valid:
+            raise NotYetImplementedException() # must save first
+        
+        return 'files' in self.metainfo['info']
+
+
+
+    def is_merkle_torrent(self):
+        """ Returns whether this TorrentDef is a Merkle torrent. Use
+        get_create_merkle_torrent() to determine this before finalization.
+        @return Boolean """
+        if self.metainfo_valid:
+            return 'root hash' in self.metainfo['info']
+        else:
+            raise TorrentDefNotFinalizedException()
+
+    
+    def get_url(self):
+        """ Returns the URL representation of this TorrentDef. The TorrentDef
+        must be a Merkle or live torrent and must be set to URL-compatible 
+        before finalizing."""
+        
+        if self.metainfo_valid:
+            return makeurl.metainfo2p2purl(self.metainfo)
+        else:
+            raise TorrentDefNotFinalizedException()
+    
+    
+    #
+    # Internal methods
+    #
+    def get_index_of_file_in_files(self,file):
+        if not self.metainfo_valid:
+            raise NotYetImplementedException() # must save first
+
+        info = self.metainfo['info']
+
+        if file is not None and 'files' in info:
+            for i in range(len(info['files'])):
+                x = info['files'][i]
+                    
+                intorrentpath = maketorrent.pathlist2filename(x['path'])
+                if intorrentpath == file:
+                    return i
+            return ValueError("File not found in torrent")
+        else:
+            raise ValueError("File not found in single-file torrent")
+
+    #
+    # Copyable interface
+    # 
+    def copy(self):
+        input = copy.copy(self.input)
+        metainfo = copy.copy(self.metainfo)
+        infohash = self.infohash
+        t = TorrentDef(input,metainfo,infohash)
+        t.metainfo_valid = self.metainfo_valid
+        t.set_cs_keys(self.get_cs_keys_as_ders())
+        return t
diff --git a/instrumentation/next-share/BaseLib/Core/Utilities/Crypto.py b/instrumentation/next-share/BaseLib/Core/Utilities/Crypto.py
new file mode 100644
index 0000000..a66d6aa
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Utilities/Crypto.py
@@ -0,0 +1,117 @@
+# Written by Arno Bakker 
+# see LICENSE.txt for license information
+
+import sys
+import base64
+import textwrap
+import binascii
+from cStringIO import StringIO
+
+# Switch between using Python's builtin SHA1 function or M2Crypto/OpenSSL's
+# TODO: optimize such that less memory is allocated, e.g. reuse a single
+# sha() object instance (hard to do here centrally with multiple threads)
+#
+
+# Arno, 2009-06-23: The OpenSSL calls used by M2Crypto's MessageDigest have 
+# different behaviour than the Python sha class ones. In particular, OpenSSL
+# needs to make special calls to incrementally digest data (i.e., update();
+# digest();update();digest(). M2Crypto's MessageDigest doesn't make these 
+# special calls. Due to bad programming, it will actually Segmentation
+# Fault when this usage occurs. And this usage occurs during hashchecking 
+# (so when using VOD repeatedly, not during live), see StorageWrapper.
+#
+# We'll need to patch M2Crypto to work around this. In the meanwhile, I
+# disable the offloading to OpenSSL for all platforms.
+#
+USE_M2CRYPTO_SHA = False
+
+
+if USE_M2CRYPTO_SHA:
+    from M2Crypto import EVP    
+
+    class sha:
+        def __init__(self,data=None):
+            self.hash = None
+            self.md = EVP.MessageDigest('sha1')
+            if data is not None:
+                self.md.update(data)
+            
+        def update(self,data):
+            if self.hash:
+                raise ValueError("sha: Cannot update after calling digest (OpenSSL limitation)")
+            self.md.update(data)
+
+        def digest(self):
+            if not self.hash:
+                self.hash = self.md.final() 
+            return self.hash 
+        
+        def hexdigest(self):
+            d = self.digest()
+            return binascii.hexlify(d)
+else:
+    from sha import sha
+
+
+#
+# M2Crypto has no functions to read a pubkey in DER
+#
+def RSA_pub_key_from_der(der):
+    from M2Crypto import RSA,BIO
+
+    s = '-----BEGIN PUBLIC KEY-----\n'
+    b = base64.standard_b64encode(der)
+    s += textwrap.fill(b,64)
+    s += '\n'
+    s += '-----END PUBLIC KEY-----\n'
+    bio = BIO.MemoryBuffer(s)
+    return RSA.load_pub_key_bio(bio)
+
+def RSA_keypair_to_pub_key_in_der(keypair):
+    # Cannot use rsapubkey.save_key_der_bio(bio). It calls
+    # i2d_RSAPrivateKey_bio() and appears to write just the
+    # three RSA parameters, and not the extra ASN.1 stuff that 
+    # says "rsaEncryption". In detail:
+    #
+    # * pubkey.save_key_der("orig.der") gives:
+    #  0:d=0  hl=3 l= 138 cons: SEQUENCE
+    #  3:d=1  hl=2 l=   1 prim: INTEGER           :00
+    #  6:d=1  hl=3 l= 129 prim: INTEGER           :A8D3A10FF772E1D5CEA86D88B2B09CE48A8DB2E563008372F4EF02BCB4E498B8BE974F8A7CD1398C7D408DF3B85D58FF0E3835AE96AB003898511D4914DE80008962C46E199276C35E4ABB7F1507F7E9A336CED3AFDC04F4DDA7B6941E8F15C1AD071599007C1F486C1560CBB96B8E07830F8E1849612E532833B55675E1D84B
+    #138:d=1  hl=2 l=   1 prim: INTEGER           :03
+    #
+    # when run through 
+    #   $ openssl asn1parse -in origpub.der -inform DER
+    #
+    # * keypair.save_pub_key("origpub.pem"). If we pass this file through asn1parse
+    #         $ openssl asn1parse -in origpub.pem -inform PEM
+    # we get:
+    #  0:d=0  hl=3 l= 157 cons: SEQUENCE
+    #  3:d=1  hl=2 l=  13 cons: SEQUENCE
+    #  5:d=2  hl=2 l=   9 prim: OBJECT            :rsaEncryption
+    # 16:d=2  hl=2 l=   0 prim: NULL
+    # 18:d=1  hl=3 l= 139 prim: BIT STRING
+    #
+    # where the BIT STRING should contain the three params.
+    #
+    # EVP.PKey.as_der() also returns the latter, so we use that as our DER format.
+    #
+    # HOWEVER: The following code, when used inside a function as here, crashes
+    # Python, so we can't use it:
+    #
+    #pkey = EVP.PKey()
+    #pkey.assign_rsa(keypair)
+    #return pkey.as_der()
+    #
+    #
+    from M2Crypto import RSA,BIO
+
+    bio = BIO.MemoryBuffer()
+    keypair.save_pub_key_bio(bio)
+    pem = bio.read_all()
+    stream = StringIO(pem)
+    lines = stream.readlines()
+    s = ''
+    for i in range(1,len(lines)-1):
+        s += lines[i]
+    return base64.standard_b64decode(s)
+    
diff --git a/instrumentation/next-share/BaseLib/Core/Utilities/__init__.py b/instrumentation/next-share/BaseLib/Core/Utilities/__init__.py
new file mode 100644
index 0000000..395f8fb
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Utilities/__init__.py
@@ -0,0 +1,2 @@
+# Written by Arno Bakker
+# see LICENSE.txt for license information
diff --git a/instrumentation/next-share/BaseLib/Core/Utilities/timeouturlopen.py b/instrumentation/next-share/BaseLib/Core/Utilities/timeouturlopen.py
new file mode 100644
index 0000000..01c9bf1
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Utilities/timeouturlopen.py
@@ -0,0 +1,75 @@
+# Written by Feek Zindel
+# see LICENSE.txt for license information
+
+import sys
+import httplib
+import socket
+import urllib2
+
+import urllib
+import urlparse
+
+DEBUG = False
+
+def urlOpenTimeout(url,timeout=30,*data):
+    class TimeoutHTTPConnection(httplib.HTTPConnection):
+        def connect(self):
+            """Connect to the host and port specified in __init__."""
+            msg = "getaddrinfo returns an empty list"
+            for res in socket.getaddrinfo(self.host, self.port, 0,
+                                          socket.SOCK_STREAM):
+                af, socktype, proto, canonname, sa = res
+                try:
+                    self.sock = socket.socket(af,socktype, proto)
+                    self.sock.settimeout(timeout)
+                    if self.debuglevel > 0:
+                        print "connect: (%s, %s)" % (self.host, self.port)
+                    self.sock.connect(sa)
+                except socket.error, msg:
+                    if self.debuglevel > 0:
+                        print 'connect fail:', (self.host, self.port)
+                    if self.sock:
+                        self.sock.close()
+                    self.sock = None
+                    continue
+                break
+            if not self.sock:
+                raise socket.error, msg
+
+    class TimeoutHTTPHandler(urllib2.HTTPHandler):
+        def http_open(self, req):
+            return self.do_open(TimeoutHTTPConnection, req)
+
+    # Arno, 2010-03-09: ProxyHandler is implicit, so code already proxy aware.
+    opener = urllib2.build_opener(TimeoutHTTPHandler,
+                                  urllib2.HTTPDefaultErrorHandler,
+                                  urllib2.HTTPRedirectHandler)
+    return opener.open(url,*data)
+
+
+def find_proxy(url):
+    """ Returns proxy host as "host:port" string """
+    (scheme, netloc, path, pars, query, fragment) = urlparse.urlparse(url)
+    proxies = urllib.getproxies()
+    proxyhost = None
+    if scheme in proxies:
+        if '@' in netloc:
+            sidx = netloc.find('@')+1
+        else:
+            sidx = 0
+        # IPVSIX TODO: what if host is IPv6 address
+        eidx = netloc.find(':')
+        if eidx == -1:
+            eidx = len(netloc)
+        host = netloc[sidx:eidx]
+        if not (host == "127.0.0.1" or urllib.proxy_bypass(host)):
+            proxyurl = proxies[scheme]
+            proxyelems = urlparse.urlparse(proxyurl)
+            proxyhost = proxyelems[1]
+            
+    if DEBUG:
+        print >>sys.stderr,"find_proxy: Got proxies",proxies,"selected",proxyhost,"URL was",url
+    return proxyhost
+
+
+#s = urlOpenTimeout("http://www.google.com",timeout=30)
diff --git a/instrumentation/next-share/BaseLib/Core/Utilities/unicode.py b/instrumentation/next-share/BaseLib/Core/Utilities/unicode.py
new file mode 100644
index 0000000..bc18a8c
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Utilities/unicode.py
@@ -0,0 +1,83 @@
+# Written by Arno Bakker
+# see LICENSE.txt for license information
+
+import sys
+
+def bin2unicode(bin,possible_encoding='utf_8'):
+    sysenc = sys.getfilesystemencoding()
+    if possible_encoding is None:
+        possible_encoding = sysenc
+    try:
+        return bin.decode(possible_encoding)
+    except:
+        try:
+            if possible_encoding == sysenc:
+                raise
+            return bin.decode(sysenc)
+        except:
+            try:
+                return bin.decode('utf_8')
+            except:
+                try:
+                    return bin.decode('iso-8859-1')
+                except:
+                    try:
+                        return bin.decode(sys.getfilesystemencoding())
+                    except:
+                        return bin.decode(sys.getdefaultencoding(), errors = 'replace')
+
+
+def str2unicode(s):
+    try:
+        s = unicode(s)
+    except: 
+        flag = 0
+        for encoding in [sys.getfilesystemencoding(), 'utf_8', 'iso-8859-1', 'unicode-escape' ]:
+            try:
+                s = unicode(s, encoding)
+                flag = 1
+                break
+            except: 
+                pass
+        if flag == 0:
+            try:
+                s = unicode(s,sys.getdefaultencoding(), errors = 'replace')
+            except:
+                pass
+    return s
+
+def dunno2unicode(dunno):
+    newdunno = None
+    if isinstance(dunno,unicode):
+        newdunno = dunno
+    else:
+        try:
+            newdunno = bin2unicode(dunno)
+        except:
+            newdunno = str2unicode(dunno)
+    return newdunno
+
+
+def name2unicode(metadata):
+    if metadata['info'].has_key('name.utf-8'):
+        namekey = 'name.utf-8'
+    else:
+        namekey = 'name'
+    if metadata.has_key('encoding'):
+        encoding = metadata['encoding']
+        metadata['info'][namekey] = bin2unicode(metadata['info'][namekey],encoding)
+    else:
+        metadata['info'][namekey] = bin2unicode(metadata['info'][namekey])
+
+    # change metainfo['info']['name'] to metainfo['info'][namekey], just in case...
+    # roer888 TODO: Never tested the following 2 lines 
+    if namekey != 'name':
+        metadata['info']['name'] = metadata['info'][namekey ]
+
+    return namekey
+
+
+def unicode2str(s):
+    if not isinstance(s,unicode):
+        return s
+    return s.encode(sys.getfilesystemencoding())
diff --git a/instrumentation/next-share/BaseLib/Core/Utilities/utilities.py b/instrumentation/next-share/BaseLib/Core/Utilities/utilities.py
new file mode 100644
index 0000000..f965105
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Utilities/utilities.py
@@ -0,0 +1,626 @@
+# Written by Jie Yang
+# see LICENSE.txt for license information
+
+import socket
+from time import time, strftime, gmtime
+from base64 import encodestring, decodestring
+from BaseLib.Core.Utilities.Crypto import sha
+import sys
+import os
+import copy
+from types import UnicodeType, StringType, LongType, IntType, ListType, DictType
+import urlparse
+from traceback import print_exc,print_stack
+import binascii
+
+STRICT_CHECK = True
+DEBUG = False
+
+infohash_len = 20
+
+def bin2str(bin):
+    # Full BASE64-encoded 
+    return encodestring(bin).replace("\n","")
+    
+def str2bin(str):
+    return decodestring(str)
+
+def validName(name):
+    if not isinstance(name, str) and len(name) == 0:
+        raise RuntimeError, "invalid name: " + name
+    return True
+
+def validPort(port):
+    port = int(port)
+    if port < 0 or port > 65535:
+        raise RuntimeError, "invalid Port: " + str(port)
+    return True
+
+def validIP(ip):
+    try:
+        try:
+            # Is IPv4 addr?
+            socket.inet_aton(ip)
+            return True
+        except socket.error:
+            # Is hostname / IPv6?
+            socket.getaddrinfo(ip, None)
+            return True
+    except:
+        print_exc()
+    raise RuntimeError, "invalid IP address: " + ip
+
+    
+def validPermid(permid):
+    if not isinstance(permid, str):
+        raise RuntimeError, "invalid permid: " + permid
+    # Arno,2010-02-17: permid is ASN.1 encoded data that is NOT fixed length
+    return True
+
+def validInfohash(infohash):
+    if not isinstance(infohash, str):
+        raise RuntimeError, "invalid infohash " + infohash
+    if STRICT_CHECK and len(infohash) != infohash_len:
+        raise RuntimeError, "invalid length infohash " + infohash
+    return True
+    
+def isValidPermid(permid):
+    try:
+        return validPermid(permid)
+    except:
+        return False
+    
+def isValidInfohash(infohash):
+    try:
+        return validInfohash(infohash)
+    except:
+        return False
+
+def isValidPort(port):
+    try:
+        return validPort(port)
+    except:
+        return False
+    
+def isValidIP(ip):
+    try:
+        return validIP(ip)
+    except:
+        return False
+
+def isValidName(name):
+    try:
+        return validPort(name)
+    except:
+        return False
+    
+    
+def validTorrentFile(metainfo):
+    # Jie: is this function too strict? Many torrents could not be downloaded
+    if type(metainfo) != DictType:
+        raise ValueError('metainfo not dict')
+    
+    
+    if 'info' not in metainfo:
+        raise ValueError('metainfo misses key info')
+    
+    if 'announce' in metainfo and not isValidURL(metainfo['announce']):
+        raise ValueError('announce URL bad')
+    
+    # http://www.bittorrent.org/DHT_protocol.html says both announce and nodes
+    # are not allowed, but some torrents (Azureus?) apparently violate this.
+
+    #if 'announce' in metainfo and 'nodes' in metainfo:
+    #    raise ValueError('both announce and nodes present')
+    
+    if 'nodes' in metainfo:
+        nodes = metainfo['nodes']
+        if type(nodes) != ListType:
+            raise ValueError('nodes not list, but '+`type(nodes)`)
+        for pair in nodes:
+            if type(pair) != ListType and len(pair) != 2:
+                raise ValueError('node not 2-item list, but '+`type(pair)`)
+            host,port = pair
+            if type(host) != StringType:
+                raise ValueError('node host not string, but '+`type(host)`)
+            if type(port) != IntType:
+                raise ValueError('node port not int, but '+`type(port)`)
+
+    if not ('announce' in metainfo or 'nodes' in metainfo):
+        raise ValueError('announce and nodes missing')
+
+    # 04/05/10 boudewijn: with the introduction of magnet links we
+    # also allow for peer addresses to be (temporarily) stored in the
+    # metadata.  Typically these addresses are recently gathered.
+    if "initial peers" in metainfo:
+        if not isinstance(metainfo["initial peers"], list):
+            raise ValueError("initial peers not list, but %s" % type(metainfo["initial peers"]))
+        for address in metainfo["initial peers"]:
+            if not (isinstance(address, tuple) and len(address) == 2):
+                raise ValueError("address not 2-item tuple, but %s" % type(address))
+            if not isinstance(address[0], str):
+                raise ValueError("address host not string, but %s" % type(address[0]))
+            if not isinstance(address[1], int):
+                raise ValueError("address port not int, but %s" % type(address[1]))
+    
+    info = metainfo['info']
+    if type(info) != DictType:
+        raise ValueError('info not dict')
+    
+    if 'root hash' in info:
+        infokeys = ['name','piece length', 'root hash']
+    elif 'live' in info:
+        infokeys = ['name','piece length', 'live']
+    else:
+        infokeys = ['name','piece length', 'pieces']
+    for key in infokeys:
+        if key not in info:
+            raise ValueError('info misses key '+key)
+    name = info['name']
+    if type(name) != StringType:
+        raise ValueError('info name is not string but '+`type(name)`)
+    pl = info['piece length']
+    if type(pl) != IntType and type(pl) != LongType:
+        raise ValueError('info piece size is not int, but '+`type(pl)`)
+    if 'root hash' in info:
+        rh = info['root hash']
+        if type(rh) != StringType or len(rh) != 20:
+            raise ValueError('info roothash is not 20-byte string')
+    elif 'live' in info:
+        live = info['live']
+        if type(live) != DictType:
+            raise ValueError('info live is not a dict')
+        else:
+            if 'authmethod' not in live:
+                raise ValueError('info live misses key'+'authmethod')
+    else:
+        p = info['pieces']
+        if type(p) != StringType or len(p) % 20 != 0:
+            raise ValueError('info pieces is not multiple of 20 bytes')
+        
+    if 'length' in info:
+        # single-file torrent
+        if 'files' in info:
+            raise ValueError('info may not contain both files and length key')
+        
+        l = info['length']
+        if type(l) != IntType and type(l) != LongType:
+            raise ValueError('info length is not int, but '+`type(l)`)
+    else:
+        # multi-file torrent
+        if 'length' in info:
+            raise ValueError('info may not contain both files and length key')
+        
+        files = info['files']
+        if type(files) != ListType:
+            raise ValueError('info files not list, but '+`type(files)`)
+        
+        filekeys = ['path','length']
+        for file in files:
+            for key in filekeys:
+                if key not in file:
+                    raise ValueError('info files missing path or length key')
+            
+            p = file['path']
+            if type(p) != ListType:
+                raise ValueError('info files path is not list, but '+`type(p)`)
+            for dir in p:
+                if type(dir) != StringType:
+                    raise ValueError('info files path is not string, but '+`type(dir)`)
+            
+            l = file['length']
+            if type(l) != IntType and type(l) != LongType:
+                raise ValueError('info files length is not int, but '+`type(l)`)
+            
+    # common additional fields
+    if 'announce-list' in metainfo:
+        al = metainfo['announce-list']
+        if type(al) != ListType:
+            raise ValueError('announce-list is not list, but '+`type(al)`)
+        for tier in al:
+            if type(tier) != ListType:
+                raise ValueError('announce-list tier is not list '+`tier`)
+        # Jie: this limitation is not necessary
+#            for url in tier:
+#                if not isValidURL(url):
+#                    raise ValueError('announce-list url is not valid '+`url`)
+
+    if 'azureus_properties' in metainfo:
+        azprop = metainfo['azureus_properties']
+        if type(azprop) != DictType:
+            raise ValueError('azureus_properties is not dict, but '+`type(azprop)`)
+        if 'Content' in azprop:
+                content = azprop['Content']
+                if type(content) != DictType:
+                    raise ValueError('azureus_properties content is not dict, but '+`type(content)`)
+                if 'thumbnail' in content:
+                    thumb = content['thumbnail']
+                    if type(content) != StringType:
+                        raise ValueError('azureus_properties content thumbnail is not string')
+
+    # Diego: perform check on httpseeds/url-list field
+    if 'url-list' in metainfo:
+        if 'files' in metainfo['info']:
+            # Diego: only single-file mode allowed for http seeding now
+            raise ValueError("Only single-file mode supported with HTTP seeding: remove url-list")
+        elif type( metainfo['url-list'] ) != ListType:
+            raise ValueError('url-list is not list, but '+`type(metainfo['url-list'])`)
+        else:
+            for url in metainfo['url-list']:
+                if not isValidURL(url):
+                    raise ValueError("url-list url is not valid: "+`url`)
+
+    if 'httpseeds' in metainfo:
+        if 'files' in metainfo['info']:
+            # Diego: only single-file mode allowed for http seeding now
+            raise ValueError("Only single-file mode supported with HTTP seeding: remove httpseeds")
+        elif type( metainfo['httpseeds'] ) != ListType:
+            raise ValueError('httpseeds is not list, but '+`type(metainfo['httpseeds'])`)
+        else:
+            for url in metainfo['httpseeds']:
+                if not isValidURL(url):
+                    raise ValueError("httpseeds url is not valid: "+`url`)
+
+
+def isValidTorrentFile(metainfo):
+    try:
+        validTorrentFile(metainfo)
+        return True
+    except:
+        if DEBUG:
+            print_exc()
+        return False
+    
+    
+def isValidURL(url):
+    if url.lower().startswith('udp'):    # exception for udp
+        url = url.lower().replace('udp','http',1)
+    r = urlparse.urlsplit(url)
+    # if DEBUG:
+    #     print >>sys.stderr,"isValidURL:",r
+    
+    if r[0] == '' or r[1] == '':
+        return False
+    return True
+    
+def show_permid(permid):
+    # Full BASE64-encoded. Must not be abbreviated in any way. 
+    if not permid:
+        return 'None'
+    return encodestring(permid).replace("\n","")
+    # Short digest
+    ##return sha(permid).hexdigest()
+
+def show_permid_short(permid):
+    if not permid:
+        return 'None'
+    s = encodestring(permid).replace("\n","")
+    return s[-10:]
+    #return encodestring(sha(s).digest()).replace("\n","")
+
+def show_permid_shorter(permid):
+    if not permid:
+        return 'None'
+    s = encodestring(permid).replace("\n","")
+    return s[-5:]
+
+def readableBuddyCastMsg(buddycast_data,selversion):
+    """ Convert msg to readable format.
+    As this copies the original dict, and just transforms it,
+    most added info is already present and therefore logged
+    correctly. Exception is the OLPROTO_VER_EIGHTH which
+    modified the preferences list. """
+    prefxchg_msg = copy.deepcopy(buddycast_data)
+    
+    if prefxchg_msg.has_key('permid'):
+        prefxchg_msg.pop('permid')
+    if prefxchg_msg.has_key('ip'):
+        prefxchg_msg.pop('ip')
+    if prefxchg_msg.has_key('port'):
+        prefxchg_msg.pop('port')
+        
+    name = repr(prefxchg_msg['name'])    # avoid coding error
+
+    if prefxchg_msg['preferences']:
+        prefs = []
+        if selversion < 8: # OLPROTO_VER_EIGHTH: Can't use constant due to recursive import
+            for pref in prefxchg_msg['preferences']:
+                prefs.append(show_permid(pref))
+        else:
+            for preftuple in prefxchg_msg['preferences']:
+                # Copy tuple and escape infohash
+                newlist = []
+                for i in range(0,len(preftuple)):
+                    if i == 0:
+                        val = show_permid(preftuple[i])
+                    else:
+                        val = preftuple[i]
+                    newlist.append(val)
+                prefs.append(newlist)
+                    
+        prefxchg_msg['preferences'] = prefs
+
+        
+    if prefxchg_msg.get('taste buddies', []):
+        buddies = []
+        for buddy in prefxchg_msg['taste buddies']:
+            buddy['permid'] = show_permid(buddy['permid'])
+            if buddy.get('preferences', []):
+                prefs = []
+                for pref in buddy['preferences']:
+                    prefs.append(show_permid(pref))
+                buddy['preferences'] = prefs
+            buddies.append(buddy)
+        prefxchg_msg['taste buddies'] = buddies
+        
+    if prefxchg_msg.get('random peers', []):
+        peers = []
+        for peer in prefxchg_msg['random peers']:
+            peer['permid'] = show_permid(peer['permid'])
+            peers.append(peer)
+        prefxchg_msg['random peers'] = peers
+        
+    return prefxchg_msg
+    
+def print_prefxchg_msg(prefxchg_msg):
+    def show_permid(permid):
+        return permid
+    print "------- preference_exchange message ---------"
+    print prefxchg_msg
+    print "---------------------------------------------"
+    print "permid:", show_permid(prefxchg_msg['permid'])
+    print "name", prefxchg_msg['name']
+    print "ip:", prefxchg_msg['ip']
+    print "port:", prefxchg_msg['port']
+    print "preferences:"
+    if prefxchg_msg['preferences']:
+        for pref in prefxchg_msg['preferences']:
+            print "\t", pref#, prefxchg_msg['preferences'][pref]
+    print "taste buddies:"
+    if prefxchg_msg['taste buddies']:
+        for buddy in prefxchg_msg['taste buddies']:
+            print "\t permid:", show_permid(buddy['permid'])
+            #print "\t permid:", buddy['permid']
+            print "\t ip:", buddy['ip']
+            print "\t port:", buddy['port']
+            print "\t age:", buddy['age']
+            print "\t preferences:"
+            if buddy['preferences']:
+                for pref in buddy['preferences']:
+                    print "\t\t", pref#, buddy['preferences'][pref]
+            print
+    print "random peers:"
+    if prefxchg_msg['random peers']:
+        for peer in prefxchg_msg['random peers']:
+            print "\t permid:", show_permid(peer['permid'])
+            #print "\t permid:", peer['permid']
+            print "\t ip:", peer['ip']
+            print "\t port:", peer['port']
+            print "\t age:", peer['age']
+            print    
+            
+def print_dict(data, level=0):
+    if isinstance(data, dict):
+        print
+        for i in data:
+            print "  "*level, str(i) + ':',
+            print_dict(data[i], level+1)
+    elif isinstance(data, list):
+        if not data:
+            print "[]"
+        else:
+            print
+        for i in xrange(len(data)):
+            print "  "*level, '[' + str(i) + ']:',
+            print_dict(data[i], level+1)
+    else:
+        print data
+        
+def friendly_time(old_time):
+    curr_time = time()
+    try:
+        old_time = int(old_time)
+        assert old_time > 0
+        diff = int(curr_time - old_time)
+    except:
+        if isinstance(old_time, str):
+            return old_time
+        else:
+            return '?'
+    if diff < 0:
+        return '?'
+    elif diff < 2:
+        return str(diff) + " sec. ago"
+    elif diff < 60:
+        return str(diff) + " secs. ago"
+    elif diff < 120:
+        return "1 min. ago"
+    elif diff < 3600:
+        return str(int(diff/60)) + " mins. ago"
+    elif diff < 7200:
+        return "1 hour ago"
+    elif diff < 86400:
+        return str(int(diff/3600)) + " hours ago"
+    elif diff < 172800:
+        return "Yesterday"
+    elif diff < 259200:
+        return str(int(diff/86400)) + " days ago"
+    else:
+        return strftime("%d-%m-%Y", gmtime(old_time))
+        
+def sort_dictlist(dict_list, key, order='increase'):
+    
+    aux = []
+    for i in xrange(len(dict_list)):
+        #print >>sys.stderr,"sort_dictlist",key,"in",dict_list[i].keys(),"?"
+        if key in dict_list[i]:
+            aux.append((dict_list[i][key],i))
+    aux.sort()
+    if order == 'decrease' or order == 1:    # 0 - increase, 1 - decrease
+        aux.reverse()
+    return [dict_list[i] for x, i in aux]
+
+
+def dict_compare(a, b, keys):
+    for key in keys:
+        order = 'increase'
+        if type(key) == tuple:
+            skey, order = key
+        else:
+            skey = key
+
+        if a.get(skey) > b.get(skey):
+            if order == 'decrease' or order == 1:
+                return -1
+            else:
+                return 1
+        elif a.get(skey) < b.get(skey):
+            if order == 'decrease' or order == 1:
+                return 1
+            else:
+                return -1
+
+    return 0
+
+
+def multisort_dictlist(dict_list, keys):
+
+    listcopy = copy.copy(dict_list)
+    cmp = lambda a, b: dict_compare(a, b, keys)
+    listcopy.sort(cmp=cmp)
+    return listcopy
+
+
+def find_content_in_dictlist(dict_list, content, key='infohash'):
+    title = content.get(key)
+    if not title:
+        print 'Error: content had no content_name'
+        return False
+    for i in xrange(len(dict_list)):
+        if title == dict_list[i].get(key):
+            return i
+    return -1
+
+def remove_torrent_from_list(list, content, key = 'infohash'):
+    remove_data_from_list(list, content, key)
+
+def remove_data_from_list(list, content, key = 'infohash'):
+    index = find_content_in_dictlist(list, content, key)
+    if index != -1:
+        del list[index]
+    
+def sortList(list_to_sort, list_key, order='decrease'):
+        aux = zip(list_key, list_to_sort)
+        aux.sort()
+        if order == 'decrease':
+            aux.reverse()
+        return [i for k, i in aux]    
+
+def getPlural( n):
+        if n == 1:
+            return ''
+        else:
+            return 's'
+
+
+def find_prog_in_PATH(prog):
+    envpath = os.path.expandvars('${PATH}')
+    if sys.platform == 'win32':
+        splitchar = ';'
+    else:
+        splitchar = ':'
+    paths = envpath.split(splitchar)
+    foundat = None
+    for path in paths:
+        fullpath = os.path.join(path,prog)
+        if os.access(fullpath,os.R_OK|os.X_OK):
+            foundat = fullpath
+            break
+    return foundat
+    
+def hostname_or_ip2ip(hostname_or_ip):
+    # Arno: don't DNS resolve always, grabs lock on most systems
+    ip = None
+    try:
+        # test that hostname_or_ip contains a xxx.xxx.xxx.xxx string
+        socket.inet_aton(hostname_or_ip)
+        ip = hostname_or_ip
+
+    except:
+        try:
+            # dns-lookup for hostname_or_ip into an ip address
+            ip = socket.gethostbyname(hostname_or_ip)
+            if not hostname_or_ip.startswith("superpeer"):
+                print >>sys.stderr,"hostname_or_ip2ip: resolved ip from hostname, an ip should have been provided", hostname_or_ip
+
+        except:
+            print >>sys.stderr,"hostname_or_ip2ip: invalid hostname", hostname_or_ip
+            print_exc()
+
+    return ip
+
+
+def get_collected_torrent_filename(infohash):
+    # Arno: Better would have been the infohash in hex.
+    filename = sha(infohash).hexdigest()+'.torrent'    # notice: it's sha1-hash of infohash
+    return filename
+    # exceptions will be handled by got_metadata()
+    
+
+def uintToBinaryString(uint, length=4):
+    '''
+    Converts an unsigned integer into its binary representation.
+    
+    @type uint: int
+    @param uint: un unsigned intenger to convert into binary data.
+    
+    @type length: int
+    @param length: the number of bytes the the resulting binary
+                   string should have
+                   
+    @rtype: a binary string
+    @return: a binary string. Each element in the string is one byte 
+            of data.
+                   
+    @precondition: uint >= 0 and uint < 2**(length*8)
+    '''
+    assert 0 <= uint < 2**(length*8), "Cannot represent string"
+    hexlen = length*2
+    hexString =  "{0:0>{1}}".format(hex(uint)[2:], hexlen)
+    if hexString.endswith('L'):
+        hexString = hexString[:-1]
+    
+    binaryString = binascii.unhexlify(hexString)
+    return binaryString
+
+def binaryStringToUint(bstring):
+    '''
+    Converts a binary string into an unsigned integer
+    
+    @param bstring: a string of binary data
+    
+    @return a non-negative integer representing the 
+            value of the binary data interpreted as an
+            unsigned integer 
+    '''
+    hexstr = binascii.hexlify(bstring)
+    intval = int(hexstr,16)
+    return intval
+
+
+    
+
+
+if __name__=='__main__':
+
+    torrenta = {'name':'a', 'swarmsize' : 12}
+    torrentb = {'name':'b', 'swarmsize' : 24}
+    torrentc = {'name':'c', 'swarmsize' : 18, 'Web2' : True}
+    torrentd = {'name':'b', 'swarmsize' : 36, 'Web2' : True}
+
+    torrents = [torrenta, torrentb, torrentc, torrentd]
+    print multisort_dictlist(torrents, ["Web2", ("swarmsize", "decrease")])
+
+
+    #d = {'a':1,'b':[1,2,3],'c':{'c':2,'d':[3,4],'k':{'c':2,'d':[3,4]}}}
+    #print_dict(d)    
diff --git a/instrumentation/next-share/BaseLib/Core/Utilities/win32regchecker.py b/instrumentation/next-share/BaseLib/Core/Utilities/win32regchecker.py
new file mode 100644
index 0000000..04ec788
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Utilities/win32regchecker.py
@@ -0,0 +1,113 @@
+# Written by ABC authors and Arno Bakker
+# see LICENSE.txt for license information
+
+import sys
+import os
+from traceback import print_exc
+
+if (sys.platform == 'win32'):
+    import _winreg
+
+    # short for PyHKEY from "_winreg" module
+    HKCR = _winreg.HKEY_CLASSES_ROOT
+    HKLM = _winreg.HKEY_LOCAL_MACHINE
+    HKCU = _winreg.HKEY_CURRENT_USER
+else:
+    HKCR = 0
+    HKLM = 1
+    HKCU = 2
+
+DEBUG = False
+
+class Win32RegChecker:
+    def __init__(self):
+        pass
+
+    def readRootKey(self,key_name,value_name=""):
+        return self.readKey(HKCR,key_name,value_name)
+        
+    def readKey(self,hkey,key_name,value_name=""):
+        if (sys.platform != 'win32'):
+            return None
+            
+        try:
+            # test that shell/open association with ABC exist
+            if DEBUG:
+                print >>sys.stderr,"win32regcheck: Opening",key_name,value_name
+            full_key = _winreg.OpenKey(hkey, key_name, 0, _winreg.KEY_READ)
+            
+            if DEBUG:
+                print >>sys.stderr,"win32regcheck: Open returned",full_key
+            
+            value_data, value_type = _winreg.QueryValueEx(full_key, value_name)
+            if DEBUG:
+                print >>sys.stderr,"win32regcheck: Read",value_data,value_type
+            _winreg.CloseKey(full_key)
+                    
+            return value_data
+        except:
+            print_exc(file=sys.stderr)
+            # error, test failed, key don't exist
+            # (could also indicate a unicode error)
+            return None
+
+
+    def readKeyRecursively(self,hkey,key_name,value_name=""):
+        if (sys.platform != 'win32'):
+            return None
+            
+        lasthkey = hkey
+        try:
+            toclose = []
+            keyparts = key_name.split('\\')
+            print >>sys.stderr,"win32regcheck: keyparts",keyparts
+            for keypart in keyparts:
+                if keypart == '':
+                    continue
+                if DEBUG:
+                    print >>sys.stderr,"win32regcheck: Opening",keypart
+                full_key = _winreg.OpenKey(lasthkey, keypart, 0, _winreg.KEY_READ)
+                lasthkey = full_key
+                toclose.append(full_key)
+            
+            if DEBUG:
+                print >>sys.stderr,"win32regcheck: Open returned",full_key
+            
+            value_data, value_type = _winreg.QueryValueEx(full_key, value_name)
+            if DEBUG:
+                print >>sys.stderr,"win32regcheck: Read",value_data,value_type
+            for hkey in toclose:
+                _winreg.CloseKey(hkey)
+                    
+            return value_data
+        except:
+            print_exc()
+            # error, test failed, key don't exist
+            # (could also indicate a unicode error)
+            return None
+
+
+    def writeKey(self,hkey,key_name,value_name,value_data,value_type):
+        try:
+            # kreate desired key in Windows register
+            full_key = _winreg.CreateKey(hkey, key_name)
+        except EnvironmentError:
+            return False;
+        # set desired value in created Windows register key
+        _winreg.SetValueEx(full_key, value_name, 0, value_type, value_data)
+        # close Windows register key
+        _winreg.CloseKey(full_key)
+            
+        return True
+
+
+
+if __name__ == "__main__":
+    w = Win32RegChecker()
+    winfiletype = w.readRootKey(".wmv")
+    playkey = winfiletype+"\shell\play\command"
+    urlplay = w.readRootKey(playkey)
+    print urlplay
+    openkey = winfiletype+"\shell\open\command"
+    urlopen = w.readRootKey(openkey)
+    print urlopen
diff --git a/instrumentation/next-share/BaseLib/Core/Video/LiveSourceAuth.py b/instrumentation/next-share/BaseLib/Core/Video/LiveSourceAuth.py
new file mode 100644
index 0000000..65fdc5d
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Video/LiveSourceAuth.py
@@ -0,0 +1,477 @@
+# written by Arno Bakker
+# see LICENSE.txt for license information
+
+import sys
+from traceback import print_exc
+from cStringIO import StringIO
+import struct
+import time
+import array
+
+from BaseLib.Core.Utilities.Crypto import sha,RSA_pub_key_from_der
+from BaseLib.Core.osutils import *
+from M2Crypto import EC
+from BaseLib.Core.osutils import *
+from types import StringType
+
+DEBUG = False
+
+class Authenticator:
+    
+    def __init__(self,piecelen,npieces):
+        self.piecelen = piecelen
+        self.npieces = npieces
+        self.seqnum = 0L
+    
+    def get_piece_length(self):
+        return self.piecelen
+    
+    def get_npieces(self):
+        return self.npieces
+    
+    def get_content_blocksize(self):
+        pass
+    
+    def sign(self,content):
+        pass
+    
+    def verify(self,piece):
+        pass
+    
+    def get_content(self,piece):
+        pass
+
+    def get_source_seqnum(self):
+        return self.seqnum
+
+    def set_source_seqnum(self,seqnum):
+        self.seqnum = seqnum
+
+
+class NullAuthenticator(Authenticator):
+    
+    def __init__(self,piecelen,npieces):
+        Authenticator.__init__(self,piecelen,npieces)
+        self.contentblocksize = piecelen
+    
+    def get_content_blocksize(self):
+        return self.contentblocksize
+    
+    def sign(self,content):
+        return [content]
+    
+    def verify(self,piece):
+        return True
+    
+    def get_content(self,piece):
+        return piece
+
+
+class ECDSAAuthenticator(Authenticator):
+    """ Authenticator who places a ECDSA signature in the last part of a
+    piece. In particular, the sig consists of:
+    - an 8 byte sequence number
+    - an 8 byte real-time timestamp
+    - a 1 byte length field followed by
+    - a variable-length ECDSA signature in ASN.1, (max 64 bytes)  
+    - optionally 0x00 padding bytes, if the ECDSA sig is less than 64 bytes,
+    to give a total of 81 bytes.
+    """
+    
+    SEQNUM_SIZE = 8
+    RTSTAMP_SIZE = 8
+    LENGTH_SIZE = 1
+    MAX_ECDSA_ASN1_SIGSIZE = 64
+    EXTRA_SIZE = SEQNUM_SIZE + RTSTAMP_SIZE
+    # = seqnum + rtstamp + 1 byte length + MAX_ECDSA, padded
+    # put seqnum + rtstamp directly after content, so we calc the sig directly 
+    # from the received buffer.
+    OUR_SIGSIZE = EXTRA_SIZE+LENGTH_SIZE+MAX_ECDSA_ASN1_SIGSIZE 
+    
+    def __init__(self,piecelen,npieces,keypair=None,pubkeypem=None):
+        
+        print >>sys.stderr,"ECDSAAuth: npieces",npieces
+        
+        Authenticator.__init__(self,piecelen,npieces)
+        self.contentblocksize = piecelen-self.OUR_SIGSIZE
+        self.keypair = keypair
+        if pubkeypem is not None:
+            #print >>sys.stderr,"ECDSAAuth: pubkeypem",`pubkeypem`
+            self.pubkey = EC.pub_key_from_der(pubkeypem)
+        else:
+            self.pubkey = None
+        self.startts = None
+
+    def get_content_blocksize(self):
+        return self.contentblocksize
+    
+    def sign(self,content):
+        rtstamp = time.time()
+        #print >>sys.stderr,"ECDSAAuth: sign: ts %.5f s" % rtstamp
+        
+        extra = struct.pack('>Qd', self.seqnum,rtstamp)
+        self.seqnum += 1L
+
+        sig = ecdsa_sign_data(content,extra,self.keypair)
+        # The sig returned is either 64 or 63 bytes long (62 also possible I 
+        # guess). Therefore we transmit size as 1 bytes and fill to 64 bytes.
+        lensig = chr(len(sig))
+        if len(sig) != self.MAX_ECDSA_ASN1_SIGSIZE:
+            # Note: this is not official ASN.1 padding. Also need to modify
+            # the header length for that I assume.
+            diff = self.MAX_ECDSA_ASN1_SIGSIZE-len(sig)
+            padding = '\x00' * diff 
+            return [content,extra,lensig,sig,padding]
+        else:
+            return [content,extra,lensig,sig]
+        
+    def verify(self,piece,index):
+        """ A piece is valid if:
+        - the signature is correct,
+        - the seqnum % npieces == piecenr.
+        - the seqnum is no older than self.seqnum - npieces
+        @param piece The piece data as received from peer
+        @param index The piece number as received from peer
+        @return Boolean
+        """
+        try:
+            # Can we do this without memcpy?
+            #print >>sys.stderr,"ECDSAAuth: verify",len(piece)
+            extra = piece[-self.OUR_SIGSIZE:-self.OUR_SIGSIZE+self.EXTRA_SIZE]
+            lensig = ord(piece[-self.OUR_SIGSIZE+self.EXTRA_SIZE])
+            if lensig > self.MAX_ECDSA_ASN1_SIGSIZE:
+                print >>sys.stderr,"ECDSAAuth: @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ failed piece",index,"lensig wrong",lensig
+                return False
+            #print >>sys.stderr,"ECDSAAuth: verify lensig",lensig
+            diff = lensig-self.MAX_ECDSA_ASN1_SIGSIZE
+            if diff == 0:
+                sig = piece[-self.OUR_SIGSIZE+self.EXTRA_SIZE+self.LENGTH_SIZE:]
+            else:
+                sig = piece[-self.OUR_SIGSIZE+self.EXTRA_SIZE+self.LENGTH_SIZE:diff]
+            content = piece[:-self.OUR_SIGSIZE]
+            if DEBUG:
+                print >>sys.stderr,"ECDSAAuth: verify piece",index,"sig",`sig`
+                print >>sys.stderr,"ECDSAAuth: verify dig",sha(content).hexdigest()
+        
+            ret = ecdsa_verify_data_pubkeyobj(content,extra,self.pubkey,sig)
+            if ret:
+                (seqnum, rtstamp) = self._decode_extra(piece)
+                
+                if DEBUG:
+                    print >>sys.stderr,"ECDSAAuth: verify piece",index,"seq",seqnum,"ts %.5f s" % rtstamp,"ls",lensig
+                
+                mod = seqnum % self.get_npieces()
+                thres = self.seqnum - self.get_npieces()/2
+                if seqnum <= thres:
+                    print >>sys.stderr,"ECDSAAuth: @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ failed piece",index,"old seqnum",seqnum,"<<",self.seqnum
+                    return False
+                elif mod != index:
+                    print >>sys.stderr,"ECDSAAuth: @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ failed piece",index,"expected",mod
+                    return False 
+                elif self.startts is not None and rtstamp < self.startts:
+                    print >>sys.stderr,"ECDSAAuth: @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ failed piece",index,"older than oldest known ts",rtstamp,self.startts
+                    return False
+                else:
+                    self.seqnum = max(self.seqnum,seqnum)
+                    if self.startts is None:
+                        self.startts = rtstamp-300.0 # minus 5 min in case we read piece N+1 before piece N
+                        print >>sys.stderr,"ECDSAAuth: @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@: startts",self.startts
+            else:
+                print >>sys.stderr,"ECDSAAuth: @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ piece",index,"failed sig"
+            
+            return ret
+        except:
+            print_exc()
+            return False 
+
+    def get_content(self,piece):
+        return piece[:-self.OUR_SIGSIZE]
+
+    # Extra fields
+    def get_seqnum(self,piece):
+        (seqnum, rtstamp) = self._decode_extra(piece)
+        return seqnum
+
+    def get_rtstamp(self,piece):
+        (seqnum, rtstamp) = self._decode_extra(piece)
+        return rtstamp
+        
+    def _decode_extra(self,piece):
+        extra = piece[-self.OUR_SIGSIZE:-self.OUR_SIGSIZE+self.EXTRA_SIZE]
+        if type(extra) == array.array:
+            extra = extra.tostring()
+        return struct.unpack('>Qd',extra)
+
+    
+def ecdsa_sign_data(plaintext,extra,ec_keypair):
+    digester = sha(plaintext)
+    digester.update(extra)
+    digest = digester.digest()
+    return ec_keypair.sign_dsa_asn1(digest)
+    
+def ecdsa_verify_data_pubkeyobj(plaintext,extra,pubkey,blob):
+    digester = sha(plaintext)
+    digester.update(extra)
+    digest = digester.digest()
+    return pubkey.verify_dsa_asn1(digest,blob)
+    
+
+
+
+class RSAAuthenticator(Authenticator):
+    """ Authenticator who places a RSA signature in the last part of a piece. 
+    In particular, the sig consists of:
+    - an 8 byte sequence number
+    - an 8 byte real-time timestamp
+    - a variable-length RSA signature, length equivalent to the keysize in bytes  
+    to give a total of 16+(keysize/8) bytes.
+    """
+    
+    SEQNUM_SIZE = 8
+    RTSTAMP_SIZE = 8
+    EXTRA_SIZE = SEQNUM_SIZE + RTSTAMP_SIZE
+    # put seqnum + rtstamp directly after content, so we calc the sig directly 
+    # from the received buffer.
+    def our_sigsize(self):
+        return self.EXTRA_SIZE+self.rsa_sigsize() 
+    
+    def rsa_sigsize(self):
+        return len(self.pubkey)/8
+    
+    def __init__(self,piecelen,npieces,keypair=None,pubkeypem=None):
+        Authenticator.__init__(self,piecelen,npieces)
+        self.keypair = keypair
+        if pubkeypem is not None:
+            #print >>sys.stderr,"ECDSAAuth: pubkeypem",`pubkeypem`
+            self.pubkey = RSA_pub_key_from_der(pubkeypem)
+        else:
+            self.pubkey = self.keypair
+        self.contentblocksize = piecelen-self.our_sigsize()
+        self.startts = None
+
+    def get_content_blocksize(self):
+        return self.contentblocksize
+    
+    def sign(self,content):
+        rtstamp = time.time()
+        #print >>sys.stderr,"ECDSAAuth: sign: ts %.5f s" % rtstamp
+        
+        extra = struct.pack('>Qd', self.seqnum,rtstamp)
+        self.seqnum += 1L
+
+        sig = rsa_sign_data(content,extra,self.keypair)
+        return [content,extra,sig]
+        
+    def verify(self,piece,index):
+        """ A piece is valid if:
+        - the signature is correct,
+        - the seqnum % npieces == piecenr.
+        - the seqnum is no older than self.seqnum - npieces
+        @param piece The piece data as received from peer
+        @param index The piece number as received from peer
+        @return Boolean
+        """
+        try:
+            # Can we do this without memcpy?
+            #print >>sys.stderr,"ECDSAAuth: verify",len(piece)
+            extra = piece[-self.our_sigsize():-self.our_sigsize()+self.EXTRA_SIZE]
+            sig = piece[-self.our_sigsize()+self.EXTRA_SIZE:]
+            content = piece[:-self.our_sigsize()]
+            #if DEBUG:
+            #    print >>sys.stderr,"RSAAuth: verify piece",index,"sig",`sig`
+            #    print >>sys.stderr,"RSAAuth: verify dig",sha(content).hexdigest()
+        
+            ret = rsa_verify_data_pubkeyobj(content,extra,self.pubkey,sig)
+            if ret:
+                (seqnum, rtstamp) = self._decode_extra(piece)
+                
+                if DEBUG:
+                    print >>sys.stderr,"RSAAuth: verify piece",index,"seq",seqnum,"ts %.5f s" % rtstamp
+                
+                mod = seqnum % self.get_npieces()
+                thres = self.seqnum - self.get_npieces()/2
+                if seqnum <= thres:
+                    print >>sys.stderr,"RSAAuth: @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ failed piece",index,"old seqnum",seqnum,"<<",self.seqnum
+                    return False
+                elif mod != index:
+                    print >>sys.stderr,"RSAAuth: @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ failed piece",index,"expected",mod
+                    return False
+                elif self.startts is not None and rtstamp < self.startts:
+                    print >>sys.stderr,"RSAAuth: @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ failed piece",index,"older than oldest known ts",rtstamp,self.startts
+                    return False
+                else:
+                    self.seqnum = max(self.seqnum,seqnum)
+                    if self.startts is None:
+                        self.startts = rtstamp-300.0 # minus 5 min in case we read piece N+1 before piece N
+                        print >>sys.stderr,"RSAAuth: @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@: startts",self.startts
+            else:
+                print >>sys.stderr,"RSAAuth: @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ piece",index,"failed sig"
+            
+            return ret
+        except:
+            print_exc()
+            return False 
+
+    def get_content(self,piece):
+        return piece[:-self.our_sigsize()]
+
+    # Extra fields
+    def get_seqnum(self,piece):
+        (seqnum, rtstamp) = self._decode_extra(piece)
+        return seqnum
+
+    def get_rtstamp(self,piece):
+        (seqnum, rtstamp) = self._decode_extra(piece)
+        return rtstamp
+        
+    def _decode_extra(self,piece):
+        extra = piece[-self.our_sigsize():-self.our_sigsize()+self.EXTRA_SIZE]
+        if type(extra) == array.array:
+            extra = extra.tostring()
+        return struct.unpack('>Qd',extra)
+
+
+def rsa_sign_data(plaintext,extra,rsa_keypair):
+    digester = sha(plaintext)
+    digester.update(extra)
+    digest = digester.digest()
+    return rsa_keypair.sign(digest)
+    
+def rsa_verify_data_pubkeyobj(plaintext,extra,pubkey,sig):
+    digester = sha(plaintext)
+    digester.update(extra)
+    digest = digester.digest()
+    
+    # The type of sig is array.array() at this point (why?), M2Crypto RSA verify
+    # will complain if it is not a string or Unicode object. Check if this is a
+    # memcpy. 
+    s = sig.tostring()
+    return pubkey.verify(digest,s)
+
+
+
+
+
+
+    
+class AuthStreamWrapper:
+    """ Wrapper around the stream returned by VideoOnDemand/MovieOnDemandTransporter
+    that strips of the signature info
+    """
+    
+    def __init__(self,inputstream,authenticator):
+        self.inputstream = inputstream
+        self.buffer = StringIO()
+        self.authenticator = authenticator
+        self.piecelen = authenticator.get_piece_length()
+        self.last_rtstamp = None
+
+    def read(self,numbytes=None):
+        rawdata = self._readn(self.piecelen)
+        if len(rawdata) == 0:
+            # EOF
+            return rawdata
+        content = self.authenticator.get_content(rawdata)
+        self.last_rtstamp = self.authenticator.get_rtstamp(rawdata)
+        if numbytes is None or numbytes < 0:
+            raise ValueError('Stream has unlimited size, read all not supported.')
+        elif numbytes < len(content):
+            # TODO: buffer unread data for next read
+            raise ValueError('reading less than piecesize not supported yet')
+        else:
+            return content
+
+    def get_generation_time(self):
+        """ Returns the time at which the last read piece was generated at the source. """
+        return self.last_rtstamp
+    
+    def seek(self,pos,whence=os.SEEK_SET):
+        if pos == 0 and whence == os.SEEK_SET:
+            print >>sys.stderr,"authstream: seek: Ignoring seek 0 in live"
+        else:
+            raise ValueError("authstream does not support seek")
+
+    def close(self):
+        self.inputstream.close()
+
+    def available(self):
+        return self.inputstream.available()
+
+
+    # Internal method
+    def _readn(self,n):
+        """ read exactly n bytes from inputstream, block if unavail """
+        nwant = n
+        while True:
+            data = self.inputstream.read(nwant)
+            if len(data) == 0:
+                return data
+            nwant -= len(data)
+            self.buffer.write(data)
+            if nwant == 0:
+                break
+        self.buffer.seek(0)
+        data = self.buffer.read(n)
+        self.buffer.seek(0)
+        return data
+        
+
+
+class VariableReadAuthStreamWrapper:
+    """ Wrapper around AuthStreamWrapper that allows reading of variable
+    number of bytes. TODO: optimize whole stack of AuthWrapper, 
+    MovieTransportWrapper, MovieOnDemandTransporter
+    """
+    
+    def __init__(self,inputstream,piecelen):
+        self.inputstream = inputstream
+        self.buffer = ''
+        self.piecelen = piecelen
+
+    def read(self,numbytes=None):
+        if numbytes is None or numbytes < 0:
+            raise ValueError('Stream has unlimited size, read all not supported.')
+        return self._readn(numbytes)
+
+    def get_generation_time(self):
+        """ Returns the time at which the last read piece was generated at the source. """
+        return self.inputstream.get_generation_time()
+    
+    def seek(self,pos,whence=os.SEEK_SET):
+        return self.inputstream.seek(pos,whence=whence)
+        
+    def close(self):
+        self.inputstream.close()
+
+    def available(self):
+        return self.inputstream.available()
+
+    # Internal method
+    def _readn(self,nwant):
+        """ read *at most* nwant bytes from inputstream """
+        
+        if len(self.buffer) == 0:
+            # Must read fixed size blocks from authwrapper
+            data = self.inputstream.read(self.piecelen)
+            #print >>sys.stderr,"varread: Got",len(data),"want",nwant
+            if len(data) == 0:
+                return data
+            self.buffer = data
+
+        lenb = len(self.buffer)
+        tosend = min(nwant,lenb)
+            
+        if tosend == lenb:
+            #print >>sys.stderr,"varread: zero copy 2 lenb",lenb
+            pre = self.buffer
+            post = ''
+        else:
+            #print >>sys.stderr,"varread: copy",tosend,"lenb",lenb
+            pre = self.buffer[0:tosend]
+            post = self.buffer[tosend:]
+            
+        self.buffer = post
+        #print >>sys.stderr,"varread: Returning",len(pre)
+        return pre
+    
+        
diff --git a/instrumentation/next-share/BaseLib/Core/Video/MovieTransport.py b/instrumentation/next-share/BaseLib/Core/Video/MovieTransport.py
new file mode 100644
index 0000000..d0bef70
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Video/MovieTransport.py
@@ -0,0 +1,81 @@
+# Written by Jan David Mol, Arno Bakker
+# see LICENSE.txt for license information
+
+
+import os,sys
+
+from BaseLib.Core.osutils import *
+
+DEBUG = False
+
+class MovieTransport:
+    
+    def __init__(self):
+        pass
+        
+    def start( self, bytepos = 0 ):
+        pass
+    
+    def size(self ):
+        pass
+
+    def read(self):
+        pass
+        
+    def stop(self):
+        pass
+
+    def done(self):
+        pass
+    
+    def get_mimetype(self):
+        pass
+ 
+    def set_mimetype(self,mimetype):
+        pass
+
+    def available(self):
+        pass
+    
+ 
+class MovieTransportStreamWrapper:
+    """ Provide a file-like interface """
+    def __init__(self,mt):
+        self.mt = mt
+        self.started = False
+
+    def read(self,numbytes=None):
+        if DEBUG:
+            print >>sys.stderr,"MovieTransportStreamWrapper: read",numbytes
+
+        if not self.started:
+            self.mt.start(0)
+            self.started = True
+        if self.mt.done():
+            return ''
+        data = self.mt.read(numbytes)
+        if data is None:
+            print >>sys.stderr,"MovieTransportStreamWrapper: mt read returns None"
+            data = ''
+        return data
+
+    def seek(self,pos,whence=os.SEEK_SET):
+        # TODO: shift play_pos in PiecePicking + interpret whence
+        if DEBUG:
+            print >>sys.stderr,"MovieTransportStreamWrapper: seek:",pos,"whence",whence
+        self.mt.seek(pos,whence=whence)
+        # Arno, 2010-01-08: seek also means we've started.
+        self.started = True
+    
+    def close(self):
+        if DEBUG:
+            print >>sys.stderr,"MovieTransportStreamWrapper: close"
+        self.mt.stop()
+
+    def available(self):
+        return self.mt.available()
+    
+    def get_generation_time(self):
+        # Overrriden by AuthStreamWrapper normally. Added to give sane warning
+        # when playing unauthenticated stream as if it had auth.
+        raise ValueError("This is an unauthenticated stream that provides no timestamp")
diff --git a/instrumentation/next-share/BaseLib/Core/Video/PiecePickerSVC.py b/instrumentation/next-share/BaseLib/Core/Video/PiecePickerSVC.py
new file mode 100644
index 0000000..e0507cd
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Video/PiecePickerSVC.py
@@ -0,0 +1,652 @@
+# wRIsten by Jan David Mol, Arno Bakker, Riccardo Petrocco, George Milescu
+# see LICENSE.txt for license information
+
+import sys
+import time
+import random
+from traceback import print_exc
+
+from BaseLib.Core.BitTornado.BT1.PiecePicker import PiecePicker 
+if __debug__:
+    from BaseLib.Core.BitTornado.BT1.Downloader import print_chunks
+
+# percent piece loss to emulate -- we just don't request this percentage of the pieces
+# only implemented for live streaming
+PIECELOSS = 0
+
+DEBUG = False
+DEBUG_CHUNKS = False
+DEBUGPP = False
+
+def rarest_first( has_dict, rarity_list, filter = lambda x: True ):
+    """ Select the rarest of pieces in has_dict, according
+        to the rarities in rarity_list. Breaks ties uniformly
+        at random. Additionally, `filter' is applied to select
+        the pieces we can return. """
+
+    """ Strategy:
+        - `choice' is the choice so far
+        - `n' is the number of pieces we could choose from so far
+        - `rarity' is the rarity of the choice so far
+
+        Every time we see a rarer piece, we reset our choice.
+        Every time we see a piece of the same rarity we're looking for,
+        we select it (overriding the previous choice) with probability 1/n.
+        This leads to a uniformly selected piece in one pass, be it that
+        we need more random numbers than when doing two passes. """
+
+    choice = None
+    rarity = None
+    n = 0
+
+    for k in (x for x in has_dict if filter(x)):
+        r = rarity_list[k]
+
+        if rarity is None or r < rarity:
+            rarity = r
+            n = 1
+            choice = k
+        elif r == rarity:
+            n += 1
+            if random.uniform(0,n) == 0: # uniform selects from [0,n)
+                choice = k
+
+    return choice
+
+class PiecePickerSVC(PiecePicker):
+    """ Implements piece picking for streaming video. Keeps track of playback
+        point and avoids requesting obsolete pieces. """
+
+    # order of initialisation and important function calls
+    #   PiecePicker.__init__              (by BitTornado.BT1Download.__init__)
+    #   PiecePicker.complete              (by hash checker, for pieces on disk)
+    #   MovieSelector.__init__
+    #   PiecePicker.set_download_range    (indirectly by MovieSelector.__init__)
+    #   MovieOnDemandTransporter.__init__ (by BitTornado.BT1Download.startEngine)
+    #   PiecePicker.set_bitrate           (by MovieOnDemandTransporter)
+    #   PiecePicker.set_transporter       (by MovieOnDemandTransporter)
+    #
+    #   PiecePicker._next                 (once connections are set up)
+    #
+    #   PiecePicker.complete              (by hash checker, for pieces received)
+
+    # relative size of mid-priority set
+    MU = 4
+
+    def __init__(self, numpieces,
+                 rarest_first_cutoff = 1, rarest_first_priority_cutoff = 3,
+                 priority_step = 20, helper = None, coordinator = None, rate_predictor = None, piecesize = 0):
+        PiecePicker.__init__( self, numpieces, rarest_first_cutoff, rarest_first_priority_cutoff,
+                              priority_step, helper, coordinator, rate_predictor )
+
+        # maximum existing piece number, to avoid scanning beyond it in next()
+        self.maxhave = 0
+
+        # some statistics
+        self.stats = {}
+        self.stats["high"] = 0
+        self.stats["mid"] = 0
+        self.stats["low"] = 0
+
+        # playback module
+        self.transporter = None
+
+        # self.outstanding_requests contains (piece-id, begin,
+        # length):timestamp pairs for each outstanding request.
+        self.outstanding_requests = {}
+
+        # The playing_delay and buffering_delay give three values
+        # (min, max, offeset) in seconds.
+        #
+        # The min tells how long before the cancel policy is allowed
+        # to kick in. We can not expect to receive a piece instantly,
+        # so we have to wait this time before having a download speed
+        # estimation.
+        #
+        # The max tells how long before we cancel the request. The
+        # request may also be canceled because the chunk will not be
+        # completed given the current download speed.
+        #
+        # The offset gives a grace period that is taken into account
+        # when choosing to cancel a request. For instance, when the
+        # peer download speed is to low to receive the chunk within 10
+        # seconds, a grace offset of 15 would ensure that the chunk is
+        # NOT canceled (usefull while buffering)
+        self.playing_delay = (5, 20, -0.5)
+        self.buffering_delay = (7.5, 30, 10)
+        
+    def set_transporter(self, transporter):
+        self.transporter = transporter
+
+        # update its information -- pieces read from disk
+        download_range = self.videostatus.download_range()
+        for x in range(len(download_range)):
+            (f,l) = download_range[x]
+            for i in xrange(f, l):
+                if self.has[i]:
+                    self.transporter.complete( i, downloaded=False )
+
+    def set_videostatus(self,videostatus):
+        """ Download in a wrap-around fashion between pieces [0,numpieces).
+            Look at most delta pieces ahead from download_range[0].
+        """
+        self.videostatus = videostatus
+        videostatus.add_playback_pos_observer( self.change_playback_pos )
+
+    def is_interesting(self,piece):
+        if PIECELOSS and piece % 100 < PIECELOSS:
+            return False
+
+        if self.has[piece]:
+            return False
+
+        if not self.videostatus or self.videostatus.in_download_range( piece ):
+            return True
+
+        return False
+
+    def change_playback_pos(self, oldpos, newpos):
+        if oldpos is None:
+            # (re)initialise
+            valid = self.is_interesting
+
+            for d in self.peer_connections.values():
+                interesting = {}
+                has = d["connection"].download.have
+                for i in xrange(self.videostatus.first_piece,self.videostatus.last_piece+1):
+                    if has[i] and valid(i):
+                        interesting[i] = 1
+
+                d["interesting"] = interesting
+        else:
+            # playback position incremented -- remove timed out piece
+            for d in self.peer_connections.values():
+                d["interesting"].pop(oldpos,0)
+
+    def got_have(self, piece, connection=None):
+        # if DEBUG:
+        #     print >>sys.stderr,"PiecePickerStreaming: got_have:",piece
+        self.maxhave = max(self.maxhave,piece)
+        PiecePicker.got_have( self, piece, connection )
+        if self.transporter:
+            self.transporter.got_have( piece )
+
+        if self.is_interesting(piece):
+            self.peer_connections[connection]["interesting"][piece] = 1
+
+    def got_seed(self):
+        self.maxhave = self.numpieces
+        PiecePicker.got_seed( self )
+
+    def lost_have(self, piece):
+        PiecePicker.lost_have( self, piece )
+
+    def got_peer(self, connection):
+        PiecePicker.got_peer( self, connection )
+
+        self.peer_connections[connection]["interesting"] = {}
+
+    def lost_peer(self, connection):
+        PiecePicker.lost_peer( self, connection )
+
+    def got_piece(self, *request):
+        if request in self.outstanding_requests:
+            del self.outstanding_requests[request]
+        if self.transporter:
+            self.transporter.got_piece(*request)
+
+    def complete(self, piece):
+        # if DEBUG:
+        #     print >>sys.stderr,"PiecePickerStreaming: complete:",piece
+        PiecePicker.complete( self, piece )
+        if self.transporter:
+            self.transporter.complete( piece )
+
+        for request in self.outstanding_requests.keys():
+            if request[0] == piece:
+                del self.outstanding_requests[request]
+
+        # don't consider this piece anymore
+        for d in self.peer_connections.itervalues():
+            d["interesting"].pop(piece,0)
+
+    def num_nonempty_neighbours(self):
+        # return #neighbours who have something
+        return len( [c for c in self.peer_connections if c.download.have.numfalse < c.download.have.length] )
+
+    def pos_is_sustainable(self,fudge=2):
+        """
+            Returns whether we have enough data around us to support the current playback position.
+            If not, playback should pause, stall or reinitialised when pieces are lost.
+        """
+        vs = self.videostatus
+
+        # only holds for live streaming for now. theoretically, vod can have the same problem
+        # since data can be seeded in a 'live' fashion
+        if not vs.live_streaming:
+            if DEBUG:
+                print >>sys.stderr, "PiecePickerStreaming: pos is sustainable: not streaming live"
+            return True
+
+        # We assume the maximum piece number that is available at at least half of the neighbours
+        # to be sustainable. Although we only need a fixed number of neighbours with enough bandwidth,
+        # such neighbours may depart, hence we choose a relative trade-off.
+
+        # this means that our current playback position is sustainable if any future piece
+        # is owned by at least half of the peers
+
+        # ignore peers which have nothing
+        numconn = self.num_nonempty_neighbours()
+
+        if not numconn:
+            # not sustainable, but nothing we can do. Return True to avoid pausing
+            # and getting out of sync.
+            if DEBUG:
+                print >>sys.stderr, "PiecePickerStreaming: pos is sustainable: no neighbours with pieces"
+            return True
+
+        half = max( 1, numconn/2 )
+        skip = fudge # ignore the first 'fudge' pieces
+
+        for x in vs.generate_range( vs.download_range() ):
+            if skip > 0:
+                skip -= 1
+            elif self.numhaves[x] >= half:
+                if DEBUG:
+                    print >>sys.stderr, "PiecePickerStreaming: pos is sustainable: piece %s @ %s>%s peers (fudge=%s)" % (x,self.numhaves[x],half,fudge)
+                return True
+            else:
+                pass
+
+        if DEBUG:
+            print >>sys.stderr, "PiecePickerStreaming: pos is NOT sustainable playpos=%s fudge=%s numconn=%s half=%s numpeers=%s %s" % (vs.playback_pos,fudge,numconn,half,len(self.peer_connections),[x.get_ip() for x in self.peer_connections])
+
+        # too few neighbours own the future pieces. it's wise to pause and let neighbours catch up
+        # with us
+        return False
+
+
+    # next: selects next piece to download. adjusts wantfunc with filter for streaming; calls
+    #   _next: selects next piece to download. completes partial downloads first, if needed, otherwise calls
+    #     next_new: selects next piece to download. override this with the piece picking policy
+
+    def next(self, haves, wantfunc, sdownload, complete_first = False, helper_con = False, slowpieces=[], willrequest=True,connection=None,proxyhave=None):
+        def newwantfunc( piece ):
+            #print >>sys.stderr,"S",self.streaming_piece_filter( piece ),"!sP",not (piece in slowpieces),"w",wantfunc( piece )
+            return not (piece in slowpieces) and wantfunc( piece )
+
+        # fallback: original piece picker
+        p = PiecePicker.next(self, haves, newwantfunc, sdownload, complete_first, helper_con, slowpieces=slowpieces, willrequest=willrequest,connection=connection)
+        if DEBUGPP and self.videostatus.prebuffering:
+            print >>sys.stderr,"PiecePickerStreaming: original PP.next returns",p
+        if p is None and not self.videostatus.live_streaming:
+            # When the file we selected from a multi-file torrent is complete,
+            # we won't request anymore pieces, so the normal way of detecting 
+            # we're done is not working and we won't tell the video player 
+            # we're playable. Do it here instead.
+            self.transporter.notify_playable()
+        return p
+
+    def _next(self, haves, wantfunc, complete_first, helper_con, willrequest=True, connection=None):
+        """ First, complete any partials if needed. Otherwise, select a new piece. """
+
+        #print >>sys.stderr,"PiecePickerStreaming: complete_first is",complete_first,"started",self.started
+
+        # cutoff = True:  random mode
+        #          False: rarest-first mode
+        cutoff = self.numgot < self.rarest_first_cutoff
+
+        # whether to complete existing partials first -- do so before the
+        # cutoff, or if forced by complete_first, but not for seeds.
+        #complete_first = (complete_first or cutoff) and not haves.complete()
+        complete_first = (complete_first or cutoff)
+
+        # most interesting piece
+        best = None
+
+        # interest level of best piece
+        bestnum = 2 ** 30
+
+        # select piece we started to download with best interest index.
+        for i in self.started:
+# 2fastbt_
+            if haves[i] and wantfunc(i) and (self.helper is None or helper_con or not self.helper.is_ignored(i)):
+# _2fastbt
+                if self.level_in_interests[i] < bestnum:
+                    best = i
+                    bestnum = self.level_in_interests[i]
+
+        if best is not None:
+            # found a piece -- return it if we are completing partials first
+            # or if there is a cutoff
+            if complete_first or (cutoff and len(self.interests) > self.cutoff):
+                return best
+
+        p = self.next_new(haves, wantfunc, complete_first, helper_con,willrequest=willrequest,connection=connection)
+        # if DEBUG:
+        #     print >>sys.stderr,"PiecePickerStreaming: next_new returns",p
+        return p
+
+    def check_outstanding_requests(self, downloads):
+        if not self.transporter:
+            return
+        
+        now = time.time()
+        cancel_requests = []
+        in_high_range = self.videostatus.in_high_range
+        playing_mode = self.videostatus.playing and not self.videostatus.paused
+        piece_due = self.transporter.piece_due
+        
+        if playing_mode:
+            # playing mode
+            min_delay, max_delay, offset_delay = self.playing_delay
+        else:
+            # buffering mode
+            min_delay, max_delay, offset_delay = self.buffering_delay
+
+        for download in downloads:
+            total_length = 0
+            download_rate = download.get_short_term_rate()
+            for piece_id, begin, length in download.active_requests:
+                # select policy for this piece
+        
+                try:
+                    time_request = self.outstanding_requests[(piece_id, begin, length)]
+                except KeyError:
+                    continue
+                
+                # add the length of this chunk to the total of bytes
+                # that needs to be downloaded
+                total_length += length
+        
+                # each request must be allowed at least some
+                # minimal time to be handled
+                if now < time_request + min_delay:
+                    continue
+
+                # high-priority pieces are eligable for
+                # cancelation. Others are not. They will eventually be
+                # eligable as they become important for playback.
+                if in_high_range(piece_id):
+                    if download_rate == 0:
+                        # we have not received anything in the last min_delay seconds
+                        if DEBUG: print >>sys.stderr, "PiecePickerStreaming: download not started yet for piece", piece_id, "chunk", begin, "on", download.ip
+                        cancel_requests.append((piece_id, begin, length))
+                        download.bad_performance_counter += 1
+
+                    else:
+                        if playing_mode:
+                            time_until_deadline = min(piece_due(piece_id), time_request + max_delay - now)
+                        else:
+                            time_until_deadline = time_request + max_delay - now
+                        time_until_download = total_length / download_rate
+
+                        # we have to cancel when the deadline can not be met
+                        if time_until_deadline < time_until_download - offset_delay:
+                            if DEBUG: print >>sys.stderr, "PiecePickerStreaming: download speed too slow for piece", piece_id, "chunk", begin, "on", download.ip, "Deadline in", time_until_deadline, "while estimated download in", time_until_download
+                            cancel_requests.append((piece_id, begin, length))
+                
+        # Cancel all requests that are too late
+        if cancel_requests:
+            try:
+                self.downloader.cancel_requests(cancel_requests)
+            except:
+                print_exc()
+
+        if __debug__:
+            if DEBUG_CHUNKS:
+                print_chunks(self.downloader, list(self.videostatus.generate_high_range()), compact=False)
+
+    def requested(self, *request):
+        self.outstanding_requests[request] = time.time()
+        return PiecePicker.requested(self, *request)
+        
+    def next_new(self, haves, wantfunc, complete_first, helper_con, willrequest=True, connection=None):
+        """ Determine which piece to download next from a peer.
+
+        haves:          set of pieces owned by that peer
+        wantfunc:       custom piece filter
+        complete_first: whether to complete partial pieces first
+        helper_con:
+        willrequest:    whether the returned piece will actually be requested
+
+        """
+        vs = self.videostatus
+
+        def pick_first( f, t ): # no shuffle
+            for i in vs.generate_range([(f,t)]):
+                # Is there a piece in the range the peer has?
+                # Is there a piece in the range we don't have?
+                if not haves[i] or self.has[i]: 
+                    continue
+
+                if not wantfunc(i): # Is there a piece in the range we want? 
+                    continue
+
+                if self.helper is None or helper_con or not self.helper.is_ignored(i):
+                    return i
+
+            return None
+
+        def pick_rarest_loop_over_small_range(f,t,shuffle=True):
+            # Arno: pick_rarest is way expensive for the midrange thing,
+            # therefore loop over the list of pieces we want and see
+            # if it's avail, rather than looping over the list of all
+            # pieces to see if one falls in the (f,t) range.
+            #
+            xr = vs.generate_range([(f,t)])
+            #xr = xrl[0]
+            r = None
+            if shuffle:
+                # xr is an xrange generator, need real values to shuffle
+                r = []
+                r.extend(xr)
+                random.shuffle(r)
+            else:
+                r = xr
+            for i in r:
+                #print >>sys.stderr,"H",
+                if not haves[i] or self.has[i]:
+                    continue
+
+                #print >>sys.stderr,"W",
+                if not wantfunc(i):
+                    continue
+
+                if self.helper is None or helper_con or not self.helper.is_ignored(i):
+                    return i
+
+            return None
+
+        def pick_rarest_small_range(f,t):
+            #print >>sys.stderr,"choice small",f,t
+            d = vs.dist_range(f,t)
+            
+            for level in xrange(len(self.interests)):
+                piecelist  = self.interests[level]
+                
+                if len(piecelist) > d:
+                #if level+1 == len(self.interests):
+                    # Arno: Lowest level priorities / long piecelist.
+                    # This avoids doing a scan that goes over the entire list 
+                    # of pieces when we already have the hi and/or mid ranges.
+                    
+                    # Arno, 2008-05-21: Apparently, the big list is not always
+                    # at the lowest level, hacked distance metric to determine
+                    # whether to use slow or fast method.
+                    
+                    #print >>sys.stderr,"choice QUICK"
+                    return pick_rarest_loop_over_small_range(f,t)
+                    #print >>sys.stderr,"choice Q",diffstr,"l",level,"s",len(piecelist) 
+                else:
+                    # Higher priorities / short lists
+                    for i in piecelist:
+                        if not vs.in_range( [(f, t)], i ):
+                            continue
+    
+                        #print >>sys.stderr,"H",
+                        if not haves[i] or self.has[i]:
+                            continue
+    
+                        #print >>sys.stderr,"W",
+                        if not wantfunc(i):
+                            continue
+    
+                        if self.helper is None or helper_con or not self.helper.is_ignored(i):
+                            return i
+
+            return None
+
+        def pick_rarest(f,t): #BitTorrent already shuffles the self.interests for us
+            for piecelist in self.interests:
+                for i in piecelist:
+                    if not vs.in_range( f, t, i ):
+                        continue
+
+                    #print >>sys.stderr,"H",
+                    if not haves[i] or self.has[i]:
+                        continue
+
+                    #print >>sys.stderr,"W",
+                    if not wantfunc(i):
+                        continue
+
+                    if self.helper is None or helper_con or not self.helper.is_ignored(i):
+                        return i
+
+            return None
+
+        def pieces_in_buffer_completed(high_range):
+            for i in vs.generate_range(high_range):
+                if not self.has[i]:            
+                    return False
+            return True
+            
+        # Ric: mod for new set of high priority
+        download_range = vs.download_range()
+        first, _ = download_range[0]
+        last = vs.get_highest_piece(download_range)
+        high_range = vs.get_high_range()
+        
+        #print >>sys.stderr , "wwwwwwwwwwwwww", download_range, high_range, first, last300.13KB/s
+        priority_first, _ = high_range[0]
+        priority_last = vs.get_highest_piece(high_range)
+        
+        if priority_first != priority_last:
+            first = priority_first
+            highprob_cutoff = vs.normalize(priority_last + 1)
+            # TODO
+            midprob_cutoff = vs.normalize(first + self.MU * vs.get_small_range_length(first, last))
+        else:
+            highprob_cutoff = last
+            midprob_cutoff = vs.normalize(first + self.MU * vs.high_prob_curr_pieces)
+
+        # for VOD playback consider peers to be bad when they miss the deadline 1 time
+        allow_based_on_performance = connection.download.bad_performance_counter < 1
+
+        # Ric: only prebuffering of the base layer
+        if vs.prebuffering:
+            f = first
+            t = vs.normalize( first + self.transporter.max_prebuf_packets )
+            choice = pick_rarest_small_range(f,t)
+            type = "high"
+        else:
+            choice = None
+
+        if choice is None:
+            for i in high_range:
+                f, l = i
+                choice = pick_first( f, l )
+                if choice != None:
+                    # TODO bad hack
+                    break
+
+            # once there are no more pieces to pick in the current high
+            # priority set, increase the quality and run the code again
+            if choice is None and vs.quality < vs.available_qualities-1:
+                # We increase the quality only if we already recieved all
+                # the pieces from the current high priority set
+                if pieces_in_buffer_completed(high_range):
+                    if DEBUG:
+                        print >>sys.stderr, "vod: Enough pieces of the current quality have been downloaded. Increase the quality!"
+                    vs.quality += 1
+                    self.next_new(haves, wantfunc, complete_first, helper_con, willrequest, connection)
+                
+
+            type = "high"
+
+        # it is possible that the performance of this peer prohibits
+        # us from selecting this piece...
+        if not allow_based_on_performance:
+            high_priority_choice = choice
+            choice = None
+
+        if choice is None:
+            temp_range = vs.get_respective_range( (highprob_cutoff, midprob_cutoff) )
+            for i in range(vs.quality + 1):
+                f, l = temp_range[i]
+                choice = pick_rarest_small_range( f, l )
+                if choice != None:
+                    # TODO bad hack
+                    break
+            type = "mid"
+
+        if choice is None:
+            temp_range = vs.get_respective_range( (midprob_cutoff, last) )
+            for i in temp_range:
+                f, l = i
+                choice = pick_rarest( f, l )
+                if choice != None:
+                    # TODO bad hack
+                    break
+            type = "low"
+            
+        if choice and willrequest:
+            self.stats[type] += 1
+
+        if DEBUG:
+            # TODO
+            print >>sys.stderr,"vod: picked piece %s [type=%s] [%d,%d,%d,%d]" % (`choice`,type,first,highprob_cutoff,midprob_cutoff,last)
+            #print >>sys.stderr,"vod: picked piece %s [type=%s] [%d,%d]" % (`choice`,type,first,highprob_cutoff)
+
+        # 12/05/09, boudewijn: (1) The bad_performance_counter is
+        # incremented whenever a piece download failed and decremented
+        # whenever is succeeds. (2) A peer with a positive
+        # bad_performance_counter is only allowd to pick low-priority
+        # pieces. (Conclusion) When all low-priority pieces are
+        # downloaded the client hangs when one or more high-priority
+        # pieces are required and if all peers have a positive
+        # bad_performance_counter.
+        if choice is None and not allow_based_on_performance:
+            # ensure that there is another known peer with a
+            # non-positive bad_performance_counter that has the piece
+            # that we would pick from the high-priority set for this
+            # connection.
+
+            if high_priority_choice:
+                availability = 0
+                for download in self.downloader.downloads:
+                    if download.have[high_priority_choice] and not download.bad_performance_counter:
+                        availability += 1
+
+                if not availability:
+                    # no other connection has it... then ignore the
+                    # bad_performance_counter advice and attempt to
+                    # download it from this connection anyway
+                    if DEBUG: print >>sys.stderr, "vod: the bad_performance_counter says this is a bad peer... but we have nothing better... requesting piece", high_priority_choice, "regardless."
+                    choice = high_priority_choice
+
+        return choice
+
+    def is_valid_piece(self,piece):
+        return self.videostatus.in_valid_range(piece)
+   
+    def get_valid_range_iterator(self):
+
+        #print >>sys.stderr,"PiecePickerStreaming: Live hooked in, or VOD, valid range set to subset"
+        download_range = self.videostatus.download_range()
+#        first,last = self.videostatus.download_range()
+#        return self.videostatus.generate_range((first,last))
+        return self.videostatus.generate_range(download_range)
+            
+
diff --git a/instrumentation/next-share/BaseLib/Core/Video/PiecePickerStreaming.py b/instrumentation/next-share/BaseLib/Core/Video/PiecePickerStreaming.py
new file mode 100644
index 0000000..a9925d8
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Video/PiecePickerStreaming.py
@@ -0,0 +1,700 @@
+# Written by Jan David Mol, Arno Bakker, George Milescu
+# see LICENSE.txt for license information
+
+import sys
+import time
+import random
+from traceback import print_exc,print_stack
+
+from BaseLib.Core.BitTornado.BT1.PiecePicker import PiecePicker 
+
+if __debug__:
+    from BaseLib.Core.BitTornado.BT1.Downloader import print_chunks
+
+# percent piece loss to emulate -- we just don't request this percentage of the pieces
+# only implemented for live streaming
+#PIECELOSS = 0
+TEST_VOD_OVERRIDE = False
+
+DEBUG = False
+DEBUG_CHUNKS = False # set DEBUG_CHUNKS in BT1.Downloader to True
+DEBUGPP = False
+
+def rarest_first( has_dict, rarity_list, filter = lambda x: True ):
+    """ Select the rarest of pieces in has_dict, according
+        to the rarities in rarity_list. Breaks ties uniformly
+        at random. Additionally, `filter' is applied to select
+        the pieces we can return. """
+
+    """ Strategy:
+        - `choice' is the choice so far
+        - `n' is the number of pieces we could choose from so far
+        - `rarity' is the rarity of the choice so far
+
+        Every time we see a rarer piece, we reset our choice.
+        Every time we see a piece of the same rarity we're looking for,
+        we select it (overriding the previous choice) with probability 1/n.
+        This leads to a uniformly selected piece in one pass, be it that
+        we need more random numbers than when doing two passes. """
+
+    choice = None
+    rarity = None
+    n = 0
+
+    for k in (x for x in has_dict if filter(x)):
+        r = rarity_list[k]
+
+        if rarity is None or r < rarity:
+            rarity = r
+            n = 1
+            choice = k
+        elif r == rarity:
+            n += 1
+            if random.uniform(0,n) == 0: # uniform selects from [0,n)
+                choice = k
+
+    return choice
+
+class PiecePickerStreaming(PiecePicker):
+    """ Implements piece picking for streaming video. Keeps track of playback
+        point and avoids requesting obsolete pieces. """
+
+    # order of initialisation and important function calls
+    #   PiecePicker.__init__              (by BitTornado.BT1Download.__init__)
+    #   PiecePicker.complete              (by hash checker, for pieces on disk)
+    #   MovieSelector.__init__
+    #   PiecePicker.set_download_range    (indirectly by MovieSelector.__init__)
+    #   MovieOnDemandTransporter.__init__ (by BitTornado.BT1Download.startEngine)
+    #   PiecePicker.set_bitrate           (by MovieOnDemandTransporter)
+    #   PiecePicker.set_transporter       (by MovieOnDemandTransporter)
+    #
+    #   PiecePicker._next                 (once connections are set up)
+    #
+    #   PiecePicker.complete              (by hash checker, for pieces received)
+
+    # relative size of mid-priority set
+    MU = 4
+
+    def __init__(self, numpieces,
+                 rarest_first_cutoff = 1, rarest_first_priority_cutoff = 3,
+                 priority_step = 20, helper = None, coordinator = None, rate_predictor = None, piecesize = 0):
+        PiecePicker.__init__( self, numpieces, rarest_first_cutoff, rarest_first_priority_cutoff,
+                              priority_step, helper, coordinator, rate_predictor)
+
+        # maximum existing piece number, to avoid scanning beyond it in next()
+        self.maxhave = 0
+
+        # some statistics
+        self.stats = {}
+        self.stats["high"] = 0
+        self.stats["mid"] = 0
+        self.stats["low"] = 0
+
+        # playback module
+        self.transporter = None
+
+        # self.outstanding_requests contains (piece-id, begin,
+        # length):timestamp pairs for each outstanding request.
+        self.outstanding_requests = {}
+
+        # The playing_delay and buffering_delay give three values
+        # (min, max, offeset) in seconds.
+        #
+        # The min tells how long before the cancel policy is allowed
+        # to kick in. We can not expect to receive a piece instantly,
+        # so we have to wait this time before having a download speed
+        # estimation.
+        #
+        # The max tells how long before we cancel the request. The
+        # request may also be canceled because the chunk will not be
+        # completed given the current download speed.
+        #
+        # The offset gives a grace period that is taken into account
+        # when choosing to cancel a request. For instance, when the
+        # peer download speed is too low to receive the chunk within 10
+        # seconds, a grace offset of 15 would ensure that the chunk is
+        # NOT canceled (useful while buffering)
+        self.playing_delay = (5, 20, -0.5)
+        self.buffering_delay = (7.5, 30, 10)
+
+        # Arno, 2010-04-20: STBSPEED: is_interesting is now a variable.
+        self.is_interesting  = self.is_interesting_normal
+
+    def set_transporter(self, transporter):
+        self.transporter = transporter
+
+        """
+        Arno, 2010-04-20: STBSPEED: Replaced by transporter.complete_from_persistent_state()
+        # update its information -- pieces read from disk
+        if not self.videostatus.live_streaming:
+            for i in xrange(self.videostatus.first_piece,self.videostatus.last_piece+1):
+                if self.has[i]:
+                    self.transporter.complete( i, downloaded=False )
+        """
+
+    def set_videostatus(self,videostatus):
+        """ Download in a wrap-around fashion between pieces [0,numpieces).
+            Look at most delta pieces ahead from download_range[0].
+        """
+        self.videostatus = videostatus
+        
+        if self.videostatus.live_streaming:
+            self.is_interesting  = self.is_interesting_live
+        else:
+            self.is_interesting  = self.is_interesting_vod
+        videostatus.add_playback_pos_observer( self.change_playback_pos )
+
+    def is_interesting_live(self,piece):
+        return self.videostatus.in_download_range( piece ) and not self.has[piece]
+
+    def is_interesting_vod(self,piece):
+        return (self.videostatus.first_piece <= piece <= self.videostatus.last_piece) and not self.has[piece]
+
+    def is_interesting_normal(self,piece):
+        return not self.has[piece]
+
+
+    def change_playback_pos(self, oldpos, newpos):
+        if oldpos is None:
+            # (re)initialise
+            valid = self.is_interesting
+
+            for d in self.peer_connections.values():
+                interesting = {}
+                has = d["connection"].download.have
+                
+                # Arno, 2009-11-07: STBSPEED: iterator over just valid range, that's
+                # what we'll be interested in.
+                #for i in xrange(self.videostatus.first_piece,self.videostatus.last_piece+1):
+                for i in self.get_valid_range_iterator():
+                    if has[i] and valid(i):
+                        interesting[i] = 1
+
+                d["interesting"] = interesting
+        else:
+            # playback position incremented -- remove timed out piece
+            for d in self.peer_connections.values():
+                d["interesting"].pop(oldpos,0)
+
+    def got_have(self, piece, connection=None):
+        # if DEBUG:
+        #     print >>sys.stderr,"PiecePickerStreaming: got_have:",piece
+        self.maxhave = max(self.maxhave,piece)
+
+        # Arno, 2010-04-15: STBSPEED Disabled, does nothing but stats.
+        #if self.transporter:
+        #    self.transporter.got_have( piece )
+        PiecePicker.got_have(self,piece,connection)
+
+        if self.is_interesting(piece):
+            self.peer_connections[connection]["interesting"][piece] = 1
+
+
+    def got_seed(self):
+        self.maxhave = self.numpieces
+        PiecePicker.got_seed( self )
+
+    def lost_have(self, piece):
+        PiecePicker.lost_have( self, piece )
+
+    def got_peer(self, connection):
+        PiecePicker.got_peer( self, connection )
+
+        self.peer_connections[connection]["interesting"] = {}
+
+    def lost_peer(self, connection):
+        PiecePicker.lost_peer( self, connection )
+
+    def got_piece(self, *request):
+        if request in self.outstanding_requests:
+            del self.outstanding_requests[request]
+        if self.transporter:
+            self.transporter.got_piece(*request)
+
+    def complete(self, piece):
+        if DEBUG:
+            print >>sys.stderr,"PiecePickerStreaming: complete:",piece
+        
+        PiecePicker.complete( self, piece )
+        if self.transporter:
+            self.transporter.complete( piece )
+
+        for request in self.outstanding_requests.keys():
+            if request[0] == piece:
+                del self.outstanding_requests[request]
+
+        # don't consider this piece anymore
+        for d in self.peer_connections.itervalues():
+            d["interesting"].pop(piece,0)
+
+    def num_nonempty_neighbours(self):
+        # return #neighbours who have something
+        return len( [c for c in self.peer_connections if c.download.have.numfalse < c.download.have.length] )
+
+    def pos_is_sustainable(self,fudge=2):
+        """
+            Returns whether we have enough data around us to support the current playback position.
+            If not, playback should pause, stall or reinitialised when pieces are lost.
+        """
+        vs = self.videostatus
+
+        # only holds for live streaming for now. theoretically, vod can have the same problem
+        # since data can be seeded in a 'live' fashion
+        if not vs.live_streaming:
+            if DEBUG:
+                print >>sys.stderr, "PiecePickerStreaming: pos is sustainable: not streaming live"
+            return True
+
+        # We assume the maximum piece number that is available at at least half of the neighbours
+        # to be sustainable. Although we only need a fixed number of neighbours with enough bandwidth,
+        # such neighbours may depart, hence we choose a relative trade-off.
+
+        # this means that our current playback position is sustainable if any future piece
+        # is owned by at least half of the peers
+
+        # ignore peers which have nothing
+        numconn = self.num_nonempty_neighbours()
+
+        if not numconn:
+            # not sustainable, but nothing we can do. Return True to avoid pausing
+            # and getting out of sync.
+            if DEBUG:
+                print >>sys.stderr, "PiecePickerStreaming: pos is sustainable: no neighbours with pieces"
+            return True
+
+        half = max( 1, numconn/2 )
+        skip = fudge # ignore the first 'fudge' pieces
+
+        for x in vs.generate_range( vs.download_range() ):
+            if skip > 0:
+                skip -= 1
+            elif self.numhaves[x] >= half:
+                if DEBUG:
+                    print >>sys.stderr, "PiecePickerStreaming: pos is sustainable: piece %s @ %s>%s peers (fudge=%s)" % (x,self.numhaves[x],half,fudge)
+                return True
+            else:
+                pass
+
+        if DEBUG:
+            print >>sys.stderr, "PiecePickerStreaming: pos is NOT sustainable playpos=%s fudge=%s numconn=%s half=%s numpeers=%s %s" % (vs.playback_pos,fudge,numconn,half,len(self.peer_connections),[x.get_ip() for x in self.peer_connections])
+
+        # too few neighbours own the future pieces. it's wise to pause and let neighbours catch up
+        # with us
+        return False
+
+
+    # next: selects next piece to download. adjusts wantfunc with filter for streaming; calls
+    #   _next: selects next piece to download. completes partial downloads first, if needed, otherwise calls
+    #     next_new: selects next piece to download. override this with the piece picking policy
+
+    def next(self, haves, wantfunc, sdownload, complete_first = False, helper_con = False, slowpieces=[], willrequest=True,connection=None,proxyhave=None):
+        def newwantfunc( piece ):
+            #print >>sys.stderr,"S",self.streaming_piece_filter( piece ),"!sP",not (piece in slowpieces),"w",wantfunc( piece )
+            return not (piece in slowpieces) and wantfunc( piece )
+
+        # fallback: original piece picker
+        p = PiecePicker.next(self, haves, newwantfunc, sdownload, complete_first, helper_con, slowpieces=slowpieces, willrequest=willrequest,connection=connection)
+        if DEBUGPP and self.videostatus.prebuffering:
+            print >>sys.stderr,"PiecePickerStreaming: original PP.next returns",p
+        # Arno, 2010-03-11: Njaal's CS something causes this to return None
+        # when we're not complete: added check
+        if p is None and not self.videostatus.live_streaming and self.am_I_complete() or TEST_VOD_OVERRIDE:
+            # When the file we selected from a multi-file torrent is complete,
+            # we won't request anymore pieces, so the normal way of detecting 
+            # we're done is not working and we won't tell the video player 
+            # we're playable. Do it here instead.
+            self.transporter.notify_playable()
+        return p
+
+    def _next(self, haves, wantfunc, complete_first, helper_con, willrequest=True, connection=None):
+        """ First, complete any partials if needed. Otherwise, select a new piece. """
+
+        #print >>sys.stderr,"PiecePickerStreaming: complete_first is",complete_first,"started",self.started
+
+        # cutoff = True:  random mode
+        #          False: rarest-first mode
+        cutoff = self.numgot < self.rarest_first_cutoff
+
+        # whether to complete existing partials first -- do so before the
+        # cutoff, or if forced by complete_first, but not for seeds.
+        #complete_first = (complete_first or cutoff) and not haves.complete()
+        complete_first = (complete_first or cutoff)
+
+        # most interesting piece
+        best = None
+
+        # interest level of best piece
+        bestnum = 2 ** 30
+
+        # select piece we started to download with best interest index.
+        for i in self.started:
+# 2fastbt_
+            if haves[i] and wantfunc(i) and (self.helper is None or helper_con or not self.helper.is_ignored(i)):
+# _2fastbt
+                if self.level_in_interests[i] < bestnum:
+                    best = i
+                    bestnum = self.level_in_interests[i]
+
+        if best is not None:
+            # found a piece -- return it if we are completing partials first
+            # or if there is a cutoff
+            if complete_first or (cutoff and len(self.interests) > self.cutoff):
+                return best
+
+        p = self.next_new(haves, wantfunc, complete_first, helper_con,willrequest=willrequest,connection=connection)
+        if DEBUG:
+             print >>sys.stderr,"PiecePickerStreaming: next_new returns",p
+        return p
+
+    def check_outstanding_requests(self, downloads):
+        if not self.transporter:
+            return
+        
+        now = time.time()
+        cancel_requests = []
+        in_high_range = self.videostatus.in_high_range
+        playing_mode = self.videostatus.playing and not self.videostatus.paused
+        piece_due = self.transporter.piece_due
+        
+        if playing_mode:
+            # playing mode
+            min_delay, max_delay, offset_delay = self.playing_delay
+        else:
+            # buffering mode
+            min_delay, max_delay, offset_delay = self.buffering_delay
+
+        for download in downloads:
+
+            total_length = 0
+            download_rate = download.get_short_term_rate()
+            for piece_id, begin, length in download.active_requests:
+                # select policy for this piece
+                try:
+                    time_request = self.outstanding_requests[(piece_id, begin, length)]
+                except KeyError:
+                    continue
+                
+                # add the length of this chunk to the total of bytes
+                # that needs to be downloaded
+                total_length += length
+
+                # each request must be allowed at least some
+                # minimal time to be handled
+                if now < time_request + min_delay:
+                    continue
+
+                # high-priority pieces are eligable for
+                # cancelation. Others are not. They will eventually be
+                # eligable as they become important for playback.
+                if in_high_range(piece_id):
+                    if download_rate == 0:
+                        # we have not received anything in the last min_delay seconds
+                        if DEBUG: print >>sys.stderr, "PiecePickerStreaming: download not started yet for piece", piece_id, "chunk", begin, "on", download.ip
+                        cancel_requests.append((piece_id, begin, length))
+                        download.bad_performance_counter += 1
+
+                    else:
+                        if playing_mode:
+                            time_until_deadline = min(piece_due(piece_id), time_request + max_delay - now)
+                        else:
+                            time_until_deadline = time_request + max_delay - now
+                        time_until_download = total_length / download_rate
+
+                        # we have to cancel when the deadline can not be met
+                        if time_until_deadline < time_until_download - offset_delay:
+                            if DEBUG: print >>sys.stderr, "PiecePickerStreaming: download speed too slow for piece", piece_id, "chunk", begin, "on", download.ip, "Deadline in", time_until_deadline, "while estimated download in", time_until_download
+                            cancel_requests.append((piece_id, begin, length))
+                
+        # Cancel all requests that are too late
+        if cancel_requests:
+            try:
+                self.downloader.cancel_requests(cancel_requests)
+            except:
+                print_exc()
+
+        if __debug__:
+            if DEBUG_CHUNKS:
+                print_chunks(self.downloader, list(self.videostatus.generate_high_range()), compact=False)
+
+    def requested(self, *request):
+        self.outstanding_requests[request] = time.time()
+        return PiecePicker.requested(self, *request)
+        
+    def next_new(self, haves, wantfunc, complete_first, helper_con, willrequest=True, connection=None):
+        """ Determine which piece to download next from a peer.
+
+        haves:          set of pieces owned by that peer
+        wantfunc:       custom piece filter
+        complete_first: whether to complete partial pieces first
+        helper_con:
+        willrequest:    whether the returned piece will actually be requested
+
+        """
+
+        vs = self.videostatus
+
+        if vs.live_streaming:
+            # first, make sure we know where to start downloading
+            if vs.live_startpos is None:
+                self.transporter.calc_live_startpos( self.transporter.max_prebuf_packets, False )
+                print >>sys.stderr,"vod: pp: determined startpos of",vs.live_startpos
+
+            # select any interesting piece, rarest first
+            if connection:
+                # Without 'connection', we don't know who we will request from.
+                
+                #print >>sys.stderr,"PiecePickerStreaming: pp",connection.get_ip(),"int",self.peer_connections[connection]["interesting"]
+                
+                return rarest_first( self.peer_connections[connection]["interesting"], self.numhaves, wantfunc )
+
+        def pick_first( f, t ): # no shuffle
+            for i in vs.generate_range((f,t)):
+                # Is there a piece in the range the peer has?
+                # Is there a piece in the range we don't have?
+                if not haves[i] or self.has[i]: 
+                    continue
+
+                if not wantfunc(i): # Is there a piece in the range we want? 
+                    continue
+
+                if self.helper is None or helper_con or not self.helper.is_ignored(i):
+                    return i
+
+            return None
+
+        def pick_rarest_loop_over_small_range(f,t,shuffle=True):
+            # Arno: pick_rarest is way expensive for the midrange thing,
+            # therefore loop over the list of pieces we want and see
+            # if it's avail, rather than looping over the list of all
+            # pieces to see if one falls in the (f,t) range.
+            #
+            xr = vs.generate_range((f,t))
+            r = None
+            if shuffle:
+                # xr is an xrange generator, need real values to shuffle
+                r = []
+                r.extend(xr)
+                random.shuffle(r)
+            else:
+                r = xr
+            for i in r:
+                #print >>sys.stderr,"H",
+                if not haves[i] or self.has[i]:
+                    continue
+
+                #print >>sys.stderr,"W",
+                if not wantfunc(i):
+                    continue
+
+                if self.helper is None or helper_con or not self.helper.is_ignored(i):
+                    return i
+
+            return None
+
+        def pick_rarest_small_range(f,t):
+            #print >>sys.stderr,"choice small",f,t
+            d = vs.dist_range(f,t)
+            
+            for level in xrange(len(self.interests)):
+                piecelist  = self.interests[level]
+                
+                if len(piecelist) > d:
+                #if level+1 == len(self.interests):
+                    # Arno: Lowest level priorities / long piecelist.
+                    # This avoids doing a scan that goes over the entire list 
+                    # of pieces when we already have the hi and/or mid ranges.
+                    
+                    # Arno, 2008-05-21: Apparently, the big list is not always
+                    # at the lowest level, hacked distance metric to determine
+                    # whether to use slow or fast method.
+                    
+                    #print >>sys.stderr,"choice QUICK"
+                    return pick_rarest_loop_over_small_range(f,t)
+                    #print >>sys.stderr,"choice Q",diffstr,"l",level,"s",len(piecelist) 
+                else:
+                    # Higher priorities / short lists
+                    for i in piecelist:
+                        if not vs.in_range( f, t, i ):
+                            continue
+    
+                        #print >>sys.stderr,"H",
+                        if not haves[i] or self.has[i]:
+                            continue
+    
+                        #print >>sys.stderr,"W",
+                        if not wantfunc(i):
+                            continue
+    
+                        if self.helper is None or helper_con or not self.helper.is_ignored(i):
+                            return i
+
+            return None
+
+        def pick_rarest(f,t): #BitTorrent already shuffles the self.interests for us
+            for piecelist in self.interests:
+                for i in piecelist:
+                    if not vs.in_range( f, t, i ):
+                        continue
+
+                    #print >>sys.stderr,"H",
+                    if not haves[i] or self.has[i]:
+                        continue
+
+                    #print >>sys.stderr,"W",
+                    if not wantfunc(i):
+                        continue
+
+                    if self.helper is None or helper_con or not self.helper.is_ignored(i):
+                        return i
+
+            return None
+
+        first, last = vs.download_range()
+        priority_first, priority_last = vs.get_high_range()
+        if priority_first != priority_last:
+            first = priority_first
+            highprob_cutoff = vs.normalize(priority_last + 1)
+            # Arno, 2010-08-10: Errr, mid = MU * high
+            midprob_cutoff = vs.normalize(first + self.MU * vs.get_range_length(first, highprob_cutoff))
+        else:
+            highprob_cutoff = last
+            midprob_cutoff = vs.normalize(first + self.MU * vs.high_prob_curr_pieces)
+        # h = vs.time_to_pieces( self.HIGH_PROB_SETSIZE )
+        # highprob_cutoff = vs.normalize(first + max(h, self.HIGH_PROB_MIN_PIECES))
+        # midprob_cutoff = vs.normalize(first + max(self.MU * h, self.HIGH_PROB_MIN_PIECES))
+
+        # print >>sys.stderr, "Prio %s:%s:%s" % (first, highprob_cutoff, midprob_cutoff), highprob_cutoff - first, midprob_cutoff - highprob_cutoff
+
+        # first,last = vs.download_range()
+        # if vs.wraparound:
+        #     max_lookahead = vs.wraparound_delta
+        # else:
+        #     max_lookahead = vs.last_piece - vs.playback_pos
+
+        # highprob_cutoff = vs.normalize( first + min( h, max_lookahead ) )
+        # midprob_cutoff  = vs.normalize( first + min( h + self.MU * h, max_lookahead ) )
+
+        if vs.live_streaming:
+            # for live playback consider peers to be bad when they miss the deadline 5 times
+            allow_based_on_performance = connection.download.bad_performance_counter < 5
+        else:
+            # for VOD playback consider peers to be bad when they miss the deadline 1 time
+            # Diego : patch from Jan
+            if connection:
+                allow_based_on_performance = connection.download.bad_performance_counter < 1
+            else:
+                allow_based_on_performance = True
+
+        if vs.prebuffering:
+            f = first
+            t = vs.normalize( first + self.transporter.max_prebuf_packets )
+            choice = pick_rarest_small_range(f,t)
+            type = "high"
+        else:
+            choice = None
+
+        if choice is None:
+            if vs.live_streaming:
+                choice = pick_rarest_small_range( first, highprob_cutoff )
+            else:
+                choice = pick_first( first, highprob_cutoff )
+            type = "high"
+
+        # it is possible that the performance of this peer prohibits
+        # us from selecting this piece...
+        if not allow_based_on_performance:
+            high_priority_choice = choice
+            choice = None
+
+        if choice is None:
+            choice = pick_rarest_small_range( highprob_cutoff, midprob_cutoff )
+            type = "mid"
+
+        if choice is None:
+            if vs.live_streaming:
+                # Want: loop over what peer has avail, respecting piece priorities
+                # (could ignore those for live).
+                #
+                # Attempt 1: loop over range (which is 25% of window (see 
+                # VideoStatus), ignoring priorities, no shuffle.
+                #print >>sys.stderr,"vod: choice low RANGE",midprob_cutoff,last
+                #choice = pick_rarest_loop_over_small_range(midprob_cutoff,last,shuffle=False)
+                pass
+            else:
+                choice = pick_rarest( midprob_cutoff, last )
+            type = "low"
+            
+        if choice and willrequest:
+            self.stats[type] += 1
+
+        if DEBUG:
+            print >>sys.stderr,"vod: pp: picked piece %s [type=%s] [%d,%d,%d,%d]" % (`choice`,type,first,highprob_cutoff,midprob_cutoff,last)
+
+        # 12/05/09, boudewijn: (1) The bad_performance_counter is
+        # incremented whenever a piece download failed and decremented
+        # whenever is succeeds. (2) A peer with a positive
+        # bad_performance_counter is only allowd to pick low-priority
+        # pieces. (Conclusion) When all low-priority pieces are
+        # downloaded the client hangs when one or more high-priority
+        # pieces are required and if all peers have a positive
+        # bad_performance_counter.
+        if choice is None and not allow_based_on_performance:
+            # ensure that there is another known peer with a
+            # non-positive bad_performance_counter that has the piece
+            # that we would pick from the high-priority set for this
+            # connection.
+
+            if high_priority_choice:
+                availability = 0
+                for download in self.downloader.downloads:
+                    if download.have[high_priority_choice] and not download.bad_performance_counter:
+                        availability += 1
+
+                if not availability:
+                    # no other connection has it... then ignore the
+                    # bad_performance_counter advice and attempt to
+                    # download it from this connection anyway
+                    if DEBUG: print >>sys.stderr, "vod: pp: the bad_performance_counter says this is a bad peer... but we have nothing better... requesting piece", high_priority_choice, "regardless."
+                    choice = high_priority_choice
+
+        if not vs.live_streaming:
+            if choice is None and not self.am_I_complete():
+                # Arno, 2010-02-24:
+                # VOD + seeking: we seeked into the future and played till end, 
+                # there is a gap between the old playback and the seek point
+                # which we didn't download, and otherwise never will.
+                #
+                secondchoice = pick_rarest(vs.first_piece,vs.last_piece)
+                if secondchoice is not None:
+                    if DEBUG:
+                        print >>sys.stderr,"vod: pp: Picking skipped-over piece",secondchoice 
+                    return secondchoice
+
+        return choice
+
+    def is_valid_piece(self,piece):
+        return self.videostatus.in_valid_range(piece)
+   
+    def get_valid_range_iterator(self):
+        if self.videostatus.live_streaming and self.videostatus.get_live_startpos() is None:
+            # Not hooked in, so cannot provide a sensible download range
+            #print >>sys.stderr,"PiecePickerStreaming: Not hooked in, valid range set to total"
+            return PiecePicker.get_valid_range_iterator(self)
+            
+        #print >>sys.stderr,"PiecePickerStreaming: Live hooked in, or VOD, valid range set to subset"
+        first,last = self.videostatus.download_range()
+        return self.videostatus.generate_range((first,last))
+
+    def get_live_source_have(self):
+        for d in self.peer_connections.values():
+            if d["connection"].is_live_source():
+                return d["connection"].download.have
+        return None
+    
+            
+            
+    def am_I_complete(self):
+        return self.done and not TEST_VOD_OVERRIDE
+            
+            
+PiecePickerVOD = PiecePickerStreaming
diff --git a/instrumentation/next-share/BaseLib/Core/Video/SVCTransporter.py b/instrumentation/next-share/BaseLib/Core/Video/SVCTransporter.py
new file mode 100644
index 0000000..3c7f18c
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Video/SVCTransporter.py
@@ -0,0 +1,1275 @@
+# Written by Jan David Mol, Arno Bakker, Riccardo Petrocco
+# see LICENSE.txt for license information
+
+import sys
+from math import ceil
+from threading import Condition,currentThread
+from traceback import print_exc
+from tempfile import mkstemp
+import collections
+import os
+import base64
+import os,sys,time
+import re
+
+from BaseLib.Core.BitTornado.CurrentRateMeasure import Measure
+from BaseLib.Core.Video.MovieTransport import MovieTransport,MovieTransportStreamWrapper
+from BaseLib.Core.simpledefs import *
+from BaseLib.Core.osutils import *
+from BaseLib.Core.Video.VideoOnDemand import *
+
+# pull all video data as if a video player was attached
+FAKEPLAYBACK = False
+
+DEBUG = True
+DEBUGPP = False
+
+class SVCTransporter(MovieOnDemandTransporter):
+    """ Takes care of providing a bytestream interface based on the available pieces. """
+
+    # seconds to prebuffer if bitrate is known (always for SVC)
+    PREBUF_SEC_VOD  = 10
+
+    # max number of seconds in queue to player
+    # Arno: < 2008-07-15: St*pid vlc apparently can't handle lots of data pushed to it
+    # Arno: 2008-07-15: 0.8.6h apparently can
+    BUFFER_TIME = 5.0
+    
+    # polling interval to refill buffer
+    #REFILL_INTERVAL = BUFFER_TIME * 0.75
+    # Arno: there's is no guarantee we got enough (=BUFFER_TIME secs worth) to write to output bug!
+    REFILL_INTERVAL = 0.1
+
+    # amount of time (seconds) to push a packet into
+    # the player queue ahead of schedule
+    VLC_BUFFER_SIZE = 0
+    PIECE_DUE_SKEW = 0.1 + VLC_BUFFER_SIZE
+
+    # Arno: If we don't know playtime and FFMPEG gave no decent bitrate, this is the minimum
+    # bitrate (in KByte/s) that the playback birate-estimator must have to make us
+    # set the bitrate in movieselector.
+    MINPLAYBACKRATE = 32*1024
+
+    # maximum delay between pops before we force a restart (seconds)
+    MAX_POP_TIME = 60
+
+    def __init__(self,bt1download,videostatus,videoinfo,videoanalyserpath,vodeventfunc):
+
+        # dirty hack to get the Tribler Session
+        from BaseLib.Core.Session import Session
+        session = Session.get_instance()
+
+        if session.get_overlay():
+            # see comment in else section on importing...
+            from BaseLib.Core.CacheDB.SqliteVideoPlaybackStatsCacheDB import VideoPlaybackDBHandler
+            self._playback_stats = VideoPlaybackDBHandler.get_instance()
+        else:
+            # hack: we should not import this since it is not part of
+            # the core nor should we import here, but otherwise we
+            # will get import errors
+            from BaseLib.Player.Reporter import VideoPlaybackReporter
+            self._playback_stats = VideoPlaybackReporter.get_instance()
+            
+        # add an event to indicate that the user wants playback to
+        # start
+        def set_nat(nat):
+            self._playback_stats.add_event(self._playback_key, "nat:%s" % nat)
+        self._playback_key = base64.b64encode(os.urandom(20))
+        self._playback_stats.add_event(self._playback_key, "play-init")
+        self._playback_stats.add_event(self._playback_key, "piece-size:%d" % videostatus.piecelen)
+        self._playback_stats.add_event(self._playback_key, "num-pieces:%d" % videostatus.movie_numpieces)
+        self._playback_stats.add_event(self._playback_key, "bitrate:%d" % videostatus.bitrate)
+        self._playback_stats.add_event(self._playback_key, "nat:%s" % session.get_nat_type(callback=set_nat))
+
+
+        self._complete = False
+        self.videoinfo = videoinfo
+        self.bt1download = bt1download
+        self.piecepicker = bt1download.picker
+        self.rawserver = bt1download.rawserver
+        self.storagewrapper = bt1download.storagewrapper
+        self.fileselector = bt1download.fileselector
+
+        self.vodeventfunc = vodeventfunc
+        self.videostatus = vs = videostatus
+        
+        # Add quotes around path, as that's what os.popen() wants on win32
+        if sys.platform == "win32" and videoanalyserpath is not None and videoanalyserpath.find(' ') != -1:
+            self.video_analyser_path='"'+videoanalyserpath+'"'
+        else:
+            self.video_analyser_path=videoanalyserpath
+
+        # counter for the sustainable() call. Every X calls the
+        # buffer-percentage is updated.
+        self.sustainable_counter = sys.maxint
+
+        # boudewijn: because we now update the downloadrate for each
+        # received chunk instead of each piece we do not need to
+        # average the measurement over a 'long' period of time. Also,
+        # we only update the downloadrate for pieces that are in the
+        # high priority range giving us a better estimation on how
+        # likely the pieces will be available on time.
+        self.overall_rate = Measure(10)
+        self.high_range_rate = Measure(2)
+
+        # buffer: a link to the piecepicker buffer
+        self.has = self.piecepicker.has
+
+        # number of pieces in buffer
+        self.pieces_in_buffer = 0
+
+        self.data_ready = Condition()
+        
+        # Arno: Call FFMPEG only if the torrent did not provide the 
+        # bitrate and video dimensions. This is becasue FFMPEG 
+        # sometimes hangs e.g. Ivaylo's Xvid Finland AVI, for unknown 
+        # reasons
+        
+        # Arno: 2007-01-06: Since we use VideoLan player, videodimensions not important
+        assert vs.bitrate_set
+        self.doing_ffmpeg_analysis = False
+        self.doing_bitrate_est = False
+        self.videodim = None #self.movieselector.videodim
+
+        self.player_opened_with_width_height = False
+        self.ffmpeg_est_bitrate = None
+        
+        prebufsecs = self.PREBUF_SEC_VOD
+
+        # assumes first piece is whole (first_piecelen == piecelen)
+        piecesneeded = vs.time_to_pieces( prebufsecs )
+        bytesneeded = piecesneeded * vs.piecelen
+
+        self.max_prebuf_packets = min(vs.movie_numpieces, piecesneeded)
+
+        if self.doing_ffmpeg_analysis and DEBUG:
+            print >>sys.stderr,"vod: trans: Want",self.max_prebuf_packets,"pieces for FFMPEG analysis, piecesize",vs.piecelen
+
+        if DEBUG:
+            print >>sys.stderr,"vod: trans: Want",self.max_prebuf_packets,"pieces for prebuffering"
+
+        self.nreceived = 0
+        
+        if DEBUG:
+            print >>sys.stderr,"vod: trans: Setting MIME type to",self.videoinfo['mimetype']
+        
+        self.set_mimetype(self.videoinfo['mimetype'])
+
+        # some statistics
+        self.stat_playedpieces = 0 # number of pieces played successfully
+        self.stat_latepieces = 0 # number of pieces that arrived too late
+        self.stat_droppedpieces = 0 # number of pieces dropped
+        self.stat_stalltime = 0.0 # total amount of time the video was stalled
+        self.stat_prebuffertime = 0.0 # amount of prebuffer time used
+        self.stat_pieces = PieceStats() # information about each piece
+
+        # start periodic tasks
+        self.curpiece = ""
+        self.curpiece_pos = 0
+        # The outbuf keeps only the pieces from the base layer.. We play if we 
+        # have at least a piece from the base layer!
+        self.outbuf = []
+        #self.last_pop = None # time of last pop
+        self.reset_bitrate_prediction()
+
+        self.lasttime=0
+        # For DownloadState
+        self.prebufprogress = 0.0
+        self.prebufstart = time.time()
+        self.playable = False
+        self.usernotified = False
+        
+        self.outbuflen = None
+
+        # LIVESOURCEAUTH
+        self.authenticator = None
+
+        self.refill_rawserv_tasker()
+        self.tick_second()
+
+        # link to others (last thing to do)
+        self.piecepicker.set_transporter( self )
+        #self.start()
+
+        if FAKEPLAYBACK:
+            import threading
+            
+            class FakeReader(threading.Thread):
+                def __init__(self,movie):
+                    threading.Thread.__init__(self)
+                    self.movie = movie
+                    
+                def run(self):
+                    self.movie.start()
+                    while not self.movie.done():
+                        self.movie.read()
+            
+            t = FakeReader(self)
+            t.start()
+          
+        #self.rawserver.add_task( fakereader, 0.0 )
+
+
+    def parse_video(self):
+        """ Feeds the first max_prebuf_packets to ffmpeg to determine video bitrate. """
+        vs = self.videostatus
+        width = None
+        height = None
+
+        # Start ffmpeg, let it write to a temporary file to prevent 
+        # blocking problems on Win32 when FFMPEG outputs lots of
+        # (error) messages.
+        #
+        [loghandle,logfilename] = mkstemp()
+        os.close(loghandle)
+        if sys.platform == "win32":
+            # Not "Nul:" but "nul" is /dev/null on Win32
+            sink = 'nul'
+        else:
+            sink = '/dev/null'
+        # DON'T FORGET 'b' OTHERWISE WE'RE WRITING BINARY DATA IN TEXT MODE!
+        (child_out,child_in) = os.popen2( "%s -y -i - -vcodec copy -acodec copy -f avi %s > %s 2>&1" % (self.video_analyser_path, sink, logfilename), 'b' )
+        """
+        # If the path is "C:\Program Files\bla\bla" (escaping left out) and that file does not exist
+        # the output will say something cryptic like "vod: trans: FFMPEG said C:\Program" suggesting an
+        # error with the double quotes around the command, but that's not it. Be warned!
+        cmd = self.video_analyser_path+' -y -i - -vcodec copy -acodec copy -f avi '+sink+' > '+logfilename+' 2>&1'
+        print >>sys.stderr,"vod: trans: Video analyser command is",cmd
+        (child_out,child_in) = os.popen2(cmd,'b')  # DON'T FORGET 'b' OTHERWISE THINGS GO WRONG!
+        """
+
+        # feed all the pieces
+        download_range = vs.download_range()
+        # We get the bitrate from the base layer and determine the rest based on this
+        first, last = download_range[0]
+
+        for i in xrange(first,last):
+            piece = self.get_piece( i )
+            if piece is None:
+                break
+
+            try:
+                child_out.write( piece )
+            except IOError:
+                print_exc(file=sys.stderr)
+                break
+
+        child_out.close()
+        child_in.close()
+
+        logfile = open(logfilename, 'r')
+
+        # find the bitrate in the output
+        bitrate = None
+
+        r = re.compile( "bitrate= *([0-9.]+)kbits/s" )
+        r2 = re.compile( "Video:.* ([0-9]+x[0-9]+)," )    # video dimensions WIDTHxHEIGHT
+
+        founddim = False
+        for x in logfile.readlines():
+            if DEBUG:
+                print >>sys.stderr,"vod: trans: FFMPEG said:",x
+            occ = r.findall( x )
+            if occ:
+                # use the latest mentioning of bitrate
+                bitrate = float( occ[-1] ) * 1024 / 8
+                if DEBUG:
+                    if bitrate is not None:
+                        print >>sys.stderr,"vod: trans: Bitrate according to FFMPEG: %.2f KByte/s" % (bitrate/1024)
+                    else:
+                        print >>sys.stderr,"vod: trans: Bitrate could not be determined by FFMPEG"
+            occ = r2.findall( x )
+            if occ and not founddim:
+                # use first occurence
+                dim = occ[0]
+                idx = dim.find('x')
+                width = int(dim[:idx])
+                height = int(dim[idx+1:])
+                founddim = True
+                
+                if DEBUG:
+                    print >>sys.stderr,"vod: width",width,"heigth",height
+        logfile.close()
+        try:
+            os.remove(logfilename)
+        except:
+            pass
+
+        return [bitrate,width,height]
+
+    def update_prebuffering(self,received_piece=None):
+        """ Update prebuffering process. 'received_piece' is a hint that we just received this piece;
+            keep at 'None' for an update in general. """
+
+        if DEBUG: print >>sys.stderr, "vod: Updating prebuffer. Received piece: ", received_piece
+        vs = self.videostatus
+
+        if not vs.prebuffering:
+            return
+
+        if received_piece:
+            self.nreceived += 1
+        
+        # for the prebuffer we keep track only of the base layer
+        high_range = vs.generate_base_high_range()
+        high_range_length = vs.get_base_high_range_length()
+
+        # Arno, 2010-01-13: This code is only used when *pre*buffering, not
+        # for in-playback buffering. See refill_buffer() for that.
+        # Restored original code here that looks at max_prebuf_packets
+        # and not highrange. The highrange solution didn't allow the prebuf
+        # time to be varied independently of highrange width. 
+        #
+        wantprebuflen = min(self.max_prebuf_packets,high_range_length)
+        high_range_list = list(high_range)
+        wantprebuflist = high_range_list[:wantprebuflen]
+        
+        missing_pieces = filter(lambda i: not self.have_piece(i), wantprebuflist)
+        gotall = not missing_pieces
+        if high_range_length:
+            self.prebufprogress = min(1, float(wantprebuflen - len(missing_pieces)) / max(1, wantprebuflen))
+        else:
+            self.prebufprogress = 1.0
+        
+        if DEBUG:
+            print >>sys.stderr,"vod: trans: Already got",(self.prebufprogress*100.0),"% of prebuffer"
+        
+        if not gotall and DEBUG:
+            print >>sys.stderr,"vod: trans: Still need pieces",missing_pieces,"for prebuffering/FFMPEG analysis"
+
+        if vs.dropping:
+            if not self.doing_ffmpeg_analysis and not gotall and not (0 in missing_pieces) and self.nreceived > self.max_prebuf_packets:
+                perc = float(self.max_prebuf_packets)/10.0
+                if float(len(missing_pieces)) < perc or self.nreceived > (2*len(missing_pieces)):
+                    # If less then 10% of packets missing, or we got 2 times the packets we need already,
+                    # force start of playback
+                    gotall = True
+                    if DEBUG:
+                        print >>sys.stderr,"vod: trans: Forcing stop of prebuffering, less than",perc,"missing, or got 2N packets already"
+
+        if gotall and self.doing_ffmpeg_analysis:
+
+            [bitrate,width,height] = self.parse_video()
+            self.doing_ffmpeg_analysis = False
+            if DEBUG:
+                print >>sys.stderr,"vod: trans: after parse",bitrate,self.doing_bitrate_est
+            if bitrate is None or round(bitrate)== 0:
+                if self.doing_bitrate_est:
+                    # Errr... there was no playtime info in the torrent
+                    # and FFMPEG can't tell us...
+                    #bitrate = (1*1024*1024/8) # 1mbps
+                    # Ric: in svc every piece should be 2,56 sec.
+                    bitrate = vs.piecelen / 2.56
+                    if DEBUG:
+                        print >>sys.stderr,"vod: trans: No bitrate info avail, wild guess: %.2f KByte/s" % (bitrate/1024)
+
+                    vs.set_bitrate(bitrate)
+                    self._playback_stats.add_event(self._playback_key, "bitrate-guess:%d" % bitrate)
+            else:
+                if self.doing_bitrate_est:
+                    # There was no playtime info in torrent, use what FFMPEG tells us
+                    self.ffmpeg_est_bitrate = bitrate
+                    bitrate *= 1.1  # Make FFMPEG estimation 10% higher
+                    if DEBUG:
+                        print >>sys.stderr,"vod: trans: Estimated bitrate: %.2f KByte/s" % (bitrate/1024)
+
+                    vs.set_bitrate(bitrate)
+                    self._playback_stats.add_event(self._playback_key, "bitrate-ffmpeg:%d" % bitrate)
+
+            if width is not None and height is not None:
+                diff = False
+                if self.videodim is None:
+                    self.videodim = (width,height)
+                    self.height = height
+                elif self.videodim[0] != width or self.videodim[1] != height:
+                    diff =  True
+                if not self.player_opened_with_width_height or diff:
+                    #self.user_setsize(self.videodim)
+                    pass
+
+        # # 10/03/09 boudewijn: For VOD we will wait for the entire
+        # # buffer to fill (gotall) before we start playback. For live
+        # # this is unlikely to happen and we will therefore only wait
+        # # until we estimate that we have enough_buffer.
+        # if (gotall or vs.live_streaming) and self.enough_buffer():
+        if gotall and self.enough_buffer():
+            # enough buffer and could estimated bitrate - start streaming
+            if DEBUG:
+                print >>sys.stderr,"vod: trans: Prebuffering done",currentThread().getName()
+            self.data_ready.acquire()
+            vs.prebuffering = False
+            self.stat_prebuffertime = time.time() - self.prebufstart
+            self.notify_playable()
+            self.data_ready.notify()
+            self.data_ready.release()
+
+        elif DEBUG:
+            if self.doing_ffmpeg_analysis:
+                print >>sys.stderr,"vod: trans: Prebuffering: waiting to obtain the first %d packets" % (self.max_prebuf_packets)
+            else:
+                print >>sys.stderr,"vod: trans: Prebuffering: %.2f seconds left" % (self.expected_buffering_time())
+        
+        
+    def got_have(self,piece):
+        vs = self.videostatus
+
+        # update stats
+        self.stat_pieces.set( piece, "known", time.time() )
+        """
+        if vs.playing and vs.wraparound:
+            # check whether we've slipped back too far
+            d = vs.wraparound_delta
+            n = max(1,self.piecepicker.num_nonempty_neighbours()/2)
+            if self.piecepicker.numhaves[piece] > n and d/2 < (piece - vs.playback_pos) % vs.movie_numpieces < d:
+                # have is confirmed by more than half of the neighours and is in second half of future window
+                print >>sys.stderr,"vod: trans: Forcing restart. Am at playback position %d but saw %d at %d>%d peers." % (vs.playback_pos,piece,self.piecepicker.numhaves[piece],n)
+
+                self.start(force=True)
+        """
+
+    def got_piece(self, piece_id, begin, length):
+        """
+        Called when a chunk has been downloaded. This information can
+        be used to estimate download speed.
+        """
+        if self.videostatus.in_high_range(piece_id):
+            self.high_range_rate.update_rate(length)
+            if DEBUG: print >>sys.stderr, "vod: high priority rate:", self.high_range_rate.get_rate()
+    
+    def complete(self,piece,downloaded=True):
+        """ Called when a movie piece has been downloaded or was available from the start (disk). """
+
+        vs = self.videostatus
+ 
+        if vs.in_high_range(piece):
+            self._playback_stats.add_event(self._playback_key, "hipiece:%d" % piece)
+        else:
+            self._playback_stats.add_event(self._playback_key, "piece:%d" % piece)
+
+        if not self._complete and self.piecepicker.am_I_complete():
+            self._complete = True
+            self._playback_stats.add_event(self._playback_key, "complete")
+            self._playback_stats.flush()
+
+        self.stat_pieces.set( piece, "complete", time.time() )
+
+        if DEBUG:
+            print >>sys.stderr,"vod: trans: Completed",piece
+
+        if downloaded: 
+            self.overall_rate.update_rate( vs.piecelen )
+        if vs.in_download_range( piece ):
+            self.pieces_in_buffer += 1
+        else:
+            if DEBUG:
+                print >>sys.stderr,"vod: piece %d too late [pos=%d]" % (piece,vs.playback_pos)
+            self.stat_latepieces += 1
+
+        if vs.playing and vs.playback_pos == piece:
+            # we were delaying for this piece
+            self.refill_buffer()
+
+        self.update_prebuffering( piece )
+
+    def set_pos(self,pos):
+        """ Update the playback position. Called when playback is started (depending
+        on requested offset). """
+
+        vs = self.videostatus
+
+        oldpos = vs.playback_pos
+        vs.playback_pos = pos
+
+        # fast forward
+        for i in xrange(oldpos,pos+1):
+            if self.has[i]:
+                self.pieces_in_buffer -= 1
+
+        # fast rewind
+        for i in xrange(pos,oldpos+1):
+            if self.has[i]:
+                self.pieces_in_buffer += 1
+
+    def inc_pos(self):
+        vs = self.videostatus
+
+        if self.has[vs.playback_pos]:
+            self.pieces_in_buffer -= 1
+
+        vs.inc_playback_pos()
+        
+    
+    def expected_download_time(self):
+        """ Expected download time left. """
+        vs = self.videostatus
+        if vs.wraparound:
+            return float(2 ** 31)
+
+        # Ric: TODO for the moment keep track only of the base layer. Afterwards we will send
+        # different signals depending on the buffer layer
+        pieces_left = vs.last_piece - vs.playback_pos - self.pieces_in_buffer
+        if pieces_left <= 0:
+            return 0.0
+
+        # list all pieces from the high priority set that have not
+        # been completed
+        uncompleted_pieces = filter(self.storagewrapper.do_I_have, vs.generate_high_range())
+
+        # when all pieces in the high-range have been downloaded,
+        # we have an expected download time of zero
+        if not uncompleted_pieces:
+            return 0.0
+
+        # the download time estimator is very inacurate when we only
+        # have a few chunks left. therefore, we will put more emphesis
+        # on the overall_rate as the number of uncompleted_pieces does
+        # down.
+        total_length = vs.get_high_range_length()
+        uncompleted_length = len(uncompleted_pieces)
+        expected_download_speed = self.high_range_rate.get_rate() * (1 - float(uncompleted_length) / total_length) + \
+                                  self.overall_rate.get_rate() * uncompleted_length / total_length
+        if expected_download_speed < 0.1:
+            return float(2 ** 31)
+
+        return pieces_left * vs.piecelen / expected_download_speed
+
+    def expected_playback_time(self):
+        """ Expected playback time left. """
+
+        vs = self.videostatus
+
+        pieces_to_play = vs.last_piece - vs.playback_pos + 1
+
+        if pieces_to_play <= 0:
+            return 0.0
+
+        if not vs.bitrate:
+            return float(2 ** 31)
+
+        return pieces_to_play * vs.piecelen / vs.bitrate
+
+    def expected_buffering_time(self):
+        """ Expected time required for buffering. """
+        download_time = self.expected_download_time()
+        playback_time = self.expected_playback_time()
+        #print >>sys.stderr,"EXPECT",self.expected_download_time(),self.expected_playback_time()
+        # Infinite minus infinite is still infinite
+        if download_time > float(2 ** 30) and playback_time > float(2 ** 30):
+            return float(2 ** 31)
+        return abs(download_time - playback_time)
+
+    def enough_buffer(self):
+        """ Returns True if we can safely start playback without expecting to run out of
+            buffer. """
+        return max(0.0, self.expected_download_time() - self.expected_playback_time()) == 0.0
+
+    def tick_second(self):
+        self.rawserver.add_task( self.tick_second, 1.0 )
+
+        vs = self.videostatus
+
+        # Adjust estimate every second, but don't display every second
+        display = False # (int(time.time()) % 5) == 0
+        if DEBUG: # display
+            print >>sys.stderr,"vod: Estimated download time: %5.1fs [priority: %7.2f Kbyte/s] [overall: %7.2f Kbyte/s]" % (self.expected_download_time(), self.high_range_rate.get_rate()/1024, self.overall_rate.get_rate()/1024)
+
+        if vs.playing and round(self.playbackrate.rate) > self.MINPLAYBACKRATE and not vs.prebuffering:
+            if self.doing_bitrate_est:
+                if display:
+                    print >>sys.stderr,"vod: Estimated playback time: %5.0fs [%7.2f Kbyte/s], doing estimate=%d" % (self.expected_playback_time(),self.playbackrate.rate/1024, self.ffmpeg_est_bitrate is None)
+                if self.ffmpeg_est_bitrate is None:
+                    vs.set_bitrate( self.playbackrate.rate )
+
+        if display:
+            sys.stderr.flush()
+
+    #
+    # MovieTransport interface
+    #
+    # WARNING: these methods will be called by other threads than NetworkThread!
+    #
+    def size( self ):
+        # Ric: returning the size of the base layer
+        return self.videostatus.selected_movie[0]["size"]
+
+    def read(self,numbytes=None):
+        """ Read a set of pieces. The return data will be 
+            a byte for the pieces presence and a set of pieces
+            depending on the available quality. Return None in
+            case of an error or end-of-stream. """
+        vs = self.videostatus
+        # keep track in the base layer
+        if not self.curpiece:
+            # curpiece_pos could be set to something other than 0! 
+            # for instance, a seek request sets curpiece_pos but does not
+            # set curpiece.
+
+            base_layer_piece = self.pop()
+            if base_layer_piece is None:
+                return None
+
+            piecenr,self.curpiece = base_layer_piece
+            relatives = vs.get_respective_piece(piecenr)
+                        
+            
+            if DEBUG:
+                print >>sys.stderr,"vod: trans: popped piece %d to transport to player," % piecenr, "relative pieces are", relatives
+
+
+        curpos = self.curpiece_pos
+        left = len(self.curpiece) - curpos
+
+
+        if numbytes is None:
+            # default on one piece per read
+            numbytes = left
+
+        # TODO ask, we could leave it like this
+        if left > numbytes:
+            # piece contains enough -- return what was requested
+            data = self.curpiece[curpos:curpos+numbytes]
+            self.curpiece_pos += numbytes
+        else:
+            # TODO add get_bitrate method in SVC status to see how many 
+            # pieces we need from the different layers!
+
+            header = str(vs.piecelen)
+            data = header            
+            # return remainder of the piece, could be less than numbytes
+            data += self.curpiece[curpos:]
+
+            for i in relatives:
+                if self.has[i]:
+                    if DEBUG: print>>sys.stderr, "vod: trans: filling stream with piece %d from an enhancement layer" % i
+                    data += self.get_piece(i)
+            #print>>sys.stderr, "vod: trans: filling stream with piece %d from an enhancement layer" % i, len(data)
+            self.curpiece = ""
+            self.curpiece_pos = 0
+
+        return data
+
+    def start( self, bytepos = 0, force = False ):
+        """ Initialise to start playing at position `bytepos'. """
+        self._playback_stats.add_event(self._playback_key, "play")
+
+        # ARNOTODO: we don't use start(bytepos != 0) at the moment. See if we 
+        # should. Also see if we need the read numbytes here, or that it
+        # is better handled at a higher layer. For live it is currently
+        # done at a higher level, see VariableReadAuthStreamWrapper because
+        # we have to strip the signature. Hence the self.curpiece buffer here
+        # is superfluous. Get rid off it or check if 
+        #
+        #    curpiece[0:piecelen]
+        #
+        # returns curpiece if piecelen has length piecelen == optimize for
+        # piecesized case.
+        #
+        # For VOD seeking we may use the numbytes facility to seek to byte offsets
+        # not just piece offsets.
+        #
+        vs = self.videostatus
+
+        if vs.playing and not force:
+            return
+
+        # lock before changing startpos or any other playing variable
+        self.data_ready.acquire()
+        try:
+            # Determine piece number and offset
+            if bytepos < vs.piecelen:
+                piece = vs.first_piece
+                offset = bytepos
+            else:
+                newbytepos = bytepos - vs.first_piecelen
+
+                piece  = vs.first_piece + newbytepos / vs.piecelen + 1
+                offset = newbytepos % vs.piecelen
+
+            if DEBUG:
+                print >>sys.stderr,"vod: trans: === START, START, START, START, START, START, START, START, START, START, START, START, START,START"
+                print >>sys.stderr,"vod: trans: === START at offset %d (piece %d) (forced: %s) ===" % (bytepos,piece,force)
+
+            # Initialise all playing variables
+            self.curpiece = "" # piece currently being popped
+            self.curpiece_pos = offset
+            # TODO
+            self.set_pos( piece )
+            self.outbuf = []
+            #self.last_pop = time.time()
+            self.reset_bitrate_prediction()
+            vs.playing = True
+            self.playbackrate = Measure( 60 )
+        finally:
+            self.data_ready.release()
+
+        # ARNOTODO: start is called by non-NetworkThreads, these following methods
+        # are usually called by NetworkThread.
+        #
+        # We now know that this won't be called until notify_playable() so
+        # perhaps this can be removed?
+        #
+        # CAREFUL: if we use start() for seeking... that's OK. User won't be
+        # able to seek before he got his hands on the stream, so after 
+        # notify_playable()
+        
+        # See what we can do right now
+        self.update_prebuffering()
+        self.refill_buffer()
+
+    def stop( self ):
+        """ Playback is stopped. """
+        self._playback_stats.add_event(self._playback_key, "stop")
+
+        vs = self.videostatus
+        if DEBUG:
+            print >>sys.stderr,"vod: trans: === STOP  = player closed conn === "
+        if not vs.playing:
+            return
+        vs.playing = False
+
+        # clear buffer and notify possible readers
+        self.data_ready.acquire()
+        self.outbuf = []
+        #self.last_pop = None
+        vs.prebuffering = False
+        self.data_ready.notify()
+        self.data_ready.release()
+
+    def pause( self, autoresume = False ):
+        """ Pause playback. If `autoresume' is set, playback is expected to be
+        resumed automatically once enough data has arrived. """
+        self._playback_stats.add_event(self._playback_key, "pause")
+
+        vs = self.videostatus
+        if not vs.playing or not vs.pausable:
+            return
+
+        if vs.paused:
+            vs.autoresume = autoresume
+            return
+
+        if DEBUG:
+            print >>sys.stderr,"vod: trans: paused (autoresume: %s)" % (autoresume,)
+
+        vs.paused = True
+        vs.autoresume = autoresume
+        self.paused_at = time.time()
+        #self.reset_bitrate_prediction()
+        self.videoinfo["usercallback"](VODEVENT_PAUSE,{ "autoresume": autoresume })
+
+    def resume( self ):
+        """ Resume paused playback. """
+        self._playback_stats.add_event(self._playback_key, "resume")
+
+        vs = self.videostatus
+
+        if not vs.playing or not vs.paused or not vs.pausable:
+            return
+
+        if DEBUG:
+            print >>sys.stderr,"vod: trans: resumed"
+
+        vs.paused = False
+        vs.autoresume = False
+        self.stat_stalltime += time.time() - self.paused_at
+        self.addtime_bitrate_prediction( time.time() - self.paused_at )
+        self.videoinfo["usercallback"](VODEVENT_RESUME,{})
+
+        self.update_prebuffering()
+        self.refill_buffer()
+
+    def autoresume( self, testfunc = lambda: True ):
+        """ Resumes if testfunc returns True. If not, will test every second. """
+
+        vs = self.videostatus
+
+        if not vs.playing or not vs.paused or not vs.autoresume:
+            return
+
+        if not testfunc():
+            self.rawserver.add_task( lambda: self.autoresume( testfunc ), 1.0 )
+            return
+
+        if DEBUG:
+            print >>sys.stderr,"vod: trans: Resuming, since we can maintain this playback position"
+        self.resume()
+
+    def done( self ):
+        vs = self.videostatus
+
+        if not vs.playing:
+            return True
+
+        if vs.wraparound:
+            return False
+
+        return vs.playback_pos == vs.last_piece+1 and self.curpiece_pos >= len(self.curpiece)
+
+    def seek(self,pos,whence=os.SEEK_SET):
+        """ Seek to the given position, a number in bytes relative to both
+        the "whence" reference point and the file being played.
+        
+        We currently actually seek at byte level, via the start() method.
+        We support all forms of seeking, including seeking past the current
+        playback pos. Note this may imply needing to prebuffer again or 
+        being paused.
+        
+        vs.playback_pos in NetworkThread domain. Does data_ready lock cover 
+        that? Nope. However, this doesn't appear to be respected in any
+        of the MovieTransport methods, check all.
+        
+        Check
+        * When seeking reset other buffering, e.g. read()'s self.curpiece
+           and higher layers.
+        
+        """
+        vs = self.videostatus
+        length = self.size()
+
+        # lock before changing startpos or any other playing variable
+        self.data_ready.acquire()
+        try:
+            if whence == os.SEEK_SET:
+                abspos = pos
+            elif whence == os.SEEK_END:
+                if pos > 0:
+                    raise ValueError("seeking beyond end of stream")
+                else:
+                    abspos = size+pos
+            else: # SEEK_CUR
+                raise ValueError("seeking does not currently support SEEK_CUR")
+            
+            self.stop()
+            self.start(pos)
+        finally:
+            self.data_ready.release()
+
+
+
+    def get_mimetype(self):
+        return self.mimetype
+
+    def set_mimetype(self,mimetype):
+        self.mimetype = mimetype
+    #
+    # End of MovieTransport interface
+    #
+
+    def have_piece(self,piece):
+        return self.piecepicker.has[piece]
+
+    def get_piece(self,piece):
+        """ Returns the data of a certain piece, or None. """
+
+        vs = self.videostatus
+
+        if not self.have_piece( piece ):
+            return None
+
+        begin = 0
+        length = vs.piecelen
+
+        data = self.storagewrapper.do_get_piece(piece, 0, length)
+        if data is None:
+            return None
+        return data.tostring()
+
+    def reset_bitrate_prediction(self):
+        self.start_playback = None
+        self.last_playback = None
+        self.history_playback = collections.deque()
+
+    def addtime_bitrate_prediction(self,seconds):
+        if self.start_playback is not None:
+            self.start_playback["local_ts"] += seconds
+
+    def valid_piece_data(self,i,piece):
+        if not piece:
+            return False
+
+        if not self.start_playback or self.authenticator is None:
+            # no check possible
+            return True
+
+        s = self.start_playback
+
+        seqnum = self.authenticator.get_seqnum( piece )
+        source_ts = self.authenticator.get_rtstamp( piece )
+
+        if seqnum < s["absnr"] or source_ts < s["source_ts"]:
+            # old packet???
+            print >>sys.stderr,"vod: trans: **** INVALID PIECE #%s **** seqnum=%d but we started at seqnum=%d" % (i,seqnum,s["absnr"])
+            return False
+
+        return True
+
+
+    def update_bitrate_prediction(self,i,piece):
+        """ Update the rate prediction given that piece i has just been pushed to the buffer. """
+
+        if self.authenticator is not None:
+            seqnum = self.authenticator.get_seqnum( piece )
+            source_ts = self.authenticator.get_rtstamp( piece )
+        else:
+            seqnum = i
+            source_ts = 0
+
+        d = {
+            "nr": i,
+            "absnr": seqnum,
+            "local_ts": time.time(),
+            "source_ts": source_ts,
+        }
+
+        # record 
+        if self.start_playback is None:
+            self.start_playback = d
+
+        if self.last_playback and self.last_playback["absnr"] > d["absnr"]:
+            # called out of order
+            return
+
+        self.last_playback = d
+
+        # keep a recent history
+        MAX_HIST_LEN = 10*60 # seconds
+
+        self.history_playback.append( d )
+
+        # of at most 10 entries (or minutes if we keep receiving pieces)
+        while source_ts - self.history_playback[0]["source_ts"] > MAX_HIST_LEN:
+            self.history_playback.popleft()
+
+        if DEBUG:
+            vs = self.videostatus
+            first, last = self.history_playback[0], self.history_playback[-1]
+            
+            if first["source_ts"] and first != last:
+                divd = (last["source_ts"] - first["source_ts"])
+                if divd == 0:
+                    divd = 0.000001
+                bitrate = "%.2f kbps" % (8.0 / 1024 * (vs.piecelen - vs.sigsize) * (last["absnr"] - first["absnr"]) / divd,)
+            else:
+                bitrate = "%.2f kbps (external info)" % (8.0 / 1024 * vs.bitrate)
+
+            print >>sys.stderr,"vod: trans: %i: pushed at t=%.2f, age is t=%.2f, bitrate = %s" % (i,d["local_ts"]-self.start_playback["local_ts"],d["source_ts"]-self.start_playback["source_ts"],bitrate)
+
+    def piece_due(self,i):
+        """ Return the time when we expect to have to send a certain piece to the player. For
+        wraparound, future pieces are assumed. """
+
+        if self.start_playback is None:
+            return float(2 ** 31) # end of time
+
+        s = self.start_playback
+        l = self.last_playback
+        vs = self.videostatus
+
+        if not vs.wraparound and i < l["nr"]:
+            # should already have arrived!
+            return time.time()
+
+        # assume at most one wrap-around between l and i
+        piecedist = (i - l["nr"]) % vs.movie_numpieces
+
+        if s["source_ts"]:
+            # ----- we have timing information from the source
+            first, last = self.history_playback[0], self.history_playback[-1]
+
+            if first != last:
+                # we have at least two recent pieces, so can calculate average bitrate. use the recent history
+                # do *not* adjust for sigsize since we don't want the actual video speed but the piece rate
+                bitrate = 1.0 * vs.piecelen * (last["absnr"] - first["absnr"]) / (last["source_ts"] - first["source_ts"])
+            else:
+                # fall-back to bitrate predicted from torrent / ffmpeg
+                bitrate = vs.bitrate
+           
+            # extrapolate with the average bitrate so far
+            return s["local_ts"] + l["source_ts"] - s["source_ts"] + piecedist * vs.piecelen / bitrate - self.PIECE_DUE_SKEW
+        else:
+            # ----- no timing information from pieces, so do old-fashioned methods
+            if vs.live_streaming:
+                # Arno, 2008-11-20: old-fashioned method is well bad,
+                # ignore.
+                return time.time() + 60.0
+            else:
+                i =  piecedist + (l["absnr"] - s["absnr"])
+                
+                if s["nr"] == vs.first_piece:
+                    bytepos = vs.first_piecelen + (i-1) * vs.piecelen
+                else:
+                    bytepos = i * vs.piecelen
+                
+                return s["local_ts"] + bytepos / vs.bitrate - self.PIECE_DUE_SKEW
+            
+
+    def max_buffer_size( self ):
+        vs = self.videostatus
+        return max(256*1024, vs.piecelen * 4, self.BUFFER_TIME * vs.bitrate)
+
+
+    def refill_buffer( self ):
+        """ Push pieces (from the base layer) into the player FIFO when needed and able. 
+            This counts as playing the pieces as far as playback_pos is concerned."""
+
+        self.data_ready.acquire()
+
+        vs = self.videostatus
+
+        if vs.prebuffering or not vs.playing:
+            self.data_ready.release()
+            return
+
+        if vs.paused:
+            self.data_ready.release()
+            return
+
+        mx = self.max_buffer_size()
+        self.outbuflen = sum( [len(d) for (p,d) in self.outbuf] )
+        now = time.time()
+
+        def buffer_underrun():
+            return self.outbuflen == 0 and self.start_playback and now - self.start_playback["local_ts"] > 1.0
+
+        if buffer_underrun():
+            # TODO
+            def sustainable():
+
+                self.sustainable_counter += 1
+                if self.sustainable_counter > 10:
+                    self.sustainable_counter = 0
+                    
+                    base_high_range_length = vs.get_base_high_range_length()
+                    have_length = len(filter(lambda n:self.has[n], vs.generate_base_high_range()))
+
+                    # progress                                                                                  
+                    self.prebufprogress = min(1.0, float(have_length) / max(1, base_high_range_length))
+
+                    return have_length >= base_high_range_length
+
+                else:
+                    num_immediate_packets = 0
+                    base_high_range_length = vs.get_base_high_range_length()
+
+                    for piece in vs.generate_base_high_range(): 
+                        if self.has[piece]:
+                            num_immediate_packets += 1
+                            if num_immediate_packets >= base_high_range_length:
+                                break
+                        else:
+                            break
+                    else:
+                        # progress                                                                              
+                        self.prebufprogress = 1.0
+                        # completed loop without breaking, so we have everything we need                        
+                        return True
+
+                    return num_immediate_packets >= base_high_range_length
+
+            sus = sustainable()
+            if vs.pausable and not sus:
+                if DEBUG:
+                    print >>sys.stderr,"vod: trans:                        BUFFER UNDERRUN -- PAUSING"
+                self.pause( autoresume = True )
+                self.autoresume( sustainable )
+
+                # boudewijn: increase the minimum buffer size
+                vs.increase_high_range()
+
+                self.data_ready.release()
+                return
+            elif sus:
+                if DEBUG:
+                    print >>sys.stderr,"vod: trans:                        BUFFER UNDERRUN -- IGNORING, rate is sustainable"
+            else:
+                if DEBUG:
+                    print >>sys.stderr,"vod: trans:                         BUFFER UNDERRUN -- STALLING, cannot pause player to fall back some, so just wait for more pieces"
+                self.data_ready.release()
+                return
+                    
+        def push( i, data ):
+            # push packet into queue
+            if DEBUG:
+                print >>sys.stderr,"vod: trans: %d: pushed l=%d" % (vs.playback_pos,piece)
+
+            # update predictions based on this piece
+            self.update_bitrate_prediction( i, data )
+
+            self.stat_playedpieces += 1
+            self.stat_pieces.set( i, "tobuffer", time.time() )
+                    
+            self.outbuf.append( (vs.playback_pos,data) )
+            self.outbuflen += len(data)
+
+            self.data_ready.notify()
+            self.inc_pos()
+
+        def drop( i ):
+            # drop packet
+            if DEBUG:
+                print >>sys.stderr,"vod: trans: %d: dropped pos=%d; deadline expired %.2f sec ago !!!!!!!!!!!!!!!!!!!!!!" % (piece,vs.playback_pos,time.time()-self.piece_due(i))
+
+            self.stat_droppedpieces += 1
+            self.stat_pieces.complete( i )
+            self.inc_pos()
+
+        # We push in queue only pieces from the base layer 
+        download_range = vs.download_range()
+        base_range = download_range[0]
+        for piece in vs.generate_range( [base_range] ): 
+            ihavepiece = self.has[piece]
+            forcedrop = False
+
+            # check whether we have room to store it
+            if self.outbuflen > mx:
+                # buffer full
+                break
+
+            # final check for piece validity
+            if ihavepiece:
+                data = self.get_piece( piece )
+                if not self.valid_piece_data( piece, data ):
+                    # I should have the piece, but I don't: WAAAAHH!
+                    forcedrop = True
+                    ihavepiece = False
+
+            if ihavepiece:
+                # have piece - push it into buffer
+                if DEBUG:
+                    print >>sys.stderr,"vod: trans:                        BUFFER STATUS (max %.0f): %.0f kbyte" % (mx/1024.0,self.outbuflen/1024.0)
+
+                # piece found -- add it to the queue
+                push( piece, data )
+            else:
+                # don't have piece, or forced to drop
+                if not vs.dropping and forcedrop:
+                    print >>sys.stderr,"vod: trans: DROPPING INVALID PIECE #%s, even though we shouldn't drop anything." % piece
+                if vs.dropping or forcedrop:
+                    if time.time() >= self.piece_due( piece ) or buffer_underrun() or forcedrop:
+                        # piece is too late or we have an empty buffer (and future data to play, otherwise we would have paused) -- drop packet
+                        drop( piece )
+                    else:
+                        # we have time to wait for the piece and still have data in our buffer -- wait for packet
+                        if DEBUG:
+                            print >>sys.stderr,"vod: trans: %d: due in %.2fs  pos=%d" % (piece,self.piece_due(piece)-time.time(),vs.playback_pos)
+                    break
+                else: # not dropping
+                    if self.outbuflen == 0:
+                        print >>sys.stderr,"vod: trans: SHOULD NOT HAPPEN: missing piece but not dropping. should have paused. pausable=",vs.pausable,"player reading too fast looking for I-Frame?"
+                    else:
+                        if DEBUG:
+                            print >>sys.stderr,"vod: trans: prebuffering done, but could not fill buffer."
+                    break
+
+        self.data_ready.release()
+
+    def refill_rawserv_tasker( self ):
+        self.refill_buffer()
+
+        self.rawserver.add_task( self.refill_rawserv_tasker, self.REFILL_INTERVAL )
+
+    def pop( self ):
+        self.data_ready.acquire()
+        vs = self.videostatus
+
+        while vs.prebuffering and not self.done():
+            # wait until done prebuffering
+            self.data_ready.wait()
+
+        while not self.outbuf and not self.done():
+            # wait until a piece is available
+            #if DEBUG:
+            #    print >>sys.stderr,"vod: trans: Player waiting for data"
+            self.data_ready.wait()
+
+        if not self.outbuf:
+            piece = None
+        else:
+            piece = self.outbuf.pop( 0 ) # nr,data pair
+            self.playbackrate.update_rate( len(piece[1]) )
+
+        #self.last_pop = time.time()
+
+        self.data_ready.release()
+
+        if piece:
+            self.stat_pieces.set( piece[0], "toplayer", time.time() )
+            self.stat_pieces.complete( piece[0] )
+
+        return piece
+
+    def notify_playable(self):
+        """ Tell user he can play the media, 
+        cf. BaseLib.Core.DownloadConfig.set_vod_event_callback()
+        """
+        #if self.bufferinfo:
+        #    self.bufferinfo.set_playable()
+        #self.progressinf.bufferinfo_updated_callback()
+
+        # triblerAPI
+        if self.usernotified:
+            return
+        self.usernotified = True
+        self.prebufprogress = 1.0
+        self.playable = True
+        
+        #print >>sys.stderr,"vod: trans: notify_playable: Calling usercallback to tell it we're ready to play",self.videoinfo['usercallback']
+        
+        # MIME type determined normally in LaunchManyCore.network_vod_event_callback
+        # However, allow for recognition by videoanalyser
+        mimetype = self.get_mimetype()
+        complete = self.piecepicker.am_I_complete()
+
+        if complete:
+            stream = None
+            filename = self.videoinfo["outpath"]
+        else:
+            endstream = MovieTransportStreamWrapper(self)
+            filename = None 
+            
+        print >>sys.stderr,"3.3", self.size(), endstream, self.vodeventfunc, complete, self.size()
+        # Call user callback
+        #print >>sys.stderr,"vod: trans: notify_playable: calling:",self.vodeventfunc
+        self.vodeventfunc( self.videoinfo, VODEVENT_START, {
+            "complete":  complete,
+            "filename":  filename,
+            "mimetype":  mimetype,
+            "stream":    endstream,
+            "length":      self.size(),
+        } )
+
+
+    #
+    # Methods for DownloadState to extract status info of VOD mode.
+    #
+    def get_stats(self):
+        """ Returns accumulated statistics. The piece data is cleared after this call to save memory. """
+        """ Called by network thread """
+        s = { "played": self.stat_playedpieces,
+              "late": self.stat_latepieces,
+              "dropped": self.stat_droppedpieces,
+              "stall": self.stat_stalltime,
+              "pos": self.videostatus.playback_pos,
+              "prebuf": self.stat_prebuffertime,
+              "pp": self.piecepicker.stats,
+              "pieces": self.stat_pieces.pop_completed(), }
+        return s
+
+    def get_prebuffering_progress(self):
+        """ Called by network thread """
+        return self.prebufprogress
+    
+    def is_playable(self):
+        """ Called by network thread """
+        if not self.playable or self.videostatus.prebuffering:
+            self.playable = (self.prebufprogress == 1.0 and self.enough_buffer())
+        return self.playable
+        
+    def get_playable_after(self):
+        """ Called by network thread """
+        return self.expected_buffering_time()
+    
+    def get_duration(self):
+        return 1.0 * self.videostatus.selected_movie[0]["size"] / self.videostatus.bitrate
+        
+        
diff --git a/instrumentation/next-share/BaseLib/Core/Video/SVCVideoStatus.py b/instrumentation/next-share/BaseLib/Core/Video/SVCVideoStatus.py
new file mode 100644
index 0000000..4872271
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Video/SVCVideoStatus.py
@@ -0,0 +1,427 @@
+# Written by Jan David Mol, Arno Bakker, Riccardo Petrocco
+# see LICENSE.txt for license information
+
+import sys
+from math import ceil
+from sets import Set
+
+from BaseLib.Core.simpledefs import *
+
+# live streaming means wrapping around
+LIVE_WRAPAROUND = False
+
+DEBUG = False
+
+class SVCVideoStatus:
+    """ Info about the selected video and status of the playback. """
+
+    # TODO: thread safety? PiecePicker, MovieSelector and MovieOnDemandTransporter all interface this
+
+    def __init__(self,piecelen,fileinfo,videoinfo,authparams):
+        """
+            piecelen = length of BitTorrent pieces
+            fileinfo = list of (name,length) pairs for all files in the torrent,
+                       in their recorded order
+            videoinfo = videoinfo object from download engine
+        """
+        self.piecelen = piecelen # including signature, if any
+        self.sigsize = 0
+        self.fileinfo = fileinfo
+        self.videoinfo = videoinfo
+        self.authparams = authparams
+        self.selected_movie = []
+
+        # size of high probability set, in seconds (piecepicker varies
+        # between the minmax values depending on network performance,
+        # performance, increases and decreases with step (min,max,step)
+        self.high_prob_curr_time = 10
+        self.high_prob_curr_time_limit = (10, 180,10)
+
+        # minimal size of high probability set, in pieces (piecepicker
+        # varies between the limit values depending on network
+        # performance, increases and decreases with step (min,max,step)
+        self.high_prob_curr_pieces = 5
+        self.high_prob_curr_pieces_limit = (5, 50, 5)
+
+        # Ric: keeps track of the current layer
+        self.quality = 0
+
+        # ----- locate selected movie in fileinfo
+        indexes = self.videoinfo['index']
+        
+        # the available layers in the torrent
+        self.available_qualities = len(indexes)
+        
+        if DEBUG: print >>sys.stderr, "VideoStatus: indexes of ordered layer [base, enhance1, enhance2,....] in the torrent: ", indexes
+        # Ric: first index is the base layer
+        index = indexes[0]
+
+        base_offset = sum( (filesize for (_,filesize) in fileinfo[:index] if filesize) )
+        base_name = fileinfo[index][0]
+        base_size = fileinfo[index][1]
+        
+        # Ric: ordered list of info about the layers
+        self.selected_movie = []
+        
+
+        #enhancementIdx = indexes[1::]
+        #print >>sys.stderr, "enhancementIdx", enhancementIdx
+
+        for idx in indexes:
+            #field = "enhancement" + str(enhancementIdx.index(idx))
+            name = fileinfo[idx][0]
+            size = fileinfo[idx][1]
+            offset = sum( (filesize for (_,filesize) in fileinfo[:idx] if filesize) )
+            self.selected_movie.append( {"name": name, "size": size, "offset": offset} )
+
+        print >> sys.stderr, self.selected_movie
+        
+        self.playback_pos_observers = []
+        # da rimuovere serve a video on demand
+        self.live_streaming = videoinfo['live']
+
+        self.first_piecelen = 0
+        self.last_piecelen = 0
+
+        
+        # Ric: derive generic layers parameters
+        # TODO check if we can assume piece bounderies
+        self.layer_info = []
+        for layer in self.selected_movie:
+            movie_begin = layer["offset"]
+            movie_end = layer["offset"] + layer["size"] - 1
+
+            # movie_range = (bpiece,offset),(epiece,offset), inclusive
+            movie_range = ( (movie_begin/piecelen, movie_begin%piecelen),
+                                 (movie_end/piecelen, movie_end%piecelen) )
+            # first_piecelen = piecelen - movie_range[0][1]
+            # last_piecelen  = movie_range[1][1]
+            first_piece = movie_range[0][0]
+            last_piece = movie_range[1][0]
+            movie_numpieces = last_piece - first_piece + 1
+            self.layer_info.append( {"movie_begin": movie_begin, "movie_end": movie_end, "movie_range": movie_range, "first_piece": first_piece, "last_piece": last_piece, "movie_numpieces": movie_numpieces } )
+
+        if videoinfo['bitrate']:
+            self.set_bitrate( videoinfo['bitrate'] )
+        else:
+        # Ric: TODO
+            self.set_bitrate( 512*1024/8 ) # default to 512 Kbit/s
+            self.bitrate_set = False
+
+
+        # TODO keep first piece for observer
+        self.first_piece = self.layer_info[0]["first_piece"]
+        self.movie_numpieces = self.layer_info[0]["movie_numpieces"]
+        # last piece of the base layer.. to control
+        self.last_piece = self.layer_info[0]["last_piece"]
+        # we are not in live sit. We don't drop
+        self.dropping = False 
+        # for live
+        self.wraparound = False
+        print >>sys.stderr, self.first_piece
+
+        # ----- set defaults for dynamic positions
+        self.playing = False     # video has started playback
+        self.paused = False      # video is paused
+        self.autoresume = False  # video is paused but will resume automatically
+        self.prebuffering = True # video is prebuffering
+        self.playback_pos = self.first_piece
+
+        self.pausable = (VODEVENT_PAUSE in videoinfo["userevents"]) and (VODEVENT_RESUME in videoinfo["userevents"])
+# TODO
+    def add_playback_pos_observer( self, observer ):
+        """ Add a function to be called when the playback position changes. Is called as follows:
+            observer( oldpos, newpos ). In case of initialisation: observer( None, startpos ). """
+        self.playback_pos_observers.append( observer )
+
+# TODO see if needed
+    def real_piecelen( self, x ):
+        if x == self.first_piece:
+            return self.first_piecelen
+        elif x == self.last_piece:
+            return self.last_piecelen
+        else:
+            return self.piecelen
+
+    def set_bitrate( self, bitrate ):
+        self.bitrate_set = True
+        self.bitrate = bitrate
+        self.sec_per_piece = 1.0 * bitrate / self.piecelen
+
+    # the following functions work with absolute piece numbers,
+    # so they all function within the range [first_piece,last_piece]
+
+    # the range of pieces to download is
+    # [playback_pos,numpieces) for normal downloads and
+    # [playback_pos,playback_pos+delta) for wraparound
+
+    def generate_range( self, download_range ):
+
+        for i in range(len(download_range)):
+            (f,t) = download_range[i]
+            for x in xrange (f,t):
+                #print >> sys.stderr, "ttttttttttttttttttttttttttt", x
+                yield x    
+
+    def dist_range(self, f, t):
+        """ Returns the distance between f and t """
+        if f > t:
+            return self.last_piece-f + t-self.first_piece 
+        else:
+            return t - f
+
+    # TODO same method with diff param, see if need it!
+    def in_small_range( self, f, t, x ):
+        return f <= x < t
+        
+    def in_range(self, download_range, x):
+        for i in download_range:
+            f, l = i
+            if self.in_small_range(f, l, x):
+                return True
+        return False
+
+    def inc_playback_pos( self ):
+        oldpos = self.playback_pos
+        self.playback_pos += 1
+
+        if self.playback_pos > self.last_piece:
+            if self.wraparound:
+                self.playback_pos = self.first_piece
+            else:
+                self.playback_pos = self.last_piece
+
+        for o in self.playback_pos_observers:
+            o( oldpos, self.playback_pos )
+
+    def in_download_range( self, x ):
+
+        for i in range(self.quality + 1):
+            f = self.layer_info[i]["first_piece"]
+            l = self.layer_info[i]["last_piece"]
+
+            if f <= x <= l:
+                return True
+
+        return False
+        
+    # TODO just keep for the moment
+    def in_valid_range(self,piece):
+        return self.in_download_range( piece )
+        
+    def get_range_diff(self,oldrange,newrange):
+        """ Returns the diff between oldrange and newrange as a Set.
+        """
+        oldset = range2set(oldrange,self.movie_numpieces)
+        newset = range2set(newrange,self.movie_numpieces)
+        return oldset - newset
+    
+    def normalize( self, x ):
+        """ Caps or wraps a piece number. """
+
+        if self.in_download_range(x):
+            return x
+
+        return max( self.first_piece, min( x, self.get_highest_piece(self.range) ) )
+
+    def time_to_pieces( self, sec ):
+        """ Returns the piece number that contains data for a few seconds down the road. """
+
+        # TODO: take first and last piece into account, as they can have a different size
+        return int(ceil(sec * self.sec_per_piece))
+
+    def download_range( self ):
+        """ Returns the range [(first,last),(first,last)] of pieces we like to download from the layers. """
+        download_range = []
+        pos = self.playback_pos
+        # Ric: the pieces of difference
+        play_offset = pos - self.first_piece
+
+        #for i in range(self.quality + 1):
+        for i in range(self.available_qualities):
+            # Ric: if they have the same bitrate they have the same size TODO
+            if self.selected_movie[0]["size"] / self.selected_movie[i]["size"] == 1:
+                f = self.layer_info[i]["first_piece"]
+                position = f + play_offset
+                l = self.layer_info[i]["last_piece"]
+                download_range.append((position,l)) # should I add + 1  to the last?
+            else:
+                # TODO case of different bitrates
+                pass
+        # Ric: for global use like first and last piece
+        self.range = download_range
+        return download_range
+            
+            
+    def get_wraparound(self):
+        return self.wraparound
+
+    def increase_high_range(self, factor=1):
+        """
+        Increase the high priority range (effectively enlarging the buffer size)
+        """
+        assert factor > 0
+        self.high_prob_curr_time += factor * self.high_prob_curr_time_limit[2]
+        if self.high_prob_curr_time > self.high_prob_curr_time_limit[1]:
+            self.high_prob_curr_time = self.high_prob_curr_time_limit[1]
+        
+        self.high_prob_curr_pieces += int(factor * self.high_prob_curr_pieces_limit[2])
+        if self.high_prob_curr_pieces > self.high_prob_curr_pieces_limit[1]:
+            self.high_prob_curr_pieces = self.high_prob_curr_pieces_limit[1]
+
+        if DEBUG: print >>sys.stderr, "VideoStatus:increase_high_range", self.high_prob_curr_time, "seconds or", self.high_prob_curr_pieces, "pieces"
+
+    def decrease_high_range(self, factor=1):
+        """
+        Decrease the high priority range (effectively reducing the buffer size)
+        """
+        assert factor > 0
+        self.high_prob_curr_time -= factor * self.high_prob_curr_time_limit[2]
+        if self.high_prob_curr_time < self.high_prob_curr_time_limit[0]:
+            self.high_prob_curr_time = self.high_prob_curr_time_limit[0]
+        
+        self.high_prob_curr_pieces -= int(factor * self.high_prob_curr_pieces_limit[2])
+        if self.high_prob_curr_pieces < self.high_prob_curr_pieces_limit[0]:
+            self.high_prob_curr_pieces = self.high_prob_curr_pieces_limit[0]
+
+        if DEBUG: print >>sys.stderr, "VideoStatus:decrease_high_range", self.high_prob_curr_time, "seconds or", self.high_prob_curr_pieces, "pieces"
+
+    def set_high_range(self, seconds=None, pieces=None):
+        """
+        Set the minimum size of the high priority range. Can be given
+        in seconds of pieces.
+        """
+        if seconds: self.high_prob_curr_time = seconds
+        if pieces: self.high_prob_curr_pieces = pieces
+
+    def get_high_range(self):
+        """
+        Returns [(first, last), (first, last), ..] list of tuples
+        """
+        download_range = self.download_range()
+        number_of_pieces = self.time_to_pieces(self.high_prob_curr_time)
+
+        high_range = []
+        for i in range(self.quality + 1):
+
+            if i == 0:
+                # the other layers will align to the last piece of 
+                # the first one
+                f, _ = download_range[0]
+                l = min(self.last_piece,                                                          # last piece
+                         1 + f + max(number_of_pieces, self.high_prob_curr_pieces), # based on time OR pieces
+                         1 + f + self.high_prob_curr_pieces_limit[1])               # hard-coded buffer maximum
+
+                high_range.append((f, l))
+
+            # Ric: for higher layers the initial piece is ahead 
+            # in time regarding the previous layer            
+            else:
+                base_f, base_l = high_range[0]
+                align = self.get_respective_range( (base_f, base_l) )
+                new_b, new_e = align[i]
+                # We increase of one piece the start of the high range for the following layer
+                new_b += i
+                high_range.append( (new_b, new_e) )
+            
+        return high_range
+
+    def in_high_range(self, piece):
+        """
+        Returns True when PIECE is in the high priority range.
+        """
+        high_range = self.get_high_range()
+        return self.in_range(high_range, piece)
+
+    def get_range_length(self, download_range):
+        res = 0
+        for i in range(self.quality + 1):
+            f, l = download_range[i]
+            res = res + self.get_small_range_length(f, l) 
+        return res
+
+    def get_small_range_length(self, first, last):
+        return last - first
+
+    def get_high_range_length(self):
+        high_range = self.get_high_range()
+        return self.get_range_length(high_range)
+
+    # Needed to detect if the buffer undeflow is sustainable
+    def get_base_high_range_length(self):
+        high_range = self.get_high_range()
+        f, l = high_range[0]
+        return self.get_small_range_length(f, l) 
+    
+    def generate_high_range(self):
+        """
+        Returns the high current high priority range in piece_ids
+        """
+        high_range = self.get_high_range()
+        return self.generate_range(high_range)
+        
+    def generate_base_high_range(self):
+        """
+        Returns the high current high priority range in piece_ids
+        """
+        high_range = self.get_high_range()
+        base_high_range = [high_range[0]]
+        return self.generate_range(base_high_range)
+    
+    def get_highest_piece(self, list_of_ranges):
+        highest = 0
+        for i in range(self.quality + 1):
+            (f,l) = list_of_ranges[i]
+            if l > highest:
+                highest = l
+        return highest
+
+    def get_respective_range(self, (f,l)):
+        ret = []
+
+        for i in range(self.quality + 1):
+            if i == 0:
+                # for the first layer just copy the input
+                ret.append((f,l))
+            else:
+                # Ric: if they have the same bitrate they have the same size TODO
+                if self.selected_movie[0]["size"] / self.selected_movie[i]["size"] == 1:              
+                    bdiff = f - self.first_piece
+                    ediff = l - self.first_piece
+                    beg = self.layer_info[i]["first_piece"]
+                    new_beg = beg + bdiff
+                    new_end = beg + ediff
+                    ret.append((new_beg, new_end))
+                else:
+                    # TODO case of different bitrates
+                    pass
+        return ret
+
+    # returns a list of pieces that represent the same moment in the stream from all the layers
+    def get_respective_piece(self, piece):
+        ret = []
+
+        for i in range(self.available_qualities):
+            if i == 0:
+                pass
+                #ret.append(piece)
+            else:
+                # Ric: if they have the same bitrate they have the same size TODO
+                if self.selected_movie[0]["size"] / self.selected_movie[i]["size"] == 1:
+                    diff = piece - self.first_piece
+                    beg = self.layer_info[i]["first_piece"]
+                    res = beg + diff
+                    ret.append(res)
+                else:
+                    # TODO case of different bitrates
+                    pass
+        return ret
+
+def range2set(range,maxrange):    
+    if range[0] <= range[1]:
+        set = Set(xrange(range[0],range[1]))
+    else:
+        set = Set(xrange(range[0],maxrange)) | Set(xrange(0,range[1]))
+    return set
+    
+    
diff --git a/instrumentation/next-share/BaseLib/Core/Video/VideoOnDemand.py b/instrumentation/next-share/BaseLib/Core/Video/VideoOnDemand.py
new file mode 100644
index 0000000..7fc6595
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Video/VideoOnDemand.py
@@ -0,0 +1,1814 @@
+# Written by Jan David Mol, Arno Bakker
+# see LICENSE.txt for license information
+
+import sys
+from math import ceil
+from threading import Condition,currentThread
+from traceback import print_exc,print_stack
+from tempfile import mkstemp
+import collections
+import os
+import base64
+import os,sys,time
+import re
+from base64 import b64encode
+
+from BaseLib.Core.BitTornado.CurrentRateMeasure import Measure
+from BaseLib.Core.Video.MovieTransport import MovieTransport,MovieTransportStreamWrapper
+from BaseLib.Core.simpledefs import *
+from BaseLib.Core.osutils import *
+from BaseLib.Core.Statistics.Status.Status import get_status_holder
+
+# pull all video data as if a video player was attached
+FAKEPLAYBACK = False
+
+DEBUG = False
+DEBUG_HOOKIN = True
+
+class PieceStats:
+    """ Keeps track of statistics for each piece as it flows through the system. """
+
+    def __init__(self):
+        self.pieces = {}
+        self.completed = {}
+
+    def set(self,piece,stat,value,firstonly=True):
+        if piece not in self.pieces:
+            self.pieces[piece] = {}
+
+        if firstonly and stat in self.pieces[piece]:
+            return
+
+        self.pieces[piece][stat] = value
+
+    def complete(self,piece):
+        self.completed[piece] = 1
+
+    def reset(self):
+        for x in self.completed:
+            self.pieces.pop(x,0)
+
+        self.completed = {}
+
+    def pop_completed(self):
+        completed = {}
+
+        for x in self.completed.keys():
+            completed[x] = self.pieces.pop(x,{})
+
+        self.completed = {}
+        return completed
+
+class MovieOnDemandTransporter(MovieTransport):
+    """ Takes care of providing a bytestream interface based on the available pieces. """
+
+    # seconds to prebuffer if bitrate is known
+    PREBUF_SEC_LIVE = 10.0
+    PREBUF_SEC_VOD  = 10.0
+    
+    # Arno, 2010-01-14: prebuf is controlled with the above params. Buffering
+    # while playback for VOD is handled by increasing the high range, see
+    # VideoStatus self.high_prob_curr_*
+
+    # max number of seconds in queue to player
+    # Arno: < 2008-07-15: St*pid vlc apparently can't handle lots of data pushed to it
+    # Arno: 2008-07-15: 0.8.6h apparently can
+    BUFFER_TIME = 5.0
+    
+    # polling interval to refill buffer
+    #REFILL_INTERVAL = BUFFER_TIME * 0.75
+    # Arno: there is no guarantee we got enough (=BUFFER_TIME secs worth) to write to output bug!
+    REFILL_INTERVAL = 0.1
+
+    # amount of time (seconds) to push a packet into
+    # the player queue ahead of schedule
+    VLC_BUFFER_SIZE = 0
+    PIECE_DUE_SKEW = 0.1 + VLC_BUFFER_SIZE
+
+    # Arno: If we don't know playtime and FFMPEG gave no decent bitrate, this is the minimum
+    # bitrate (in KByte/s) that the playback birate-estimator must have to make us
+    # set the bitrate in movieselector.
+    MINPLAYBACKRATE = 32*1024
+    
+    # If not yet playing and if the difference between the peer's first chosen
+    # hookin time and the newly calculated hookin time is larger than 
+    # PREBUF_REHOOKIN_SECS (because his peer environment changed), then rehookin. 
+    PREBUF_REHOOKIN_SECS = 5.0
+
+    # maximum delay between pops when live streaming before we force a restart (seconds)
+    MAX_POP_TIME = 10
+
+    def __init__(self,bt1download,videostatus,videoinfo,videoanalyserpath,vodeventfunc,httpsupport=None):
+
+        # dirty hack to get the Tribler Session
+        from BaseLib.Core.Session import Session
+        session = Session.get_instance()
+
+        # hack: we should not import this since it is not part of the
+        # core nor should we import here, but otherwise we will get
+        # import errors
+        #
+        # _event_reporter stores events that are logged somewhere...
+        # from BaseLib.Core.Statistics.StatusReporter import get_reporter_instance
+        self._event_reporter = get_status_holder("LivingLab")
+        self.b64_infohash = b64encode(bt1download.infohash)
+            
+        # add an event to indicate that the user wants playback to
+        # start
+        def set_nat(nat):
+            self._event_reporter.create_and_add_event("nat", [self.b64_infohash, nat])
+        self._event_reporter.create_and_add_event("play-init", [self.b64_infohash])
+        self._event_reporter.create_and_add_event("piece-size", [self.b64_infohash, videostatus.piecelen])
+        self._event_reporter.create_and_add_event("num-pieces", [self.b64_infohash, videostatus.movie_numpieces])
+        self._event_reporter.create_and_add_event("bitrate", [self.b64_infohash, videostatus.bitrate])
+        self._event_reporter.create_and_add_event("nat", [self.b64_infohash, session.get_nat_type(callback=set_nat)])
+
+        # self._complete = False
+        self.videoinfo = videoinfo
+        self.bt1download = bt1download
+        self.piecepicker = bt1download.picker
+        self.rawserver = bt1download.rawserver
+        self.storagewrapper = bt1download.storagewrapper
+        self.fileselector = bt1download.fileselector
+
+        self.vodeventfunc = vodeventfunc
+        self.videostatus = vs = videostatus
+        # Diego : Http seeding video support
+        self.http_support = httpsupport
+        self.traker_peers_report = None
+        self.http_first_run = None
+        
+        # Add quotes around path, as that's what os.popen() wants on win32
+        if sys.platform == "win32" and videoanalyserpath is not None and videoanalyserpath.find(' ') != -1:
+            self.video_analyser_path='"'+videoanalyserpath+'"'
+        else:
+            self.video_analyser_path=videoanalyserpath
+
+        # counter for the sustainable() call. Every X calls the
+        # buffer-percentage is updated.
+        self.sustainable_counter = sys.maxint
+
+        # boudewijn: because we now update the downloadrate for each
+        # received chunk instead of each piece we do not need to
+        # average the measurement over a 'long' period of time. Also,
+        # we only update the downloadrate for pieces that are in the
+        # high priority range giving us a better estimation on how
+        # likely the pieces will be available on time.
+        self.overall_rate = Measure(10)
+        self.high_range_rate = Measure(2)
+
+        # buffer: a link to the piecepicker buffer
+        self.has = self.piecepicker.has
+
+        # number of pieces in buffer
+        self.pieces_in_buffer = 0
+
+        self.data_ready = Condition()
+        
+        # Arno: Call FFMPEG only if the torrent did not provide the 
+        # bitrate and video dimensions. This is becasue FFMPEG 
+        # sometimes hangs e.g. Ivaylo's Xvid Finland AVI, for unknown 
+        # reasons
+        
+        # Arno: 2007-01-06: Since we use VideoLan player, videodimensions not important
+        if vs.bitrate_set:
+            self.doing_ffmpeg_analysis = False
+            self.doing_bitrate_est = False
+            self.videodim = None #self.movieselector.videodim
+        else:
+            self.doing_ffmpeg_analysis = True
+            self.doing_bitrate_est = True
+            self.videodim = None
+
+        self.player_opened_with_width_height = False
+        self.ffmpeg_est_bitrate = None
+        
+        # number of packets required to preparse the video
+        # I say we need 128 KB to sniff size and bitrate
+        
+        # Arno: 2007-01-04: Changed to 1MB. It appears ffplay works better with some
+        # decent prebuffering. We should replace this with a timing based thing, 
+        
+        if not self.doing_bitrate_est:
+            if vs.live_streaming:
+                prebufsecs = self.PREBUF_SEC_LIVE
+            else:
+                prebufsecs = self.PREBUF_SEC_VOD
+
+            if vs.bitrate <=  (256 * 1024 / 8):
+                print >>sys.stderr,"vod: trans: Increasing prebuffer for low-bitrate feeds"
+                prebufsecs += 5.0
+            # assumes first piece is whole (first_piecelen == piecelen)
+            piecesneeded = vs.time_to_pieces( prebufsecs )
+        else:
+            # Arno, 2010-01-13: For torrents with unknown bitrate, prebuf more
+            # following Boudewijn's heuristics
+            piecesneeded = 2 * vs.get_high_range_length()
+
+        if vs.wraparound:
+            self.max_prebuf_packets = min(vs.wraparound_delta, piecesneeded)
+        else:
+            self.max_prebuf_packets = min(vs.movie_numpieces, piecesneeded)
+            
+        if DEBUG:
+            if self.doing_ffmpeg_analysis:
+                print >>sys.stderr,"vod: trans: Want",self.max_prebuf_packets,"pieces for FFMPEG analysis, piecesize",vs.piecelen
+            else:
+                print >>sys.stderr,"vod: trans: Want",self.max_prebuf_packets,"pieces for prebuffering"
+
+        self.nreceived = 0
+        
+        if DEBUG:
+            print >>sys.stderr,"vod: trans: Setting MIME type to",self.videoinfo['mimetype']
+        
+        self.set_mimetype(self.videoinfo['mimetype'])
+
+        # some statistics
+        self.stat_playedpieces = 0 # number of pieces played successfully
+        self.stat_latepieces = 0 # number of pieces that arrived too late
+        self.stat_droppedpieces = 0 # number of pieces dropped
+        self.stat_stalltime = 0.0 # total amount of time the video was stalled
+        self.stat_prebuffertime = 0.0 # amount of prebuffer time used
+        self.stat_pieces = PieceStats() # information about each piece
+
+        # start periodic tasks
+        self.curpiece = ""
+        self.curpiece_pos = 0
+        self.outbuf = []
+        self.outbuflen = None
+        self.last_pop = None # time of last pop
+        self.rehookin = False
+        self.reset_bitrate_prediction()
+
+        self.lasttime=0
+        # For DownloadState
+        self.prebufprogress = 0.0
+        self.prebufstart = time.time()
+        self.playable = False
+        self.usernotified = False
+        
+
+        # LIVESOURCEAUTH
+        if vs.live_streaming:
+            from BaseLib.Core.Video.LiveSourceAuth import ECDSAAuthenticator,RSAAuthenticator
+
+            if vs.authparams['authmethod'] == LIVE_AUTHMETHOD_ECDSA:
+                self.authenticator = ECDSAAuthenticator(vs.first_piecelen,vs.movie_numpieces,pubkeypem=vs.authparams['pubkey'])
+                vs.sigsize = vs.piecelen - self.authenticator.get_content_blocksize()
+            elif vs.authparams['authmethod'] == LIVE_AUTHMETHOD_RSA:
+                self.authenticator = RSAAuthenticator(vs.first_piecelen,vs.movie_numpieces,pubkeypem=vs.authparams['pubkey'])
+                vs.sigsize = vs.piecelen - self.authenticator.get_content_blocksize()
+            else:
+                self.authenticator = None
+                vs.sigsize = 0
+        else:
+            self.authenticator = None
+
+        self.video_refillbuf_rawtask()
+        if False:
+            self.video_printstats_tick_second()
+
+        # link to others (last thing to do)
+        self.piecepicker.set_transporter( self )
+        if not vs.live_streaming:
+            self.complete_from_persistent_state(self.storagewrapper.get_pieces_on_disk_at_startup())
+        #self.start()
+
+        if FAKEPLAYBACK:
+            import threading
+            
+            class FakeReader(threading.Thread):
+                def __init__(self,movie):
+                    threading.Thread.__init__(self)
+                    self.movie = movie
+                    
+                def run(self):
+                    self.movie.start()
+                    while not self.movie.done():
+                        self.movie.read()
+            
+            t = FakeReader(self)
+            t.start()
+          
+        #self.rawserver.add_task( fakereader, 0.0 )
+
+        if self.videostatus.live_streaming:
+            self.live_streaming_timer()
+
+        self.update_prebuffering()
+
+    def calc_live_startpos(self,prebufsize=2,have=False):
+        """ When watching a live stream, determine where to 'hook in'. Adjusts 
+            self.download_range[0] accordingly, never decreasing it. If 'have' 
+            is true, we need to have the data ourself. If 'have' is false, we 
+            look at availability at our neighbours.
+
+            Return True if successful, False if more data has to be collected. 
+            
+            This is called periodically until playback has started. After that,
+            the hookin point / playback position is either fixed, or determined 
+            by active pausing and resuming of the playback by this class, see
+            refill_buffer().
+        """
+
+        if DEBUG_HOOKIN:
+            print >>sys.stderr,"vod: calc_live_startpos: prebuf",prebufsize,"have",have
+
+        # ----- determine highest known piece number
+        if have:
+            # I am already hooked in and downloaded stuff, now do the final
+            # hookin based on what I have.
+            numseeds = 0
+            numhaves = self.piecepicker.has 
+            totalhaves = self.piecepicker.numgot
+            sourcehave = None
+
+            threshold = 1
+        else:
+            # Check neighbours playback pos to see where I should hookin.
+            numseeds = self.piecepicker.seeds_connected
+            numhaves = self.piecepicker.numhaves # excludes seeds
+            totalhaves = self.piecepicker.totalcount # excludes seeds
+            sourcehave = self.piecepicker.get_live_source_have()
+            
+            if DEBUG and DEBUG_HOOKIN:
+                if sourcehave is not None:
+                    print >>sys.stderr,"vod: calc_live_offset: DEBUG: testing for multiple clients at source IP (forbidden!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!, but used in CS testing). Source have numtrue",sourcehave.get_numtrue()
+                    if sourcehave.get_numtrue() < 100:
+                        print >>sys.stderr,"vod: calc_live_offset: Source sent near empty BITFIELD, CS testing bug? Not tuning in as source"
+                        sourcehave = None
+
+            numconns = self.piecepicker.num_nonempty_neighbours()
+            if sourcehave is None:
+                # We don't have a connection to the source, must get at least 2 votes 
+                threshold = max( 2, numconns/2 )
+            else:
+                if DEBUG_HOOKIN:
+                    print >>sys.stderr,"vod: calc_live_offset: Connected to source, hookin on that" 
+                    #print >>sys.stderr,"vod: calc_live_offset: sourcehave",`sourcehave.tostring()`
+                threshold = 1
+
+        # FUDGE: number of pieces we subtract from maximum known/have,
+        # to start playback with some buffer present. We need enough
+        # pieces to do pass the prebuffering phase. when still
+        # requesting pieces, FUDGE can probably be a bit low lower,
+        # since by the time they arrive, we will have later pieces anyway.
+        # NB: all live torrents have the bitrate set.
+        FUDGE = prebufsize #self.max_prebuf_packets
+
+        if numseeds == 0 and totalhaves == 0:
+            # optimisation: without seeds or pieces, just wait
+            if DEBUG_HOOKIN:
+                print >>sys.stderr,"vod: calc_live_offset: no pieces"
+            return False
+
+        # pieces are known, so we can determine where to start playing
+        vs = self.videostatus
+
+        bpiece = vs.first_piece
+        epiece = vs.last_piece
+
+        if not vs.wraparound:
+            if numseeds > 0 or numhaves[epiece] > 0:
+                # special: if full video is available, do nothing and enter VoD mode
+                if DEBUG_HOOKIN:
+                    print >>sys.stderr,"vod: calc_live_offset: vod mode"
+                vs.set_live_startpos( 0 )
+                return True
+
+        # maxnum = highest existing piece number owned by more than half of the neighbours
+        maxnum = None
+        if sourcehave is None:
+            # Look at peer neighbourhood
+            inspecthave = numhaves
+        else:
+            # Just look at source
+            inspecthave = sourcehave
+        
+        for i in xrange(epiece,bpiece-1,-1):
+            #if DEBUG_HOOKIN:
+            #    if 0 < inspecthave[i] < threshold:
+            #        print >>sys.stderr,"vod: calc_live_offset: discarding piece %d as it is owned by only %d<%d neighbours" % (i,inspecthave[i],threshold)
+
+            if inspecthave[i] >= threshold:
+                maxnum = i
+                if not have:
+                    if inspecthave == numhaves:
+                        print >>sys.stderr,"vod: calc_live_offset: chosing piece %d as it is owned by %d>=%d neighbours (prewrap)" % (i,inspecthave[i],threshold)
+                    else:
+                        print >>sys.stderr,"vod: calc_live_offset: chosing piece %d as it is owned by the source (prewrap)" % (i)
+                else:
+                    print >>sys.stderr,"vod: calc_live_offset: chosing piece %d as it is owned by me (prewrap)" % (i)
+                        
+                break
+
+        if maxnum is None:
+            print >>sys.stderr,"vod: calc_live_offset: Failed to find quorum for any piece"
+            return False
+
+        # if there is wraparound, newest piece may actually have wrapped
+        if vs.wraparound and maxnum > epiece - vs.wraparound_delta:
+            delta_left = vs.wraparound_delta - (epiece-maxnum)
+
+            for i in xrange( vs.first_piece+delta_left-1, vs.first_piece-1, -1 ):
+                if inspecthave[i] >= threshold:
+                    maxnum = i
+                    if DEBUG_HOOKIN:
+                        if not have:
+                            if inspecthave == numhaves:
+                                print >>sys.stderr,"vod: calc_live_offset: chosing piece %d as it is owned by %d>=%d neighbours (wrap)" % (i,inspecthave[i],threshold)
+                            else:
+                                print >>sys.stderr,"vod: calc_live_offset: chosing piece %d as it is owned by the source (wrap)" % (i)
+                        else:
+                            print >>sys.stderr,"vod: calc_live_offset: chosing piece %d as it is owned by me (wrap)" % (i)
+                            
+                    break
+
+        print >>sys.stderr,"vod: calc_live_offset: hookin candidate (unfudged)",maxnum
+        
+        # start watching from maximum piece number, adjusted by fudge.
+        if vs.wraparound:
+            maxnum = vs.normalize( maxnum - FUDGE )
+            #f = bpiece + (maxnum - bpiece - FUDGE) % (epiece-bpiece)
+            #t = bpiece + (f - bpiece + vs.wraparound_delta) % (epiece-bpiece)
+
+            # start at a piece known to exist to avoid waiting for something that won't appear
+            # for another round. guaranteed to succeed since we would have bailed if noone had anything
+            while not inspecthave[maxnum]:
+                maxnum = vs.normalize( maxnum + 1 )
+        else:
+            maxnum = max( bpiece, maxnum - FUDGE )
+
+            if maxnum == bpiece:
+                # video has just started -- watch from beginning
+                return True
+
+        # If we're connected to the source, and already hooked in,
+        # don't change the hooking point unless it is really far off
+        oldstartpos = vs.get_live_startpos()
+        if not have and threshold == 1 and oldstartpos is not None:
+            diff = vs.dist_range(oldstartpos,maxnum)
+            diffs = float(diff) * float(vs.piecelen) / vs.bitrate
+            print >>sys.stderr,"vod: calc_live_offset: m o",maxnum,oldstartpos,"diff",diff,"diffs",diffs
+            if diffs < self.PREBUF_REHOOKIN_SECS: 
+                return True
+            
+
+        print >>sys.stderr,"vod: === HOOKING IN AT PIECE %d (based on have: %s) ===" % (maxnum,have)
+
+        (toinvalidateset,toinvalidateranges) = vs.set_live_startpos( maxnum )
+        #print >>sys.stderr,"vod: invalidateset is",`toinvalidateset`
+        mevirgin = oldstartpos is None
+        if len(toinvalidateranges) == 0 or (len(toinvalidateranges) == 0 and not mevirgin): # LAST condition is bugcatch
+            for piece in toinvalidateset:
+                self.live_invalidate_piece_globally(piece,mevirgin)
+        else:
+            self.live_invalidate_piece_ranges_globally(toinvalidateranges,toinvalidateset)
+
+        try:
+            self._event_reporter.create_and_add_event("live-hookin", [self.b64_infohash, maxnum])
+        except:
+            print_exc()
+
+        return True
+    
+
+    def live_streaming_timer(self):
+        """ Background 'thread' to check where to hook in if live streaming. """
+        
+        if DEBUG:
+            print >>sys.stderr,"vod: live_streaming_timer: Checking hookin"
+        nextt = 1
+        try:
+            try:
+                vs = self.videostatus
+            
+                if vs.playing and not self.rehookin:
+                    # Stop adjusting the download range via this mechanism, see 
+                    # refill_buffer() for the new pause/resume mechanism.
+                    
+                    # Arno, 2010-03-04: Reactivate protection for live.
+                    if vs.live_streaming and self.last_pop is not None and time.time() - self.last_pop > self.MAX_POP_TIME:
+                        # Live: last pop too long ago, rehook-in
+                        print >>sys.stderr,"vod: live_streaming_timer: Live stalled too long, reaanounce and REHOOK-in !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+                        self.last_pop = time.time()
+                        
+                        # H4x0r: if conn to source was broken, reconnect
+                        self.bt1download.reannounce()
+                        
+                        # Give some time to connect to new peers, so rehookin in 2 seconds
+                        nextt = 2 
+                        self.rehookin = True
+                    return
+                    
+                # JD:keep checking correct playback pos since it can change if we switch neighbours
+                # due to faulty peers etc
+                if self.calc_live_startpos( self.max_prebuf_packets, False ):
+                    self.rehookin = False
+            except:
+                print_exc()
+        finally:
+            # Always executed
+            self.rawserver.add_task( self.live_streaming_timer, nextt )
+
+    def parse_video(self):
+        """ Feeds the first max_prebuf_packets to ffmpeg to determine video bitrate. """
+
+        vs = self.videostatus
+        width = None
+        height = None
+
+        # Start ffmpeg, let it write to a temporary file to prevent 
+        # blocking problems on Win32 when FFMPEG outputs lots of
+        # (error) messages.
+        #
+        [loghandle,logfilename] = mkstemp()
+        os.close(loghandle)
+        if sys.platform == "win32":
+            # Not "Nul:" but "nul" is /dev/null on Win32
+            sink = 'nul'
+        else:
+            sink = '/dev/null'
+        # DON'T FORGET 'b' OTHERWISE WE'RE WRITING BINARY DATA IN TEXT MODE!
+        (child_out,child_in) = os.popen2( "%s -y -i - -vcodec copy -acodec copy -f avi %s > %s 2>&1" % (self.video_analyser_path, sink, logfilename), 'b' )
+        """
+        # If the path is "C:\Program Files\bla\bla" (escaping left out) and that file does not exist
+        # the output will say something cryptic like "vod: trans: FFMPEG said C:\Program" suggesting an
+        # error with the double quotes around the command, but that's not it. Be warned!
+        cmd = self.video_analyser_path+' -y -i - -vcodec copy -acodec copy -f avi '+sink+' > '+logfilename+' 2>&1'
+        print >>sys.stderr,"vod: trans: Video analyser command is",cmd
+        (child_out,child_in) = os.popen2(cmd,'b')  # DON'T FORGET 'b' OTHERWISE THINGS GO WRONG!
+        """
+
+        # feed all the pieces
+        first,last = vs.download_range()
+        for i in xrange(first,last):
+            piece = self.get_piece( i )
+
+            if piece is None:
+                break
+
+            # remove any signatures etc
+            if self.authenticator is not None:
+                piece = self.authenticator.get_content( piece )
+
+            try:
+                child_out.write( piece )
+            except IOError:
+                print_exc(file=sys.stderr)
+                break
+
+        child_out.close()
+        child_in.close()
+
+        logfile = open(logfilename, 'r')
+
+        # find the bitrate in the output
+        bitrate = None
+
+        r = re.compile( "bitrate= *([0-9.]+)kbits/s" )
+        r2 = re.compile( "Video:.* ([0-9]+x[0-9]+)," )    # video dimensions WIDTHxHEIGHT
+
+        founddim = False
+        for x in logfile.readlines():
+            if DEBUG:
+                print >>sys.stderr,"vod: trans: FFMPEG said:",x
+            occ = r.findall( x )
+            if occ:
+                # use the latest mentioning of bitrate
+                bitrate = float( occ[-1] ) * 1024 / 8
+                if DEBUG:
+                    if bitrate is not None:
+                        print >>sys.stderr,"vod: trans: Bitrate according to FFMPEG: %.2f KByte/s" % (bitrate/1024)
+                    else:
+                        print >>sys.stderr,"vod: trans: Bitrate could not be determined by FFMPEG"
+            occ = r2.findall( x )
+            if occ and not founddim:
+                # use first occurence
+                dim = occ[0]
+                idx = dim.find('x')
+                width = int(dim[:idx])
+                height = int(dim[idx+1:])
+                founddim = True
+                
+                if DEBUG:
+                    print >>sys.stderr,"vod: width",width,"heigth",height
+        logfile.close()
+        try:
+            os.remove(logfilename)
+        except:
+            pass
+
+        return [bitrate,width,height]
+
+    def peers_from_tracker_report( self, num_peers ):
+        #print >>sys.stderr,"DIEGO DEBUG Got from tracker : ", num_peers
+        if self.traker_peers_report is None:
+            self.traker_peers_report = num_peers
+            self.update_prebuffering()
+        else:
+            self.traker_peers_report += num_peers
+
+    def update_prebuffering(self,received_piece=None):
+        """ Update prebuffering process. 'received_piece' is a hint that we just received this piece;
+            keep at 'None' for an update in general. """
+
+        vs = self.videostatus
+
+        if not vs.prebuffering:
+            return
+        else:
+            if self.http_support is not None:
+                # Diego : Give the possibility to other peers to give bandwidth.
+                # Wait a few seconds depending on the tracker response and
+                # on the number of peers connected
+                if self.http_first_run is None:
+                    if self.traker_peers_report is None:
+                        self.rawserver.add_task( lambda: self.peers_from_tracker_report(0), 1 ) # wait 1 second for tracker response
+                    elif self.traker_peers_report:
+                        self.http_support.start_video_support( 0, 2 ) # wait 2 seconds for connecting to peers
+                        self.http_first_run = True
+                    else:
+                        self.http_support.start_video_support( 0 ) # no peers to connect to. start immediately
+                        self.http_first_run = True
+                elif self.http_first_run:
+                    if not self.http_support.is_slow_start():
+                        # Slow start is possible only if video support is off
+                        self.http_first_run = False
+                        self.http_support.stop_video_support()
+                        num_peers = len( self.piecepicker.peer_connections )
+                        if num_peers == 0: # no peers connected. start immediately
+                            self.http_support.start_video_support( 0 )
+                        elif num_peers <= 3: # few peers connected. wait 2 second to get bandwidth. TODO : Diego : tune peer threshold
+                            self.http_support.start_video_support( 0, 2 )
+                        else: # many peers connected. wait 5 seconds to get bandwidth.
+                            self.http_support.start_video_support( 0, 5 )
+                else:
+                    self.http_support.start_video_support( 0 )
+
+        if vs.live_streaming and vs.live_startpos is None:
+            # first determine where to hook in
+            return
+
+        if received_piece:
+            self.nreceived += 1
+
+        # Arno, 2010-01-13: This code is only used when *pre*buffering, not
+        # for in-playback buffering. See refill_buffer() for that.
+        # Restored original code here that looks at max_prebuf_packets
+        # and not highrange. The highrange solution didn't allow the prebuf
+        # time to be varied independently of highrange width. 
+        #
+        f,t = vs.playback_pos, vs.normalize( vs.playback_pos + self.max_prebuf_packets )
+        prebufrange = vs.generate_range( (f, t) )
+        missing_pieces = filter( lambda i: not self.have_piece( i ), prebufrange)
+
+        gotall = not missing_pieces
+        self.prebufprogress = float(self.max_prebuf_packets-len(missing_pieces))/float(self.max_prebuf_packets)
+        
+        if DEBUG:
+            print >>sys.stderr,"vod: trans: Already got",(self.prebufprogress*100.0),"% of prebuffer"
+        
+        if not gotall and DEBUG:
+            print >>sys.stderr,"vod: trans: Still need pieces",missing_pieces,"for prebuffering/FFMPEG analysis"
+
+        if vs.dropping:
+            if not self.doing_ffmpeg_analysis and not gotall and not (0 in missing_pieces) and self.nreceived > self.max_prebuf_packets:
+                perc = float(self.max_prebuf_packets)/10.0
+                if float(len(missing_pieces)) < perc or self.nreceived > (2*self.max_prebuf_packets):
+                    # If less then 10% of packets missing, or we got 2 times the packets we need already,
+                    # force start of playback
+                    gotall = True
+                    if DEBUG:
+                        print >>sys.stderr,"vod: trans: Forcing stop of prebuffering, less than",perc,"missing, or got 2N packets already"
+
+        if gotall and self.doing_ffmpeg_analysis:
+            [bitrate,width,height] = self.parse_video()
+            self.doing_ffmpeg_analysis = False
+            if DEBUG:
+                print >>sys.stderr,"vod: trans: after parse",bitrate,self.doing_bitrate_est
+            if bitrate is None or round(bitrate)== 0:
+                if self.doing_bitrate_est:
+                    # Errr... there was no playtime info in the torrent
+                    # and FFMPEG can't tell us...
+                    bitrate = (1*1024*1024/8) # 1mbps
+                    if DEBUG:
+                        print >>sys.stderr,"vod: trans: No bitrate info avail, wild guess: %.2f KByte/s" % (bitrate/1024)
+
+                    vs.set_bitrate(bitrate)
+                    self._event_reporter.create_and_add_event("bitrate-guess", [self.b64_infohash, bitrate])
+            else:
+                if self.doing_bitrate_est:
+                    # There was no playtime info in torrent, use what FFMPEG tells us
+                    self.ffmpeg_est_bitrate = bitrate
+                    bitrate *= 1.1  # Make FFMPEG estimation 10% higher
+                    if DEBUG:
+                        print >>sys.stderr,"vod: trans: Estimated bitrate: %.2f KByte/s" % (bitrate/1024)
+
+                    vs.set_bitrate(bitrate)
+                    self._event_reporter.create_and_add_event("bitrate-ffmpeg", [self.b64_infohash, bitrate])
+
+            if width is not None and height is not None:
+                diff = False
+                if self.videodim is None:
+                    self.videodim = (width,height)
+                    self.height = height
+                elif self.videodim[0] != width or self.videodim[1] != height:
+                    diff =  True
+                if not self.player_opened_with_width_height or diff:
+                    #self.user_setsize(self.videodim)
+                    pass
+
+        # # 10/03/09 boudewijn: For VOD we will wait for the entire
+        # # buffer to fill (gotall) before we start playback. For live
+        # # this is unlikely to happen and we will therefore only wait
+        # # until we estimate that we have enough_buffer.
+        # if (gotall or vs.live_streaming) and self.enough_buffer():
+        if gotall and self.enough_buffer():
+            # enough buffer and could estimated bitrate - start streaming
+            if DEBUG:
+                print >>sys.stderr,"vod: trans: Prebuffering done",currentThread().getName()
+            self.data_ready.acquire()
+            vs.prebuffering = False
+
+            if self.http_support is not None:
+                self.http_support.stop_video_support()
+            self.stat_prebuffertime = time.time() - self.prebufstart
+            self._event_reporter.create_and_add_event("prebuf", [self.b64_infohash, self.stat_prebuffertime])
+            self.notify_playable()
+            self.data_ready.notify()
+            self.data_ready.release()
+
+        """
+        elif DEBUG:
+            if self.doing_ffmpeg_analysis:
+                print >>sys.stderr,"vod: trans: Prebuffering: waiting to obtain the first %d packets" % (self.max_prebuf_packets)
+            else:
+                print >>sys.stderr,"vod: trans: Prebuffering: %.2f seconds left" % (self.expected_buffering_time())
+        """
+
+    def got_have(self,piece):
+        # Arno, 2010-04-15: STBSPEED Not called anymore, to speedup VOD.
+        vs = self.videostatus
+
+        # update stats
+        self.stat_pieces.set( piece, "known", time.time() )
+        """
+        if vs.playing and vs.wraparound:
+            # check whether we've slipped back too far
+            d = vs.wraparound_delta
+            n = max(1,self.piecepicker.num_nonempty_neighbours()/2)
+            if self.piecepicker.numhaves[piece] > n and d/2 < (piece - vs.playback_pos) % vs.movie_numpieces < d:
+                # have is confirmed by more than half of the neighours and is in second half of future window
+                print >>sys.stderr,"vod: trans: Forcing restart. Am at playback position %d but saw %d at %d>%d peers." % (vs.playback_pos,piece,self.piecepicker.numhaves[piece],n)
+
+                self.start(force=True)
+        """
+
+    def got_piece(self, piece_id, begin, length):
+        """
+        Called when a chunk has been downloaded. This information can
+        be used to estimate download speed.
+        """
+        if self.videostatus.in_high_range(piece_id):
+            self.high_range_rate.update_rate(length)
+            # if DEBUG: print >>sys.stderr, "vod: high priority rate:", self.high_range_rate.get_rate()
+
+    def complete_from_persistent_state(self,myhavelist):
+        """ Arno, 2010-04-20: STBSPEED: Net effect of calling complete(piece,downloaded=False) 
+        for pieces available from disk """
+        vs = self.videostatus
+        for piece in myhavelist:
+            if vs.in_download_range(piece):
+                self.pieces_in_buffer += 1
+
+        self.update_prebuffering()
+
+    
+    def complete(self,piece,downloaded=True):
+        """ Called when a movie piece has been downloaded or was available from the start (disk).
+        Arno, 2010-04-20: STBSPEED: Never called anymore for available from start.  
+        """
+
+        vs = self.videostatus
+ 
+        if vs.in_high_range(piece):
+            self._event_reporter.create_and_add_event("hipiece", [self.b64_infohash, piece])
+        else:
+            self._event_reporter.create_and_add_event("piece", [self.b64_infohash, piece])
+
+        # if not self._complete and self.piecepicker.am_I_complete():
+        #     self._complete = True
+        #     self._event_reporter.create_and_add_event(self.b64_infohash, "complete")
+        #     self._event_reporter.flush()
+
+        if vs.wraparound:
+            assert downloaded
+
+        self.stat_pieces.set( piece, "complete", time.time() )
+
+        #if DEBUG:
+        #    print >>sys.stderr,"vod: trans: Completed",piece
+
+        if downloaded:
+            self.overall_rate.update_rate( vs.real_piecelen( piece ) )
+
+        # Arno, 2010-04-20: STBSPEED: vs.in_download_range( piece ) is equiv to downloaded=False
+        if vs.in_download_range( piece ):
+            self.pieces_in_buffer += 1
+        else:
+            if DEBUG:
+                print >>sys.stderr,"vod: piece %d too late [pos=%d]" % (piece,vs.playback_pos)
+            self.stat_latepieces += 1
+
+        if vs.playing and vs.playback_pos == piece:
+            # we were delaying for this piece
+            self.refill_buffer()
+
+        self.update_prebuffering( piece )
+
+    def set_pos(self,pos):
+        """ Update the playback position. Called when playback is started (depending
+        on requested offset). """
+
+        vs = self.videostatus
+
+        #print >>sys.stderr,"vod: trans: set_pos",vs.playback_pos,"->",pos
+
+        # Arno,2010-01-08: if all was pushed to buffer (!= read by user!) 
+        # then playbackpos = last+1
+        oldpos = min(vs.playback_pos,vs.last_piece) 
+        vs.playback_pos = pos
+
+        if vs.wraparound:
+            # recalculate
+            self.pieces_in_buffer = 0
+            for i in vs.generate_range( vs.download_range() ):
+                if self.has[i]:
+                    self.pieces_in_buffer += 1
+        else:
+            # fast forward
+            for i in xrange(oldpos,pos+1):
+                if self.has[i]:
+                    self.pieces_in_buffer -= 1
+
+            # fast rewind
+            for i in xrange(pos,oldpos+1):
+                if self.has[i]:
+                    self.pieces_in_buffer += 1
+
+    def inc_pos(self):
+        vs = self.videostatus
+
+        if self.has[vs.playback_pos]:
+            self.pieces_in_buffer -= 1
+
+        vs.inc_playback_pos()
+        
+        if vs.live_streaming:
+            self.live_invalidate_piece_globally(vs.live_piece_to_invalidate())
+
+#    def buffered_time_period(self):
+#        """Length of period of Buffered pieces"""
+#        if self.movieselector.bitrate is None or self.movieselector.bitrate == 0.0:
+#            return 0
+#        else:
+#            return self.pieces_in_buffer * self.movieselector.piece_length / self.movieselector.bitrate
+#    
+#    def playback_time_position(self):
+#        """Time of playback_pos and total duration
+#            Return playback_time in seconds
+#        """
+#        if self.movieselector.bitrate is None or self.movieselector.bitrate == 0.0:
+#            return 0
+#        else:
+#            return self.playback_pos * self.movieselector.piece_length / self.movieselector.bitrate
+    
+    def expected_download_time(self):
+        """ Expected download time left. """
+        vs = self.videostatus
+        if vs.wraparound:
+            return float(2 ** 31)
+
+        pieces_left = vs.last_piece - vs.playback_pos - self.pieces_in_buffer
+        if pieces_left <= 0:
+            return 0.0
+
+        # list all pieces from the high priority set that have not
+        # been completed
+        uncompleted_pieces = filter(self.storagewrapper.do_I_have, vs.generate_high_range())
+
+        # when all pieces in the high-range have been downloaded,
+        # we have an expected download time of zero
+        if not uncompleted_pieces:
+            return 0.0
+
+        # the download time estimator is very inacurate when we only
+        # have a few chunks left. therefore, we will put more emphesis
+        # on the overall_rate as the number of uncompleted_pieces goes
+        # down.
+        total_length = vs.get_high_range_length()
+        uncompleted_length = len(uncompleted_pieces)
+        expected_download_speed = self.high_range_rate.get_rate() * (1 - float(uncompleted_length) / total_length) + \
+                                  self.overall_rate.get_rate() * uncompleted_length / total_length
+        if expected_download_speed < 0.1:
+            return float(2 ** 31)
+
+        return pieces_left * vs.piecelen / expected_download_speed
+
+    def expected_playback_time(self):
+        """ Expected playback time left. """
+
+        vs = self.videostatus
+
+        if vs.wraparound:
+            return float(2 ** 31)
+
+        pieces_to_play = vs.last_piece - vs.playback_pos + 1
+
+        if pieces_to_play <= 0:
+            return 0.0
+
+        if not vs.bitrate:
+            return float(2 ** 31)
+
+        return pieces_to_play * vs.piecelen / vs.bitrate
+
+    def expected_buffering_time(self):
+        """ Expected time required for buffering. """
+        download_time = self.expected_download_time()
+        playback_time = self.expected_playback_time()
+        #print >>sys.stderr,"EXPECT",self.expected_download_time(),self.expected_playback_time()
+        # Infinite minus infinite is still infinite
+        if download_time > float(2 ** 30) and playback_time > float(2 ** 30):
+            return float(2 ** 31)
+        return abs(download_time - playback_time)
+
+    def enough_buffer(self):
+        """ Returns True if we can safely start playback without expecting to run out of
+            buffer. """
+
+        if self.videostatus.wraparound:
+            # Wrapped streaming has no (known) limited duration, so we cannot predict
+            # whether we have enough download speed. The only way is just to hope
+            # for the best, since any buffer will be emptied if the download speed
+            # is too low.
+            return True
+
+        return max(0.0, self.expected_download_time() - self.expected_playback_time()) == 0.0
+
+    def video_printstats_tick_second(self):
+        self.rawserver.add_task( self.video_printstats_tick_second, 1.0 )
+
+        vs = self.videostatus
+
+        # Adjust estimate every second, but don't display every second
+        display = True # (int(time.time()) % 5) == 0
+        if DEBUG: # display
+            print >>sys.stderr,"vod: Estimated download time: %5.1fs [priority: %7.2f Kbyte/s] [overall: %7.2f Kbyte/s]" % (self.expected_download_time(), self.high_range_rate.get_rate()/1024, self.overall_rate.get_rate()/1024)
+
+        if vs.playing and round(self.playbackrate.rate) > self.MINPLAYBACKRATE and not vs.prebuffering:
+            if self.doing_bitrate_est:
+                if display:
+                    print >>sys.stderr,"vod: Estimated playback time: %5.0fs [%7.2f Kbyte/s], doing estimate=%d" % (self.expected_playback_time(),self.playbackrate.rate/1024, self.ffmpeg_est_bitrate is None)
+                if self.ffmpeg_est_bitrate is None:
+                    vs.set_bitrate( self.playbackrate.rate )
+
+        if display:
+            sys.stderr.flush()
+
+    #
+    # MovieTransport interface
+    #
+    # WARNING: these methods will be called by other threads than NetworkThread!
+    #
+    def size( self ):
+        if self.videostatus.get_wraparound():
+            return None
+        else:
+            return self.videostatus.selected_movie["size"]
+
+    def read(self,numbytes=None):
+        """ Read at most numbytes from the stream. If numbytes is not given,
+            pieces are returned. The bytes read will be returned, or None in
+            case of an error or end-of-stream. """
+            
+        if not self.curpiece:
+            # curpiece_pos could be set to something other than 0! 
+            # for instance, a seek request sets curpiece_pos but does not
+            # set curpiece.
+
+            piecetup = self.pop()
+            if piecetup is None:
+                return None
+            
+            piecenr,self.curpiece = piecetup
+            if DEBUG:
+                print >>sys.stderr,"vod: trans: %d: popped piece to transport to player" % piecenr
+
+        curpos = self.curpiece_pos
+        left = len(self.curpiece) - curpos
+
+        if numbytes is None:
+            # default on one piece per read
+            numbytes = left
+
+        if left > numbytes:
+            # piece contains enough -- return what was requested
+            data = self.curpiece[curpos:curpos+numbytes]
+
+            self.curpiece_pos += numbytes
+        else:
+            # return remainder of the piece, could be less than numbytes
+            
+            data = self.curpiece[curpos:]
+
+            self.curpiece = ""
+            self.curpiece_pos = 0
+
+        return data
+
+    def start( self, bytepos = 0, force = False ):
+        """ Initialise to start playing at position `bytepos'. """
+        self._event_reporter.create_and_add_event("play", [self.b64_infohash])
+
+        if DEBUG:
+            print >>sys.stderr,"vod: trans: start:",bytepos
+
+        # ARNOTODO: we don't use start(bytepos != 0) at the moment. See if we 
+        # should. Also see if we need the read numbytes here, or that it
+        # is better handled at a higher layer. For live it is currently
+        # done at a higher level, see VariableReadAuthStreamWrapper because
+        # we have to strip the signature. Hence the self.curpiece buffer here
+        # is superfluous. Get rid off it or check if 
+        #
+        #    curpiece[0:piecelen]
+        #
+        # returns curpiece if piecelen has length piecelen == optimize for
+        # piecesized case.
+        #
+        # For VOD seeking we may use the numbytes facility to seek to byte offsets
+        # not just piece offsets.
+        #
+        vs = self.videostatus
+
+        if vs.playing and not force:
+            return
+
+        # lock before changing startpos or any other playing variable
+        self.data_ready.acquire()
+        try:
+            if vs.live_streaming:
+                # Determine where to start playing. There may be several seconds
+                # between starting the download and starting playback, which we'll
+                # want to skip.
+                self.calc_live_startpos( self.max_prebuf_packets, True )
+
+                # override any position request by VLC, we only have live data
+                piece = vs.playback_pos
+                offset = 0
+            else:
+                # Determine piece number and offset
+                if bytepos < vs.first_piecelen:
+                    piece = vs.first_piece
+                    offset = bytepos
+                else:
+                    newbytepos = bytepos - vs.first_piecelen
+
+                    piece  = vs.first_piece + newbytepos / vs.piecelen + 1
+                    offset = newbytepos % vs.piecelen
+
+            if DEBUG:
+                print >>sys.stderr,"vod: trans: === START at offset %d (piece %d) (forced: %s) ===" % (bytepos,piece,force),currentThread().getName()
+
+            # Initialise all playing variables
+            self.curpiece = "" # piece currently being popped
+            self.curpiece_pos = offset
+            self.set_pos( piece )
+            self.outbuf = []
+            self.last_pop = time.time()
+            self.reset_bitrate_prediction()
+            vs.playing = True
+            self.playbackrate = Measure( 60 )
+
+        finally:
+            self.data_ready.release()
+
+        # ARNOTODO: start is called by non-NetworkThreads, these following methods
+        # are usually called by NetworkThread.
+        #
+        # We now know that this won't be called until notify_playable() so
+        # perhaps this can be removed?
+        #
+        # CAREFUL: if we use start() for seeking... that's OK. User won't be
+        # able to seek before he got his hands on the stream, so after 
+        # notify_playable()
+        
+        # See what we can do right now
+        self.update_prebuffering()
+        self.refill_buffer()
+
+    def stop( self ):
+        """ Playback is stopped. """
+        self._event_reporter.create_and_add_event("stop", [self.b64_infohash])
+        # self._event_reporter.flush()
+
+        vs = self.videostatus
+        if DEBUG:
+            print >>sys.stderr,"vod: trans: === STOP  = player closed conn === ",currentThread().getName()
+        if not vs.playing:
+            return
+        vs.playing = False
+
+        # clear buffer and notify possible readers
+        self.data_ready.acquire()
+        self.outbuf = []
+        self.last_pop = None
+        vs.prebuffering = False
+        self.data_ready.notify()
+        self.data_ready.release()
+
+    def pause( self, autoresume = False ):
+        """ Pause playback. If `autoresume' is set, playback is expected to be
+        resumed automatically once enough data has arrived. """
+        self._event_reporter.create_and_add_event("pause", [self.b64_infohash])
+
+        vs = self.videostatus
+        if not vs.playing or not vs.pausable:
+            return
+
+        if vs.paused:
+            vs.autoresume = autoresume
+            return
+
+        if DEBUG:
+            print >>sys.stderr,"vod: trans: paused (autoresume: %s)" % (autoresume,)
+
+        vs.paused = True
+        vs.autoresume = autoresume
+        self.paused_at = time.time()
+        #self.reset_bitrate_prediction()
+        self.videoinfo["usercallback"](VODEVENT_PAUSE,{ "autoresume": autoresume })
+
+    def resume( self ):
+        """ Resume paused playback. """
+        self._event_reporter.create_and_add_event("resume", [self.b64_infohash])
+
+        vs = self.videostatus
+
+        if not vs.playing or not vs.paused or not vs.pausable:
+            return
+
+        if DEBUG:
+            print >>sys.stderr,"vod: trans: resumed"
+
+        vs.paused = False
+        vs.autoresume = False
+        self.stat_stalltime += time.time() - self.paused_at
+        self.addtime_bitrate_prediction( time.time() - self.paused_at )
+        self.videoinfo["usercallback"](VODEVENT_RESUME,{})
+
+        self.update_prebuffering()
+        self.refill_buffer()
+
+    def autoresume( self, testfunc = lambda: True ):
+        """ Resumes if testfunc returns True. If not, will test every second. """
+
+        vs = self.videostatus
+
+        if not vs.playing or not vs.paused or not vs.autoresume:
+            return
+
+        if not testfunc():
+            self.rawserver.add_task( lambda: self.autoresume( testfunc ), 1.0 )
+            return
+
+        if DEBUG:
+            print >>sys.stderr,"vod: trans: Resuming, since we can maintain this playback position"
+        self.resume()
+
+    def done( self ):
+        vs = self.videostatus
+
+        if not vs.playing:
+            return True
+
+        if vs.wraparound:
+            return False
+
+        # Arno, 2010-01-08: Adjusted EOF condition to work well with seeking/HTTP range queries
+        return vs.playback_pos == vs.last_piece+1 and len(self.outbuf) == 0 and self.curpiece_pos == 0 and len(self.curpiece) == 0
+
+    def seek(self,pos,whence=os.SEEK_SET):
+        """ Seek to the given position, a number in bytes relative to both
+        the "whence" reference point and the file being played.
+        
+        We currently actually seek at byte level, via the start() method.
+        We support all forms of seeking, including seeking past the current
+        playback pos. Note this may imply needing to prebuffer again or 
+        being paused.
+        
+        vs.playback_pos in NetworkThread domain. Does data_ready lock cover 
+        that? Nope. However, this doesn't appear to be respected in any
+        of the MovieTransport methods, check all.
+        
+        Check
+        * When seeking reset other buffering, e.g. read()'s self.curpiece
+           and higher layers.
+        
+        """
+        vs = self.videostatus
+        length = self.size()
+
+        # lock before changing startpos or any other playing variable
+        self.data_ready.acquire()
+        try:
+            if vs.live_streaming:
+                # Arno, 2010-07-16: Raise error on seek, is clearer to stream user.
+                raise ValueError("seeking not possible for live")
+            if whence == os.SEEK_SET:
+                abspos = pos
+            elif whence == os.SEEK_END:
+                if pos > 0:
+                    raise ValueError("seeking beyond end of stream")
+                else:
+                    abspos = length+pos
+            else: # SEEK_CUR
+                raise ValueError("seeking does not currently support SEEK_CUR")
+            
+            self.stop()
+            self.start(pos)
+        finally:
+            self.data_ready.release()
+
+
+
+    def get_mimetype(self):
+        return self.mimetype
+
+    def set_mimetype(self,mimetype):
+        self.mimetype = mimetype
+        
+    def available(self):
+        self.data_ready.acquire()
+        try:
+            return self.outbuflen    
+        finally:
+            self.data_ready.release()
+        
+        
+    #
+    # End of MovieTransport interface
+    #
+
+    def have_piece(self,piece):
+        return self.piecepicker.has[piece]
+
+    def get_piece(self,piece):
+        """ Returns the data of a certain piece, or None. """
+
+        vs = self.videostatus
+
+        if not self.have_piece( piece ):
+            return None
+
+        begin = 0
+        length = vs.piecelen
+
+        if piece == vs.first_piece:
+            begin = vs.movie_range[0][1]
+            length -= begin
+
+        if piece == vs.last_piece:
+            cutoff = vs.piecelen - (vs.movie_range[1][1] + 1)
+            length -= cutoff
+
+        #print >>sys.stderr,"get_piece",piece
+        data = self.storagewrapper.do_get_piece(piece, begin, length)
+        if data is None:
+            return None
+        return data.tostring()
+
+    def reset_bitrate_prediction(self):
+        self.start_playback = None
+        self.last_playback = None
+        self.history_playback = collections.deque()
+
+    def addtime_bitrate_prediction(self,seconds):
+        if self.start_playback is not None:
+            self.start_playback["local_ts"] += seconds
+
+    def valid_piece_data(self,i,piece):
+        if not piece:
+            return False
+
+        if not self.start_playback or self.authenticator is None:
+            # no check possible
+            return True
+
+        s = self.start_playback
+
+        seqnum = self.authenticator.get_seqnum( piece )
+        source_ts = self.authenticator.get_rtstamp( piece )
+
+        if seqnum < s["absnr"] or source_ts < s["source_ts"]:
+            # old packet???
+            print >>sys.stderr,"vod: trans: **** INVALID PIECE #%s **** seqnum=%d but we started at seqnum=%d, ts=%f but we started at %f" % (i,seqnum,s["absnr"],source_ts,s["source_ts"])
+            return False
+
+        return True
+
+
+    def update_bitrate_prediction(self,i,piece):
+        """ Update the rate prediction given that piece i has just been pushed to the buffer. """
+
+        if self.authenticator is not None:
+            seqnum = self.authenticator.get_seqnum( piece )
+            source_ts = self.authenticator.get_rtstamp( piece )
+        else:
+            seqnum = i
+            source_ts = 0
+
+        d = {
+            "nr": i,
+            "absnr": seqnum,
+            "local_ts": time.time(),
+            "source_ts": source_ts,
+        }
+
+        # record 
+        if self.start_playback is None:
+            self.start_playback = d
+
+        if self.last_playback and self.last_playback["absnr"] > d["absnr"]:
+            # called out of order
+            return
+
+        self.last_playback = d
+
+        # keep a recent history
+        MAX_HIST_LEN = 10*60 # seconds
+
+        self.history_playback.append( d )
+
+        # of at most 10 entries (or minutes if we keep receiving pieces)
+        while source_ts - self.history_playback[0]["source_ts"] > MAX_HIST_LEN:
+            self.history_playback.popleft()
+
+        if DEBUG:
+            vs = self.videostatus
+            first, last = self.history_playback[0], self.history_playback[-1]
+            
+            if first["source_ts"] and first != last:
+                divd = (last["source_ts"] - first["source_ts"])
+                if divd == 0:
+                    divd = 0.000001
+                bitrate = "%.2f kbps" % (8.0 / 1024 * (vs.piecelen - vs.sigsize) * (last["absnr"] - first["absnr"]) / divd,)
+            else:
+                bitrate = "%.2f kbps (external info)" % (8.0 / 1024 * vs.bitrate)
+
+            print >>sys.stderr,"vod: trans: %i: pushed at t=%.2f, age is t=%.2f, bitrate = %s" % (i,d["local_ts"]-self.start_playback["local_ts"],d["source_ts"]-self.start_playback["source_ts"],bitrate)
+
+    def piece_due(self,i):
+        """ Return the time when we expect to have to send a certain piece to the player. For
+        wraparound, future pieces are assumed. """
+
+        if self.start_playback is None:
+            return float(2 ** 31) # end of time
+
+        s = self.start_playback
+        l = self.last_playback
+        vs = self.videostatus
+
+        if not vs.wraparound and i < l["nr"]:
+            # should already have arrived!
+            return time.time()
+
+        # assume at most one wrap-around between l and i
+        piecedist = (i - l["nr"]) % vs.movie_numpieces
+
+        if s["source_ts"]:
+            # ----- we have timing information from the source
+            first, last = self.history_playback[0], self.history_playback[-1]
+
+            if first != last:
+                # we have at least two recent pieces, so can calculate average bitrate. use the recent history
+                # do *not* adjust for sigsize since we don't want the actual video speed but the piece rate
+                bitrate = 1.0 * vs.piecelen * (last["absnr"] - first["absnr"]) / (last["source_ts"] - first["source_ts"])
+            else:
+                # fall-back to bitrate predicted from torrent / ffmpeg
+                bitrate = vs.bitrate
+           
+            # extrapolate with the average bitrate so far
+            return s["local_ts"] + l["source_ts"] - s["source_ts"] + piecedist * vs.piecelen / bitrate - self.PIECE_DUE_SKEW
+        else:
+            # ----- no timing information from pieces, so do old-fashioned methods
+            if vs.live_streaming:
+                # Arno, 2008-11-20: old-fashioned method is well bad,
+                # ignore.
+                return time.time() + 60.0
+            else:
+                i =  piecedist + (l["absnr"] - s["absnr"])
+                
+                if s["nr"] == vs.first_piece:
+                    bytepos = vs.first_piecelen + (i-1) * vs.piecelen
+                else:
+                    bytepos = i * vs.piecelen
+                
+                return s["local_ts"] + bytepos / vs.bitrate - self.PIECE_DUE_SKEW
+            
+
+    def max_buffer_size( self ):
+        vs = self.videostatus
+        if vs.dropping:
+            # live
+            # Arno: 1/2 MB or based on bitrate if that is above 5 Mbps
+            return max( 0*512*1024, self.BUFFER_TIME * vs.bitrate )
+        else:
+            # VOD
+            # boudewijn: 1/4 MB, bitrate, or 2 pieces (wichever is higher)
+            return max(256*1024, vs.piecelen * 2, self.BUFFER_TIME * vs.bitrate)
+
+
+    def refill_buffer( self ):
+        """ Push pieces into the player FIFO when needed and able. This counts as playing
+            the pieces as far as playback_pos is concerned."""
+            
+        # HttpSeed policy: if the buffer is underrun we start asking pieces from http (always available)
+        # till the next refill_buffer call. As soon as refill_buffer is called again we stop asking http
+        # and start again in case the buffer is still underrun. We never stop asking till prefuffering.
+
+        self.data_ready.acquire()
+
+        vs = self.videostatus
+
+        if vs.prebuffering or not vs.playing:
+            self.data_ready.release()
+            return
+
+        if vs.paused:
+            self.data_ready.release()
+            return
+
+        mx = self.max_buffer_size()
+        self.outbuflen = sum( [len(d) for (p,d) in self.outbuf] )
+        now = time.time()
+
+        if self.http_support is not None:
+            if self.outbuflen < ( mx / 4 ):
+                if not self.done(): # TODO : Diego : correct end test?
+                    self.http_support.start_video_support( 0 )
+                else:
+                    self.http_support.stop_video_support() # Download finished
+            else:
+                self.http_support.stop_video_support()
+
+        def buffer_underrun():
+            return self.outbuflen == 0 and self.start_playback and now - self.start_playback["local_ts"] > 1.0
+
+        # Arno, 2010-04-16: STBSPEED: simplified. If we cannot pause, we
+        # just push everything we got to the player this buys us time to 
+        # retrieve more data.
+        if buffer_underrun() and vs.pausable:
+
+            if vs.dropping: # live
+                def sustainable():
+                    # buffer underrun -- check for available pieces
+                    num_future_pieces = 0
+                    for piece in vs.generate_range( vs.download_range() ):
+                        if self.has[piece]:
+                            num_future_pieces += 1
+
+                    goal = mx / 2
+                    # progress
+                    self.prebufprogress = min(1.0,float(num_future_pieces * vs.piecelen) / float(goal))
+                    
+                    # enough future data to fill the buffer
+                    return num_future_pieces * vs.piecelen >= goal
+            else: # vod
+                def sustainable():
+                    # num_immediate_packets = 0
+                    # for piece in vs.generate_range( vs.download_range() ):
+                    #     if self.has[piece]:
+                    #         num_immediate_packets += 1
+                    #     else:
+                    #         break
+                    # else:
+                    #     # progress                                                                              
+                    #     self.prebufprogress = 1.0
+                    #     # completed loop without breaking, so we have everything we need                        
+                    #     return True
+                    #
+                    # # progress                                                                                  
+                    # self.prebufprogress = min(1.0,float(num_immediate_packets) / float(self.max_prebuf_packets))
+                    #
+                    # return num_immediate_packets >= self.max_prebuf_packets
+
+                    self.sustainable_counter += 1
+                    if self.sustainable_counter > 10:
+                        self.sustainable_counter = 0
+                        
+                        high_range_length = vs.get_high_range_length()
+                        have_length = len(filter(lambda n:self.has[n], vs.generate_high_range()))
+
+                        # progress                                                                                  
+                        self.prebufprogress = min(1.0, float(have_length) / max(1, high_range_length))
+
+                        return have_length >= high_range_length
+
+                    else:
+                        num_immediate_packets = 0
+                        high_range_length = vs.get_high_range_length()
+                        # for piece in vs.generate_range(vs.download_range()): 
+                        for piece in vs.generate_high_range(): 
+                            if self.has[piece]:
+                                num_immediate_packets += 1
+                                if num_immediate_packets >= high_range_length:
+                                    break
+                            else:
+                                break
+                        else:
+                            # progress                                                                              
+                            self.prebufprogress = 1.0
+                            # completed loop without breaking, so we have everything we need                        
+                            return True
+
+                        return num_immediate_packets >= high_range_length
+
+            sus = sustainable()
+            if not sus:
+                if DEBUG:
+                    print >>sys.stderr,"vod: trans:                        BUFFER UNDERRUN -- PAUSING"
+                    
+                # TODO : Diego : The Http support level should be tuned according to the sustainability level
+                if self.http_support is not None:
+                    self.http_support.start_video_support( 0 ) # TODO : Diego : still needed? here the buffer is 0 so already asking for support
+                self.pause( autoresume = True )
+                self.autoresume( sustainable )
+
+                # boudewijn: increase the minimum buffer size
+                vs.increase_high_range()
+                        
+                self.data_ready.release()
+                return
+                    
+        def push( i, data ):
+            # force buffer underrun:
+            #if self.start_playback and time.time()-self.start_playback["local_ts"] > 60:
+            #    # hack: dont push after 1 minute
+            #    return
+
+            # push packet into queue
+            if DEBUG:
+                print >>sys.stderr,"vod: trans: %d: pushed l=%d" % (vs.playback_pos,piece)
+
+            # update predictions based on this piece
+            self.update_bitrate_prediction( i, data )
+
+            self.stat_playedpieces += 1
+            self.stat_pieces.set( i, "tobuffer", time.time() )
+                    
+            self.outbuf.append( (vs.playback_pos,data) )
+            self.outbuflen += len(data)
+
+            self.data_ready.notify()
+            self.inc_pos()
+
+        def drop( i ):
+            # drop packet
+            if DEBUG:
+                print >>sys.stderr,"vod: trans: %d: dropped pos=%d; deadline expired %.2f sec ago !!!!!!!!!!!!!!!!!!!!!!" % (piece,vs.playback_pos,time.time()-self.piece_due(i))
+
+            self.stat_droppedpieces += 1
+            self.stat_pieces.complete( i )
+            self.inc_pos()
+
+        for piece in vs.generate_range( vs.download_range() ): 
+            ihavepiece = self.has[piece]
+            forcedrop = False
+
+            # check whether we have room to store it
+            if self.outbuflen > mx:
+                # buffer full
+                break
+
+            # final check for piece validity
+            if ihavepiece:
+                data = self.get_piece( piece )
+                if not self.valid_piece_data( piece, data ):
+                    # I should have the piece, but I don't: WAAAAHH!
+                    forcedrop = True
+                    ihavepiece = False
+
+            if ihavepiece:
+                # have piece - push it into buffer
+                if DEBUG:
+                    print >>sys.stderr,"vod: trans:                        BUFFER STATUS (max %.0f): %.0f kbyte" % (mx/1024.0,self.outbuflen/1024.0)
+
+                # piece found -- add it to the queue
+                push( piece, data )
+            else:
+                # don't have piece, or forced to drop
+                if not vs.dropping and forcedrop:
+                    print >>sys.stderr,"vod: trans: DROPPING INVALID PIECE #%s, even though we shouldn't drop anything." % piece
+                if vs.dropping or forcedrop:
+                    if time.time() >= self.piece_due( piece ) or (vs.pausable and buffer_underrun()) or forcedrop:
+                        # piece is too late or we have an empty buffer (and future data to play, otherwise we would have paused) -- drop packet
+                        drop( piece )
+                    else:
+                        # we have time to wait for the piece and still have data in our buffer -- wait for packet
+                        if DEBUG:
+                            print >>sys.stderr,"vod: trans: %d: due in %.2fs  pos=%d" % (piece,self.piece_due(piece)-time.time(),vs.playback_pos)
+                    break
+                else: # not dropping
+                    if DEBUG:
+                        print >>sys.stderr,"vod: trans: %d: not enough pieces to fill buffer." % (piece)
+                    break
+
+        self.data_ready.release()
+
+    def video_refillbuf_rawtask( self ):
+        self.refill_buffer()
+
+        self.rawserver.add_task( self.video_refillbuf_rawtask, self.REFILL_INTERVAL )
+
+    def pop( self ):
+        self.data_ready.acquire()
+        vs = self.videostatus
+
+        while vs.prebuffering and not self.done():
+            # wait until done prebuffering
+            self.data_ready.wait()
+
+        while not self.outbuf and not self.done():
+            # wait until a piece is available
+            #if DEBUG:
+            #    print >>sys.stderr,"vod: trans: Player waiting for data"
+            self.data_ready.wait()
+
+        if not self.outbuf:
+            piecetup = None
+        else:
+            piecetup = self.outbuf.pop( 0 ) # nr,data pair
+            # Arno, 2010-02-01: Grrrr...
+            self.outbuflen -= len(piecetup[1])
+            self.playbackrate.update_rate( len(piecetup[1]) )
+
+        self.last_pop = time.time()
+
+        lenoutbuf = len(self.outbuf)
+
+        self.data_ready.release()
+
+        if piecetup:
+            self.stat_pieces.set( piecetup[0], "toplayer", time.time() )
+            self.stat_pieces.complete( piecetup[0] )
+
+        # Arno, 2010-02-11: STBSPEEDMERGE: Do we want this for STB?
+        if vs.pausable:
+            # 23/06/09 Boudewijn: because of vlc buffering the self.outbuf
+            # almost always gets emptied. This results in periodic (every
+            # few seconds) pause signals to VLC. 
+            #
+            # To 'solve' this we delay the delivery to VLC based on the
+            # current buffer size. More delay when there is less data
+            # available.
+            #
+            # 24/06/09 Boudewijn: smaller is (much) better for live as
+            # they never get over a certain amount of outstanding
+            # pieces. So this will need to be made dependent. VOD can be
+            # higher than live.
+            if lenoutbuf < 5:
+                if lenoutbuf > 0:
+                    delay = min(0.1, 3 * 0.1 / lenoutbuf)
+                else:
+                    delay = 0.1
+                if DEBUG: print >>sys.stderr, "Vod: Delaying pop to VLC by", delay, "seconds"
+                time.sleep(delay)
+
+        return piecetup
+
+
+    def notify_playable(self):
+        """ Tell user he can play the media, 
+        cf. BaseLib.Core.DownloadConfig.set_vod_event_callback()
+        """
+        #if self.bufferinfo:
+        #    self.bufferinfo.set_playable()
+        #self.progressinf.bufferinfo_updated_callback()
+        
+        # triblerAPI
+        if self.usernotified:
+            return
+        self.usernotified = True
+        self.prebufprogress = 1.0
+        self.playable = True
+        
+        #print >>sys.stderr,"vod: trans: notify_playable: Calling usercallback to tell it we're ready to play",self.videoinfo['usercallback']
+        
+        # MIME type determined normally in LaunchManyCore.network_vod_event_callback
+        # However, allow for recognition by videoanalyser
+        mimetype = self.get_mimetype()
+        complete = self.piecepicker.am_I_complete()
+        if complete:
+            endstream = None
+            filename = self.videoinfo["outpath"]
+        else:
+            stream = MovieTransportStreamWrapper(self)
+            if self.videostatus.live_streaming and self.videostatus.authparams['authmethod'] != LIVE_AUTHMETHOD_NONE:
+                from BaseLib.Core.Video.LiveSourceAuth import AuthStreamWrapper,VariableReadAuthStreamWrapper
+
+                intermedstream = AuthStreamWrapper(stream,self.authenticator)
+                endstream = VariableReadAuthStreamWrapper(intermedstream,self.authenticator.get_piece_length()) 
+            else:
+                endstream = stream
+            filename = None 
+            
+        # Call user callback
+        print >>sys.stderr,"vod:::::::::: trans: notify_playable: calling:",self.vodeventfunc
+        try:
+            self.vodeventfunc( self.videoinfo, VODEVENT_START, {
+                "complete":  complete,
+                "filename":  filename,
+                "mimetype":  mimetype,
+                "stream":    endstream,
+                "length":      self.size(),
+                "bitrate":   self.videostatus.bitrate,
+            } )
+        except:
+            print_exc()
+
+
+    #
+    # Methods for DownloadState to extract status info of VOD mode.
+    #
+    def get_stats(self):
+        """ Returns accumulated statistics. The piece data is cleared after this call to save memory. """
+        """ Called by network thread """
+        
+        s = { "played": self.stat_playedpieces,
+              "late": self.stat_latepieces,
+              "dropped": self.stat_droppedpieces,
+              "stall": self.stat_stalltime,
+              "pos": self.videostatus.playback_pos,
+              "prebuf": self.stat_prebuffertime,
+              "pp": self.piecepicker.stats,
+              "pieces": self.stat_pieces.pop_completed(),
+              "firstpiece":self.videostatus.first_piece,
+              "npieces":self.videostatus.movie_numpieces}
+        return s
+
+    def get_prebuffering_progress(self):
+        """ Called by network thread """
+        return self.prebufprogress
+    
+    def is_playable(self):
+        """ Called by network thread """
+        if not self.playable or self.videostatus.prebuffering:
+            self.playable = (self.prebufprogress == 1.0 and self.enough_buffer())
+        return self.playable
+        
+    def get_playable_after(self):
+        """ Called by network thread """
+        return self.expected_buffering_time()
+    
+    def get_duration(self):
+        return 1.0 * self.videostatus.selected_movie["size"] / self.videostatus.bitrate
+
+    #
+    # Live streaming
+    #
+    def live_invalidate_piece_globally(self, piece, mevirgin=False):
+        """ Make piece disappear from this peer's view of BT world.
+        mevirgen indicates whether we already downloaded stuff,
+        skipping some cleanup if not.
+        """
+        #print >>sys.stderr,"vod: trans: live_invalidate",piece
+                 
+        self.piecepicker.invalidate_piece(piece)
+        self.piecepicker.downloader.live_invalidate(piece, mevirgin)
+
+    def live_invalidate_piece_ranges_globally(self,toinvalidateranges,toinvalidateset):
+        # STBSPEED optimization
+        # v = Set()
+        for s,e in toinvalidateranges:
+            for piece in xrange(s,e+1):
+                # v.add(piece)
+                self.piecepicker.invalidate_piece(piece)
+                
+        """
+        diffleft = v.difference(toinvalidateset)
+        diffright = toinvalidateset.difference(v)
+        print >>sys.stderr,"vod: live_invalidate_piece_ranges_globally: diff: in v",diffleft,"in invset",diffright
+        assert v == toinvalidateset
+        """
+        self.piecepicker.downloader.live_invalidate_ranges(toinvalidateranges,toinvalidateset)
+
+
+    # LIVESOURCEAUTH
+    def piece_from_live_source(self,index,data):
+        if self.authenticator is not None:
+            return self.authenticator.verify(data,index=index)
+        else:
+            return True
+    
diff --git a/instrumentation/next-share/BaseLib/Core/Video/VideoSource.py b/instrumentation/next-share/BaseLib/Core/Video/VideoSource.py
new file mode 100644
index 0000000..7836aae
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Video/VideoSource.py
@@ -0,0 +1,284 @@
+# written by Jan David Mol
+# see LICENSE.txt for license information
+#
+# Represent a source of video (other than a BitTorrent swarm), which can inject
+# pieces into the downloading engine.
+
+# We assume we are the sole originator of these pieces, i.e. none of the pieces
+# injected are already obtained from another source or requested from some peer.
+
+import sys
+from threading import RLock,Thread
+from traceback import print_exc
+from time import sleep
+from BaseLib.Core.BitTornado.BT1.PiecePicker import PiecePicker
+from BaseLib.Core.simpledefs import *
+from BaseLib.Core.Video.LiveSourceAuth import NullAuthenticator,ECDSAAuthenticator,RSAAuthenticator
+from BaseLib.Core.Utilities.Crypto import sha
+
+
+DEBUG = True
+
+class SimpleThread(Thread):
+    """ Wraps a thread around a single function. """
+
+    def __init__(self,runfunc):
+        Thread.__init__(self)
+        self.setDaemon(True)
+        self.setName("VideoSourceSimple"+self.getName())
+        self.runfunc = runfunc
+
+    def run(self):
+        self.runfunc()
+
+
+class VideoSourceTransporter:
+    """ Reads data from an external source and turns it into BitTorrent chunks. """
+
+    def __init__(self, stream, bt1download, authconfig,restartstatefilename):
+        self.stream = stream
+        self.bt1download = bt1download
+        self.restartstatefilename = restartstatefilename
+        self.exiting = False
+
+        # shortcuts to the parts we use
+        self.storagewrapper = bt1download.storagewrapper
+        self.picker = bt1download.picker
+        self.rawserver = bt1download.rawserver
+        self.connecter = bt1download.connecter
+        self.fileselector = bt1download.fileselector
+
+        # generic video information
+        self.videostatus = bt1download.videostatus
+
+        # buffer to accumulate video data
+        self.buffer = []
+        self.buflen = 0
+        self.bufferlock = RLock()
+        self.handling_pieces = False
+        self.readlastseqnum = False
+
+        # LIVESOURCEAUTH
+        if authconfig.get_method() == LIVE_AUTHMETHOD_ECDSA:
+            self.authenticator = ECDSAAuthenticator(self.videostatus.piecelen,self.bt1download.len_pieces,keypair=authconfig.get_keypair())
+        elif authconfig.get_method() == LIVE_AUTHMETHOD_RSA:
+            self.authenticator = RSAAuthenticator(self.videostatus.piecelen,self.bt1download.len_pieces,keypair=authconfig.get_keypair())
+        else:
+            self.authenticator = NullAuthenticator(self.videostatus.piecelen,self.bt1download.len_pieces)
+
+    def start(self):
+        """ Start transporting data. """
+
+        self.input_thread_handle = SimpleThread(self.input_thread)
+        self.input_thread_handle.start()
+
+    def _read(self,length):
+        """ Called by input_thread. """
+        return self.stream.read(length)
+
+    def input_thread(self):
+        """ A thread reading the stream and buffering it. """
+
+        print >>sys.stderr,"VideoSource: started input thread"
+
+        # we can't set the playback position from this thread, so
+        # we assume all pieces are vs.piecelen in size.
+
+        contentbs = self.authenticator.get_content_blocksize()
+        try:
+            while not self.exiting:
+                data = self._read(contentbs)
+                if not data:
+                    break
+
+                if DEBUG:
+                    print >>sys.stderr,"VideoSource: read %d bytes" % len(data)
+
+                self.process_data(data)
+        except IOError:
+            if DEBUG:
+                print_exc()
+
+        self.shutdown()
+
+    def shutdown(self):
+        """ Stop transporting data. """
+
+        print >>sys.stderr,"VideoSource: shutting down"
+
+        if self.exiting:
+            return
+
+        self.exiting = True
+
+        try:
+            self.stream.close()
+        except IOError:
+            # error on closing, nothing we can do
+            pass
+
+    def process_data(self,data):
+        """ Turn data into pieces and queue them for insertion. """
+        """ Called by input thread. """
+
+        vs = self.videostatus
+
+        self.bufferlock.acquire()
+        try:
+            # add data to buffer
+            self.buffer.append( data )
+            self.buflen += len( data )
+
+            if not self.handling_pieces:
+                # signal to network thread that data has arrived
+                self.rawserver.add_task( self.create_pieces )
+                self.handling_pieces = True
+        finally:
+            self.bufferlock.release()
+
+    def create_pieces(self):
+        """ Process the buffer and create pieces when possible.
+        Called by network thread """
+
+        def handle_one_piece():
+            vs = self.videostatus
+
+            # LIVESOURCEAUTH
+            # Arno: make room for source auth info
+            contentbs = self.authenticator.get_content_blocksize()
+            
+            if self.buflen < contentbs:
+                return False
+
+            if len(self.buffer[0]) == contentbs:
+                content = self.buffer[0]
+                del self.buffer[0]
+            else:
+                if DEBUG:
+                    print >>sys.stderr,"VideoSource: JOIN ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"
+                buffer = "".join(self.buffer)
+                self.buffer = [buffer[contentbs:]]
+                content = buffer[:contentbs]
+            self.buflen -= contentbs
+            
+            datas = self.authenticator.sign(content)
+
+            piece = "".join(datas)
+            
+            # add new piece
+            self.add_piece(vs.playback_pos,piece)
+
+            # invalidate old piece
+            self.del_piece( vs.live_piece_to_invalidate() )
+
+            try:
+                lastseqnum = self.authenticator.get_source_seqnum()
+                f = open(self.restartstatefilename,"wb")
+                f.write(str(lastseqnum))
+                f.close()
+            except:
+                print_exc()
+
+            # advance pointer
+            vs.inc_playback_pos()
+            
+            return True
+
+        if not self.readlastseqnum:
+            self.readlastseqnum = True
+            try:
+                f = open(self.restartstatefilename,"rb")
+                data = f.read()
+                f.close()
+                lastseqnum = int(data)
+                
+                print >>sys.stderr,"VideoSource: Restarting stream at abs.piece",lastseqnum
+                
+                # Set playback pos of source and absolute piece nr.
+                lastpiecenum = lastseqnum % self.authenticator.get_npieces()
+                self.authenticator.set_source_seqnum(lastseqnum+1L)
+                
+                self.videostatus.set_live_startpos(lastpiecenum)
+                self.videostatus.inc_playback_pos()
+            except:
+                print_exc()
+            
+        self.bufferlock.acquire()
+        try:
+            while handle_one_piece():
+                pass
+
+            self.handling_pieces = False
+        finally:
+            self.bufferlock.release()
+
+    def add_piece(self,index,piece):
+        """ Push one piece into the BitTorrent system. """
+
+        # Modelled after BitTornado.BT1.Downloader.got_piece
+        # We don't need most of that function, since this piece
+        # was never requested from another peer.
+
+        if DEBUG:
+            print >>sys.stderr,"VideoSource: created piece #%d" % index
+            # ECDSA
+            #print >>sys.stderr,"VideoSource: sig",`piece[-64:]`
+            #print >>sys.stderr,"VideoSource: dig",sha(piece[:-64]).hexdigest()
+            # RSA, 768 bits
+            #print >>sys.stderr,"VideoSource: sig",`piece[-96:]`
+            #print >>sys.stderr,"VideoSource: dig",sha(piece[:-112]).hexdigest()
+
+
+        # act as if the piece was requested and just came in
+        # do this in chunks, as StorageWrapper expects to handle
+        # a request for each chunk
+        chunk_size = self.storagewrapper.request_size
+        length = min( len(piece), self.storagewrapper._piecelen(index) )
+        x = 0
+        while x < length:
+            self.storagewrapper.new_request( index )
+            self.storagewrapper.piece_came_in( index, x, [], piece[x:x+chunk_size], min(chunk_size,length-x) )
+            x += chunk_size
+
+        # also notify the piecepicker
+        self.picker.complete( index )
+
+        # notify our neighbours
+        self.connecter.got_piece( index )
+
+    def del_piece(self,piece):
+        if DEBUG:
+            print >>sys.stderr,"VideoSource: del_piece",piece
+        # See Tribler/Core/Video/VideoOnDemand.py, live_invalidate_piece_globally
+        self.picker.invalidate_piece(piece)
+        self.picker.downloader.live_invalidate(piece)
+
+
+class RateLimitedVideoSourceTransporter(VideoSourceTransporter):
+    """ Reads from the stream at a certain byte rate.
+
+        Useful for creating live streams from file. """
+
+    def __init__( self, ratelimit, *args, **kwargs ):
+        """@param ratelimit: maximum rate in bps"""
+        VideoSourceTransporter.__init__( self, *args, **kwargs )
+
+        self.ratelimit = int(ratelimit)
+
+    def _read(self,length):
+        # assumes reads and processing data is instant, so
+        # we know how long to sleep
+        sleep(1.0 * length / self.ratelimit)
+        return VideoSourceTransporter._read(self,length)
+
+
+class PiecePickerSource(PiecePicker):
+    """ A special piece picker for the source, which never
+        picks any pieces. Used to prevent the injection
+        of corrupted pieces at the source. """
+
+    def next(self,*args,**kwargs):
+        # never pick any pieces
+        return None
+
+    
diff --git a/instrumentation/next-share/BaseLib/Core/Video/VideoStatus.py b/instrumentation/next-share/BaseLib/Core/Video/VideoStatus.py
new file mode 100644
index 0000000..851e044
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Video/VideoStatus.py
@@ -0,0 +1,371 @@
+# Written by Jan David Mol, Arno Bakker
+# see LICENSE.txt for license information
+
+import sys
+import time
+from math import ceil
+from sets import Set
+
+from BaseLib.Core.simpledefs import *
+
+# live streaming means wrapping around
+LIVE_WRAPAROUND = True
+
+DEBUG = False
+
+class VideoStatus:
+    """ Info about the selected video and status of the playback. """
+
+    # TODO: thread safety? PiecePicker, MovieSelector and MovieOnDemandTransporter all interface this
+
+    def __init__(self,piecelen,fileinfo,videoinfo,authparams):
+        """
+            piecelen = length of BitTorrent pieces
+            fileinfo = list of (name,length) pairs for all files in the torrent,
+                       in their recorded order
+            videoinfo = videoinfo object from download engine
+        """
+        self.piecelen = piecelen # including signature, if any
+        self.sigsize = 0
+        self.fileinfo = fileinfo
+        self.videoinfo = videoinfo
+        self.authparams = authparams
+
+        # size of high probability set, in seconds (piecepicker varies
+        # between the limit values depending on network performance,
+        # increases and decreases are in the specified step (min,max,step)
+        self.high_prob_curr_time = 10
+        self.high_prob_curr_time_limit = (10, 180, 10)
+
+        # size of high probability set, in pieces (piecepicker
+        # varies between the limit values depending on network
+        # performance, increases and decreases are in the specified step 
+        # (min,max,step).
+        # Arno, 2010-03-10: max 50 pieces too little for 32K piece-sized
+        # VOD streams.
+        #
+        self.high_prob_curr_pieces = 5
+        self.high_prob_curr_pieces_limit = (5, 1800 ,5) # Arno, 2010-03-11: with 32K pieces and 3 mbps we're talking 10 pieces / sec
+
+        # ----- locate selected movie in fileinfo
+        index = self.videoinfo['index']
+        if index == -1:
+            index = 0
+
+        movie_offset = sum( (filesize for (_,filesize) in fileinfo[:index] if filesize) )
+        movie_name = fileinfo[index][0]
+        movie_size = fileinfo[index][1]
+
+        self.selected_movie = {
+          "offset": movie_offset,
+          "name": movie_name,
+          "size": movie_size,
+        }
+
+        # ----- derive generic movie parameters
+        movie_begin = movie_offset
+        movie_end = movie_offset + movie_size - 1
+
+        # movie_range = (bpiece,offset),(epiece,offset), inclusive
+        self.movie_range = ( (movie_begin/piecelen, movie_begin%piecelen),
+                             (movie_end/piecelen, movie_end%piecelen) )
+        self.first_piecelen = piecelen - self.movie_range[0][1]
+        self.last_piecelen  = self.movie_range[1][1]+1 # Arno, 2010-01-08: corrected off by one error
+        self.first_piece = self.movie_range[0][0]
+        self.last_piece = self.movie_range[1][0]
+        self.movie_numpieces = self.last_piece - self.first_piece + 1
+
+        # ----- live streaming settings
+        self.live_streaming = videoinfo['live']
+        self.live_startpos = None
+        self.playback_pos_observers = []
+        self.wraparound = self.live_streaming and LIVE_WRAPAROUND
+        # /8 means -12.5 % ... + 12.5 % = 25 % window
+        self.wraparound_delta = max(4,self.movie_numpieces/8) 
+
+        # ----- generic streaming settings
+        # whether to drop packets that come in too late
+        if self.live_streaming:
+            self.dropping = True  # drop, but we will autopause as well
+        else:
+            self.dropping = False # just wait and produce flawless playback
+
+        if videoinfo['bitrate']:
+            self.set_bitrate( videoinfo['bitrate'] )
+        else:
+            self.set_bitrate( 512*1024/8 ) # default to 512 Kbit/s
+            self.bitrate_set = False
+
+        # ----- set defaults for dynamic positions
+        self.playing = False     # video has started playback
+        self.paused = False      # video is paused
+        self.autoresume = False  # video is paused but will resume automatically
+        self.prebuffering = True # video is prebuffering
+        self.playback_pos = self.first_piece
+
+        self.pausable = (VODEVENT_PAUSE in videoinfo["userevents"]) and (VODEVENT_RESUME in videoinfo["userevents"])
+
+    def add_playback_pos_observer( self, observer ):
+        """ Add a function to be called when the playback position changes. Is called as follows:
+            observer( oldpos, newpos ). In case of initialisation: observer( None, startpos ). """
+        self.playback_pos_observers.append( observer )
+
+    def real_piecelen( self, x ):
+        if x == self.first_piece:
+            return self.first_piecelen
+        elif x == self.last_piece:
+            return self.last_piecelen
+        else:
+            return self.piecelen
+
+    def set_bitrate( self, bitrate ):
+        #print >>sys.stderr,"vodstatus: set_bitrate",bitrate
+        self.bitrate_set = True
+        self.bitrate = bitrate
+        self.sec_per_piece = 1.0 * bitrate / self.piecelen
+
+    def set_live_startpos( self, pos ):
+        if self.wraparound:
+            if self.live_startpos is None:
+                oldrange = self.first_piece,self.last_piece
+            else:
+                oldrange = self.live_get_valid_range()
+            if DEBUG:
+                print >>sys.stderr,"vodstatus: set_live_pos: old",oldrange
+        self.live_startpos = pos
+        self.playback_pos = pos
+        for o in self.playback_pos_observers:
+            o( None, pos )
+
+        if self.wraparound:
+            newrange = self.live_get_valid_range()
+            if DEBUG:
+                print >>sys.stderr,"vodstatus: set_live_pos: new",newrange
+            return self.get_range_diff(oldrange,newrange)
+        else:
+            return (Set(),[])
+
+
+    def get_live_startpos(self):
+        return self.live_startpos
+
+    # the following functions work with absolute piece numbers,
+    # so they all function within the range [first_piece,last_piece]
+
+    # the range of pieces to download is
+    # [playback_pos,numpieces) for normal downloads and
+    # [playback_pos,playback_pos+delta) for wraparound
+
+    def generate_range( self, (f, t) ):
+        if self.wraparound and f > t:
+            for x in xrange( f, self.last_piece+1 ):
+                yield x
+            for x in xrange( self.first_piece, t ):
+                yield x
+        else:
+            for x in xrange( f, t ):
+                yield x
+
+    def dist_range(self, f, t):
+        """ Returns the distance between f and t """
+        if f > t:
+            return self.last_piece-f + t-self.first_piece 
+        else:
+            return t - f
+
+    def in_range( self, f, t, x ):
+        if self.wraparound and f > t:
+            return self.first_piece <= x < t or f <= x <= self.last_piece
+        else:
+            return f <= x < t
+
+    def inc_playback_pos( self ):
+        oldpos = self.playback_pos
+        self.playback_pos += 1
+
+        if self.playback_pos > self.last_piece:
+            if self.wraparound:
+                self.playback_pos = self.first_piece
+            else:
+                # Arno, 2010-01-08: Adjusted EOF condition to work well with seeking/HTTP range queries
+                self.playback_pos = self.last_piece+1
+
+        for o in self.playback_pos_observers:
+            o( oldpos, self.playback_pos )
+
+    def in_download_range( self, x ):
+        if self.wraparound:
+            wraplen = self.playback_pos + self.wraparound_delta - self.last_piece
+            if wraplen > 0:
+                return self.first_piece <= x < self.first_piece + wraplen or self.playback_pos <= x <= self.last_piece
+
+            return self.playback_pos <= x < self.playback_pos + self.wraparound_delta
+        else:
+            return self.first_piece <= x <= self.last_piece
+
+    def in_valid_range(self,piece):
+        if self.live_streaming:
+            if self.live_startpos is None:
+                # Haven't hooked in yet
+                return True
+            else:
+                (begin,end) = self.live_get_valid_range()
+                ret = self.in_range(begin,end,piece)
+                if ret == False:
+                    print >>sys.stderr,"vod: status: NOT in_valid_range:",begin,"<",piece,"<",end
+                return ret
+        else:
+            return self.first_piece <= piece <= self.last_piece
+        
+    def live_get_valid_range(self):
+        begin = self.normalize(self.playback_pos - self.wraparound_delta)
+        end = self.normalize(self.playback_pos + self.wraparound_delta)
+        return (begin,end)
+        
+    def live_piece_to_invalidate(self):
+        #print >>sys.stderr,"vod: live_piece_to_inval:",self.playback_pos,self.wraparound_delta,self.movie_numpieces
+        return self.normalize(self.playback_pos - self.wraparound_delta)
+
+    def get_range_diff(self,oldrange,newrange):
+        """ Returns the diff between oldrange and newrange as a Set.
+        """
+        rlist = []
+        if oldrange[0] == 0 and oldrange[1] == self.movie_numpieces-1:
+            # Optimize for case where there is no playback pos yet, for STB.
+            if newrange[0] < newrange[1]:
+                # 100-500, diff is 0-99 + 501-7200
+                a = (oldrange[0],newrange[0]-1)
+                b = (newrange[1]+1,oldrange[1])
+                #print >>sys.stderr,"get_range_diff: ranges",a,b
+                rlist = [a,b]
+                return (None,rlist)
+                #return Set(range(a[0],a[1]) + range(b[0],b[1]))
+            else:
+                # 500-100, diff is 101-499
+                a = (newrange[1]+1,newrange[0]-1)
+                #print >>sys.stderr,"get_range_diff: range",a
+                rlist = [a]
+                return (None,rlist)
+                #return Set(xrange(a[0],a[1]))
+             
+        oldset = range2set(oldrange,self.movie_numpieces)
+        newset = range2set(newrange,self.movie_numpieces)
+        return (oldset - newset,rlist)
+    
+    def normalize( self, x ):
+        """ Caps or wraps a piece number. """
+
+        if self.first_piece <= x <= self.last_piece:
+            return x
+
+        if self.wraparound:
+            # in Python, -1 % 3 == 2, so modulo will do our work for us if x < first_piece
+            return (x - self.first_piece) % self.movie_numpieces + self.first_piece
+        else:
+            return max( self.first_piece, min( x, self.last_piece ) )
+
+    def time_to_pieces( self, sec ):
+        """ Returns the number of pieces that are needed to hold "sec" seconds of content. """
+
+        # TODO: take first and last piece into account, as they can have a different size
+        return int(ceil(sec * self.sec_per_piece))
+
+    def download_range( self ):
+        """ Returns the range [first,last) of pieces we like to download. """
+
+        first = self.playback_pos
+
+        if self.wraparound:
+            wraplen = first + self.wraparound_delta + 1 - self.last_piece
+            if wraplen > 0:
+                last = self.first_piece + wraplen
+            else:
+                last = first + self.wraparound_delta + 1
+        else:
+            last = self.last_piece + 1
+
+        return (first,last)
+
+    def get_wraparound(self):
+        return self.wraparound
+
+    def increase_high_range(self, factor=1):
+        """
+        Increase the high priority range (effectively enlarging the buffer size)
+        """
+        assert factor > 0
+        self.high_prob_curr_time += factor * self.high_prob_curr_time_limit[2]
+        if self.high_prob_curr_time > self.high_prob_curr_time_limit[1]:
+            self.high_prob_curr_time = self.high_prob_curr_time_limit[1]
+        
+        self.high_prob_curr_pieces += int(factor * self.high_prob_curr_pieces_limit[2])
+        if self.high_prob_curr_pieces > self.high_prob_curr_pieces_limit[1]:
+            self.high_prob_curr_pieces = self.high_prob_curr_pieces_limit[1]
+
+        if DEBUG: print >>sys.stderr, "VideoStatus:increase_high_range", self.high_prob_curr_time, "seconds or", self.high_prob_curr_pieces, "pieces"
+
+    def decrease_high_range(self, factor=1):
+        """
+        Decrease the high priority range (effectively reducing the buffer size)
+        """
+        assert factor > 0
+        self.high_prob_curr_time -= factor * self.high_prob_curr_time_limit[2]
+        if self.high_prob_curr_time < self.high_prob_curr_time_limit[0]:
+            self.high_prob_curr_time = self.high_prob_curr_time_limit[0]
+        
+        self.high_prob_curr_pieces -= int(factor * self.high_prob_curr_pieces_limit[2])
+        if self.high_prob_curr_pieces < self.high_prob_curr_pieces_limit[0]:
+            self.high_prob_curr_pieces = self.high_prob_curr_pieces_limit[0]
+
+        if DEBUG: print >>sys.stderr, "VideoStatus:decrease_high_range", self.high_prob_curr_time, "seconds or", self.high_prob_curr_pieces, "pieces"
+
+    def set_high_range(self, seconds=None, pieces=None):
+        """
+        Set the minimum size of the high priority range. Can be given
+        in seconds of pieces.
+        """
+        if seconds: self.high_prob_curr_time = seconds
+        if pieces: self.high_prob_curr_pieces = pieces
+
+    def get_high_range(self):
+        """
+        Returns (first, last) tuple
+        """
+        first, _ = self.download_range()
+        number_of_pieces = self.time_to_pieces(self.high_prob_curr_time)
+        last = min(self.last_piece,                                              # last piece
+                   1 + first + max(number_of_pieces, self.high_prob_curr_pieces), # based on time OR pieces
+                   1 + first + self.high_prob_curr_pieces_limit[1])               # hard-coded buffer maximum
+        return first, last
+
+    def in_high_range(self, piece):
+        """
+        Returns True when PIECE is in the high priority range.
+        """
+        first, last = self.get_high_range()
+        return self.in_range(first, last, piece)
+
+    def get_range_length(self, first, last):
+        if self.wraparound and first > last:
+            return self.last_piece - first + \
+                   last - self.first_piece
+        else:
+            return last - first
+
+    def get_high_range_length(self):
+        first, last = self.get_high_range()
+        return self.get_range_length(first, last)
+
+    def generate_high_range(self):
+        """
+        Returns the high current high priority range in piece_ids
+        """
+        first, last = self.get_high_range()
+        return self.generate_range((first, last))
+
+def range2set(range,maxrange):    
+    if range[0] <= range[1]:
+        set = Set(xrange(range[0],range[1]+1))
+    else:
+        set = Set(xrange(range[0],maxrange)) | Set(xrange(0,range[1]+1))
+    return set
diff --git a/instrumentation/next-share/BaseLib/Core/Video/__init__.py b/instrumentation/next-share/BaseLib/Core/Video/__init__.py
new file mode 100644
index 0000000..395f8fb
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/Video/__init__.py
@@ -0,0 +1,2 @@
+# Written by Arno Bakker
+# see LICENSE.txt for license information
diff --git a/instrumentation/next-share/BaseLib/Core/__init__.py b/instrumentation/next-share/BaseLib/Core/__init__.py
new file mode 100644
index 0000000..395f8fb
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/__init__.py
@@ -0,0 +1,2 @@
+# Written by Arno Bakker
+# see LICENSE.txt for license information
diff --git a/instrumentation/next-share/BaseLib/Core/defaults.py b/instrumentation/next-share/BaseLib/Core/defaults.py
new file mode 100644
index 0000000..45fd9c9
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/defaults.py
@@ -0,0 +1,246 @@
+# Written by Arno Bakker and Bram Cohen, George Milescu 
+# see LICENSE.txt for license information
+""" Default values for all configurarable parameters of the Core"""
+#
+# For an explanation of each parameter, see SessionConfig/DownloadConfig.py
+#
+# defaults with comments behind them are not user-setable via the 
+# *ConfigInterface classes, because they are not currently implemented (IPv6)
+# or we only use them internally.
+#
+# WARNING:  
+#    As we have release Tribler 4.5.0 you must now take into account that
+#    people have stored versions of these params on their disk. Make sure 
+#    you change the version number of the structure and provide upgrade code
+#    such that your code won't barf because we loaded an older version from
+#    disk that does not have your new fields.
+#
+import sys
+
+from simpledefs import *
+
+DEFAULTPORT=7760
+
+#
+# Session opts
+#
+# History: 
+#  Version 2: as released in Tribler 4.5.0
+#
+SESSDEFAULTS_VERSION = 2
+sessdefaults = {}
+sessdefaults['version'] = SESSDEFAULTS_VERSION
+sessdefaults['state_dir'] = None
+sessdefaults['install_dir'] = u'.'
+sessdefaults['ip'] = ''
+sessdefaults['minport'] = DEFAULTPORT
+sessdefaults['maxport'] = DEFAULTPORT
+sessdefaults['random_port'] = 1
+sessdefaults['bind'] = []
+sessdefaults['ipv6_enabled'] = 0      # allow the client to connect to peers via IPv6 (currently not supported)
+sessdefaults['ipv6_binds_v4'] = None  # set if an IPv6 server socket won't also field IPv4 connections (default = set automatically)
+sessdefaults['upnp_nat_access'] = UPNPMODE_UNIVERSAL_DIRECT
+sessdefaults['timeout'] = 300.0
+sessdefaults['timeout_check_interval'] = 60.0
+sessdefaults['eckeypairfilename'] = None
+sessdefaults['megacache'] = True
+sessdefaults['overlay'] = True
+sessdefaults['crawler'] = True
+sessdefaults['buddycast'] = True
+sessdefaults['magnetlink'] = True
+sessdefaults['start_recommender'] = True
+sessdefaults['download_help'] = True
+sessdefaults['torrent_collecting'] = True
+sessdefaults['superpeer'] = False
+sessdefaults['overlay_log'] = None
+sessdefaults['buddycast_interval'] = 15
+sessdefaults['buddycast_max_peers'] = 2500  # max number of peers to use for recommender. 
+sessdefaults['torrent_collecting_max_torrents'] = 50000
+sessdefaults['torrent_collecting_dir'] = None
+sessdefaults['torrent_collecting_rate'] = 5 * 10
+sessdefaults['torrent_checking'] = 1
+sessdefaults['torrent_checking_period'] = 31 #will be changed to min(max(86400/ntorrents, 15), 300) at runtime
+sessdefaults['dialback'] = True
+sessdefaults['dialback_active'] = True  # do active discovery (needed to disable for testing only) (0 = disabled)
+sessdefaults['dialback_trust_superpeers'] = True # trust superpeer replies (needed to disable for testing only) (0 = disabled)
+sessdefaults['socnet'] = True
+sessdefaults['rquery'] = True
+sessdefaults['stop_collecting_threshold'] = 200
+sessdefaults['internaltracker'] = True
+sessdefaults['nickname'] = '__default_name__' # is replaced with hostname in LaunchManyCore.py
+sessdefaults['mugshot'] = None
+sessdefaults['videoanalyserpath'] = None
+sessdefaults['overlay_max_message_length'] = 2 ** 23
+sessdefaults['download_help_dir'] = None
+sessdefaults['bartercast'] = True
+sessdefaults['superpeer_file'] = None
+sessdefaults['crawler_file'] = None
+sessdefaults['buddycast_collecting_solution'] = BCCOLPOLICY_SIMPLE
+sessdefaults['peer_icon_path'] = None
+sessdefaults['stop_collecting_threshold'] = 200
+sessdefaults['coopdlconfig'] = None
+sessdefaults['family_filter'] = True
+sessdefaults['nat_detect'] = True
+sessdefaults['puncturing_internal_port'] = 6700
+sessdefaults['stun_servers'] = [('stun1.tribler.org',6701),('stun2.tribler.org',6702)]
+sessdefaults['pingback_servers'] = [('pingback.tribler.org',6703),('pingback2.tribler.org',6703)]
+sessdefaults['live_aux_seeders'] = []
+sessdefaults['mainline_dht'] = True
+sessdefaults['multicast_local_peer_discovery'] = True
+sessdefaults['votecast_recent_votes']=25
+sessdefaults['votecast_random_votes']=25
+sessdefaults['channelcast_recent_own_subscriptions'] = 13
+sessdefaults['channelcast_random_own_subscriptions'] = 12 
+
+# 14-04-2010, Andrea: settings to limit the results for a remote query in channels
+# if there are too many results the gui got freezed for a considerable amount of
+# time
+sessdefaults['max_channel_query_results'] = 25
+
+# 13-04-2010 Andrea, config for subtitle dissemination subsytem
+sessdefaults['subtitles_collecting'] = False
+sessdefaults['subtitles_collecting_dir'] = None
+sessdefaults['subtitles_upload_rate'] = 1024 # KB/s 
+
+# ProxyService global config
+sessdefaults['proxyservice_status'] = PROXYSERVICE_OFF
+
+trackerdefaults = {}
+trackerdefaults['tracker_url'] = None
+trackerdefaults['tracker_dfile'] = None
+trackerdefaults['tracker_dfile_format'] = ITRACKDBFORMAT_PICKLE
+trackerdefaults['tracker_socket_timeout'] = 15
+trackerdefaults['tracker_save_dfile_interval'] = 300
+trackerdefaults['tracker_timeout_downloaders_interval'] = 2700
+trackerdefaults['tracker_reannounce_interval'] = 1800
+trackerdefaults['tracker_response_size'] = 50
+trackerdefaults['tracker_timeout_check_interval'] = 5
+trackerdefaults['tracker_nat_check'] = 3
+trackerdefaults['tracker_log_nat_checks'] = 0
+trackerdefaults['tracker_min_time_between_log_flushes'] = 3.0
+trackerdefaults['tracker_min_time_between_cache_refreshes'] = 600.0
+trackerdefaults['tracker_allowed_dir'] = None
+trackerdefaults['tracker_allowed_list'] = ''
+trackerdefaults['tracker_allowed_controls'] = 0
+trackerdefaults['tracker_multitracker_enabled'] = 0
+trackerdefaults['tracker_multitracker_allowed'] = ITRACKMULTI_ALLOW_AUTODETECT
+trackerdefaults['tracker_multitracker_reannounce_interval'] = 120
+trackerdefaults['tracker_multitracker_maxpeers'] = 20
+trackerdefaults['tracker_aggregate_forward'] = [None,None]
+trackerdefaults['tracker_aggregator'] = 0
+trackerdefaults['tracker_hupmonitor'] = 0
+trackerdefaults['tracker_multitracker_http_timeout'] = 60
+trackerdefaults['tracker_parse_dir_interval'] = 60
+trackerdefaults['tracker_show_infopage'] = 1
+trackerdefaults['tracker_infopage_redirect'] = None
+trackerdefaults['tracker_show_names'] = 1
+trackerdefaults['tracker_favicon'] = None
+trackerdefaults['tracker_allowed_ips'] = []
+trackerdefaults['tracker_banned_ips'] = []
+trackerdefaults['tracker_only_local_override_ip'] = ITRACK_IGNORE_ANNOUNCEIP_IFNONATCHECK
+
+trackerdefaults['tracker_logfile'] = None
+trackerdefaults['tracker_allow_get'] = 1
+trackerdefaults['tracker_keep_dead'] = 0
+trackerdefaults['tracker_scrape_allowed'] = ITRACKSCRAPE_ALLOW_FULL
+
+sessdefaults.update(trackerdefaults)
+
+#
+# BT per download opts
+#
+# History: 
+#  Version 2: as released in Tribler 4.5.0
+#  Version 3: 
+DLDEFAULTS_VERSION = 3
+dldefaults = {}
+dldefaults['version'] = DLDEFAULTS_VERSION
+dldefaults['max_uploads'] = 7
+dldefaults['keepalive_interval'] = 120.0
+dldefaults['download_slice_size'] = 2 ** 14
+dldefaults['upload_unit_size'] = 1460
+dldefaults['request_backlog'] = 10
+dldefaults['max_message_length'] = 2 ** 23
+dldefaults['selector_enabled'] = 1  # whether to enable the file selector and fast resume function. Arno, 2009-02-9: Must be on for checkpoints to work. 
+dldefaults['expire_cache_data'] = 10 # the number of days after which you wish to expire old cache data (0 = disabled)
+dldefaults['priority'] = []  # a list of file priorities separated by commas, must be one per file, 0 = highest, 1 = normal, 2 = lowest, -1 = download disabled'
+dldefaults['saveas'] = None # Set to get_default_destdir()
+dldefaults['max_slice_length'] = 2 ** 17
+dldefaults['max_rate_period'] = 20.0
+dldefaults['upload_rate_fudge'] = 5.0
+dldefaults['tcp_ack_fudge'] = 0.03
+dldefaults['rerequest_interval'] = 300
+dldefaults['min_peers'] = 20
+dldefaults['http_timeout'] = 60
+dldefaults['max_initiate'] = 40
+dldefaults['check_hashes'] = 1
+dldefaults['max_upload_rate'] = 0
+dldefaults['max_download_rate'] = 0
+# Arno, 2009-12-11: Sparse as default reduces CPU usage. Previously this was
+# also set, but in DownloadConfig.__init__
+if sys.platform == 'win32':
+    dldefaults['alloc_type'] = DISKALLOC_NORMAL
+else:
+    dldefaults['alloc_type'] = DISKALLOC_SPARSE
+dldefaults['alloc_rate'] = 2.0
+dldefaults['buffer_reads'] = 1
+dldefaults['write_buffer_size'] = 4
+dldefaults['breakup_seed_bitfield'] = 1
+dldefaults['snub_time'] = 30.0
+dldefaults['rarest_first_cutoff'] = 2
+dldefaults['rarest_first_priority_cutoff'] = 5
+dldefaults['min_uploads'] = 4
+dldefaults['max_files_open'] = 50
+dldefaults['round_robin_period'] = 30
+dldefaults['super_seeder'] = 0
+dldefaults['security'] = 1
+dldefaults['max_connections'] = 0
+dldefaults['auto_kick'] = 1
+dldefaults['double_check'] = 0
+dldefaults['triple_check'] = 0
+dldefaults['lock_files'] = 0
+dldefaults['lock_while_reading'] = 0
+dldefaults['auto_flush'] = 0
+#
+# Tribler per-download opts
+#
+dldefaults['coopdl_role'] = COOPDL_ROLE_COORDINATOR
+dldefaults['coopdl_coordinator_permid'] = ''
+dldefaults['proxy_mode'] = PROXY_MODE_OFF
+dldefaults['max_helpers'] = 10
+dldefaults['exclude_ips'] = ''
+dldefaults['mode'] = 0
+dldefaults['vod_usercallback'] = None
+dldefaults['vod_userevents'] = []
+dldefaults['video_source'] = None
+dldefaults['video_ratelimit'] = 0
+dldefaults['video_source_authconfig'] = None
+dldefaults['selected_files'] = []
+dldefaults['ut_pex_max_addrs_from_peer'] = 16
+# Version 3:
+dldefaults['same_nat_try_internal'] = 0
+dldefaults['unchoke_bias_for_internal'] = 0
+
+tdefdictdefaults = {}
+tdefdictdefaults['comment'] = None
+tdefdictdefaults['created by'] = None
+tdefdictdefaults['announce'] = None
+tdefdictdefaults['announce-list'] = None
+tdefdictdefaults['nodes'] = None # mainline DHT
+tdefdictdefaults['httpseeds'] = None
+tdefdictdefaults['url-list'] = None
+tdefdictdefaults['encoding'] = None
+
+tdefmetadefaults = {}
+tdefmetadefaults['version'] = 1
+tdefmetadefaults['piece length'] = 0
+tdefmetadefaults['makehash_md5'] = 0
+tdefmetadefaults['makehash_crc32'] = 0
+tdefmetadefaults['makehash_sha1'] = 0
+tdefmetadefaults['createmerkletorrent'] = 0
+tdefmetadefaults['torrentsigkeypairfilename'] = None
+tdefmetadefaults['thumb'] = None # JPEG data
+
+tdefdefaults = {}
+tdefdefaults.update(tdefdictdefaults)
+tdefdefaults.update(tdefmetadefaults)
diff --git a/instrumentation/next-share/BaseLib/Core/exceptions.py b/instrumentation/next-share/BaseLib/Core/exceptions.py
new file mode 100644
index 0000000..a704380
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/exceptions.py
@@ -0,0 +1,82 @@
+# Written by Arno Bakker 
+# see LICENSE.txt for license information
+""" The Tribler-specifc Exceptions the Core may throw. """
+
+#
+# Exceptions
+#
+class TriblerException(Exception):
+    """ Super class for all Tribler-specific Exceptions the Tribler Core 
+    throws.
+    """
+    def __init__(self,msg=None):
+        Exception.__init__(self,msg)
+
+    def __str__(self):
+        return str(self.__class__)+': '+Exception.__str__(self)
+ 
+
+class OperationNotPossibleAtRuntimeException(TriblerException):
+    """ The requested operation is not possible after the Session or Download
+    has been started.
+    """
+    def __init__(self,msg=None):
+        TriblerException.__init__(self,msg)
+
+class OperationNotPossibleWhenStoppedException(TriblerException):
+    """ The requested operation is not possible when the Download
+    has been stopped.
+    """
+    def __init__(self,msg=None):
+        TriblerException.__init__(self,msg)
+
+class OperationNotEnabledByConfigurationException(TriblerException):
+    """ The requested operation is not possible with the current
+    Session/Download configuration.
+    """
+    def __init__(self,msg=None):
+        TriblerException.__init__(self,msg)
+
+    
+class NotYetImplementedException(TriblerException):
+    """ The requested operation is not yet fully implemented. """
+    def __init__(self,msg=None):
+        TriblerException.__init__(self,msg)
+
+
+class DuplicateDownloadException(TriblerException):
+    """ The Download already exists in the Session, i.e., a Download for
+    a torrent with the same infohash already exists. """
+    def __init__(self,msg=None):
+        TriblerException.__init__(self,msg)
+
+class VODNoFileSelectedInMultifileTorrentException(TriblerException):
+    """ Attempt to download a torrent in Video-On-Demand mode that contains
+    multiple video files, but without specifying which one to play. """
+    def __init__(self,msg=None):
+        TriblerException.__init__(self,msg)
+
+class LiveTorrentRequiresUsercallbackException(TriblerException):
+    """ Attempt to download a live-stream torrent without specifying a 
+    callback function to call when the stream is ready to play. 
+    Use set_video_event_callback(usercallback) to correct this problem. """
+    def __init__(self,msg=None):
+        TriblerException.__init__(self,msg)
+
+class TorrentDefNotFinalizedException(TriblerException):
+    """ Attempt to start downloading a torrent from a torrent definition
+    that was not finalized. """
+    def __init__(self,msg=None):
+        TriblerException.__init__(self,msg)
+
+
+class TriblerLegacyException(TriblerException):
+    """ Wrapper around fatal errors that happen in the download engine,
+    but which are not reported as Exception objects for legacy reasons,
+    just as text (often containing a stringified Exception).
+    Will be phased out.
+    """
+    
+    def __init__(self,msg=None):
+        TriblerException.__init__(self,msg)
+    
diff --git a/instrumentation/next-share/BaseLib/Core/osutils.py b/instrumentation/next-share/BaseLib/Core/osutils.py
new file mode 100644
index 0000000..80dd50c
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/osutils.py
@@ -0,0 +1,352 @@
+# Written by Arno Bakker, ABC authors
+# see LICENSE.txt for license information
+"""
+OS-independent utility functions
+
+get_home_dir()      : Returns CSIDL_APPDATA i.e. App data directory on win32
+get_picture_dir()
+getfreespace(path)
+"""
+    
+#
+# Multiple methods for getting free diskspace
+#
+import sys
+import os
+import time
+import binascii
+
+if sys.platform == "win32":
+    try:
+        from win32com.shell import shell
+        def get_home_dir():
+            # http://www.mvps.org/access/api/api0054.htm
+            # CSIDL_PROFILE = &H28
+            # C:\Documents and Settings\username
+            return shell.SHGetSpecialFolderPath(0, 0x28)
+
+        def get_appstate_dir():
+            # http://www.mvps.org/access/api/api0054.htm
+            # CSIDL_APPDATA = &H1A
+            # C:\Documents and Settings\username\Application Data
+            return shell.SHGetSpecialFolderPath(0, 0x1a)
+
+        def get_picture_dir():
+            # http://www.mvps.org/access/api/api0054.htm
+            # CSIDL_MYPICTURES = &H27
+            # C:\Documents and Settings\username\My Documents\My Pictures
+            return shell.SHGetSpecialFolderPath(0, 0x27)
+
+        def get_desktop_dir():
+            # http://www.mvps.org/access/api/api0054.htm
+            # CSIDL_DESKTOPDIRECTORY = &H10
+            # C:\Documents and Settings\username\Desktop
+            return shell.SHGetSpecialFolderPath(0, 0x10)
+
+    except ImportError:
+        def get_home_dir():
+            try:
+                # when there are special unicode characters in the username,
+                # the following will fail on python 2.4, 2.5, 2.x this will
+                # always succeed on python 3.x
+                return os.path.expanduser(u"~")
+            except Exception, unicode_error:
+                pass
+
+            # non-unicode home
+            home = os.path.expanduser("~")
+            head, tail = os.path.split(home)
+
+            dirs = os.listdir(head)
+            udirs = os.listdir(unicode(head))
+
+            # the character set may be different, but the string length is
+            # still the same
+            islen = lambda dir: len(dir) == len(tail)
+            dirs = filter(islen, dirs)
+            udirs = filter(islen, udirs)
+            if len(dirs) == 1 and len(udirs) == 1:
+                return os.path.join(head, udirs[0])
+
+            # remove all dirs that are equal in unicode and non-unicode. we
+            # know that we don't need these dirs because the initial
+            # expandusers would not have failed on them
+            for dir in dirs[:]:
+                if dir in udirs:
+                    dirs.remove(dir)
+                    udirs.remove(dir)
+            if len(dirs) == 1 and len(udirs) == 1:
+                return os.path.join(head, udirs[0])
+
+            # assume that the user has write access in her own
+            # directory. therefore we can filter out any non-writable
+            # directories
+            writable_udir = [udir for udir in udirs if os.access(udir, os.W_OK)]
+            if len(writable_udir) == 1:
+                return os.path.join(head, writable_udir[0])
+
+            # fallback: assume that the order of entries in dirs is the same
+            # as in udirs
+            for dir, udir in zip(dirs, udirs):
+                if dir == tail:
+                    return os.path.join(head, udir)
+
+            # failure
+            raise unicode_error
+
+        def get_appstate_dir():
+            homedir = get_home_dir()
+            # 5 = XP, 6 = Vista
+            # [E1101] Module 'sys' has no 'getwindowsversion' member
+            # pylint: disable-msg=E1101
+            winversion = sys.getwindowsversion()
+            # pylint: enable-msg=E1101
+            if winversion[0] == 6:
+                appdir = os.path.join(homedir,u"AppData",u"Roaming")
+            else:
+                appdir = os.path.join(homedir,u"Application Data")
+            return appdir
+
+        def get_picture_dir():
+            return get_home_dir()
+
+        def get_desktop_dir():
+            home = get_home_dir()
+            return os.path.join(home,u"Desktop")
+            
+else:
+    # linux or darwin (mac)
+    def get_home_dir():
+        return os.path.expanduser(u"~")
+
+    def get_appstate_dir():
+        return get_home_dir()
+
+    def get_picture_dir():
+        return get_desktop_dir()
+
+    def get_desktop_dir():
+        home = get_home_dir()
+        desktop = os.path.join(home, "Desktop")
+        if os.path.exists(desktop):
+            return desktop
+        else:
+            return home
+
+if sys.version.startswith("2.4"):
+    os.SEEK_SET = 0
+    os.SEEK_CUR = 1
+    os.SEEK_END = 2
+
+try:
+    # Unix
+    from os import statvfs
+    import statvfs
+    def getfreespace(path):
+        s = os.statvfs(path.encode("utf-8"))
+        size = s[statvfs.F_BAVAIL] * long(s[statvfs.F_BSIZE])
+        return size
+except:
+    if (sys.platform == 'win32'):
+        try:
+            # Windows if win32all extensions are installed
+            import win32file
+            try:
+                # Win95 OSR2 and up
+                # Arno: this code was totally broken as the method returns
+                # a list of values indicating 1. free space for the user,
+                # 2. total space for the user and 3. total free space, so
+                # not a single value.
+                win32file.GetDiskFreeSpaceEx(".")
+                def getfreespace(path):
+                    # Boudewijn: the win32file module is NOT unicode
+                    # safe! We will try directories further up the
+                    # directory tree in the hopes of getting a path on
+                    # the same disk without the unicode...
+                    while True:
+                        try:
+                            return win32file.GetDiskFreeSpaceEx(path)[0]
+                        except:
+                            path = os.path.split(path)[0]
+                            if not path:
+                                raise
+            except:                
+                # Original Win95
+                # (2GB limit on partition size, so this should be
+                #  accurate except for mapped network drives)
+                # Arno: see http://aspn.activestate.com/ASPN/docs/ActivePython/2.4/pywin32/win32file__GetDiskFreeSpace_meth.html
+                def getfreespace(path):
+                    [spc, bps, nfc, tnc] = win32file.GetDiskFreeSpace(path)
+                    return long(nfc) * long(spc) * long(bps)
+                    
+        except ImportError:
+            # Windows if win32all extensions aren't installed
+            # (parse the output from the dir command)
+            def getfreespace(path):
+                try:
+                    mystdin, mystdout = os.popen2(u"dir " + u"\"" + path + u"\"")
+                    
+                    sizestring = "0"
+                
+                    for line in mystdout:
+                        line = line.strip()
+                        # Arno: FIXME: this won't work on non-English Windows, as reported by the IRT
+                        index = line.rfind("bytes free")
+                        if index > -1 and line[index:] == "bytes free":
+                            parts = line.split(" ")
+                            if len(parts) > 3:
+                                part = parts[-3]
+                                part = part.replace(",", "")
+                                sizestring = part
+                                break
+
+                    size = long(sizestring)                    
+                    
+                    if size == 0L:
+                        print >>sys.stderr,"getfreespace: can't determine freespace of ",path
+                        for line in mystdout:
+                            print >>sys.stderr,line
+                            
+                        size = 2**80L
+                except:
+                    # If in doubt, just return something really large
+                    # (1 yottabyte)
+                    size = 2**80L
+                
+                return size
+    else:
+        # Any other cases
+        # TODO: support for Mac? (will statvfs work with OS X?)
+        def getfreespace(path):
+            # If in doubt, just return something really large
+            # (1 yottabyte)
+            return 2**80L
+
+
+invalidwinfilenamechars = ''
+for i in range(32):
+    invalidwinfilenamechars += chr(i)
+invalidwinfilenamechars += '"*/:<>?\\|'
+invalidlinuxfilenamechars = '/'
+
+def fix_filebasename(name, unit=False, maxlen=255):
+    """ Check if str is a valid Windows file name (or unit name if unit is true)
+     * If the filename isn't valid: returns a corrected name
+     * If the filename is valid: returns the filename
+    """
+    if unit and (len(name) != 2 or name[1] != ':'):
+        return 'c:'
+    if not name or name == '.' or name == '..':
+        return '_'
+    
+    if unit:
+        name = name[0]
+    fixed = False
+    if len(name) > maxlen:
+        name = name[:maxlen]
+        fixed = True
+
+    fixedname = ''
+    spaces = 0
+    for c in name:
+        if sys.platform.startswith('win'):
+            invalidchars = invalidwinfilenamechars
+        else:
+            invalidchars = invalidlinuxfilenamechars
+             
+        if c in invalidchars:
+            fixedname += '_'
+            fixed = True
+        else:
+            fixedname += c
+            if c == ' ':
+                spaces += 1
+    
+    file_dir, basename = os.path.split(fixedname)
+    while file_dir != '':
+        fixedname = basename
+        file_dir, basename = os.path.split(fixedname)
+        fixed = True
+    
+    if fixedname == '':
+        fixedname = '_'
+        fixed = True
+        
+    if fixed:
+        return last_minute_filename_clean(fixedname)
+    elif spaces == len(name):
+        # contains only spaces
+        return '_'
+    else:
+        return last_minute_filename_clean(name)
+    
+def last_minute_filename_clean(name):
+    s = name.strip() # Arno: remove initial or ending space
+    if sys.platform == 'win32' and s.endswith('..'):
+        s = s[:-2]
+    return s
+
+
+def get_readable_torrent_name(infohash, raw_filename):
+    # return name__infohash.torrent
+    hex_infohash = binascii.hexlify(infohash)
+    suffix = '__' + hex_infohash + '.torrent'
+    save_name = ' ' + fix_filebasename(raw_filename, maxlen=254-len(suffix)) + suffix
+    # use a space ahead to distinguish from previous collected torrents
+    return save_name
+
+
+if sys.platform == "win32":
+    import win32pdh
+    
+    def getcpuload():
+        """ Returns total CPU usage as fraction (0..1).
+        Warning: side-effect: sleeps for 0.1 second to do diff """
+        #mempath = win32pdh.MakeCounterPath((None, "Memory", None, None, -1, "Available MBytes"))
+        cpupath = win32pdh.MakeCounterPath((None, "Processor", "_Total", None, -1, "% Processor Time"))
+        query = win32pdh.OpenQuery(None, 0)
+        counter = win32pdh.AddCounter(query, cpupath, 0)
+        
+        win32pdh.CollectQueryData(query)
+        # Collect must be called twice for CPU, see http://support.microsoft.com/kb/262938
+        time.sleep(0.1)
+        win32pdh.CollectQueryData(query)
+            
+        status, value = win32pdh.GetFormattedCounterValue(counter,win32pdh.PDH_FMT_LONG)
+             
+        return float(value)/100.0
+    
+elif sys.platform == "linux2":
+    def read_proc_stat():
+        """ Read idle and total CPU time counters from /proc/stat, see
+        man proc """
+        f = open("/proc/stat","rb")
+        try:
+            while True:
+                line = f.readline()
+                if len(line) == 0:
+                    break
+                if line.startswith("cpu "): # note space
+                    words = line.split()
+                    total = 0
+                    for i in range(1,5):
+                        total += int(words[i])
+                    idle = int(words[4])
+                    return (total,idle)
+        finally:
+            f.close()
+    
+    
+    def getcpuload():
+        """ Returns total CPU usage as fraction (0..1).
+        Warning: side-effect: sleeps for 0.1 second to do diff """
+        (total1,idle1) = read_proc_stat()
+        time.sleep(0.1)
+        (total2,idle2) = read_proc_stat()
+        total = total2 - total1
+        idle = idle2 - idle1
+        return 1.0-(float(idle))/float(total)
+else:
+    # Mac
+    def getupload():
+        raise ValueError("Not yet implemented")
diff --git a/instrumentation/next-share/BaseLib/Core/simpledefs.py b/instrumentation/next-share/BaseLib/Core/simpledefs.py
new file mode 100644
index 0000000..049060a
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/simpledefs.py
@@ -0,0 +1,177 @@
+# Written by Arno Bakker 
+# see LICENSE.txt for license information
+""" Simple definitions for the Tribler Core. """
+import os
+
+DLSTATUS_ALLOCATING_DISKSPACE = 0 # TODO: make sure this get set when in this alloc mode
+DLSTATUS_WAITING4HASHCHECK = 1
+DLSTATUS_HASHCHECKING = 2
+DLSTATUS_DOWNLOADING = 3
+DLSTATUS_SEEDING = 4
+DLSTATUS_STOPPED = 5
+DLSTATUS_STOPPED_ON_ERROR = 6
+DLSTATUS_REPEXING = 7
+
+dlstatus_strings = ['DLSTATUS_ALLOCATING_DISKSPACE',
+'DLSTATUS_WAITING4HASHCHECK', 
+'DLSTATUS_HASHCHECKING',
+'DLSTATUS_DOWNLOADING',
+'DLSTATUS_SEEDING',
+'DLSTATUS_STOPPED',
+'DLSTATUS_STOPPED_ON_ERROR',
+'DLSTATUS_REPEXING']
+
+UPLOAD = 'up'
+DOWNLOAD = 'down'
+
+DLMODE_NORMAL = 0
+DLMODE_VOD = 1
+DLMODE_SVC = 2 # Ric: added download mode for Scalable Video Coding (SVC) 
+
+PERSISTENTSTATE_CURRENTVERSION = 3
+"""
+V1 = SwarmPlayer 1.0.0
+V2 = Tribler 4.5.0: SessionConfig: Added NAT fields
+V3 = SessionConfig: Added multicast_local_peer_discovery, 
+     Removed rss_reload_frequency + rss_check_frequency.
+
+For details see API.py
+"""
+
+STATEDIR_ITRACKER_DIR = 'itracker'
+STATEDIR_DLPSTATE_DIR = 'dlcheckpoints'
+STATEDIR_PEERICON_DIR = 'icons'
+STATEDIR_TORRENTCOLL_DIR = 'collected_torrent_files'
+
+# 13-04-2010, Andrea: subtitles collecting dir default
+STATEDIR_SUBSCOLL_DIR = 'collected_subtitles_files'
+STATEDIR_SESSCONFIG = 'sessconfig.pickle'
+STATEDIR_SEEDINGMANAGER_DIR = 'seeding_manager_stats'
+DESTDIR_COOPDOWNLOAD = 'downloadhelp' 
+
+# For observer/callback mechanism, see Session.add_observer()
+   
+# subjects
+NTFY_PEERS = 'peers'
+NTFY_TORRENTS = 'torrents'
+NTFY_PREFERENCES = 'preferences'
+NTFY_SUPERPEERS = 'superpeers' # use NTFY_PEERS !!
+NTFY_FRIENDS = 'friends'       # use NTFY_PEERS !!
+NTFY_MYPREFERENCES = 'mypreferences' # currently not observable
+NTFY_BARTERCAST = 'bartercast' # currently not observable
+NTFY_MYINFO = 'myinfo'
+NTFY_SEEDINGSTATS = 'seedingstats'
+NTFY_SEEDINGSTATSSETTINGS = 'seedingstatssettings'
+NTFY_VOTECAST = 'votecast'
+NTFY_CHANNELCAST = 'channelcast'
+# this corresponds to the event of a peer advertising
+# new rich metadata available (for now just subtitles)
+NTFY_RICH_METADATA = 'rich_metadata'
+# this corresponds to the event of a subtitle file (the actual .srt)
+# received from a remote peer
+NTFY_SUBTITLE_CONTENTS = 'subtitles_in'
+NTFY_SEARCH = 'clicklogsearch' # BuddyCast 4
+NTFY_TERM= 'clicklogterm'
+
+
+# non data handler subjects
+NTFY_ACTIVITIES = 'activities' # an activity was set (peer met/dns resolved)
+NTFY_REACHABLE = 'reachable'   # the Session is reachable from the Internet
+
+# changeTypes
+NTFY_UPDATE = 'update'         # data is updated
+NTFY_INSERT = 'insert'         # new data is inserted
+NTFY_DELETE = 'delete'         # data is deleted
+NTFY_SEARCH_RESULT = 'search_result'     # new search result
+NTFY_CONNECTION = 'connection' # connection made or broken
+
+# object IDs for NTFY_ACTIVITIES subject 
+NTFY_ACT_NONE = 0
+NTFY_ACT_UPNP = 1
+NTFY_ACT_REACHABLE = 2
+NTFY_ACT_GET_EXT_IP_FROM_PEERS = 3
+NTFY_ACT_MEET = 4
+NTFY_ACT_GOT_METADATA = 5
+NTFY_ACT_RECOMMEND = 6
+NTFY_ACT_DISK_FULL = 7
+NTFY_ACT_NEW_VERSION = 8
+NTFY_ACT_ACTIVE = 9
+ 
+# Disk-allocation policies for download, see DownloadConfig.set_alloc_type
+DISKALLOC_NORMAL = 'normal'              
+DISKALLOC_BACKGROUND = 'background'      
+DISKALLOC_PREALLOCATE = 'pre-allocate'
+DISKALLOC_SPARSE = 'sparse'
+
+# UPnP modes, see SessionConfig.set_upnp_mode
+UPNPMODE_DISABLED = 0
+UPNPMODE_WIN32_HNetCfg_NATUPnP = 1
+UPNPMODE_WIN32_UPnP_UPnPDeviceFinder = 2
+UPNPMODE_UNIVERSAL_DIRECT = 3
+
+# Buddycast Collecting Policy parameters
+BCCOLPOLICY_SIMPLE = 1
+# BCCOLPOLICY_T4T = 2 # Future work
+
+# Internal tracker scrape
+ITRACKSCRAPE_ALLOW_NONE = 'none'
+ITRACKSCRAPE_ALLOW_SPECIFIC = 'specific'
+ITRACKSCRAPE_ALLOW_FULL = 'full'
+
+ITRACKDBFORMAT_BENCODE = 'bencode'
+ITRACKDBFORMAT_PICKLE= 'pickle'
+
+ITRACKMULTI_ALLOW_NONE = 'none'
+ITRACKMULTI_ALLOW_AUTODETECT = 'autodetect'
+ITRACKMULTI_ALLOW_ALL = 'all'
+
+ITRACK_IGNORE_ANNOUNCEIP_NEVER = 0
+ITRACK_IGNORE_ANNOUNCEIP_ALWAYS = 1
+ITRACK_IGNORE_ANNOUNCEIP_IFNONATCHECK = 2
+
+# Cooperative download
+COOPDL_ROLE_COORDINATOR = 'coordinator'
+COOPDL_ROLE_HELPER = 'helper'
+
+# Methods for authentication of the source in live streaming
+LIVE_AUTHMETHOD_NONE = "None"   # No auth, also no abs. piece nr. or timestamp.
+LIVE_AUTHMETHOD_ECDSA = "ECDSA" # Elliptic Curve DSA signatures
+LIVE_AUTHMETHOD_RSA = "RSA"     # RSA signatures
+
+# Video-On-Demand / live events
+VODEVENT_START = "start"
+VODEVENT_PAUSE = "pause"
+VODEVENT_RESUME = "resume"
+
+
+# Friendship messages
+F_REQUEST_MSG = "REQ"
+F_RESPONSE_MSG = "RESP"
+F_FORWARD_MSG = "FWD" # Can forward any type of other friendship message
+
+
+# States for a friend
+FS_NOFRIEND = 0
+FS_MUTUAL = 1
+FS_I_INVITED = 2
+FS_HE_INVITED = 3
+FS_I_DENIED = 4
+FS_HE_DENIED = 5
+
+P2PURL_SCHEME = "tribe" # No colon
+
+URL_MIME_TYPE = 'text/x-url'
+TSTREAM_MIME_TYPE = "application/x-ns-stream"
+
+TRIBLER_TORRENT_EXT = ".tribe" # Unused
+ 
+# Infohashes are always 20 byte binary strings
+INFOHASH_LENGTH = 20
+
+
+# ProxyService 
+PROXY_MODE_OFF = 0
+PROXY_MODE_PRIVATE = 1
+PROXY_MODE_SPEED= 2
+PROXYSERVICE_OFF = 0
+PROXYSERVICE_ON = 1
diff --git a/instrumentation/next-share/BaseLib/Core/superpeer.txt b/instrumentation/next-share/BaseLib/Core/superpeer.txt
new file mode 100644
index 0000000..8f9410d
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Core/superpeer.txt
@@ -0,0 +1,9 @@
+#ip, port, permid, [name]
+superpeer1.das2.ewi.tudelft.nl, 7001, MFIwEAYHKoZIzj0CAQYFK4EEABoDPgAEAL2I5yVc1+dWVEx3nbriRKJmOSlQePZ9LU7yYQoGABMvU1uGHvqnT9t+53eaCGziV12MZ1g2p0GLmZP9, SuperPeer1@Tribler
+superpeer2.cs.vu.nl,            7002, MFIwEAYHKoZIzj0CAQYFK4EEABoDPgAEAZNX5NBOuGH4j2kumv/9WkPLrJPVkOr5oVImhcp8AC7w7ww9eZwUF7S/Q96If4UmVX+L6HMKSOTLPoPk, SuperPeer2@Tribler
+superpeer3.tribler.org,         7003, MFIwEAYHKoZIzj0CAQYFK4EEABoDPgAEAQaLGR940aKktbAJNm6vYOTSN2P8z1P9EiQ48kJNAdrDl7oBkyrERZOq+IMMKIpu4ocsz5hxZHMTy2Fh, SuperPeer3@Tribler
+superpeer4.das2.ewi.tudelft.nl, 7004, MFIwEAYHKoZIzj0CAQYFK4EEABoDPgAEAduAK0/ZTjdg/NPd8CD9Q17J10CXqpFyHN5M8m6fAFXBQflBZT/YdH1fYwizR/hnQE4hIKCQTfvKz1pA, SuperPeer4@Tribler
+superpeer5.das2.ewi.tudelft.nl, 7005, MFIwEAYHKoZIzj0CAQYFK4EEABoDPgAEAGZomjLNDu6i/5c/ATpsatWiL0P7huV/ixgzdvwlAU8AEYHp7ppyumydUg2MnoneHJ74H58yB+pUPSdu, SuperPeer5@Tribler
+superpeer6.das2.ewi.tudelft.nl, 7006, MFIwEAYHKoZIzj0CAQYFK4EEABoDPgAEAfizxRUl7S3Fec2cl1+uML6tORwnUIod5mh3soWVANxmu+flFNp1yayiLPjB+dWQ6Va77FXbHDkw5smd, SuperPeer6@Tribler
+superpeer7.das2.ewi.tudelft.nl, 7007, MFIwEAYHKoZIzj0CAQYFK4EEABoDPgAEAFzfWHb/WPL+luFfXfUbtJGRpUnwbmyB0kH7t3UpAVSpKilym4Fzt2rS7HJTZyQ7yCI3c+xTRtMLZ0sc, SuperPeer7@Tribler
+superpeer8.das2.ewi.tudelft.nl, 7008, MFIwEAYHKoZIzj0CAQYFK4EEABoDPgAEAIV8h+eS+vQ+0uqZNv3MYYTLo5s0JP+cmkvJ7U4JAHhfRv1wCqZSKIuY7Q+3ESezhRnnmmX4pbOVhKTU, SuperPeer8@Tribler
diff --git a/instrumentation/next-share/BaseLib/Debug/__init__.py b/instrumentation/next-share/BaseLib/Debug/__init__.py
new file mode 100644
index 0000000..1485bab
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Debug/__init__.py
@@ -0,0 +1,3 @@
+# Written by Boudewijn Schoon
+# see LICENSE.txt for license information
+
diff --git a/instrumentation/next-share/BaseLib/Debug/console.py b/instrumentation/next-share/BaseLib/Debug/console.py
new file mode 100644
index 0000000..aa8e423
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Debug/console.py
@@ -0,0 +1,43 @@
+"""
+Alternate stdout and stderr with much more protection
+"""
+
+import sys 
+
+class SafePrintStream:
+    def __init__(self, stream):
+        self._stream = stream
+
+    def write(self, arg):
+        try:
+            self._stream.write(arg.encode("ASCII", "backslashreplace"))
+        except Exception, e:
+            try:
+                s = u"{%s}" % repr(arg)
+                self._stream.write(s)
+            except:
+                self._stream.write("TriblerConsole: ERROR printing\n")
+                self._stream.write(repr(e))
+                self._stream.write("\n")
+        
+    def flush(self):
+        self._stream.flush()
+
+class SafeLinePrintStream:
+    def __init__(self, stream):
+        self._stream = stream
+        self._parts = []
+
+    def write(self, arg):
+        self._parts.append(arg.encode("ASCII", "backslashreplace"))
+        if arg == "\n":
+            self._stream.write("".join(self._parts))
+            self._parts = []
+        
+    def flush(self):
+        self._stream.write("".join(self._parts))
+        self._parts = []
+        self._stream.flush()
+        
+sys.stderr = SafePrintStream(sys.stderr)
+sys.stdout = sys.stderr
diff --git a/instrumentation/next-share/BaseLib/Debug/memory.py b/instrumentation/next-share/BaseLib/Debug/memory.py
new file mode 100644
index 0000000..6d46b96
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Debug/memory.py
@@ -0,0 +1,159 @@
+#!/usr/bin/python
+# Written by Boudewijn Schoon
+# see LICENSE.txt for license information
+
+"""
+Use the garbage collector to monitor memory usage
+"""
+
+from types import *
+import gc
+import inspect
+import sys
+import thread
+import time
+
+def _get_default_footprint(obj, depth):
+    return 4
+def _get_int_footprint(obj, depth):
+    return 4
+def _get_float_footprint(obj, depth):
+    return 8
+def _get_string_footprint(obj, depth):
+    return len(obj)
+def _get_unicode_footprint(obj, depth):
+    return 2 * len(obj)
+def _get_tuple_footprint(obj, depth):
+    if depth == 0:
+        return 4 + 4 * len(obj)
+    else:
+        return 4 + 4 * len(obj) + sum(map(lambda obj:get_memory_footprint(obj, depth), obj))
+def _get_list_footprint(obj, depth):
+    if depth == 0:
+        return 8 + 4 * len(obj)
+    else:
+        if len(obj) in (2, 3):
+            print >> sys.stderr, "Len:", type(obj[0]), type(obj[1])
+            print >> sys.stderr, `obj`
+            return 42
+        print >> sys.stderr, "Len:", len(obj)
+        return 8 + 4 * len(obj) + sum(map(lambda obj:get_memory_footprint(obj, depth), obj))
+def _get_dict_footprint(obj, depth):
+    if depth == 0:
+        return 32 + 8 * len(obj)
+    else:
+        return 32 + 8 * len(obj) + sum(map(lambda obj:get_memory_footprint(obj, depth), obj.iterkeys())) + sum(map(lambda obj:get_memory_footprint(obj, depth), obj.itervalues()))
+
+memory_footprint_map = {IntType:_get_int_footprint,
+                        FloatType:_get_float_footprint,
+                        StringType:_get_float_footprint,
+                        UnicodeType:_get_unicode_footprint,
+                        TupleType:_get_tuple_footprint,
+                        ListType:_get_list_footprint,
+                        DictType:_get_dict_footprint}
+def get_memory_footprint(obj, depth=100):
+    return memory_footprint_map.get(type(obj), _get_default_footprint)(obj, depth-1)
+
+def _get_default_description(obj):
+    return type(obj)
+def _get_function_description(obj):
+    return "" % (obj.__name__, obj.__module__)
+def _get_module_description(obj):
+    return str(obj)
+def _get_frame_description(obj):
+    return "" % (obj.f_code.co_name, obj.f_code.co_filename, obj.f_code.co_firstlineno)
+
+description_map = {FunctionType:_get_function_description,
+                   ModuleType:_get_module_description,
+                   FrameType:_get_frame_description}
+def get_description(obj):
+    return description_map.get(type(obj), _get_default_description)(obj)
+
+def get_datetime():
+    return time.strftime("%Y/%m/%d %H:%M:%S")
+
+def byte_uint_to_human(i, format="%(value).1f%(unit)s"):                       
+    """Convert a number into a formatted string.                             
+                                                                             
+    format: %(value)d%(unit)s                                                
+    1           --> 1B                                                       
+    1024        --> 1KB                                                      
+    1048576     --> 1MB                                                      
+    1073741824  --> 1GB                                                      
+                                                                             
+    format: %(value).1f %(unit-long)s                                        
+    1           --> 1.0 byte                                                 
+    2           --> 2.0 bytes                                                
+                                                                             
+    todo:                                                                    
+    - uint_to_human(1025, format="%(value)d %(unit-long)s") --> '1 kilobytes'
+      however, this should result in '1 kilobyte'                            
+                                                                             
+    """                                                                      
+    assert type(i) in (int, long)                                            
+    assert i >= 0                                                            
+    assert type(format) is str                                               
+    dic = {}                                                                 
+    if i < 1024:                                                             
+        dic["value"] = i                                                     
+        dic["unit"] = "B"                                                    
+        dic["unit-long"] = (i == 1 and "byte" or "bytes")                    
+    elif i < 1048576:                                                        
+        dic["value"] = i / 1024.0                                            
+        dic["unit"] = "KB"                                                   
+        dic["unit-long"] = (i == 1024 and "kilobyte" or "kilobytes")         
+    elif i < 1073741824:                                                     
+        dic["value"] = i / 1048576.0                                         
+        dic["unit"] = "MB"                                                   
+        dic["unit-long"] = (i == 1048576 and "megabyte" or "megabytes")      
+    else:                                                                    
+        dic["value"] = i / 1073741824.0                                      
+        dic["unit"] = "GB"                                                   
+        dic["unit-long"] = (i == 1073741824 and "gigabyte" or "gigabytes")   
+                                                                             
+    return format % dic                                                      
+
+def monitor(delay=10.0, interval=60.0, min_footprint=100000):
+    def parallel():
+        time.sleep(delay)
+
+        history = [min_footprint]
+        while True:
+            high_foot = 0
+            history = history[-2:]
+            low_foot = min(history)
+            datetime = get_datetime()
+            print >> sys.stderr, "Memory:", datetime, "using minimal footprint:", byte_uint_to_human(low_foot)
+
+            gc.collect()
+            for obj in gc.get_objects():
+                if type(obj) in (TupleType, ListType, DictType, StringType, UnicodeType):
+                    try:
+                        footprint = get_memory_footprint(obj)
+                    except:
+                        print >> sys.stderr, "Memory:", datetime, "unable to get footprint for", get_description(obj)
+                    else:
+                        if footprint > high_foot:
+                            high_foot = footprint
+                        if footprint >= low_foot:
+
+                            print >> sys.stderr, "Memory:", datetime, get_description(obj), "footprint:", byte_uint_to_human(footprint)
+                            for referrer in gc.get_referrers(obj):
+                                print >> sys.stderr, "Memory:", datetime, "REF", get_description(referrer)
+                            print >> sys.stderr, "Memory"
+
+            history.append(high_foot)
+            time.sleep(interval)
+    
+    thread.start_new_thread(parallel, ())
+
+
+def main():
+    """
+    Test the memory monitor
+    """
+    monitor(1.0)
+    time.sleep(10)
+
+if __name__ == "__main__":
+    main()
diff --git a/instrumentation/next-share/BaseLib/Images/SwarmPlayerIcon.ico b/instrumentation/next-share/BaseLib/Images/SwarmPlayerIcon.ico
new file mode 100644
index 0000000..3128616
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Images/SwarmPlayerIcon.ico differ
diff --git a/instrumentation/next-share/BaseLib/Images/SwarmPlayerLogo.png b/instrumentation/next-share/BaseLib/Images/SwarmPlayerLogo.png
new file mode 100644
index 0000000..5db0cdd
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Images/SwarmPlayerLogo.png differ
diff --git a/instrumentation/next-share/BaseLib/Images/SwarmPluginIcon.ico b/instrumentation/next-share/BaseLib/Images/SwarmPluginIcon.ico
new file mode 100644
index 0000000..3128616
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Images/SwarmPluginIcon.ico differ
diff --git a/instrumentation/next-share/BaseLib/Images/SwarmPluginLogo.png b/instrumentation/next-share/BaseLib/Images/SwarmPluginLogo.png
new file mode 100644
index 0000000..5db0cdd
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Images/SwarmPluginLogo.png differ
diff --git a/instrumentation/next-share/BaseLib/Images/SwarmServerIcon.ico b/instrumentation/next-share/BaseLib/Images/SwarmServerIcon.ico
new file mode 100644
index 0000000..3128616
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Images/SwarmServerIcon.ico differ
diff --git a/instrumentation/next-share/BaseLib/Images/background.png b/instrumentation/next-share/BaseLib/Images/background.png
new file mode 100644
index 0000000..50075ad
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Images/background.png differ
diff --git a/instrumentation/next-share/BaseLib/Images/fullScreen.png b/instrumentation/next-share/BaseLib/Images/fullScreen.png
new file mode 100644
index 0000000..efe2b4b
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Images/fullScreen.png differ
diff --git a/instrumentation/next-share/BaseLib/Images/fullScreen_hover.png b/instrumentation/next-share/BaseLib/Images/fullScreen_hover.png
new file mode 100644
index 0000000..0feefe3
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Images/fullScreen_hover.png differ
diff --git a/instrumentation/next-share/BaseLib/Images/logoTribler.png b/instrumentation/next-share/BaseLib/Images/logoTribler.png
new file mode 100644
index 0000000..481116e
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Images/logoTribler.png differ
diff --git a/instrumentation/next-share/BaseLib/Images/logoTribler_small.png b/instrumentation/next-share/BaseLib/Images/logoTribler_small.png
new file mode 100644
index 0000000..af53270
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Images/logoTribler_small.png differ
diff --git a/instrumentation/next-share/BaseLib/Images/mute.png b/instrumentation/next-share/BaseLib/Images/mute.png
new file mode 100644
index 0000000..7841c6f
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Images/mute.png differ
diff --git a/instrumentation/next-share/BaseLib/Images/mute_hover.png b/instrumentation/next-share/BaseLib/Images/mute_hover.png
new file mode 100644
index 0000000..556c893
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Images/mute_hover.png differ
diff --git a/instrumentation/next-share/BaseLib/Images/pause.png b/instrumentation/next-share/BaseLib/Images/pause.png
new file mode 100644
index 0000000..6b6be23
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Images/pause.png differ
diff --git a/instrumentation/next-share/BaseLib/Images/pause_hover.png b/instrumentation/next-share/BaseLib/Images/pause_hover.png
new file mode 100644
index 0000000..754b934
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Images/pause_hover.png differ
diff --git a/instrumentation/next-share/BaseLib/Images/play.png b/instrumentation/next-share/BaseLib/Images/play.png
new file mode 100644
index 0000000..1fc173d
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Images/play.png differ
diff --git a/instrumentation/next-share/BaseLib/Images/play_hover.png b/instrumentation/next-share/BaseLib/Images/play_hover.png
new file mode 100644
index 0000000..7eecfbc
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Images/play_hover.png differ
diff --git a/instrumentation/next-share/BaseLib/Images/save.png b/instrumentation/next-share/BaseLib/Images/save.png
new file mode 100644
index 0000000..0000d9f
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Images/save.png differ
diff --git a/instrumentation/next-share/BaseLib/Images/saveDisabled.png b/instrumentation/next-share/BaseLib/Images/saveDisabled.png
new file mode 100644
index 0000000..45df000
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Images/saveDisabled.png differ
diff --git a/instrumentation/next-share/BaseLib/Images/saveDisabled_hover.png b/instrumentation/next-share/BaseLib/Images/saveDisabled_hover.png
new file mode 100644
index 0000000..45df000
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Images/saveDisabled_hover.png differ
diff --git a/instrumentation/next-share/BaseLib/Images/save_hover.png b/instrumentation/next-share/BaseLib/Images/save_hover.png
new file mode 100644
index 0000000..c4f3dff
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Images/save_hover.png differ
diff --git a/instrumentation/next-share/BaseLib/Images/sliderDot.png b/instrumentation/next-share/BaseLib/Images/sliderDot.png
new file mode 100644
index 0000000..cf77466
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Images/sliderDot.png differ
diff --git a/instrumentation/next-share/BaseLib/Images/sliderVolume.png b/instrumentation/next-share/BaseLib/Images/sliderVolume.png
new file mode 100644
index 0000000..3da46e0
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Images/sliderVolume.png differ
diff --git a/instrumentation/next-share/BaseLib/Images/splash.jpg b/instrumentation/next-share/BaseLib/Images/splash.jpg
new file mode 100644
index 0000000..a119c3f
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Images/splash.jpg differ
diff --git a/instrumentation/next-share/BaseLib/Images/torrenticon.ico b/instrumentation/next-share/BaseLib/Images/torrenticon.ico
new file mode 100644
index 0000000..aaea20e
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Images/torrenticon.ico differ
diff --git a/instrumentation/next-share/BaseLib/Images/tribler.ico b/instrumentation/next-share/BaseLib/Images/tribler.ico
new file mode 100644
index 0000000..aaea20e
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Images/tribler.ico differ
diff --git a/instrumentation/next-share/BaseLib/Images/volume.png b/instrumentation/next-share/BaseLib/Images/volume.png
new file mode 100644
index 0000000..edbc2f7
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Images/volume.png differ
diff --git a/instrumentation/next-share/BaseLib/Images/volume_hover.png b/instrumentation/next-share/BaseLib/Images/volume_hover.png
new file mode 100644
index 0000000..debc16a
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Images/volume_hover.png differ
diff --git a/instrumentation/next-share/BaseLib/LICENSE.txt b/instrumentation/next-share/BaseLib/LICENSE.txt
new file mode 100644
index 0000000..ef95946
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/LICENSE.txt
@@ -0,0 +1,970 @@
+------------------------------------------------------------------------------
+
+    Next-Share content-delivery library.
+
+    The research leading to this library has received funding from the European
+    Community's Seventh Framework Programme in the P2P-Next project under grant
+    agreement no 216217.
+
+    All library modules are free software, unless stated otherwise; you can 
+    redistribute them and/or modify them under the terms of the GNU Lesser 
+    General Public License as published by the Free Software Foundation; in 
+    particular, version 2.1 of the License.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    The following library modules are Copyright (c) 2008-2012, VTT Technical Research Centre of Finland; All rights reserved:
+
+    The following library modules are Copyright (c) 2008-2012, Norut AS; All rights reserved:
+      BaseLib/Core/Multicast/*
+      BaseLib/Core/Statistics/Status/*
+      BaseLib/Core/ClosedSwarm/*
+      BaseLib/Player/swarmplayer-njaal.py
+      BaseLib/Plugin/BackgroundProcess-njaal.py
+      BaseLib/Test/test_closedswarm.py
+      BaseLib/Test/test_status.py
+      BaseLib/Tools/createlivestream-njaal.py
+      BaseLib/Tools/createpoa.py
+      BaseLib/Tools/trial_poa_server.py
+      BaseLib/UPnP/*
+      BaseLib/Test/test_upnp.py
+
+    The following library modules are Copyright (c) 2008-2012, DACC Systems AB; All rights reserved:
+      DACC/transfer.php
+ 
+    The following library modules are Copyright (c) 2008-2012, Lancaster University; All rights reserved:
+
+    The following library modules are Copyright (c) 2008-2012, Jožef Stefan Institute; All rights reserved:
+
+    The following library modules are Copyright (c) 2008-2012, First Oversi Ltd.; All rights reserved:
+
+    The following library modules are Copyright (c) 2008-2012, TECHNISCHE UNIVERSITEIT DELFT; All rights reserved:
+      BaseLib/Core/NATFirewall/NatCheck.py
+      BaseLib/Core/NATFirewall/TimeoutCheck.py
+      BaseLib/Core/NATFirewall/NatCheckMsgHandler.py
+      BaseLib/Policies/SeedingManager.py
+      BaseLib/Core/Statistics/SeedingStatsCrawler.py
+      BaseLib/Core/CacheDB/SqliteSeedingStatsCacheDB.py
+      BaseLib/Core/BuddyCast/moderationcast.py
+      BaseLib/Core/BuddyCast/moderationcast_util.py
+      BaseLib/Core/BuddyCast/votecast.py
+      BaseLib/Core/CacheDB/maxflow.py
+      BaseLib/Core/CacheDB/SqliteVideoPlaybackStatsCacheDB.py
+      BaseLib/Core/NATFirewall/ConnectionCheck.py
+      BaseLib/Core/NATFirewall/NatTraversal.py
+      BaseLib/Core/Search/Reranking.py
+      BaseLib/Core/Statistics/tribler_videoplayback_stats.sql
+      BaseLib/Core/Statistics/VideoPlaybackCrawler.py
+      BaseLib/Core/Utilities/Crypto.py
+      BaseLib/Images/
+      BaseLib/Player/BaseApp.py
+      BaseLib/Player/EmbeddedPlayer4Frame.py
+      BaseLib/Player/PlayerVideoFrame.py
+      BaseLib/Plugin
+      BaseLib/Test/test_multicast.py
+      BaseLib/Test/test_na_extend_hs.py
+      BaseLib/Test/test_na_extend_hs.sh
+      BaseLib/Test/test_sqlitecachedbhandler.sh
+      BaseLib/Tools/dirtrackerseeder.py
+      BaseLib/Tools/pipe-babscam-h264-nosound-mencoder.sh
+      BaseLib/Tools/superpeer.py
+      BaseLib/Utilities/LinuxSingleInstanceChecker.py
+      BaseLib/Video/Images
+      BaseLib/Video/VideoFrame.py
+      reset.bat
+      reset-keepid.bat
+      BaseLib/Core/Video/PiecePickerSVC.py
+      BaseLib/Core/Video/SVCTransporter.py
+      BaseLib/Core/Video/SVCVideoStatus.py
+      BaseLib/schema_sdb_v5.sql
+      BaseLib/Core/APIImplementation/makeurl.py
+      BaseLib/Core/BuddyCast/channelcast.py
+      BaseLib/Core/DecentralizedTracking/repex.py
+      BaseLib/Core/NATFirewall/TimeoutFinder.py
+      BaseLib/Core/NATFirewall/UDPPuncture.py
+      BaseLib/Core/Statistics/RepexCrawler.py
+      BaseLib/Debug/*
+      BaseLib/Tools/createtorrent.py
+      BaseLib/Tools/pingbackserver.py
+      BaseLib/Tools/seeking.py
+      BaseLib/Tools/stunserver.py
+      lucid-xpicreate.sh
+      patentfreevlc.bat
+      BaseLib/Core/BitTornado/BT1/GetRightHTTPDownloader.py
+      BaseLib/Core/BitTornado/BT1/HoffmanHTTPDownloader.py
+      BaseLib/Core/CacheDB/MetadataDBHandler.py
+      BaseLib/Core/DecentralizedTracking/MagnetLink/*
+      BaseLib/Core/Subtitles/*
+      BaseLib/Images/SwarmServerIcon.ico
+      BaseLib/Main/Build/Ubuntu/tribler.gconf-defaults
+      BaseLib/Main/Utility/logging_util.py
+      BaseLib/Main/vwxGUI/ChannelsPanel.py
+      BaseLib/Main/vwxGUI/images/iconSaved_state4.png
+      BaseLib/schema_sdb_v5.sql
+      BaseLib/Test/Core/*
+      BaseLib/Test/extend_hs_dir/proxyservice.test.torrent
+      BaseLib/Test/subtitles_test_res
+      BaseLib/Test/test_channelcast_plus_subtitles.py
+      BaseLib/Test/test_magnetlink.py
+      BaseLib/Test/test_miscutils.py
+      BaseLib/Test/test_subtitles.bat
+      BaseLib/Test/test_subtitles_isolation.py
+      BaseLib/Test/test_subtitles_msgs.py
+      BaseLib/Test/test_subtitles.sh
+      BaseLib/Test/test_threadpool.py
+      BaseLib/Tools/dirtracker.py
+      BaseLib/Tools/duration2torrent.py
+      BaseLib/Tools/httpseeder.py
+      BaseLib/Transport/*
+      BaseLib/Video/Ogg.py
+      BaseLib/WebUI/*
+      xpitransmakedeb.sh
+      xpitransmakedist.bat
+      xpitransmakedist.sh
+      xpitransmakedistmac.sh
+      xie8transmakedist.bat
+      TUD/swift-spbackend-r1598/*
+      vlc-1.0.5-swarmplugin-switch-kcc-src-aug2010-r16968.patch (except bindings/python)
+
+    The following library modules are Copyright (c) 2008-2012, STMicroelectronics S.r.l.; All rights reserved:
+
+    The following library modules are Copyright (c) 2008-2012, Kungliga Tekniska Högskolan (The Royal Institute of Technology); All rights reserved:
+      BaseLib/Core/DecentralizedTracking/kadtracker/*
+
+    The following library modules are Copyright (c) 2008-2012, Markenfilm GmbH & Co. KG; All rights reserved:
+
+    The following library modules are Copyright (c) 2008-2012, Radiotelevizija Slovenija Javni Zavvod Ljubljana; All rights reserved:
+
+    The following library modules are Copyright (c) 2008-2012, Kendra Foundation; All rights reserved:
+
+    The following library modules are Copyright (c) 2008-2012, Universitaet Klagenfurt; All rights reserved:
+
+    The following library modules are Copyright (c) 2008-2012, AG Projects; All rights reserved:
+      AGP/sipsimple-20100621.tgz
+      
+    The following library modules are Copyright (c) 2008-2012, The British Broadcasting Corporation; All rights reserved:
+
+    The following library modules are Copyright (c) 2008-2012, Pioneer Digital Design Centre Limited; All rights reserved:
+  
+    The following library modules are Copyright (c) 2008-2012, INSTITUT FUER RUNDFUNKTECHNIK GMBH; All rights reserved:
+
+    The following library modules are Copyright (c) 2008-2012, Fabchannel BV; All rights reserved:
+
+    The following library modules are Copyright (c) 2008-2012, University Politehnica Bucharest; All rights reserved:
+      BaseLib/Core/ProxyService/*
+      BaseLib/Tools/proxy-cmdline.py
+      BaseLib/Test/test_proxyservice_as_coord.bat
+      BaseLib/Test/test_proxyservice_as_coord.py
+      BaseLib/Test/test_proxyservice_as_coord.sh
+      BaseLib/Test/test_proxyservice.bat
+      BaseLib/Test/test_proxyservice.py
+      BaseLib/Test/test_proxyservice.sh
+      BaseLib/Test/extend_hs_dir/proxyservice.test.torrent
+      
+
+    The following library modules are Copyright (c) 2008-2012, EBU-UER; All rights reserved:
+
+    The following library modules are Copyright (c) 2008-2012, Università di Roma Sapienza; All rights reserved:
+
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+    VTT Technical Research Centre of Finland, 
+    Tekniikankatu 1, 
+    FIN-33710 Tampere, 
+    Finland
+
+    Norut AS,
+    Postboks 6434 
+    Forskningsparken, 
+    9294 Tromsø,
+    Norway
+
+    DACC Systems AB
+    Glimmervägen 4, 
+    SE18734, Täby,
+    Sweden
+
+    Lancaster University, 
+    University House, 
+    Bailrigg, Lancaster, LA1 4YW
+    United Kingdom
+
+    Jožef Stefan Institute, 
+    Jamova cesta 39, 
+    1000 Ljubljana, 
+    Slovenia
+
+    First Oversi Ltd.,
+    Rishon Lezion 1,
+    Petah Tikva 49723, 
+    Israel
+
+    TECHNISCHE UNIVERSITEIT DELFT, 
+    Faculty of Electrical Engineering, Mathematics and Computer Science, 
+    Mekelweg 4, 
+    2628 CD Delft, 
+    The Netherlands
+
+    STMicroelectronics S.r.l., 
+    via C.Olivetti 2, 
+    I-20041 Agrate Brianza,
+    Italy
+
+    Kungliga Tekniska Högskolan (The Royal Institute of Technology), 
+    KTH/ICT/ECS/TSLab
+    Electrum 229
+    164 40 Kista
+    Sweden
+
+    Markenfilm GmbH & Co. KG, 
+    Schulauer Moorweg 25, 
+    22880 Wedel, 
+    Germany
+
+    Radiotelevizija Slovenija Javni Zavvod Ljubljana, 
+    Kolodvorska 2, 
+    SI-1000 Ljubljana,
+    Slovenia
+
+
+    Kendra Foundation, 
+    Meadow Barn, Holne, 
+    Newton Abbot, Devon, TQ13 7SP,
+    United Kingdom
+
+
+    Universitaet Klagenfurt, 
+    Universitaetstrasse 65-67, 
+    9020 Klagenfurt, 
+    Austria
+
+    AG Projects, 
+    Dr. Leijdsstraat 92,
+    2021RK Haarlem, 
+    The Netherlands
+
+    The British Broadcasting Corporation,
+    Broadcasting House, Portland Place, 
+    London, W1A 1AA 
+    United Kingdom
+
+    Pioneer Digital Design Centre Limited, 
+    Pioneer House, Hollybush Hill, Stoke Poges, 
+    Slough, SL2 4QP
+    United Kingdom
+
+    INSTITUT FUER RUNDFUNKTECHNIK GMBH
+    Floriansmuehlstrasse 60,
+    80939 München, 
+    Germany
+
+    Fabchannel BV, 
+    Kleine-Gartmanplantsoen 21, 
+    1017 RP Amsterdam, 
+    The Netherlands
+
+    University Politehnica Bucharest, 
+    313 Splaiul Independentei, 
+    District 6, cod 060042, Bucharest,
+    Romania
+
+    EBU-UER, 
+    L'Ancienne Route 17A, 1218
+    Grand Saconnex - Geneva, 
+    Switzerland
+
+    Università di Roma Sapienza
+    Dipartimento di Informatica e Sistemistica (DIS),
+    Via Ariosto 25, 
+    00185 Rome, 
+    Italy
+
+
+------------------------------------------------------------------------------
+
+    BaseLib content-delivery library.
+
+    Development of the BaseLib library was supported by various research 
+    grants:
+
+     - BSIK Freeband Communication I-Share project (Dutch Ministry of Economic 
+       Affairs)
+     - Netherlands Organisation for Scientific Research (NWO) grant 612.060.215
+     - Dutch Technology Foundation STW: Veni project DTC.7299
+     - European Community's Sixth Framework Programme in the P2P-FUSION project
+       under contract no 035249.
+ 
+    The following library modules are Copyright (c) 2005-2010,
+    Delft University of Technology and Vrije Universiteit Amsterdam; 
+    All rights reserved.
+
+      BaseLib/*
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+    Delft University of Technology
+    Postbus 5
+    2600 AA Delft
+    The Netherlands
+    
+    Vrije Universiteit
+    De Boelelaan 1105
+    1081 HV Amsterdam
+    The Netherlands
+
+
+ 
+-------------------------------------------------------------------------------
+
+    BuddyCast4 content-recommendation library.
+
+    The research leading to this library has received funding from the
+    European Community's Seventh Framework Programme [FP7/2007-2011] 
+    in the Petamedia project under grant agreement no. 216444
+
+    The following library modules are Copyright (c) 2008-2010,
+    Delft University of Technology and Technische Universität Berlin; 
+    All rights reserved.
+
+      BaseLib/Core/BuddyCast/buddycast.py
+
+    The following library modules are Copyright (c) 2008-2010,
+    Technische Universität Berlin; 
+    All rights reserved.
+
+      BaseLib/Core/Search/Reranking.py
+      BaseLib/Test/test_buddycast4.py
+      BaseLib/Test/test_buddycast4_stresstest.py
+      
+    All library modules are free software, unless stated otherwise; you can 
+    redistribute them and/or modify them under the terms of the GNU Lesser 
+    General Public License as published by the Free Software Foundation; in 
+    particular, version 2.1 of the License.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+    Delft University of Technology
+    Postbus 5
+    2600 AA Delft
+    The Netherlands
+    
+    Technische Universität Berlin
+    Strasse des 17. Juni 135
+    10623 Berlin
+    Germany
+
+
+------------------------------------------------------------------------------
+
+    SwarmTransport/SwarmPlayer Firefox library.
+
+    The research leading to this library has received funding from the European
+    Community's Seventh Framework Programme in the P2P-Next project under grant
+    agreement no 216217.
+
+    All library modules are free software, unless stated otherwise; you can 
+    redistribute them and/or modify them under the terms of the GNU Lesser 
+    General Public License as published by the Free Software Foundation; in 
+    particular, version 2.1 of the License.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+
+    The following library modules are Copyright (c) 2008-2012, TECHNISCHE UNIVERSITEIT DELFT; 
+    and Jan Gerber; All rights reserved:
+      BaseLib/Transport/tribeIChannel.idl
+      BaseLib/Transport/tribeISwarmTransport.idl
+      BaseLib/Transport/components/TribeChannel.js
+      BaseLib/Transport/components/TribeProtocolHandler.js
+      BaseLib/Transport/components/SwarmTransport.js
+      BaseLib/Transport/install.rdf
+      BaseLib/Transport/chrome.manifest
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+    TECHNISCHE UNIVERSITEIT DELFT, 
+    Faculty of Electrical Engineering, Mathematics and Computer Science, 
+    Mekelweg 4, 
+    2628 CD Delft, 
+    The Netherlands
+    
+    Jan Gerber
+    j@thing.net
+
+-------------------------------------------------------------------------------
+
+Unless otherwise noted, all files written by Bram Cohen, John Hoffman, Petru 
+Paler, Uoti Urpala, Ross Cohen, Tim Tucker, Choopan RATTANAPOKA, Yejun Yang,
+Myers Carpenter, Bill Bumgarner, Henry 'Pi' James, Loring Holden, 
+Dustin Pate ("noirsoldats@codemeu.com"), kratoak5, Roee Shlomo, Greg Fleming, 
+N. Goldmann ("Pir4nhaX,www.clanyakuza.com"), and Michel Hartmann is released
+under the MIT license, exceptions contain licensing information in them.
+
+Copyright (C) 2001-2002 Bram Cohen
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation files
+(the "Software"), to deal in the Software without restriction,
+including without limitation the rights to use, copy, modify, merge,
+publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+The Software is provided "AS IS", without warranty of any kind,
+express or implied, including but not limited to the warranties of
+merchantability,  fitness for a particular purpose and
+noninfringement. In no event shall the  authors or copyright holders
+be liable for any claim, damages or other liability, whether in an
+action of contract, tort or otherwise, arising from, out of or in
+connection with the Software or the use or other dealings in the
+Software.
+
+-------------------------------------------------------------------------------
+
+                  GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 2.1, February 1999
+
+ Copyright (C) 1991, 1999 Free Software Foundation, Inc.
+     51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+[This is the first released version of the Lesser GPL.  It also counts
+ as the successor of the GNU Library Public License, version 2, hence
+ the version number 2.1.]
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+Licenses are intended to guarantee your freedom to share and change
+free software--to make sure the software is free for all its users.
+
+  This license, the Lesser General Public License, applies to some
+specially designated software packages--typically libraries--of the
+Free Software Foundation and other authors who decide to use it.  You
+can use it too, but we suggest you first think carefully about whether
+this license or the ordinary General Public License is the better
+strategy to use in any particular case, based on the explanations below.
+
+  When we speak of free software, we are referring to freedom of use,
+not price.  Our General Public Licenses are designed to make sure that
+you have the freedom to distribute copies of free software (and charge
+for this service if you wish); that you receive source code or can get
+it if you want it; that you can change the software and use pieces of
+it in new free programs; and that you are informed that you can do
+these things.
+
+  To protect your rights, we need to make restrictions that forbid
+distributors to deny you these rights or to ask you to surrender these
+rights.  These restrictions translate to certain responsibilities for
+you if you distribute copies of the library or if you modify it.
+
+  For example, if you distribute copies of the library, whether gratis
+or for a fee, you must give the recipients all the rights that we gave
+you.  You must make sure that they, too, receive or can get the source
+code.  If you link other code with the library, you must provide
+complete object files to the recipients, so that they can relink them
+with the library after making changes to the library and recompiling
+it.  And you must show them these terms so they know their rights.
+
+  We protect your rights with a two-step method: (1) we copyright the
+library, and (2) we offer you this license, which gives you legal
+permission to copy, distribute and/or modify the library.
+
+  To protect each distributor, we want to make it very clear that
+there is no warranty for the free library.  Also, if the library is
+modified by someone else and passed on, the recipients should know
+that what they have is not the original version, so that the original
+author's reputation will not be affected by problems that might be
+introduced by others.
+
+
+  Finally, software patents pose a constant threat to the existence of
+any free program.  We wish to make sure that a company cannot
+effectively restrict the users of a free program by obtaining a
+restrictive license from a patent holder.  Therefore, we insist that
+any patent license obtained for a version of the library must be
+consistent with the full freedom of use specified in this license.
+
+  Most GNU software, including some libraries, is covered by the
+ordinary GNU General Public License.  This license, the GNU Lesser
+General Public License, applies to certain designated libraries, and
+is quite different from the ordinary General Public License.  We use
+this license for certain libraries in order to permit linking those
+libraries into non-free programs.
+
+  When a program is linked with a library, whether statically or using
+a shared library, the combination of the two is legally speaking a
+combined work, a derivative of the original library.  The ordinary
+General Public License therefore permits such linking only if the
+entire combination fits its criteria of freedom.  The Lesser General
+Public License permits more lax criteria for linking other code with
+the library.
+
+  We call this license the "Lesser" General Public License because it
+does Less to protect the user's freedom than the ordinary General
+Public License.  It also provides other free software developers Less
+of an advantage over competing non-free programs.  These disadvantages
+are the reason we use the ordinary General Public License for many
+libraries.  However, the Lesser license provides advantages in certain
+special circumstances.
+
+  For example, on rare occasions, there may be a special need to
+encourage the widest possible use of a certain library, so that it becomes
+a de-facto standard.  To achieve this, non-free programs must be
+allowed to use the library.  A more frequent case is that a free
+library does the same job as widely used non-free libraries.  In this
+case, there is little to gain by limiting the free library to free
+software only, so we use the Lesser General Public License.
+
+  In other cases, permission to use a particular library in non-free
+programs enables a greater number of people to use a large body of
+free software.  For example, permission to use the GNU C Library in
+non-free programs enables many more people to use the whole GNU
+operating system, as well as its variant, the GNU/Linux operating
+system.
+
+  Although the Lesser General Public License is Less protective of the
+users' freedom, it does ensure that the user of a program that is
+linked with the Library has the freedom and the wherewithal to run
+that program using a modified version of the Library.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.  Pay close attention to the difference between a
+"work based on the library" and a "work that uses the library".  The
+former contains code derived from the library, whereas the latter must
+be combined with the library in order to run.
+
+
+                  GNU LESSER GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License Agreement applies to any software library or other
+program which contains a notice placed by the copyright holder or
+other authorized party saying it may be distributed under the terms of
+this Lesser General Public License (also called "this License").
+Each licensee is addressed as "you".
+
+  A "library" means a collection of software functions and/or data
+prepared so as to be conveniently linked with application programs
+(which use some of those functions and data) to form executables.
+
+  The "Library", below, refers to any such software library or work
+which has been distributed under these terms.  A "work based on the
+Library" means either the Library or any derivative work under
+copyright law: that is to say, a work containing the Library or a
+portion of it, either verbatim or with modifications and/or translated
+straightforwardly into another language.  (Hereinafter, translation is
+included without limitation in the term "modification".)
+
+  "Source code" for a work means the preferred form of the work for
+making modifications to it.  For a library, complete source code means
+all the source code for all modules it contains, plus any associated
+interface definition files, plus the scripts used to control compilation
+and installation of the library.
+
+  Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running a program using the Library is not restricted, and output from
+such a program is covered only if its contents constitute a work based
+on the Library (independent of the use of the Library in a tool for
+writing it).  Whether that is true depends on what the Library does
+and what the program that uses the Library does.
+  
+  1. You may copy and distribute verbatim copies of the Library's
+complete source code as you receive it, in any medium, provided that
+you conspicuously and appropriately publish on each copy an
+appropriate copyright notice and disclaimer of warranty; keep intact
+all the notices that refer to this License and to the absence of any
+warranty; and distribute a copy of this License along with the
+Library.
+
+  You may charge a fee for the physical act of transferring a copy,
+and you may at your option offer warranty protection in exchange for a
+fee.
+
+
+  2. You may modify your copy or copies of the Library or any portion
+of it, thus forming a work based on the Library, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) The modified work must itself be a software library.
+
+    b) You must cause the files modified to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    c) You must cause the whole of the work to be licensed at no
+    charge to all third parties under the terms of this License.
+
+    d) If a facility in the modified Library refers to a function or a
+    table of data to be supplied by an application program that uses
+    the facility, other than as an argument passed when the facility
+    is invoked, then you must make a good faith effort to ensure that,
+    in the event an application does not supply such function or
+    table, the facility still operates, and performs whatever part of
+    its purpose remains meaningful.
+
+    (For example, a function in a library to compute square roots has
+    a purpose that is entirely well-defined independent of the
+    application.  Therefore, Subsection 2d requires that any
+    application-supplied function or table used by this function must
+    be optional: if the application does not supply it, the square
+    root function must still compute square roots.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Library,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Library, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote
+it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Library.
+
+In addition, mere aggregation of another work not based on the Library
+with the Library (or with a work based on the Library) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may opt to apply the terms of the ordinary GNU General Public
+License instead of this License to a given copy of the Library.  To do
+this, you must alter all the notices that refer to this License, so
+that they refer to the ordinary GNU General Public License, version 2,
+instead of to this License.  (If a newer version than version 2 of the
+ordinary GNU General Public License has appeared, then you can specify
+that version instead if you wish.)  Do not make any other change in
+these notices.
+
+
+  Once this change is made in a given copy, it is irreversible for
+that copy, so the ordinary GNU General Public License applies to all
+subsequent copies and derivative works made from that copy.
+
+  This option is useful when you wish to copy part of the code of
+the Library into a program that is not a library.
+
+  4. You may copy and distribute the Library (or a portion or
+derivative of it, under Section 2) in object code or executable form
+under the terms of Sections 1 and 2 above provided that you accompany
+it with the complete corresponding machine-readable source code, which
+must be distributed under the terms of Sections 1 and 2 above on a
+medium customarily used for software interchange.
+
+  If distribution of object code is made by offering access to copy
+from a designated place, then offering equivalent access to copy the
+source code from the same place satisfies the requirement to
+distribute the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  5. A program that contains no derivative of any portion of the
+Library, but is designed to work with the Library by being compiled or
+linked with it, is called a "work that uses the Library".  Such a
+work, in isolation, is not a derivative work of the Library, and
+therefore falls outside the scope of this License.
+
+  However, linking a "work that uses the Library" with the Library
+creates an executable that is a derivative of the Library (because it
+contains portions of the Library), rather than a "work that uses the
+library".  The executable is therefore covered by this License.
+Section 6 states terms for distribution of such executables.
+
+  When a "work that uses the Library" uses material from a header file
+that is part of the Library, the object code for the work may be a
+derivative work of the Library even though the source code is not.
+Whether this is true is especially significant if the work can be
+linked without the Library, or if the work is itself a library.  The
+threshold for this to be true is not precisely defined by law.
+
+  If such an object file uses only numerical parameters, data
+structure layouts and accessors, and small macros and small inline
+functions (ten lines or less in length), then the use of the object
+file is unrestricted, regardless of whether it is legally a derivative
+work.  (Executables containing this object code plus portions of the
+Library will still fall under Section 6.)
+
+  Otherwise, if the work is a derivative of the Library, you may
+distribute the object code for the work under the terms of Section 6.
+Any executables containing that work also fall under Section 6,
+whether or not they are linked directly with the Library itself.
+
+
+  6. As an exception to the Sections above, you may also combine or
+link a "work that uses the Library" with the Library to produce a
+work containing portions of the Library, and distribute that work
+under terms of your choice, provided that the terms permit
+modification of the work for the customer's own use and reverse
+engineering for debugging such modifications.
+
+  You must give prominent notice with each copy of the work that the
+Library is used in it and that the Library and its use are covered by
+this License.  You must supply a copy of this License.  If the work
+during execution displays copyright notices, you must include the
+copyright notice for the Library among them, as well as a reference
+directing the user to the copy of this License.  Also, you must do one
+of these things:
+
+    a) Accompany the work with the complete corresponding
+    machine-readable source code for the Library including whatever
+    changes were used in the work (which must be distributed under
+    Sections 1 and 2 above); and, if the work is an executable linked
+    with the Library, with the complete machine-readable "work that
+    uses the Library", as object code and/or source code, so that the
+    user can modify the Library and then relink to produce a modified
+    executable containing the modified Library.  (It is understood
+    that the user who changes the contents of definitions files in the
+    Library will not necessarily be able to recompile the application
+    to use the modified definitions.)
+
+    b) Use a suitable shared library mechanism for linking with the
+    Library.  A suitable mechanism is one that (1) uses at run time a
+    copy of the library already present on the user's computer system,
+    rather than copying library functions into the executable, and (2)
+    will operate properly with a modified version of the library, if
+    the user installs one, as long as the modified version is
+    interface-compatible with the version that the work was made with.
+
+    c) Accompany the work with a written offer, valid for at
+    least three years, to give the same user the materials
+    specified in Subsection 6a, above, for a charge no more
+    than the cost of performing this distribution.
+
+    d) If distribution of the work is made by offering access to copy
+    from a designated place, offer equivalent access to copy the above
+    specified materials from the same place.
+
+    e) Verify that the user has already received a copy of these
+    materials or that you have already sent this user a copy.
+
+  For an executable, the required form of the "work that uses the
+Library" must include any data and utility programs needed for
+reproducing the executable from it.  However, as a special exception,
+the materials to be distributed need not include anything that is
+normally distributed (in either source or binary form) with the major
+components (compiler, kernel, and so on) of the operating system on
+which the executable runs, unless that component itself accompanies
+the executable.
+
+  It may happen that this requirement contradicts the license
+restrictions of other proprietary libraries that do not normally
+accompany the operating system.  Such a contradiction means you cannot
+use both them and the Library together in an executable that you
+distribute.
+
+
+  7. You may place library facilities that are a work based on the
+Library side-by-side in a single library together with other library
+facilities not covered by this License, and distribute such a combined
+library, provided that the separate distribution of the work based on
+the Library and of the other library facilities is otherwise
+permitted, and provided that you do these two things:
+
+    a) Accompany the combined library with a copy of the same work
+    based on the Library, uncombined with any other library
+    facilities.  This must be distributed under the terms of the
+    Sections above.
+
+    b) Give prominent notice with the combined library of the fact
+    that part of it is a work based on the Library, and explaining
+    where to find the accompanying uncombined form of the same work.
+
+  8. You may not copy, modify, sublicense, link with, or distribute
+the Library except as expressly provided under this License.  Any
+attempt otherwise to copy, modify, sublicense, link with, or
+distribute the Library is void, and will automatically terminate your
+rights under this License.  However, parties who have received copies,
+or rights, from you under this License will not have their licenses
+terminated so long as such parties remain in full compliance.
+
+  9. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Library or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Library (or any work based on the
+Library), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Library or works based on it.
+
+  10. Each time you redistribute the Library (or any work based on the
+Library), the recipient automatically receives a license from the
+original licensor to copy, distribute, link with or modify the Library
+subject to these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties with
+this License.
+
+
+  11. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Library at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Library by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Library.
+
+If any portion of this section is held invalid or unenforceable under any
+particular circumstance, the balance of the section is intended to apply,
+and the section as a whole is intended to apply in other circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  12. If the distribution and/or use of the Library is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Library under this License may add
+an explicit geographical distribution limitation excluding those countries,
+so that distribution is permitted only in or among countries not thus
+excluded.  In such case, this License incorporates the limitation as if
+written in the body of this License.
+
+  13. The Free Software Foundation may publish revised and/or new
+versions of the Lesser General Public License from time to time.
+Such new versions will be similar in spirit to the present version,
+but may differ in detail to address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Library
+specifies a version number of this License which applies to it and
+"any later version", you have the option of following the terms and
+conditions either of that version or of any later version published by
+the Free Software Foundation.  If the Library does not specify a
+license version number, you may choose any version ever published by
+the Free Software Foundation.
+
+
+  14. If you wish to incorporate parts of the Library into other free
+programs whose distribution conditions are incompatible with these,
+write to the author to ask for permission.  For software which is
+copyrighted by the Free Software Foundation, write to the Free
+Software Foundation; we sometimes make exceptions for this.  Our
+decision will be guided by the two goals of preserving the free status
+of all derivatives of our free software and of promoting the sharing
+and reuse of software generally.
+
+                            NO WARRANTY
+
+  15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
+WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
+EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR
+OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY
+KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
+LIBRARY IS WITH YOU.  SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME
+THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
+WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
+AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU
+FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
+CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
+LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
+RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
+FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
+SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+
+           How to Apply These Terms to Your New Libraries
+
+  If you develop a new library, and you want it to be of the greatest
+possible use to the public, we recommend making it free software that
+everyone can redistribute and change.  You can do so by permitting
+redistribution under these terms (or, alternatively, under the terms of the
+ordinary General Public License).
+
+  To apply these terms, attach the following notices to the library.  It is
+safest to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least the
+"copyright" line and a pointer to where the full notice is found.
+
+    
+    Copyright (C)   
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+Also add information on how to contact you by electronic and paper mail.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the library, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the
+  library `Frob' (a library for tweaking knobs) written by James Random Hacker.
+
+  , 1 April 1990
+  Ty Coon, President of Vice
+
+That's all there is to it!
+
+-------------------------------------------------------------------------------
+
diff --git a/instrumentation/next-share/BaseLib/Lang/__init__.py b/instrumentation/next-share/BaseLib/Lang/__init__.py
new file mode 100644
index 0000000..84ea404
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Lang/__init__.py
@@ -0,0 +1,3 @@
+# Written by Arno Bakker
+# see LICENSE.txt for license information
+
diff --git a/instrumentation/next-share/BaseLib/Lang/english.lang b/instrumentation/next-share/BaseLib/Lang/english.lang
new file mode 100644
index 0000000..22933a8
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Lang/english.lang
@@ -0,0 +1,1429 @@
+##################################################################
+# You can change language here, this is all Tribler using variables
+# Make sure that try to keep text length as the original text
+#
+# Note: text strings can be written as either:
+# stringname = "some string"
+# or:
+# stringname_line1 = "first line of a string"
+# stringname_line2 = "second line of a string"
+#
+# (Tribler will automatically add the lines together)
+#
+##################################################################
+
+[ABC/language]
+
+# The name of the language defined in this file:
+languagename = "English"
+
+# this credit will display in aboutme dialog
+# translate = "Translator: "
+translate = ""
+
+# All ABC Variables
+#######################
+title = "Tribler"
+
+superseederrornotcompleted = "Super-seed can only be enabled for completed torrents"
+superseedmustruntorrentbefore = "This torrent must be running before using Super-Seed mode"
+
+superwarningmsg_line1 = "This option greatly reduces the torrent's efficiency."
+superwarningmsg_line2 = "Super-seed should only be used for initial seeding or"
+superwarningmsg_line3 = "for re-seeding."
+superwarningmsg_line4 = ""
+superwarningmsg_line5 = "Super-seed mode will stay in effect until the torrent"
+superwarningmsg_line6 = "is stopped."
+
+failedinvalidtorrent = "Failed : Invalid torrent file."
+failedtorrentmissing = "Failed : .torrent file does not exist or cannot be read."
+removetorrent 		= "Do you wish to delete the torrent file?"
+ok 					= "OK"
+cancel				= "Cancel"
+apply				= "Apply"
+close				= "Close"
+save				= "Save"
+saveandapply 		= "Save and Apply"
+done 				= "Done"
+
+choosefiletosaveas  = "Choose file to save as, pick a partial download to resume"
+choosedirtosaveto 	= "Choose a directory to save to (pick a partial download to resume)"
+enterurl 			= "Enter the URL for the torrent you wish to add:"
+confirmupgrademsg   = "Do you want to close Tribler and upgrade to the next version?  See release notes below"
+confirmupgrade      = "Upgrade Tribler?"
+confirmmsg 			= "Do you want to close Tribler ?"
+confirm 			= "Confirm"
+aboutabc 			= "About Tribler"
+abcpreference 		= "Preferences"
+managefriendspeers         = "Manage Friends/Encountered Peer List"
+managefriends		= "Manage Friends"
+recommendatefiles   = "Recommendation"
+addfriend	      = "Add a friend"
+editfriend	      = "Edit a friend's info"
+viewpeerlist		= "View Encountered Peers"
+addpeeradfriend         = "Add this peer as your friend"
+deletepeer              = "Delete this peer"
+deletepeerfriend        = "Remove the peer from your friends list"
+fakefile                = "Fake File"
+norating                = "No Rating"
+rankitems               = "Rank Items"
+assignrating            = "Right click on a torrent to assign a 1--5 star rating"
+showabcwindow 		= "Show Tribler Window"
+error 				= "Error"
+warning 			= "Warning"
+invalidinput			= "Invalid input"
+cantconnectwebserver_line1 = "Could not connect to update server."
+cantconnectwebserver_line2 = "It may be down or you are not connected to the Internet."
+abclatestversion 	= "Latest Version"
+nonewversion = "There is no new version available. 
 Please visit www.tribler.org for more information"
+hasnewversion = "There is a new version available. Please upgrade."
+globaluploadsetting = "Global Upload"
+downloadsetting 	= "Download Setting"
+ratelimits 			= "Rate Limiting"
+seedoptions 		= "Seeding Options"
+webinterfaceservice = "Web Interface Service"
+
+duplicatetorrent 	= "Duplicate Torrent"
+duplicatetorrentinlist  = "This torrent (or one with the same hash value) already exists in the list"
+duplicatetorrentmsg = "This torrent is a duplicate!\nAre you sure you want to replace it?"
+choosetorrentfile 	= "Choose a torrent file"
+cantgettorrentfromurl = "Can't get torrent from this URL"
+localsetting 		= "Local Settings"
+errordeletefile 	= "Error, while trying to delete file.\nFile not cannot be found or is in use"
+filenotfound 		= "File not found or cannot be accessed."
+confirmdeletefile 	= "Are you sure you want to remove this file or folder?"
+choosenewlocation 	= "Choose a new location for this torrent"
+
+extracterrorduplicatemsg_line1 = "A file with the same name already exists in the destination folder.
+extracterrorduplicatemsg_line2 = "Do you want to overwrite it?"
+extracterrorduplicate = "Duplicate file name"
+
+
+extracterrorinactive = "At least one selected torrent is active. Please deactivate before extracting."
+
+
+extracterrormoving 	= "Can't move the torrent file."
+torrentdetail 		= "Torrent Details..."
+
+moveup 				= "Move torrent up"
+movedown 			= "Move torrent down"
+movetop 			= "Move torrent to top"
+movebottom 			= "Move torrent to bottom"
+clearallcompleted 	= "Clear all completed torrents"
+pauseall 			= "Pause All"
+stopall 			= "Stop All"
+restartall			= "Restart All"
+unstopall 			= "Queue all stopped torrents"
+mode 				= "Mode Manual/Auto"
+webservice 			= "Web Service: "
+torrentfilenotfound = "Torrent file not found"
+clear 				= "Clear"
+errormovefile 		= "Error while moving files"
+
+totaldlspeed 		= "Total DL Speed:"
+totalulspeed 		= "Total UL Speed:"
+
+failbehavior1 		= "Set status to:"
+failbehavior2 		= "when a torrent fails"
+
+defaultpriority 	= "Default priority for new torrents:"
+
+################################
+# Menu
+################################
+menu_file			= "&File"
+menuaction 			= "&Action"
+menutools 			= "&Tools"
+menuversion 		= "&Help"
+menuaboutabc 		= "&About Tribler"
+menuaboutabcmsg 	= "See Credits"
+menuchecklatestversion = "&Check for updates"
+menuchecklatestversionmsg = "Check Latest Version"
+menuwebinterfaceservice = "&Web Interface Service"
+menuwebinterfaceservicemsg = "Start/Stop and Config Web Interface Service"
+menucreatetorrent 	= "&Create Torrent"
+menucreatetorrentmsg = "Create .torrent file"
+menumanagefriends     = "&Manage Friends List"
+menumyinfo		= "My &Info"
+menuexit 			= "&Exit"
+menuexitmsg 		= "Close Program"
+menuglobaluploadsetting = "&Global Upload Setting"
+menuglobaluploadsettingmsg = "Setting global upload value"
+menuabcpreference 	= "&Preferences"
+menuabcpreferencemsg = "Set preferences"
+
+menu_addtorrent = "&Add Torrent"
+menu_addtorrentfile = "Add torrent from &file"
+menu_addtorrentnondefault = "Add torrent from file (to &non-default location)"
+menu_addtorrenturl = "Add torrent from &URL"
+
+menu_pauseall = "&Pause All"
+menu_stopall = "&Stop All"
+menu_unstopall = "&Queue all stopped torrents"
+menu_clearcompleted = "&Clear Completed"
+
+#########################
+# Library Overview
+#########################
+
+playFastDisabled	= "Give high priority and play ASAP"
+playFastEnabled		= "Back to normal mode"
+playerDisabled		= "Please wait until first part is available \n(Tribler is currently giving first part high priority)"
+playerEnabled 		= "Click to play"
+boostDisabled		= "Ask friends to boost your download"
+boostEnabled		= "Boosting"
+
+
+#########################
+# ToolBar
+#########################
+addtorrentfile_short = "Add Torrent File"
+addtorrentfiletonondefault_short = "Add Torrent File to non-default location"
+addtorrenturl_short = "Add Torrent from URL"
+
+tb_play_short   = "Play video"
+tb_resume_short = "Resume torrent"
+tb_resume_long 	= "Resume/Launch torrent"
+tb_reseedresume_short = "Reseed Resume"
+tb_reseedresume_long = "Resume without hashcheck, use only for seeding/reseeding."
+tb_pause_short 	= "Pause torrent"
+tb_pause_long 	= "Pause active torrent(s) (without releasing resources)"
+tb_stop_short 	= "Stop torrent"
+tb_stop_long 	= "Stop torrent (release resources)"
+tb_queue_short 	= "Queue torrent"
+tb_queue_long 	= "Force torrent into queue"
+tb_delete_short = "Remove torrent"
+tb_delete_long 	= "Remove torrent only from Tribler list"
+tb_spy_short 	= "Current Seed/Peer"
+tb_spy_long 	= "See current number of seed/peer of torrent on the tracker"
+tb_torrentdetail_short = "Torrent Details"
+tb_buddy_short  = "Manage Friends/Encountered Peers"
+tb_file_short   = "Show Download History"
+tb_video_short  = "Video Player"
+tb_dlhelp_short  = "Download Booster"
+
+tb_urm = "URM:"
+tb_maxsim = "Active:"
+
+##########################
+# Priority
+##########################
+# These are used for display in the list
+highest 		= "Highest"
+high 			= "High"
+normal 			= "Normal"
+low 			= "Low"
+lowest 			= "Lowest"
+
+# These are used for menus
+rhighest 		= "H&ighest"
+rhigh 			= "&High"
+rnormal 		= "&Normal"
+rlow 			= "&Low"
+rlowest 		= "L&owest"
+
+###################################################
+# Seeding Setting
+###################################################
+uploadoptforcompletedfile = "Upload option for completed files"
+unlimitedupload = "Unlimited seeding"
+continueuploadfor = "Continue seeding for"
+untilratio 		= "Seeding until UL/DL ratio = "
+uploadsetting 	= "Upload Setting"
+maxuploads 		= "Maximum uploads:"
+maxuploadrate 	= "Maximum upload rate:"
+maxoveralluploadrate = "Maximum overall upload rate:"
+whendownload 	= "when downloading"
+whennodownload 	= "when no downloading"
+
+maxdownloadrate = "Maximum download rate:"
+maxoveralldownloadrate = "Maximum overall download rate:"
+
+zeroisunlimited = "(0 = Unlimited)"
+zeroisauto 		= "(0 = Auto)"
+
+uploadrateintwarning = "Only integer allowed in Maximum upload rate setting"
+uploadrateminwarning = "Minimum upload rate is 3kB/s or 0 for unlimited upload rate"
+uploadrateminwarningauto = "Minimum upload rate is 3kB/s or 0 for auto upload rate"
+
+#Common option for t4t and g2g
+default_setting = "default"
+seed_sometime = "Seeding for"
+seed_hours = "hours"
+seed_mins = "minutes"
+no_seeding = "No seeding"
+
+#Seeding option texts for tit-4-tat
+tit-4-tat = "tit-4-tat: (Forgets about uploads)"
+no_leeching = "Seed until UL/DL ratio > 1.0 (no Bittorrent leeching)"
+unlimited_seeding = "Unlimited seeding"
+
+#Seeding option texts for give-2-get
+give-2-get = "give-2-get: (Remembers every upload)"
+seed_for_large_ratio = "Seed only to peers with UL/DL ratio >"
+boost__reputation = "Unlimited seeding (Boost your reputation)"
+
+
+############################################
+# Units
+############################################
+Byte 			= "B"
+KB 				= "KB"
+MB 				= "MB"
+GB 				= "GB"
+TB 				= "TB"
+
+week 			= "W"
+day 			= "D"
+hour 			= "H"
+minute 			= "M"
+second 			= "S"
+l_week 			= "w"
+l_day 			= "d"
+l_hour 			= "h"
+l_minute 		= "m"
+l_second 		= "s"
+
+############################################
+# Tribler Tweak
+############################################
+up 				= "upload speed"
+down 			= "download speed"
+columns 		= "Columns"
+column 			= "Column :"
+displayname 	= "Column Name :"
+columnwidth 	= "Column Width :"
+eta				= "Estimated time needed to complete: "
+
+customizetoolbar = "Customize Toolbar"
+
+##############################################
+# Tribler Detail Frame
+##############################################
+
+networkinfo 	= "Network Info"
+fileinfo 		= "File Info"
+torrentinfo     = "Torrent Info"
+geoinfo         = "Geographic Info"
+helperinfo	= "Download Booster"
+
+dnumconnectedseed = "# Connected seed :"
+dseenseed 		= "# Seen seed"
+dnumconnectedpeer = "# Connected peer :"
+dseeingcopies 	= "# Seeing copies :"
+davgpeerprogress = "Avg peer progress :"
+ddownloadedsize = "Downloaded size :"
+duploadedsize 	= "Uploaded size : "
+dtotalspeed 	= "Total speed : "
+dportused 		= "Port used : "
+updateseedpeer 	= "Update #Seed/#Peer"
+manualannounce 	= "Manual Announce"
+externalannounce = "External Announce"
+finishallocation = "Finish Allocation"
+spewoptunchoke 	= "Optimistic Unchoke"
+spewIP 			= "IP"
+spewlr 			= "Local/Remote"
+spewinterested 	= "Interested"
+spewchoking 	= "Choking"
+spewinteresting = "Interesting"
+spewchoecked 	= "Choked"
+spewsnubbed 	= "Snubbed"
+spewdownloaded 	= "Downloaded"
+spewuploaded 	= "Uploaded"
+spewcompleted 	= "Completed"
+spewpeerdownloadspeed = "Peer Download Speed"
+entertrackerannounceurl = "Enter tracker anounce URL:"
+TOTALS 			= "TOTALS:"
+KICKED 			= "KICKED"
+BANNED 			= "BANNED"
+detailline1 	= "currently downloading %d pieces (%d just started), %d pieces partially retrieved"
+detailline2 	= "%d of %d pieces complete (%d just downloaded), %d failed hash check"
+
+country_name 		= "Country"
+country_code 		= "Country Code"
+city 				= "City"
+latitude 			= "Latitude"
+longitude 			= "Longitude"
+coordinate 			= "Coordinate"
+peer_active			= "Active"
+peer_inactive		= "Inactive"
+name			= "Name"
+permid			= "PermID"
+mypermid 		= "My PermID"
+pasteinvitationemail = "Your friend should provide you the following information by sending you an invitation:"
+ipaddress		= "IP"
+icon			= "Icon"
+#nickname_help           = "Input the friend's nickname or whatever you'd like to identify him/her"
+#friendsipaddr_help      = "Input the friend's IP address, e.g. 202.115.39.65"
+#friendsport_help        = "Input the friend's listening port number"
+#friendspermid_help      = "Input the friend's PermID"
+#friendsicon_help        = "Input full path of the friend's icon"
+nicknameempty_error     = "Name is empty"
+friendsport_error       = "Port is not a number"
+friendspermid_error     = "PermID must be given (in BASE64, single line)"
+fiendsiconnotfound_error= "Icon file does not exist"
+friendsiconnot32bmp_error= "Icon file is not a 32x32 BMP"
+friendsiconnotbmp_error = "Icon file is not BMP"
+myinfo			= "My information"
+myinfo_explanation	= "Copy and paste this information in an email to your friends, so they can add you to their Friends List in Tribler."
+invitation_body = "Hi,\r\n\r\nI am using Tribler (http://tribler.org) and want to ask you to do the same and add me as a friend. To do so, start Tribler, click on Friends, then click on the Add Friends button, and paste the following information:\r\n\r\n"
+invitation_subject = "Friendship invitation on Tribler"
+invitationbtn = "Invite friends"
+dlhelpdisabledstop      = "Download Booster is disabled because the torrent is stopped"
+dlhelpdisabledhelper    = "Download Booster is disabled because you are a helper"
+dlhelphowto1  		    = "You can only request mutual (two way) friends to boost your downloads."
+dlhelphowto2		    = "\nMore info: \nTo use the download booster you must make friends with other Tribler users, and they must make friends with you. To make friends, use the 'Add as friend' button in the Persons overview or the 'Invite Friends' and 'Add Friends' button in the Friends overview."
+friends			= "Friends"
+helpers 		= "Helpers"
+availcandidates         = "Available Candidates"
+requestdlhelp		= "Request Help ->"
+requestdlhelp_help	= "Ask friends to help in downloading this torrent"
+stopdlhelp		= "<- Stop Help"
+stopdlhelp_help		= "Stop friends' help"
+helping_friend		= "Helping "
+helping_stopped		= "Helping was stopped remotely, please remove torrent"
+
+#####################################################
+# Meta info frame
+#####################################################
+fileinfo0_text  = "Filename"
+fileinfo1_text  = "Size"
+fileinfo2_text  = "Progress"
+fileinfo3_text  = "MD5"
+fileinfo4_text  = "CRC-32"
+fileinfo5_text  = "SHA-1"
+fileinfo6_text  = "ED2K"
+
+encoding        = "Encoding :"
+
+filename 		= "File name :"
+destination 	= "Destination :"
+
+directoryname 	= "Directory name :"
+file 			= "File"
+progress 		= "Progress"
+infohash 		= "Info Hash :"
+pieces 			= "Pieces : "
+str1 			= "%s (%s bytes)"
+str2 			= "%i (%s bytes each)"
+announceurl 	= "Announce URL :"
+announceurls 	= "Announce URLs"
+tier 			= "Tier "
+single 			= "Single:"
+likelytracker 	= "Likely Tracker :"
+comment 		= "Comments :"
+creationdate 	= "Creation Date :"
+filesize 		= "Filesize"
+archivesize 	= "Archive Size"
+
+########################################################
+# ABCOptionDlg
+#######################################################
+networksetting 	= "Network"
+portnumber  	= "Port:"
+portsetting 	= "Ports"
+minportnumber 	= "Minimum port : "
+maxportnumber 	= "Maximum port :"
+portrangewarning = "Minimum port cannot be greater than maximum port"
+randomport 		= "Randomize Ports"
+kickban 		= "Kick/Ban clients that send you bad data"
+security 		= "Don't allow multiple connections from the same IP"
+scrape 			= "Retrieve scrape data"
+internaltrackerurl = "URL of internal tracker"
+
+scrape_hint_line1 = "Automatically retrieve the total number of seeds/peers"
+scrape_hint_line2 = "connected to the tracker"
+scrape_hint_line3 = "(rather than just the number of connected seeds/peers)"
+scrape_hint_line4 = ""
+scrape_hint_line5 = "Note: This can put an additional burden on trackers"
+scrape_hint_line6 = "      and is therefore disabled by default"
+
+global_uprate_hint_line1 = "Amount of bandwidth to distribute between"
+global_uprate_hint_line2 = "uploading torrents"
+global_uprate_hint_line3 = ""
+global_uprate_hint_line4 = "Note: Each torrent will always get a minimum"
+global_uprate_hint_line5 = "      of 3KB/s"
+
+choose_language = "Language: "
+recategorize = "Recategorize all torrents: "
+recategorize_button = "Recategorize now"
+choosevideoplayer = "Choose video player"
+choosevideoanalyser = "Locate FFMPEG"
+
+queuesetting 	= "Queue"
+maxnumsimul     = "Maximum number of active torrents"
+trignexttorrent = "Consider torrents active if they are:"
+after_downloading = "Downloading"
+after_seeding 	= "Downloading or Seeding"
+prioritizelocal = "Don't count torrents with local settings towards global limit"
+fastresume 		= "Fast Resume (also enables File Selector)"
+
+skipcheck       = "Skip hashcheck for completed torrents"
+skipcheck_hint_line1 = "Don't conduct a hashcheck for torrents"
+skipcheck_hint_line2 = "that have already completed."
+
+fastresume_hint_line1 = "Automatically resume torrents that have already"
+fastresume_hint_line2 = "conducted a hashcheck."
+fastresume_hint_line3 = ""
+fastresume_hint_line4 = "Note: This option is required in order to set"
+fastresume_hint_line5 = "      priorities for individual files within"
+fastresume_hint_line6 = "      a multi-file torrent."
+
+
+displaysetting  = "Display"
+miscsetting 	= "Misc."
+removebackuptorrent = "Remove .torrent backup file when using remove"
+confirmonexit 	= "Confirm on exit program"
+triblersetting  = "Tribler"
+corefuncsetting = "Core functionality"
+myinfosetting = "My information"
+torrentcollectsetting = "Torrent collecting"
+enablerecommender = "Enable Recommender"
+enabledlhelp    = "Enable Download Booster"
+enabledlcollecting = "Enable Torrent Collecting"
+myname		    = "My name (as broadcast to others):"
+maxntorrents    = "Max number of torrents to collect:"
+maxnpeers       = "Max number of peers to discover:"
+tc_threshold    = "Stop collecting more torrents if the disk has less than:"
+current_free_space = "current available space:"
+torrentcollectingrate = "Maximum rate of torrent collecting (Kbps):"
+myicon		   = "My Tribler icon (as broadcast to others):"
+setdefaultfolder = "Set default download folder"
+stripedlist 	= "Striped list"
+videosetting = "Video"
+
+choosedefaultdownloadfolder = "Choose a default folder for download files"
+maxsimdownloadwarning_line1 = "The maximum number of simultaneous downloading torrents"
+maxsimdownloadwarning_line2 = "must not be greater than the number of reserved ports"
+
+choosemovedir 	= "Choose a folder to move completed files to"
+movecompleted 	= "\"Clear Completed\" moves files to:"
+
+showtray 		= "Show in tray:"
+showtray_never 	= "Never"
+showtray_min 	= "When Minimized"
+showtray_always = "Always"
+showtray_only 	= "Only show in Tray"
+
+########################################################
+# ABCOptionDlg - Advanced Options
+#######################################################
+
+disksettings 	= "Disk"
+advanced 		= "Advanced"
+advsetting 		= "Advanced settings"
+changeownrisk 	= "(Under most circumstances, these settings do not need to be changed)"
+localip 		= "Local IP: "
+iptobindto 		= "IP to bind to: "
+minnumberofpeer = "Minimum number of peers: "
+diskalloctype 	= "Disk allocation type:"
+allocrate 		= "Allocation rate:"
+filelocking 	= "File locking:"
+extradatachecking = "Extra data checking:"
+maxfileopen 	= "Max files open:"
+maxpeerconnection = "Max peer connections:"
+reverttodefault = "Restore Defaults"
+bufferdisk 		= "Disk Buffering"
+buffer_read 	= "Read Cache"
+buffer_write 	= "Write Cache"
+ut_pex_maxaddrs1 = "Maximum number of addresses to accept"
+ut_pex_maxaddrs2 = "via peer exchange per client"
+flush_data 		= "Flush data to disk every"
+
+iphint_line1 	= "The IP reported to the tracker."
+iphint_line2 	= "(unless the tracker is on the same intranet as this client,"
+iphint_line3 	= " the tracker will autodetect the client's IP and ignore this"
+iphint_line4 	= " value)"
+
+bindhint_line1 	= "The IP the client will bind to."
+bindhint_line2 	= "Only useful if your machine is directly handling multiple IPs."
+bindhint_line3 	= "If you don't know what this is, leave it blank."
+
+minpeershint_line1 = "The minimum number of peers the client tries to stay connected with."
+minpeershint_line2 = ""
+minpeershint_line3 = "Do not set this higher unless you have a very fast connection and a lot of system resources."
+
+ut_pex_maxaddrs_hint_line1 = "When you meet other peers they can give you addresses of the peers they know."
+ut_pex_maxaddrs_hint_line2 = "This value sets the maximum number of gossiped addresses you accept from each peer."
+ut_pex_maxaddrs_hint_line3 = "Don't set this too high as these gossiped addresses are from an untrusted source"
+ut_pex_maxaddrs_hint_line4 = "(i.e. a random peer) and not the trustworthy tracker."
+
+alloctypehint_line1 = "How to allocate disk space:
+alloctypehint_line2 = ""
+alloctypehint_line3 = "'Normal' allocates space as data is received"
+alloctypehint_line4 = "'background' also adds space in the background"
+alloctypehint_line5 = "'pre-allocate' reserves space up front"
+alloctypehint_line6 = "'sparse' is only for filesystems that support it by default"
+
+allocratehint_line1 = "At what rate to allocate disk space when allocating in the background."
+allocratehint_line2 = ""
+allocratehint_line3 = "Set this too high on a slow filesystem and your download will slow to a crawl."
+
+lockinghint_line1 = "File locking prevents other programs (including other instances"
+lockinghint_line2 = "of BitTorrent) from accessing files you are downloading."
+
+doublecheckhint_line1 = "How much extra checking to do to make sure no data is corrupted."
+doublecheckhint_line2 = "Double-check requires higher CPU usage"
+doublecheckhint_line3 = "Triple-check also increases disk accesses"
+
+maxfileopenhint_line1 = "The maximum number of files to keep open at the same time."
+maxfileopenhint_line2 = "Please note that if this option is in effect,"
+maxfileopenhint_line3 = "files are not guaranteed to be locked."
+
+maxconnectionhint_line1 = "Some operating systems, (most notably Win9x/ME) combined"
+maxconnectionhint_line2 = "with certain network drivers, can only handle a limited"
+maxconnectionhint_line3 = "number of open ports."
+maxconnectionhint_line4 = ""
+maxconnectionhint_line5 = "If the client freezes, try setting this to 60 or below."
+
+
+
+recommendinstructions = "Double click on a torrent to start downloading; right click to delete or manually check health of the torrent"
+recommendfilter = "Don't show torrents with recommendation value less than"
+recommendfilterall = "(set to 0.0 to see all known torrents)"
+
+############################################################
+# BTMakeTorrentGUI
+############################################################
+btfilemakertitle 	= "Create Torrent"
+btmaketorrenttitle 	= "Make Torrent"
+maketorrentof 		= "Source :"
+dir 				= "Dir"
+add 				= "Add"
+remove 				= "Remove"
+announce			= "Tracker"
+announcelist 		= "Announce list :"
+copyannouncefromtorrent = "Copy tracker from torrent"
+createdby			= "Created By :"
+
+trackerinfo			= "Tracker Info"
+miscinfo			= "Misc. Info"
+
+selectdir			= "Select a directory"
+
+multiannouncehelp_line1="(A list of announces separated by commas or whitespace.
+multiannouncehelp_line2=" Trackers on the same line will be tried randomly."
+multiannouncehelp_line3=" All the trackers on one line will be tried before the trackers on the next.)"
+
+httpseeds		= "HTTP Seeds :"
+httpseedshelp = "(A list of HTTP seeds separated by commas or whitespace.)
+
+saveasdefaultconfig = "Save as default config"
+maketorrent 		= "Make Torrent"
+
+choosefiletouse 	= "Choose file or directory to use"
+choosedottorrentfiletouse = "Choose .torrent file to use"
+youmustselectfileordir = "You must select a\n file or directory"
+
+dirnotice_line1 	= "Do you want to make a separate .torrent"
+dirnotice_line2 	= "for every item in this directory?"
+yes 				= "Yes"
+yestoall			= "Yes to All"
+no 					= "No"
+notoall				= "No to All"
+playtime			= "Duration of video ([hh:]mm:ss)"
+addthumbnail		= "Thumbnail"
+useinternaltracker = "Use internal tracker"
+manualtrackerconfig = "Use additional trackers (you must add internal tracker URL)"
+
+###########################################################
+# BTcompletedirgui
+###########################################################
+directorytomake     = "Directory to build :"
+select              = "Select"
+piecesize           = "Piece size :"
+make                = "Make"
+errormustselectdir 	= "You must select a directory"
+btmakedirtitle 		= "Make Directory"
+checkfilesize 		= "Checking file sizes"
+building 			= "Building "
+
+#################################################
+# Timeouts
+#################################################
+timeout 			= "Timeouts"
+schedulerrulemsg 	= "Set timeout rules for torrents"
+setrule_line1 		= "Reduce a torrent's priority and force it into queue so"
+setrule_line2 		= "other torrents in queue won't be blocked when:"
+timeout_tracker  	= "Torrent can't connect for:"
+timeout_download 	= "Torrent can't download for:"
+timeout_upload   	= "Seeding torrent doesn't upload for:"
+minute_long 		= "Minutes"
+hour_long 			= "Hours"
+time 				= "Time"
+
+################################################################################################
+#(-Right-) Click Menu
+################################################################################################
+rHashCheck 			= "&Hash Check"
+rResume 			= "&Resume Torrent"
+rPlay				= "   &Play Video"
+rStop 				= "&Stop"
+rPause 				= "P&ause"
+rQueue 				= "&Queue"
+rRemoveTorrent 		= "Remove Torrent"
+rRemoveTorrentandFile = "Remove Torrent and File(s)"
+
+rChangeViewModusThumb= "Thumbnail view"
+rChangeViewModusList= "List view"
+
+
+############# FILE and LIBRARY
+rOptions            = "Options:"
+rDownloadSecretly   = "   Download and hide this from other Tribler users"
+rDownloadOpenly     = "   Download"
+rModerate           = "   Change info..."
+rModerateCat        = "   Change category"
+rRecommend          = "   Recommend to a friend..."
+rAdvancedInfo		= "   Advanced info..."
+
+# Arno: categories must be completely defined by category.conf,
+# not in the code
+
+############# LIBRARY
+rLibraryOptions     = "Library options:"
+rOpenfilename 	    = "   Open file"
+rOpenfiledestination= "   Open destination"
+rRemoveFromList	    = "   Remove from library"
+rRemoveFromListAndHD= "   Remove from library and harddisk"
+
+############# PERSONS and FRIENDS
+rAddAsFriend		= "   Add as friend"
+rRemoveAsFriend		= "   Remove this friend"
+rChangeInfo			= "   Change friend info"
+
+############# FRIENDS
+rFriendsOptions     = "Friends options:"
+rSendAMessage       = "   Send a message..."
+
+############# SUBSCRIPTIONS
+rChangeSubscrTitle  = "   Change title"
+rRemoveSubscr       = "   Remove subscription"
+
+
+################################################################################################
+# Mouse roll over
+################################################################################################
+
+############# FILE
+rNumberOfSeeders    = "Number of current uploaders (seeders)
+rNumberOfLeechers   = "Number of current downloaders (leechers)
+
+
+
+rcopyfilename       = "&Copy Filename"
+rcopypath           = "Copy &Path"
+
+rcopyfromlist       = "&Copy from list..."
+rexportfromlist     = "&Export torrent"
+rextractfromlist 	= "&Extract from List..."
+rclearmessage 		= "&Clear Message"
+rtorrentdetail 		= "&Torrent Details..."
+rcurrentseedpeer 	= "Current Seed/Peer"
+rchangedownloaddest = "Change Download Destination..."
+ropenfiledest 		= "Open &File..."
+ropendest 			= "&Open Destination..."
+rsuperseedmode 		= "Use Super-seed &Mode"
+rpriosetting 		= "&Priority Setting"
+rlocaluploadsetting = "&Local Settings..."
+
+openfiledest 	    = "Open File"
+opendest 			= "Open Destination"
+
+################################
+# BT status
+################################
+completed 			= "completed"
+completedseeding 	= "completed/sharing"
+working 			= "downloading"
+superseeding 		= "super-seeding"
+waiting 			= "waiting.."
+pause 				= "pause"
+queue 				= "queue"
+stopping 			= "stopping.."
+stop 				= "stopped"
+checkingdata		= "checking existing data"
+allocatingspace		= "allocating disk space"
+movingdata			= "moving data"
+connectingtopeers	= "connecting"
+
+##############################################
+# Web Interface Service
+#############################################
+cantopensocket 		= "Can't open socket"
+socketerror 		= "Socket Error!"
+inactive 			= "Webservice: Inactive"
+active 				= "Webservice: Active"
+toolbar_webservice  = "Webservice"
+webinterfacetitle 	= "Web Interface Service (version 3.0)"
+webip 				= "IP :"
+webport 			= "Port :"
+uniquekey 			= "Unique Key :"
+commandpermission 	= "Command Permissions"
+webquery 			= "Query"
+webdelete 			= "Delete"
+webadd 				= "Add"
+webqueue 			= "Queue"
+webstop 			= "Stop"
+webpause            = "Pause/Unpause"
+webresume 			= "Resume"
+websetparam			= "Set Parameters"
+webgetparam			= "Get Parameters"
+priority 			= "Priority"
+webclearallcompleted = "Clear all completed"
+webautostart 		= "Auto start web service when launching Tribler"
+startservice 		= "Start Service"
+stopservice 		= "Stop Service"
+warningportunder1024_line1 = "Ports below 1024 are normally used for system services"
+warningportunder1024_line2 = "Do you really want to use this port?"
+cantconnectabcwebinterface = "Unable to connect to Tribler web service"
+
+##############################################
+# Scrape Dialog
+##############################################
+cantreadmetainfo 	= "Can't read metainfo"
+cantgetdatafromtracker = "Can't get data from tracker"
+noannouncetrackerinmeta = "No announce tracker in your metainfo"
+warningscrapelessthanmin = "Please don't scrape more than once per minute."
+trackernoscrape     = "Tracker does not support scraping"
+seed 				= "Seed :"
+peer 				= "Peer :"
+status 				= "Status :"
+scraping 			= "Scraping..."
+scrapingdone 		= "Scraping done"
+
+##############################################
+# Upload Rate Maximizer
+##############################################
+autostart_threshold	= "Start a new torrent if upload is more than"
+autostart_delay		= "below the global limit for at least"
+
+activetorrents      = "Active Torrents"
+autostart           = "Auto Start"
+
+dynmaxuprate 		= "Adjust upload rate for network overhead"
+dynrate 			= "(Dynamic Rate = Global Upload Rate - DownCalc - ConnectCalc)"
+downcalc_left 		= "DownCalc = "
+downcalc_top 		= "Download Rate"
+downcalc_bottom 	= " * Download Rate + "
+connectcalc_left 	= "ConnectCalc = "
+connectcalc_top 	= "(Seeds + Peers)"
+connectcalc_bottom 	= " * (Seeds + Peers) + "
+
+errorlanguagefile_line1 = "Your language file is missing at least one string."
+errorlanguagefile_line2 = "Please check to see if an updated version is available."
+restartabc 			= "(takes effect next time Tribler is opened)"
+
+messagelog 			= "Message Log"
+clearlog 			= "Clear Log"
+date 				= "Date"
+
+close_title 		= "Closing"
+
+noportavailable 	= "Couldn't find an available port to listen on"
+tryotherport        = "Would you like Tribler to try using another port?"
+
+column4_text 		= "Title"
+column5_text 		= "Progress"
+column6_text 		= "BT Status"
+column7_text 		= "Priority"
+column8_text 		= "ETA"
+column9_text 		= "Size"
+column10_text 		= "DL Speed"
+column11_text 		= "UL Speed"
+column12_text 		= "%U/D Size"
+column13_text 		= "Message"
+column14_text 		= "Seeds"
+column15_text 		= "Peers"
+column16_text 		= "Copies"
+column17_text 		= "Peer Avg Progress"
+column18_text 		= "DL Size"
+column19_text 		= "UL Size"
+column20_text 		= "Total Speed"
+column21_text 		= "Torrent Name"
+column22_text 		= "Destination"
+column23_text 		= "Seeding Time"
+column24_text 		= "Connections"
+column25_text		= "Seeding Option"
+
+savecolumnwidth 	= "Save column widths when resizing"
+showearthpanel		= "Show worldmap in detail window (higher CPU load)"
+
+errorinactivesingle_line1   = "Torrent must be inactive before proceeding"
+errorinactivesingle_line2	= "Stop this torrent?"
+
+errorinactivemultiple_line1   = "Torrents must be inactive before proceeding"
+errorinactivemultiple_line2	 = "Stop torrents?"
+
+disabletimeout 		= "Disable timeouts for this torrent"
+
+forcenewdir 		= "Always create new directory for multi-file torrents"
+
+forcenewdir_hint_line1 = "If this is enabled, a multi-file torrent will always"
+forcenewdir_hint_line2 = "be placed within its own directory."
+forcenewdir_hint_line3 = ""
+forcenewdir_hint_line4 = "If this is disabled, a multi-file torrent will be"
+forcenewdir_hint_line5 = "placed in its own directory only if no pieces"
+forcenewdir_hint_line6 = "of the file are already present to resume from."
+
+upnp 				= "UPnP"
+upnp_0 				= "Disabled"
+upnp_1 				= "Mode 1 (indirect via Windows)"
+upnp_2 				= "Mode 2 (indirect via Windows)"
+upnp_3				= "Mode 3 (direct via network)"
+tribler_warning			= "Tribler Warning"
+tribler_information		= "Tribler Information"
+tribler_startup_nonfatalerror	= "A non-fatal error occured during Tribler startup, you may need to change the network Preferences:  \n\n"
+tribler_upnp_error_intro	= "An error occured while trying to open the listen port "
+tribler_upnp_error_intro_postfix= " on the firewall."
+tribler_upnp_error1		= "request to the firewall failed."
+tribler_upnp_error2		= "request to firewall returned:  '"
+tribler_upnp_error2_postfix     = "'. "
+tribler_upnp_error3		= "was enabled, but initialization failed."
+tribler_upnp_error_extro	= " This will hurt the performance of Tribler.\n\nTo fix this, configure your firewall/router/modem or try setting a different listen port or UPnP mode in (advanced) network Preferences."
+tribler_unreachable_explanation = "Others cannot contact you over the Internet. This will hurt the performance of Tribler.\n\nTo fix this, configure your firewall/router/modem or try different UPnP settings in the advanced network preferences."
+currentdiscoveredipaddress = "Your discovered IP address"
+
+associate 			= "Associate with .torrent files"
+notassociated_line1 = "Tribler is not currently associated with .torrent files"
+notassociated_line2 = "Do you wish to use Tribler to open .torrent files?"
+errorassociating    = "Error associating Tribler with .torrent files"
+
+savelog				= "Save Log"
+savelogas			= "Save log file as..."
+error_savelog		= "Error writing log file"
+
+download_normal 	= "Download &Normally"
+download_never 		= "Download Ne&ver"
+download_later 		= "Download &Later"
+download_first 		= "Download &First"
+download_start		= "Start downloading"
+click_and_download	= "Click and Download"
+delete_torrent		= "The associated torrent file %s is not found on disk. Do you want to delete this entry from the Tribler database?"
+delete_dead_torrent = "Remove Torrent"
+
+###
+# Abbreviations in the status bar:
+###
+
+reachable_tooltip	= "Others can reach you, i.e. you are not firewalled. This is good"
+restart_tooltip		= "Please restart Tribler for your changes to take place"
+connecting_tooltip      = "Your current firewall status is being checked ..."
+unknownreach_tooltip	= "Others cannot reach you. This is not good. Click to learn more."
+abbrev_loaded		= "L:"
+abbrev_running		= "R:"
+abbrev_pause		= "P:"
+abbrev_downloading	= "D:"
+abbrev_seeding		= "S:"
+abbrev_connections	= "CX:"
+abbrev_down			= "D:"
+abbrev_up 			= "U:"
+discover_peer   = "# Peers:"
+discover_file   = "# Files:"
+
+
+alloc_normal		= "normal"
+alloc_background	= "background"
+alloc_prealloc		= "pre-allocate"
+alloc_sparse		= "sparse"
+
+lock_never			= "no locking"
+lock_writing		= "lock while writing"
+lock_always			= "lock always"
+
+check_none			= "no extra checking"
+check_double		= "double-check"
+check_triple		= "triple-check"
+
+nolimit				= "no limit"
+
+automatic			= "Automatic"
+loopback			= "Loop Back"
+
+move_up				= "Move Up"
+move_down			= "Move Down"
+
+interfacemode		= "Interface mode:"
+mode_simple			= "Simple"
+mode_intermediate	= "Intermediate"
+mode_expert			= "Expert"
+
+spew0_text          = "Optimistic Unchoke"
+spew1_text          = "IP"
+spew2_text          = "Local/Remote"
+spew3_text          = "Up"
+spew4_text          = "Interested"
+spew5_text          = "Choking"
+spew6_text          = "Down"
+spew7_text          = "Interesting"
+spew8_text          = "Choked"
+spew9_text          = "Snubbed"
+spew10_text         = "Downloaded"
+spew11_text         = "Uploaded"
+spew12_text         = "Completed"
+spew13_text         = "Peer Download Speed"
+spew14_text	    = "PermID"
+
+spew_direction_local = "L"
+spew_direction_remote = "R"
+
+color_startup		= "Not active"
+color_disconnected  = "Can't contact server"
+color_noconnections = "No connections"
+color_noincoming    = "No incoming connections"
+color_nocomplete    = "No complete copies"
+color_good          = "All good"
+
+color_stripe		= "Stripe color"
+
+torrentcolors		= "Torrent Colors"
+
+more				= "More..."
+
+trackererror_problemconnecting = "Problem connecting to tracker"
+trackererror_rejected          = "Rejected by tracker"
+trackererror_baddata           = "Bad data from tracker"
+
+###################
+#Rename Torrent Dlg
+###################
+rrenametorrent="Rename torrent"
+renametorrent="Rename torrent : "
+edittorname="Edit torrent name :"
+usenamefrom="Use name from"
+currenttorname="Current torrent name :"
+
+originalname="Original name :"
+torrentfilename=".torrent file name :"
+othername = "Other :"
+
+destname="Destination name :"
+
+copybtn="Copy"
+rendestwithtor="Also rename destination"
+rtwd = "Rename torrent with destination by default"
+
+###
+#######################################################
+# Change destination dialog
+#######################################################
+choosedowndest="Change download destination..."
+downdestloc="Set download directory location"
+downdirname="Set download directory name"
+downfilename="Set download file name"
+choosenewdestloc="Choose new download directory location"
+choosenewdirname="Choose new download directory name :"
+choosenewfilename="Choose new download file name :"
+totalsize="total size :"
+updatetorname="Rename torrent"
+choosenewdest="New download destination :"
+browsebtn="Browse"
+
+rentorwithdest="Also change title in list"
+
+#errors:
+errorinvalidpath="Invalid syntax in the path. \nTry to add a \\"
+errorinvalidwinunitname="This name cannot be used as a Windows unit name."
+
+suggestedname="Suggested corrected name :"
+invalidwinname="This name cannot be used as a Windows file or folder name."
+iconbadformat="The icon you selected is not in a supported format"
+
+#########
+#Other
+#########
+warningopenfile = "Torrent is not completed yet, are you sure you want to open it?"
+upgradeabc = "Your software is outdated. Would you like to visit http://tribler.org to upgrade?"
+upgradeabctitle = "Update to Tribler "
+mainpage = "Tribler Main Page"
+sharing_reputation_information_title = "Sharing reputation information"
+sharing_reputation_information_message = "This progress bar shows your sharing reputation. You will have faster video playback by sharing more. Leaving Tribler running will improve your sharing reputation."
+sharing_reputation_poor = "Your current sharing reputation is low! This could affect your download speed. Please leave Tribler running to improve this."
+
+#############
+#Make Torrent
+#############
+savedtofolderwithsource = "Torrent will be saved to folder containing source"
+notadir="The default download directory is a file"
+savetor="Torrent location"
+savetordefault="Save to default folder :"
+savetorsource="Save to folder containing source"
+savetorask="Ask where to save to"
+choosetordeffolder="Choose a default folder to save torrents"
+
+torrentfileswildcard = ".torrent files"
+allfileswildcard = "All Files"
+logfileswildcard = "Log Files"
+
+listfont = "List font:"
+choosefont = "Choose Font..."
+sampletext = "Sample Text, 0123456789"
+
+startnow = "Start seeding immediately"
+makehash_md5 = "MD5"
+makehash_crc32 = "CRC-32"
+makehash_sha1 = "SHA-1"
+makehash_optional = "Optional hashes:"
+createmerkletorrent = "Create Merkle torrent (Tribler-only feature)"
+createtorrentsig = "Create signature (only if PermIDs enabled)"
+
+diskfull = "Error: Not enough space left on the destination disk"
+diskfullthreshold = "Stop torrents if destination has less than:"
+
+changetitle = "Change title to"
+
+separator          = "Separator"
+buttons_available  = "Available toolbar buttons:"
+buttons_current    = "Current toolbar buttons:"
+buttons_add        = "Add"
+buttons_remove     = "Remove"
+buttons_update	   = "Update"
+buttons_edit	   = "Edit"
+
+customizecontextmenu      = "Customize Context Menu"
+menu_available  = "Available menu items:"
+menu_current    = "Current menu items:"
+
+lowuploadstart1 = "Start next torrent if upload speed stays"
+lowuploadstart2 = "below global limit for at least"
+
+
+#############
+#Torrent List
+#############
+torrent0_text	= "Torrent Name"
+torrent1_text	= "Content Name"
+torrent2_text	= "Recommendation"
+torrent3_text	= "Sources"
+torrent4_text	= "Leechers"
+torrent5_text	= "Seeders"
+torrent6_text	= "Injected"
+torrent7_text	= "Size"
+torrent8_text	= "Files"
+torrent9_text	= "Tracker"
+torrent10_text  = "Category"
+
+#############
+#My Preference List
+#############
+mypref0_text	= "Torrent Name"
+mypref1_text	= "Content Name"
+mypref2_text	= "Rank"
+mypref3_text	= "Size"
+mypref4_text	= "Last Seen"
+
+#############
+#Taste Buddy List
+#############
+buddy0_text	= "Friend"
+buddy1_text = "Name"
+buddy2_text = "IP"
+buddy3_text = "Similarity"
+buddy4_text = "Last Seen"
+buddy5_text = "Downloads"
+buddy6_text = "Connnected"
+buddy7_text = "Exchanged"
+
+#############
+#Tribler UI
+#############
+configcolumns = "Configure Columns"
+file_list_title = "Recommended Torrents"
+mypref_list_title = "My Download History"
+click_download = "Click and Download"
+start_downloading = "Start downloading "
+add_friend_notes = "Right click on a peer to add as a friend or delete it"
+delete = "Delete"
+download = "Download"
+checkstatus = "Check health"
+loading = "Loading ..."
+#############
+# Tribler activities
+#############
+act_upnp = "Opening firewall (if any) via UPnP"
+act_reachable = "Seeing if not firewalled"
+act_get_ext_ip_from_peers = "Asking peers for my IP address"
+act_meet = "Person connected: "
+act_got_metadata = "File discovered:"
+act_recommend = "Discovered more persons and files from"
+act_disk_full = "Disk is full to collect more torrents. Please change your preferences or free space on "
+act_new_version = "New version of Tribler available"
+
+#############
+#Tribler UI - ContentFrontPanel, Tribler 3.6
+#############
+item = "item"
+person_item = "person"
+page = "page"
+order_by = "Order by"
+swarmsize = "Popular"
+swarmsize_tool = "Order content by the number people in the swarm"
+recommended = "Recommended"
+recommendation = "Recommendation"
+recommendation_tool = "Order the content by how it's related to your taste"
+myhistory_tool = "Show the files you have recently downloaded"
+categories = "Categories"
+leecher = "leecher"
+leecher_tool = "%d downloaders"
+seeder = "seeder"
+seeder_tool = "%d uploaders"
+swarm_outdated_tool = "The tracker status is unknown"
+swarm_unavailable_tool = "The swarm status could not be queried"
+no_info = "No info"
+refresh = "Refresh info"
+refresh_tool = "Refresh the number of seeders and leechers in the swarm"
+size = "Size"
+size_tool = "Total size of content"
+tracker = "Tracker"
+created = "Created"
+last_checked = "Last checked"
+refreshing = "Refreshing"
+swarm = "Swarm"
+no_information = "No information"
+searching_content = "Searching for Tribler content..."
+delete_sure = "Are you sure you want to delete %s"
+delete_mypref_sure = "Are you sure you want to remove %s from your download history"
+recomm_relevance = "How much is it related to your taste"
+torrent_files = "Included files(%d)"
+
+#################
+# Tribler Video #
+#################
+videoplayererrortitle = "Tribler Video Error"
+videoplayerstartfailure = "Problem while starting video player:"
+videoplayernotfound = "Could not find video player:"
+videoplayernotfoundfor = "Could not find video player for file:"
+videoanalysernotfound = "Could not find video analyser:"
+
+# PREFERENCES/VIDEO MENU DOES NOT EXIST ANYMORE
+# videoanalyserwhereset = "Set it to FFMPEG in the Preferences / Video menu"
+videoanalyserwhereset = ""
+
+videonotcomplete = "The video cannot yet be played as it has not been completely downloaded:"
+notvideotorrent = "Nothing to play, no video files found in torrent"
+videoplaycontentnotfound = "Cannot find video file on disk"
+selectvideofiletitle = "Select video file"
+selectvideofile = "Select which video file to play:\n"
+playback_section = "Playback options"
+analysis_section = "Video-analysis options"
+videoplayer_default_path = "Path to external video player:"
+videoanalyserpath = "Path to the FFMPEG video analyser:"
+playback_mode = "Which video player to use: "
+playback_external_default = "Use external player specified below"
+playback_internal = "Use internal player (recommended)"
+playback_external_mime = "Use default Windows player"
+selectbandwidthtitle = "Enter your Internet speed"
+selectdlulbwprompt = "Your download/upload bandwidth is"
+selectdlulbwexplan = "For optimal performance, Tribler needs to know your Internet connection speed. Please specify it below. 'xxxx' means any, so if you have 512/256 kbps subscription, select 'xxxx/256 kbps'"
+savemedia = "Save content as"
+vodwarntitle = "Play As Soon As Possible"
+vodwarngeneral = "Be warned that Tribler Video-On-Demand unfortunately only works if you have high upload bandwidth and/or a lot of people are offering the video for download. It also won't work for some file types (e.g. .mov) as they are meant to be played from disk and not incrementally from the network as Tribler VOD does, sorry. But please give it a spin!"
+livewarntitle = "Play Live Stream"
+livewarngeneral = "You are about to play a live video stream that probably needs all your upload bandwidth"
+vodwarnbitrateunknown = ""
+vodwarnbitrateinsufficient = ""
+vodwarnbitrateinsufficientmeasured = ""
+vodwarnmov = ""
+vodwarnconclusionno = ""
+vodwarnbitratesufficient = ""
+vodwarnconclusionyes = ""
+
+vodwarntitle_old = "Experimental Feature Warning"
+vodwarngeneral_old = "Tribler Video-On-Demand is a highly experimental feature that allows you to watch videos while they are downloading, given you have sufficient upload bandwidth and/or a lot of people are offering the video for download. "
+vodwarnbitrateunknown_old = "The video you have selected has a unknown bitrate. "
+vodwarnbitrateinsufficient_old = "The video you have selected has a bitrate of %s KB/s, and your upload bandwidth is just %s. "
+vodwarnbitrateinsufficientmeasured_old = "The video you have selected has a bitrate of %s KB/s, and your best measured upload bandwidth is just %s. "
+vodwarnmov_old = "The selected video is a .MOV which usually cannot be played on demand. "
+vodwarnconclusionno_old = "So it's not clear whether there is enough bandwidth to watch it."
+vodwarnbitratesufficient_old = "The video you have selected has a bitrate of %s KB/s, and your upload bandwidth is %s. "
+vodwarnconclusionyes_old = "So you should be able to play it, but keep in mind this is highly experimental!"
+
+vodwhataboutothertorrents = "What to do with other downloads? \n"
+vodrestartothertorrents = "Stop all others and resume them afterwards (recommended)"
+vodstopothertorrents = "Stop all other downloads"
+vodleaveothertorrents = "Leave other downloads running"
+
+vodwarnprompt = "Continue?"
+vodwarnprompt_old = "Would you like to continue?"
+
+
+unlimited = "unlimited"
+bitrateprompt = "Bitrate:"
+unknown = "unknown"
+doesnotapply = "n/a"
+videoposition = "Position:"
+videoprogress = "Progress:"
+playprompt = "Play"
+pauseprompt = "Pause"
+fullscreen = "Fullscreen"
+volumeprompt = "Volume:"
+backtocontentview = "Back to Content View"
+vodprogress = "Progress:"
+launchvideoplayer = "Launch Video Player"
+videoserverservefailure = "Error serving video to player, probably the player does not understand the video format or cannot play it from the network."
+videoserverservefailureadvice = "Please wait until the download is complete and try again, or select a different player in Preferences/Video."
+downloading = "Active"
+
+#############
+#Tribler UI - Profile View, Tribler 4.0.0
+#############
+nothingToDo = "You are optimal here!!"
+profileDetails_Overall_description = "You are a: -current level- \n- Beginner\n- Experienced\n- Top User\n- Master"
+# --- Recommendation quality
+profileDetails_Quality_description = "Based on the files you have downloaded over time, Tribler recommends other files that are likely to be interesting to you. \n\nSo far you have%s downloaded %s files."
+profileDetails_Quality_descriptionz_onlyword = " only"
+profileDetails_Quality_improve = "* Download more files to increase the quality of Tribler recommendations."
+# --- Discoverd Files
+profileDetails_Files_description = "So far, you have discovered %s files."
+profileDetails_Files_improve = "* Stay online longer to discover more files. \n\n* You have set your maximum to %s files. If you have reached this limit please set it higher."
+# --- Discoverd Persons
+profileDetails_Persons_description = "So far, you have discovered %s people."
+profileDetails_Persons_improve = "* Stay online longer and you will discover more people."
+# --- Optimal Download Speed
+profileDetails_Download_info = "You are not using your download speed optimally. To increase, follow the instructions."
+profileDetails_Download_UpSpeed = "Your upload speed is set to %d KB/s. Limiting your upload speed also limits your download speed."
+profileDetails_Download_UpSpeedMax = "Your upload speed is set to 'unlimited'. That's good."
+profileDetails_Download_UpSpeed_improve = "* Increase the upload speed limit in your preferences (for -Play ASAP- mode you need at least 64 KB/s). "
+profileDetails_Download_UpSpeedMax_improve = "* For an improved performance, you can also increase the number of upload slots in Preferences. "
+#  profileDetails_Download_UpSlots = "You set up a number of %d slots for upload."
+#  profileDetails_Download_UpSlotsMax = "You set up an unlimited number of slots for upload. That's good."
+#  profileDetails_Download_DlSpeed = "Your download speed is set to %d KB/s."
+#  profileDetails_Download_DlSpeedMax = "Your download speed is set to unlimited. That's good."
+profileDetails_Download_Friends = "At the moment you have %d friends. If you make more friends you can help in boosting each others download speeds."
+profileDetails_Download_Friends_improve = "* Invite your friends, family, and colleagues by e-mail, to start tribler too and let them add you as a friend."
+profileDetails_Download_VisibleYes = "You are currently accessible by other people."
+profileDetails_Download_VisibleYes_improve = "* Your friends should also be accessible. For that, please guide them to www.tribler.org for instructions."
+#profileDetails_Download_VisibleNo = "Other users are not able to connect to you, because your modem/router blocks them."
+profileDetails_Download_VisibleNo = "Other users are not able to connect to you, because your modem/router (%s) blocks them."
+profileDetails_Download_VisibleNo_improve = "* You have to open a port on your modem/router to enable other users to connect to you. This will almost double your possible download speed. Read more on www.tribler.org for instructions."
+# --- Network Reach
+profileDetails_Presence_info = "If you want to increase your network reach, follow the instructions."
+#profileDetails_Presence_Friends = profileDetails_Download_Friends
+#profileDetails_Presence_Friends_improve = profileDetails_Download_Friends_improve
+profileDetails_Presence_Sharingratio = "Your overall sharing ratio is %d. This means that you download more from others than you upload to them."
+profileDetails_Presence_Sharingratio_improve = "* To reach a fair sharing ratio, you should share your files longer. "
+profileDetails_Presence_VersionNewer = "You are using a newer version of Tribler (%s) than on website (%s)."
+profileDetails_Presence_VersionNewer_improve = "* Check the website for news and updates at %s"
+profileDetails_Presence_VersionOlder = "You are using an old version of Tribler (%s) and not taking advantage of the new features available. "
+profileDetails_Presence_VersionOlder_improve = "* Update to the newest version %s at %s"
+profileDetails_Presence_VersionCurrent = "You are up to date! The current version client is %s."
+profileDetails_Presence_VersionCurrent_improve = "* Check the website for news and updates at %s"
+profileDetails_Presence_VersionUnknown = "unknown"
+profileDetails_Presence_VersionError = "Your current client version is %s."
+profileDetails_Presence_VersionError_improve = "* Check the website for news and updates at %s"
+
+###############
+# Tribler UI - persons.py, Tribler 3.7
+##############
+peer_status_tooltip = "Status of person based on last time seen"
+peer_friend_tooltip = "This person is a friend of yours. Click to remove friendship."
+peer_nofriend_tooltip = "Click to make this person your friend."
+peer_connected_times_tooltip = "Successful connections made to this person."
+peer_buddycast_times_tooltip = "Specific Tribler messages exchanged with this person."
+peer_similarity_tooltip = "Similarity between you and this person based on the download history."
+commonFiles    = "   Common files (%d)"
+alsoDownloaded = "Also downloaded (%d/%s)"
+peer_common_files_tooltip = "Files that you and this person have in common."
+peer_other_files_tooltip = "Other files that this person has downloaded."
+
+#################
+# Notification  #
+#################
+notification_download_complete = "Download Complete"
+notification_finished_seeding  = "Finished Seeding"
+
+#############
+#Tribler UI - Persons View, Tribler 4.0.0
+#############
+persons_view_no_data = "No people encountered yet"
+
+torrentcollectsleep = "Seconds between downloading torrents from RSS:"
+buddycastsubscription = "Discover content via other Tribler users"
+web2subscription = "Discover content from YouTube and LiveLeak"
+filesdefaultsearchweb2txt = "search files, YouTube and LiveLeak"
+filesdefaultsearchtxt = "search all files"
+rssurldefaulttxt = "Paste your RSS link here"
+
+vlc_linux_start_bug_title = "No flash video streaming on Ubuntu Linux with VLC"
+vlc_linux_start_bug = "The current Ubuntu version of the VLC video player cannot stream Youtube.com movies. So be warned, they will not start playing until they have been completely downloaded. We have submitted a patch to Ubuntu."
+going_search = "        Results: %d"
+#going_search = "Searching for '%s'... (%d results)"
+finished_search = "Finished search '%s'. (%d results)"
+search_web2 = "Web movies (%d results)"
+search_torrent = "Discovered files (%d results)"
+search_peers = "Discovered persons (%d results)"
+search_friends = "Friends (%d results)"
+search_library = "Library files (%d results)"
+search_remote = "Tribler network (%d results)"
+# search buttons
+searchStop       = "stop searching"
+searchStopEnabled= "stopped searching"
+searchClear      = "clear results and browse all discovered files"
+help             = "Current sharing reputation : %2.2f"
+
+################
+#Tribler UI - Column headers Tribler 4.1.0
+#################
+# FILES
+C_filename    	= "Name of the file"
+C_filesize		= "Total size"
+C_popularity    = "Popularity of the file"
+C_creationdate	= "Creation date"
+C_uploaders		= "Number of uploaders (seeders)"
+C_downloaders 	= "Number of downloaders (leechers)"
+C_recommfiles	= "Fit to your taste (top20 of discovered files)"
+C_source		= "Source of file"
+# PERSONS
+C_personname	= "Name of the persons"
+C_status		= "Last time you connected with this person"
+C_discfiles		= "Number of files discovered by this person"
+C_discpersons	= "Number of persons discovered by this person"
+C_recommpersons	= "Fit to your taste (top20 of discovered persons)"
+C_friends		= "Friends of yours"
+# LIBRARY
+C_progress		= "Progress of downloads"
+C_downspeed		= "Download speed"
+C_upspeed		= "Upload speed"
+C_downupspeed		= "Current download and upload speed"
+C_message		= "Status of downloads (no sorting)"
+C_info			= "Other info (no sorting)"
+# FRIENDS
+C_friendname	= "Name of your friends"
+C_friendstatus	= "Last time you connected with your friends"
+C_helping		= "Whether friend is boosting your downloads (no sorting)"
+C_remove		= "Remove file from Library and Disk"
+
+# TopNList discovered peers in profile view - Tribler 4.1.0
+
+totalUp         = "Up: %s"
+totalDown       = "Down: %s"
+
+# Core download status
+DLSTATUS_ALLOCATING_DISKSPACE = "initializing"
+DLSTATUS_WAITING4HASHCHECK = "initializing"
+DLSTATUS_HASHCHECKING = "checking old data"
+DLSTATUS_DOWNLOADING = "downloading"
+DLSTATUS_SEEDING = "completed/sharing"
+DLSTATUS_STOPPED = "stopped"
+DLSTATUS_STOPPED_ON_ERROR = "stopped/error"
+
+duplicate_download_msg = "You are already downloading this torrent, see the My Files section."
+duplicate_download_title = "Duplicate download"
+
+invalid_torrent_no_playable_files_msg = "You are attempting to play files from a torrent that does not contain any playable files."
+invalid_torrent_no_playable_files_title = "Invalid torrent"
+
+#
+# Friendship
+#
+question = 'Question'
+addfriendfillin = "Do you want to add\n%s\nas your friend?'
+
+################
+#Tribler UI - Upload tab
+#################
+peer_ip = "Peer IP"
+tribler_name = "Tribler name"
+curr_ul_rate = "Current upload rate"
+ul_amount = "Amount of MBytes uploaded"
+
diff --git a/instrumentation/next-share/BaseLib/Lang/lang.py b/instrumentation/next-share/BaseLib/Lang/lang.py
new file mode 100644
index 0000000..a632879
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Lang/lang.py
@@ -0,0 +1,209 @@
+# Written by ABC authors and Arno Bakker
+# see LICENSE.txt for license information
+import wx
+import sys
+import os
+
+from traceback import print_exc, print_stack
+from cStringIO import StringIO
+
+from BaseLib.__init__ import LIBRARYNAME
+from BaseLib.Utilities.configreader import ConfigReader
+from BaseLib.Core.BitTornado.__init__ import version_id
+
+################################################################
+#
+# Class: Lang
+#
+# Keep track of language strings.
+#
+# Lookups occur in the following order:
+# 1. See if the string is in user.lang
+# 2. See if the string is in the local language file
+# 3. See if the string is in english.lang
+#
+################################################################
+class Lang:
+    def __init__(self, utility):
+        self.utility = utility
+        
+        filename = self.utility.config.Read('language_file')
+        
+        
+        langpath = os.path.join(self.utility.getPath(), LIBRARYNAME,  "Lang")
+        
+        sys.stdout.write("Setting up languages\n")
+        sys.stdout.write("Language file: " + str(filename) + "\n")
+        
+        # Set up user language file (stored in user's config directory)
+        self.user_lang = None
+        user_filepath = os.path.join(self.utility.getConfigPath(), 'user.lang')
+        self.user_lang = ConfigReader(user_filepath, "ABC/language")
+
+        # Set up local language file
+        self.local_lang_filename = None
+        self.local_lang = None
+        local_filepath = os.path.join(langpath, filename)
+        
+        if filename != 'english.lang' and existsAndIsReadable(local_filepath):
+            self.local_lang_filename = filename
+            # Modified
+            self.local_lang = wx.FileConfig(localFilename = local_filepath)
+            self.local_lang.SetPath("ABC/language")
+            #self.local_lang = ConfigReader(local_filepath, "ABC/language")
+        
+        # Set up english language file
+        self.english_lang = None
+        english_filepath = os.path.join(langpath, 'english.lang')
+        if existsAndIsReadable(english_filepath):
+            self.english_lang = ConfigReader(english_filepath, "ABC/language")
+        
+        self.cache = {}
+        
+        self.langwarning = False
+        
+    def flush(self):
+        if self.user_lang is not None:
+            try:
+                self.user_lang.DeleteEntry("dummyparam", False)
+            except:
+                pass
+            self.user_lang.Flush()
+        self.cache = {}
+              
+    # Retrieve a text string
+    def get(self, label, tryuser = True, trylocal = True, tryenglish = True, giveerror = True):        
+        if tryuser and trylocal and tryenglish:
+            tryall = True
+        else:
+            tryall = False
+    
+        if tryall and label in self.cache:
+            return self.expandEnter(self.cache[label])
+    
+        if (label == 'version'):
+            return version_id
+        if (label == 'build'):
+            return "Build 17078"
+        if (label == 'build_date'):
+            return "Aug 27, 2010"
+        # see if it exists in 'user.lang'
+        if tryuser:
+            text, found = self.getFromLanguage(label, self.user_lang)
+            if found:
+                if tryall:
+                    self.cache[label] = text
+                return self.expandEnter(text)
+
+        # see if it exists in local language
+        if trylocal and self.local_lang is not None:
+            text, found = self.getFromLanguage(label, self.local_lang, giveerror = True)
+            if found:
+                if tryall:
+                    self.cache[label] = text
+                return self.expandEnter(text)
+
+        # see if it exists in 'english.lang'
+        if tryenglish:
+            text, found = self.getFromLanguage(label, self.english_lang)
+            if found:
+                if tryall:
+                    self.cache[label] = text
+                return self.expandEnter(text)
+
+        # if we get to this point, we weren't able to read anything
+        if giveerror:
+            sys.stdout.write("Language file: Got an error finding: "+label)
+            self.error(label)
+        return ""
+        
+    def expandEnter(self, text):
+        text = text.replace("\\r","\n")
+        text = text.replace("\\n","\n")
+        return text
+        
+    def getFromLanguage(self, label, langfile, giveerror = False):
+        try:
+            if langfile is not None:
+                if langfile.Exists(label):
+                    return self.getSingleline(label, langfile), True
+                if langfile.Exists(label + "_line1"):
+                    return self.getMultiline(label, langfile), True
+                
+                if giveerror:
+                    self.error(label, silent = True)
+        except:
+            fileused = ""
+            langfilenames = { "user.lang": self.user_lang, 
+                              self.local_lang_filename: self.local_lang, 
+                              "english.lang": self.english_lang }
+            for name in langfilenames:
+                if langfilenames[name] == langfile:
+                    fileused = name
+                    break
+            sys.stderr.write("Error reading language file: (" + fileused + "), label: (" + label + ")\n")
+            data = StringIO()
+            print_exc(file = data)
+            sys.stderr.write(data.getvalue())            
+                
+        return "", False
+        
+    def getSingleline(self, label, langfile):
+        return langfile.Read(label)
+    
+    def getMultiline(self, label, langfile):
+        i = 1
+        text = ""
+        while (langfile.Exists(label + "_line" + str(i))):
+            if (i != 1):
+                text+= "\n"
+            text += langfile.Read(label + "_line" + str(i))
+            i += 1
+        if not text:
+            sys.stdout.write("Language file: Got an error reading multiline string\n")
+            self.error(label)
+        return text
+        
+    def writeUser(self, label, text):
+        change = False
+        
+        text_user = self.get(label, trylocal = False, tryenglish = False, giveerror = False)
+        text_nonuser = self.get(label, tryuser = False, giveerror = False)
+               
+        user_lang = self.user_lang
+        
+        # The text string is the default string
+        if text == text_nonuser:
+            # If there was already a user string, delete it
+            # (otherwise, do nothing)
+            if text_user != "":
+                user_lang.Write("exampleparam", "example value")
+                user_lang.DeleteEntry(label)
+                change = True
+        elif text != text_user:
+            # Only need to update if the text string differs
+            # from what was already stored
+            user_lang.Write(label, text)
+            change = True
+        
+        return change
+        
+    def error(self, label, silent = False):
+        # Display a warning once that the language file doesn't contain all the values
+        if (not self.langwarning):
+            self.langwarning = True
+            error_title = self.get('error')
+            error_text = self.get('errorlanguagefile')
+            if (error_text == ""):
+                error_text = "Your language file is missing at least one string.\nPlease check to see if an updated version is available."
+            # Check to see if the frame has been created yet
+            if not silent and hasattr(self.utility, 'frame'):
+                # For the moment don't do anything if we can't display the error dialog
+                dlg = wx.MessageDialog(None, error_text, error_title, wx.ICON_ERROR)
+                dlg.ShowModal()
+                dlg.Destroy()
+        sys.stderr.write("\nError reading language file!\n")
+        sys.stderr.write("  Cannot find value for variable: " + label + "\n")
+
+def existsAndIsReadable(filename):
+    return os.access(filename, os.F_OK) and os.access(filename, os.R_OK)
diff --git a/instrumentation/next-share/BaseLib/Player/BaseApp.py b/instrumentation/next-share/BaseLib/Player/BaseApp.py
new file mode 100644
index 0000000..173a83c
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/BaseApp.py
@@ -0,0 +1,735 @@
+# Written by Arno Bakker, Choopan RATTANAPOKA, Jie Yang
+# see LICENSE.txt for license information
+""" Base class for Player and Plugin Background process. See swarmplayer.py """
+
+#
+# TODO: set 'download_slice_size' to 32K, such that pieces are no longer
+# downloaded in 2 chunks. This particularly avoids a bad case where you
+# kick the source: you download chunk 1 of piece X
+# from lagging peer and download chunk 2 of piece X from source. With the piece
+# now complete you check the sig. As the first part of the piece is old, this
+# fails and we kick the peer that gave us the completing chunk, which is the 
+# source.
+#
+# Note that the BT spec says: 
+# "All current implementations use 2 15 , and close connections which request 
+# an amount greater than 2 17." http://www.bittorrent.org/beps/bep_0003.html
+#
+# So it should be 32KB already. However, the BitTorrent (3.4.1, 5.0.9), 
+# BitTornado and Azureus all use 2 ** 14 = 16KB chunks.
+
+import os
+import sys
+import time
+import shutil
+from sets import Set
+
+from base64 import encodestring
+from threading import enumerate,currentThread,RLock
+from traceback import print_exc
+# Ric: added svc ext  
+from BaseLib.Video.utils import svcextdefaults
+
+if sys.platform == "darwin":
+    # on Mac, we can only load VLC/OpenSSL libraries
+    # relative to the location of tribler.py
+    os.chdir(os.path.abspath(os.path.dirname(sys.argv[0])))
+try:
+    import wxversion
+    wxversion.select('2.8')
+except:
+    pass
+import wx
+
+from BaseLib.__init__ import LIBRARYNAME
+from BaseLib.Core.API import *
+from BaseLib.Policies.RateManager import UserDefinedMaxAlwaysOtherwiseEquallyDividedRateManager
+from BaseLib.Utilities.Instance2Instance import *
+
+from BaseLib.Player.systray import *
+# from BaseLib.Player.Reporter import Reporter
+from BaseLib.Player.UtilityStub import UtilityStub
+from BaseLib.Core.Statistics.Status.Status import get_status_holder
+
+DEBUG = False
+RATELIMITADSL = False
+DOWNLOADSPEED = 300
+DISKSPACE_LIMIT = 5L * 1024L * 1024L * 1024L  # 5 GB
+DEFAULT_MAX_UPLOAD_SEED_WHEN_SEEDING = 75 # KB/s
+
+class BaseApp(wx.App,InstanceConnectionHandler):
+    def __init__(self, redirectstderrout, appname, appversion, params, single_instance_checker, installdir, i2iport, sport):
+        self.appname = appname
+        self.appversion = appversion
+        self.params = params
+        self.single_instance_checker = single_instance_checker
+        self.installdir = installdir
+        self.i2iport = i2iport
+        self.sport = sport
+        self.error = None
+        self.s = None
+        self.tbicon = None
+        
+        self.downloads_in_vodmode = Set() # Set of playing Downloads, one for SP, many for Plugin
+        self.ratelimiter = None
+        self.ratelimit_update_count = 0
+        self.playermode = DLSTATUS_DOWNLOADING
+        self.getpeerlistcount = 2 # for research Reporter
+        self.shuttingdown = False
+        
+        InstanceConnectionHandler.__init__(self,self.i2ithread_readlinecallback)
+        wx.App.__init__(self, redirectstderrout)
+
+        
+    def OnInitBase(self):
+        """ To be wrapped in a OnInit() method that returns True/False """
+        
+        # Normal startup
+        # Read config
+        state_dir = Session.get_default_state_dir('.'+self.appname)
+        
+        self.utility = UtilityStub(self.installdir,state_dir)
+        self.utility.app = self
+        print >>sys.stderr,self.utility.lang.get('build')
+        self.iconpath = os.path.join(self.installdir,LIBRARYNAME,'Images',self.appname+'Icon.ico')
+        self.logopath = os.path.join(self.installdir,LIBRARYNAME,'Images',self.appname+'Logo.png')
+
+        
+        # Start server for instance2instance communication
+        self.i2is = Instance2InstanceServer(self.i2iport,self,timeout=(24.0*3600.0)) 
+
+
+        # The playerconfig contains all config parameters that are not
+        # saved by checkpointing the Session or its Downloads.
+        self.load_playerconfig(state_dir)
+
+        # Install systray icon
+        # Note: setting this makes the program not exit when the videoFrame
+        # is being closed.
+        self.tbicon = PlayerTaskBarIcon(self,self.iconpath)
+        
+        # Start Tribler Session
+        cfgfilename = Session.get_default_config_filename(state_dir)
+        
+        if DEBUG:
+            print >>sys.stderr,"main: Session config",cfgfilename
+        try:
+            self.sconfig = SessionStartupConfig.load(cfgfilename)
+            
+            print >>sys.stderr,"main: Session saved port",self.sconfig.get_listen_port(),cfgfilename
+        except:
+            print_exc()
+            self.sconfig = SessionStartupConfig()
+            self.sconfig.set_install_dir(self.installdir)
+            self.sconfig.set_state_dir(state_dir)
+            self.sconfig.set_listen_port(self.sport)
+            self.configure_session()    
+
+        self.s = Session(self.sconfig)
+        self.s.set_download_states_callback(self.sesscb_states_callback)
+
+        # self.reporter = Reporter( self.sconfig )
+
+        if RATELIMITADSL:
+            self.ratelimiter = UserDefinedMaxAlwaysOtherwiseEquallyDividedRateManager()
+            self.ratelimiter.set_global_max_speed(DOWNLOAD,DOWNLOADSPEED)
+            self.ratelimiter.set_global_max_speed(UPLOAD,90)
+
+
+        # Arno: For extra robustness, ignore any errors related to restarting
+        try:
+            # Load all other downloads in cache, but in STOPPED state
+            self.s.load_checkpoint(initialdlstatus=DLSTATUS_STOPPED)
+        except:
+            print_exc()
+
+        # Start remote control
+        self.i2is.start()
+
+        # report client version
+        # from BaseLib.Core.Statistics.StatusReporter import get_reporter_instance
+        reporter = get_status_holder("LivingLab")
+        reporter.create_and_add_event("client-startup-version", [self.utility.lang.get("version")])
+        reporter.create_and_add_event("client-startup-build", [self.utility.lang.get("build")])
+        reporter.create_and_add_event("client-startup-build-date", [self.utility.lang.get("build_date")])
+
+    def configure_session(self):
+        # No overlay
+        self.sconfig.set_overlay(False)
+        self.sconfig.set_megacache(False)
+
+
+    def _get_poa(self, tdef):
+        """Try to load a POA - possibly trigger a GUI-thing or should the plugin handle getting it if none is already available?"""
+        
+        from BaseLib.Core.ClosedSwarm import ClosedSwarm,PaymentIntegration
+        print >>sys.stderr, "Swarm_id:",encodestring(tdef.infohash).replace("\n","")
+        try:
+            poa = ClosedSwarm.trivial_get_poa(self.s.get_state_dir(),
+                                              self.s.get_permid(),
+                                              tdef.infohash)
+            
+            poa.verify()
+            if not poa.torrent_id == tdef.infohash:
+                raise Exception("Bad POA - wrong infohash")
+            print >> sys.stderr,"Loaded poa from ",self.s.get_state_dir()
+        except:
+            # Try to get it or just let the plugin handle it?
+            swarm_id = encodestring(tdef.infohash).replace("\n","")
+            my_id = encodestring(self.s.get_permid()).replace("\n", "")
+            try:
+                # TODO: Support URLs from torrents?
+                poa = PaymentIntegration.wx_get_poa(None,
+                                                    swarm_id,
+                                                    my_id,
+                                                    swarm_title=tdef.get_name())
+            except Exception,e:
+                print >> sys.stderr, "Failed to get POA:",e
+                poa = None
+
+        try:
+            ClosedSwarm.trivial_save_poa(self.s.get_state_dir(),
+                                         self.s.get_permid(),
+                                         tdef.infohash,
+                                         poa)
+        except Exception,e:
+            print >> sys.stderr,"Failed to save POA",e
+            
+        if poa:
+            if not poa.torrent_id == tdef.infohash:
+                raise Exception("Bad POA - wrong infohash")
+
+        return poa
+
+
+    def start_download(self,tdef,dlfile,poa=None,supportedvodevents=None):
+        """ Start download of torrent tdef and play video file dlfile from it """
+        if poa:
+            from BaseLib.Core.ClosedSwarm import ClosedSwarm
+            if not poa.__class__ == ClosedSwarm.POA:
+                raise InvalidPOAException("Not a POA")
+            
+        # Free diskspace, if needed
+        destdir = self.get_default_destdir()
+        if not os.access(destdir,os.F_OK):
+            os.mkdir(destdir)
+
+        # Arno: For extra robustness, ignore any errors related to restarting
+        # TODO: Extend code such that we can also delete files from the 
+        # disk cache, not just Downloads. This would allow us to keep the
+        # parts of a Download that we already have, but that is being aborted
+        # by the user by closing the video window. See remove_playing_*
+        try:
+            if not self.free_up_diskspace_by_downloads(tdef.get_infohash(),tdef.get_length([dlfile])):
+                print >>sys.stderr,"main: Not enough free diskspace, ignoring"
+        except:
+            print_exc()
+        
+        # Setup how to download
+        dcfg = DownloadStartupConfig()
+
+        # CLOSED SWARMS
+        if poa:
+            dcfg.set_poa(poa)
+            print >> sys.stderr,"POA:",dcfg.get_poa()
+        else:
+            dcfg.set_poa(None)
+            
+        # Delegate processing to VideoPlayer
+        if supportedvodevents is None:
+            supportedvodevents = self.get_supported_vod_events()
+            
+        print >>sys.stderr,"bg: VOD EVENTS",supportedvodevents
+        dcfg.set_video_events(supportedvodevents)
+        
+        # Ric: added svc
+        if tdef.is_multifile_torrent():
+            svcdlfiles = self.is_svc(dlfile, tdef)
+
+            if svcdlfiles is not None:
+                dcfg.set_video_event_callback(self.sesscb_vod_event_callback, dlmode=DLMODE_SVC)
+                # Ric: svcdlfiles is an ordered list of svc layers
+                dcfg.set_selected_files(svcdlfiles)
+            else:
+                # Normal multi-file torrent
+                dcfg.set_video_event_callback(self.sesscb_vod_event_callback)
+                dcfg.set_selected_files([dlfile])
+        else:
+            dcfg.set_video_event_callback(self.sesscb_vod_event_callback)
+            # Do not set selected file
+                    
+
+        dcfg.set_dest_dir(destdir)
+        
+        # Arno: 2008-7-15: commented out, just stick with old ABC-tuned 
+        # settings for now
+        #dcfg.set_max_conns_to_initiate(300)
+        #dcfg.set_max_conns(300)
+        
+        # Cap at 1 MB/s
+        print >>sys.stderr,"bg: Capping Download speed to 1 MByte/s"
+        dcfg.set_max_speed(DOWNLOAD,1024)
+        
+        
+        # Stop all non-playing, see if we're restarting one
+        infohash = tdef.get_infohash()
+        newd = None
+        for d in self.s.get_downloads():
+            if d.get_def().get_infohash() == infohash:
+                # Download already exists.
+                # One safe option is to remove it (but not its downloaded content)
+                # so we can start with a fresh DownloadStartupConfig. However,
+                # this gives funky concurrency errors and could prevent a
+                # Download from starting without hashchecking (as its checkpoint
+                # was removed) 
+                # Alternative is to set VOD callback, etc. at Runtime:
+                print >>sys.stderr,"main: Reusing old duplicate Download",`infohash`
+                newd = d
+                                    
+                # If we have a POA, we add it to the existing download
+                if poa:
+                    d.set_poa(poa)
+
+            if d not in self.downloads_in_vodmode:
+                d.stop()
+
+        self.s.lm.h4xor_reset_init_conn_counter()
+
+        # ARNOTODO: does this work with Plugin's duplicate download facility?
+
+        self.playermode = DLSTATUS_DOWNLOADING
+        if newd is None:
+            print >>sys.stderr,"main: Starting new Download",`infohash`
+            newd = self.s.start_download(tdef,dcfg)
+        # Ric: added restart of an svc download
+        else:
+            newd.set_video_events(self.get_supported_vod_events())
+
+            svcdlfiles = self.is_svc(dlfile, tdef)
+            if svcdlfiles is not None:
+                newd.set_video_event_callback(self.sesscb_vod_event_callback, dlmode = DLMODE_SVC)
+                # Ric: svcdlfiles is an ordered list of svc layers
+                newd.set_selected_files(svcdlfiles)
+            else:
+                newd.set_video_event_callback(self.sesscb_vod_event_callback)
+                if tdef.is_multifile_torrent():
+                    newd.set_selected_files([dlfile])
+
+            print >>sys.stderr,"main: Restarting existing Download",`infohash`
+            newd.restart()
+
+        self.downloads_in_vodmode.add(newd)
+
+        print >>sys.stderr,"main: Saving content to",newd.get_dest_files()
+        return newd
+
+
+    def sesscb_vod_event_callback(self,d,event,params):
+        pass
+        
+    def get_supported_vod_events(self):
+        pass
+
+
+    #
+    # DownloadCache
+    #
+    def free_up_diskspace_by_downloads(self,infohash,needed):
+        
+        if DEBUG:
+            print >> sys.stderr,"main: free_up: needed",needed,DISKSPACE_LIMIT
+        if needed > DISKSPACE_LIMIT:
+            # Not cleaning out whole cache for bigguns
+            if DEBUG:
+                print >> sys.stderr,"main: free_up: No cleanup for bigguns"
+            return True 
+        
+        inuse = 0L
+        timelist = []
+        dlist = self.s.get_downloads()
+        for d in dlist:
+            hisinfohash = d.get_def().get_infohash()
+            if infohash == hisinfohash:
+                # Don't delete the torrent we want to play
+                continue
+            destfiles = d.get_dest_files()
+            if DEBUG:
+                print >> sys.stderr,"main: free_up: Downloaded content",`destfiles`
+            
+            dinuse = 0L
+            for (filename,savepath) in destfiles:
+                stat = os.stat(savepath)
+                dinuse += stat.st_size
+            inuse += dinuse
+            timerec = (stat.st_ctime,dinuse,d)
+            timelist.append(timerec)
+            
+        if inuse+needed < DISKSPACE_LIMIT:
+            # Enough available, done.
+            if DEBUG:
+                print >> sys.stderr,"main: free_up: Enough avail",inuse
+            return True
+        
+        # Policy: remove oldest till sufficient
+        timelist.sort()
+        if DEBUG:
+            print >> sys.stderr,"main: free_up: Found",timelist,"in dest dir"
+        
+        got = 0L
+        for ctime,dinuse,d in timelist:
+            print >> sys.stderr,"main: free_up: Removing",`d.get_def().get_name_as_unicode()`,"to free up diskspace, t",ctime
+            self.s.remove_download(d,removecontent=True)
+            got += dinuse
+            if got > needed:
+                return True
+        # Deleted all, still no space:
+        return False
+        
+        
+    #
+    # Process periodically reported DownloadStates
+    #
+    def sesscb_states_callback(self,dslist):
+        """ Called by Session thread """
+
+        #print >>sys.stderr,"bg: sesscb_states_callback",currentThread().getName()
+
+        # Display some stats
+        if (int(time.time()) % 5) == 0:
+            for ds in dslist:
+                d = ds.get_download()
+                print >>sys.stderr, '%s %s %5.2f%% %s up %8.2fKB/s down %8.2fKB/s' % \
+                    (d.get_def().get_name(), \
+                     dlstatus_strings[ds.get_status()], \
+                     ds.get_progress() * 100, \
+                     ds.get_error(), \
+                     ds.get_current_speed(UPLOAD), \
+                     ds.get_current_speed(DOWNLOAD))
+        
+        # Arno: we want the prebuf stats every second, and we want the
+        # detailed peerlist, needed for research stats. Getting them every
+        # second may be too expensive, so get them every 10.
+        #
+        self.getpeerlistcount += 1
+        getpeerlist = (self.getpeerlistcount % 10) == 0
+        haspeerlist =  (self.getpeerlistcount % 10) == 1
+
+        # Arno: delegate to GUI thread. This makes some things (especially
+        #access control to self.videoFrame easier
+        #self.gui_states_callback(dslist)
+        #print >>sys.stderr,"bg: sesscb_states_callback: calling GUI",currentThread().getName()
+        wx.CallAfter(self.gui_states_callback_wrapper,dslist,haspeerlist)
+        
+        #print >>sys.stderr,"main: SessStats:",self.getpeerlistcount,getpeerlist,haspeerlist
+        return (1.0,getpeerlist) 
+
+
+    def gui_states_callback_wrapper(self,dslist,haspeerlist):
+        try:
+            self.gui_states_callback(dslist,haspeerlist)
+        except:
+            print_exc()
+
+
+    def gui_states_callback(self,dslist,haspeerlist):
+        """ Called by *GUI* thread.
+        CAUTION: As this method is called by the GUI thread don't to any 
+        time-consuming stuff here! """
+        
+        #print >>sys.stderr,"main: Stats:"
+        if self.shuttingdown:
+            return ([],0,0)
+        
+        # See which Download is currently playing
+        playermode = self.playermode
+
+        totalspeed = {}
+        totalspeed[UPLOAD] = 0.0
+        totalspeed[DOWNLOAD] = 0.0
+        totalhelping = 0
+
+        # When not playing, display stats for all Downloads and apply rate control.
+        if playermode == DLSTATUS_SEEDING:
+            if DEBUG:
+                for ds in dslist:
+                    print >>sys.stderr,"main: Stats: Seeding: %s %.1f%% %s" % (dlstatus_strings[ds.get_status()],100.0*ds.get_progress(),ds.get_error())
+            self.ratelimit_callback(dslist)
+            
+        # Calc total dl/ul speed and find DownloadStates for playing Downloads
+        playing_dslist = []
+        for ds in dslist:
+            if ds.get_download() in self.downloads_in_vodmode:
+                playing_dslist.append(ds)
+            elif DEBUG and playermode == DLSTATUS_DOWNLOADING:
+                print >>sys.stderr,"main: Stats: Waiting: %s %.1f%% %s" % (dlstatus_strings[ds.get_status()],100.0*ds.get_progress(),ds.get_error())
+            
+            for dir in [UPLOAD,DOWNLOAD]:
+                totalspeed[dir] += ds.get_current_speed(dir)
+            totalhelping += ds.get_num_peers()
+
+        # Report statistics on all downloads to research server, every 10 secs
+        # if haspeerlist:
+        #     try:
+        #         for ds in dslist:
+        #             self.reporter.report_stat(ds)
+        #     except:
+        #         print_exc()
+
+        # Set systray icon tooltip. This has limited size on Win32!
+        txt = self.appname+' '+self.appversion+'\n\n'
+        txt += 'DL: %.1f\n' % (totalspeed[DOWNLOAD])
+        txt += 'UL:   %.1f\n' % (totalspeed[UPLOAD])
+        txt += 'Helping: %d\n' % (totalhelping) 
+        #print >>sys.stderr,"main: ToolTip summary",txt
+        self.OnSetSysTrayTooltip(txt)
+
+        # No playing Downloads        
+        if len(playing_dslist) == 0:
+            return ([],0,0)
+        elif DEBUG and playermode == DLSTATUS_DOWNLOADING:
+            for ds in playing_dslist:
+                print >>sys.stderr,"main: Stats: DL: %s %.1f%% %s dl %.1f ul %.1f n %d" % (dlstatus_strings[ds.get_status()],100.0*ds.get_progress(),ds.get_error(),ds.get_current_speed(DOWNLOAD),ds.get_current_speed(UPLOAD),ds.get_num_peers())
+
+        # If we're done playing we can now restart any previous downloads to 
+        # seed them.
+        if playermode != DLSTATUS_SEEDING:
+            playing_seeding_count = 0
+            for ds in playing_dslist:
+                 if ds.get_status() == DLSTATUS_SEEDING:
+                    playing_seeding_count += 1
+            if len(playing_dslist) == playing_seeding_count: 
+                    self.restart_other_downloads()
+
+        # cf. 25 Mbps cap to reduce CPU usage and improve playback on slow machines
+        # Arno: on some torrents this causes VLC to fail to tune into the video
+        # although it plays audio???
+        #ds.get_download().set_max_speed(DOWNLOAD,1500)
+    
+        
+        return (playing_dslist,totalhelping,totalspeed) 
+
+
+    def OnSetSysTrayTooltip(self,txt):         
+        if self.tbicon is not None:
+            self.tbicon.set_icon_tooltip(txt)
+
+    #
+    # Download Management
+    #
+    def restart_other_downloads(self):
+        """ Called by GUI thread """
+        if self.shuttingdown:
+            return
+        print >>sys.stderr,"main: Restarting other downloads"
+        self.playermode = DLSTATUS_SEEDING
+        self.ratelimiter = UserDefinedMaxAlwaysOtherwiseEquallyDividedRateManager()
+        self.set_ratelimits()
+
+        dlist = self.s.get_downloads()
+        for d in dlist:
+            if d not in self.downloads_in_vodmode:
+                d.set_mode(DLMODE_NORMAL) # checkpointed torrents always restarted in DLMODE_NORMAL, just make extra sure
+                d.restart() 
+
+
+    def remove_downloads_in_vodmode_if_not_complete(self):
+        print >>sys.stderr,"main: Removing playing download if not complete"
+        for d in self.downloads_in_vodmode:
+            d.set_state_callback(self.sesscb_remove_playing_callback)
+        
+    def sesscb_remove_playing_callback(self,ds):
+        """ Called by SessionThread """
+        
+        print >>sys.stderr,"main: sesscb_remove_playing_callback: status is",dlstatus_strings[ds.get_status()],"progress",ds.get_progress()
+        
+        d = ds.get_download()
+        name = d.get_def().get_name()
+        if (ds.get_status() == DLSTATUS_DOWNLOADING and ds.get_progress() >= 0.9) or ds.get_status() == DLSTATUS_SEEDING:
+            pass
+            print >>sys.stderr,"main: sesscb_remove_playing_callback: voting for KEEPING",`name`            
+        else:
+            print >>sys.stderr,"main: sesscb_remove_playing_callback: voting for REMOVING",`name`
+            if self.shuttingdown:
+                # Arno, 2010-04-23: Do it now ourselves, wx won't do it anymore. Saves
+                # hashchecking on sparse file on Linux.
+                self.remove_playing_download(d)
+                
+            wx.CallAfter(self.remove_playing_download,d)
+        
+        return (-1.0,False)
+        
+
+    def remove_playing_download(self,d):
+        """ Called by MainThread """
+        if self.s is not None:
+            print >>sys.stderr,"main: Removing incomplete download",`d.get_def().get_name_as_unicode()`
+            try:
+                self.s.remove_download(d,removecontent=True)
+                self.downloads_in_vodmode.remove(d)
+            except:
+                print_exc()
+
+    def stop_playing_download(self,d):
+        """ Called by MainThread """
+        print >>sys.stderr,"main: Stopping download",`d.get_def().get_name_as_unicode()`
+        try:
+            d.stop()
+            self.downloads_in_vodmode.remove(d)
+        except:
+            print_exc()
+
+
+    #
+    # Rate limiter
+    #
+    def set_ratelimits(self):
+        uploadrate = float(self.playerconfig['total_max_upload_rate'])
+        print >>sys.stderr,"main: set_ratelimits: Setting max upload rate to",uploadrate
+        if self.ratelimiter is not None:
+            self.ratelimiter.set_global_max_speed(UPLOAD,uploadrate)
+            self.ratelimiter.set_global_max_seedupload_speed(uploadrate)
+
+    def ratelimit_callback(self,dslist):
+        """ When the player is in seeding mode, limit the used upload to
+        the limit set by the user via the options menu. 
+        Called by *GUI* thread """
+        if self.ratelimiter is None:
+            return
+
+        # Adjust speeds once every 4 seconds
+        adjustspeeds = False
+        if self.ratelimit_update_count % 4 == 0:
+            adjustspeeds = True
+        self.ratelimit_update_count += 1
+        
+        if adjustspeeds:
+            self.ratelimiter.add_downloadstatelist(dslist)
+            self.ratelimiter.adjust_speeds()
+
+
+    #
+    # Player config file
+    # 
+    def load_playerconfig(self,state_dir):
+        self.playercfgfilename = os.path.join(state_dir,'playerconf.pickle')
+        self.playerconfig = None
+        try:
+            f = open(self.playercfgfilename,"rb")
+            self.playerconfig = pickle.load(f)
+            f.close()
+        except:
+            print_exc()
+            self.playerconfig = {}
+            self.playerconfig['total_max_upload_rate'] = DEFAULT_MAX_UPLOAD_SEED_WHEN_SEEDING # KB/s
+
+    def save_playerconfig(self):
+        try:
+            f = open(self.playercfgfilename,"wb")
+            pickle.dump(self.playerconfig,f)
+            f.close()
+        except:
+            print_exc()
+            
+    def set_playerconfig(self,key,value):
+        self.playerconfig[key] = value
+        
+        if key == 'total_max_upload_rate':
+            try:
+                self.set_ratelimits()
+            except:
+                print_exc()
+    
+    def get_playerconfig(self,key):
+        return self.playerconfig[key]
+
+
+    #
+    # Shutdown
+    #
+    def OnExit(self):
+        print >>sys.stderr,"main: ONEXIT",currentThread().getName()
+        self.shuttingdown = True
+        self.remove_downloads_in_vodmode_if_not_complete()
+
+        # To let Threads in Session finish their business before we shut it down.
+        time.sleep(2) 
+        
+        if self.s is not None:
+            self.s.shutdown(hacksessconfcheckpoint=False)
+        
+        if self.tbicon is not None:
+            self.tbicon.RemoveIcon()
+            self.tbicon.Destroy()
+
+        ts = enumerate()
+        for t in ts:
+            print >>sys.stderr,"main: ONEXIT: Thread still running",t.getName(),"daemon",t.isDaemon()
+        
+        self.ExitMainLoop()
+
+    
+    def clear_session_state(self):
+        """ Try to fix apps by doing hard reset. Called from systray menu """
+        try:
+            if self.s is not None:
+                dlist = self.s.get_downloads()
+                for d in dlist:
+                    self.s.remove_download(d,removecontent=True)
+        except:
+            print_exc()
+        time.sleep(1) # give network thread time to do stuff
+        try:
+                dldestdir = self.get_default_destdir()
+                shutil.rmtree(dldestdir,True) # ignore errors
+        except:
+            print_exc()
+        try:
+                dlcheckpointsdir = os.path.join(self.s.get_state_dir(),STATEDIR_DLPSTATE_DIR)
+                shutil.rmtree(dlcheckpointsdir,True) # ignore errors
+        except:
+            print_exc()
+        try:
+                cfgfilename = os.path.join(self.s.get_state_dir(),STATEDIR_SESSCONFIG)
+                os.remove(cfgfilename)
+        except:
+            print_exc()
+
+        self.s = None # HARD EXIT
+        #self.OnExit()
+        sys.exit(0) # DIE HARD 4.0
+
+
+    def show_error(self,msg):
+        dlg = wx.MessageDialog(None, msg, self.appname+" Error", wx.OK|wx.ICON_ERROR)
+        result = dlg.ShowModal()
+        dlg.Destroy()
+        
+    
+    def get_default_destdir(self):
+        return os.path.join(self.s.get_state_dir(),'downloads')
+
+    
+    def is_svc(self, dlfile, tdef):
+        """ Ric: check if it as an SVC download. If it is add the enhancement 
+        layers to the dlfiles
+        """
+        svcfiles = None
+        
+        if tdef.is_multifile_torrent():
+            enhancement = tdef.get_files(exts=svcextdefaults)
+            # Ric: order the enhancement layer in the svcfiles list
+            # if the list of enhancements is not empty
+            if enhancement:
+                enhancement.sort()
+                if tdef.get_length(enhancement[0]) == tdef.get_length(dlfile):
+                    svcfiles = [dlfile]
+                    svcfiles.extend(enhancement)
+                
+        return svcfiles
+
+    #
+    # InstanceConnectionHandler
+    #
+    def i2ithread_readlinecallback(self,ic,cmd):    
+        pass
+    
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Mac/Info.plist b/instrumentation/next-share/BaseLib/Player/Build/Mac/Info.plist
new file mode 100644
index 0000000..b1fb4ff
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/Build/Mac/Info.plist
@@ -0,0 +1,57 @@
+
+
+
+
+	CFBundleDevelopmentRegion
+	English
+	CFBundleDocumentTypes
+	
+      
+         CFBundleTypeExtensions
+         
+            tstream
+         
+         CFBundleTypeIconFile
+         SwarmPlayerDoc
+         CFBundleTypeMIMETypes
+         
+            application/x-tribler-stream
+         
+         CFBundleTypeName
+         Tribler Stream Meta-Info
+         CFBundleTypeOSTypes
+         
+            BTMF
+         
+         CFBundleTypeRole
+         Viewer
+         NSDocumentClass
+         DownloadDocument
+      
+		
+			CFBundleTypeOSTypes
+			
+				****
+				fold
+				disk
+			
+			CFBundleTypeRole
+			Viewer
+		
+	
+	CFBundleExecutable
+	SwarmPlayer
+	CFBundleIconFile
+	swarmplayer.icns
+	CFBundleIdentifier
+	SwarmPlayer
+	CFBundleInfoDictionaryVersion
+	6.0
+	CFBundleName
+	SwarmPlayer
+	CFBundlePackageType
+	APPL
+	CFBundleSignature
+	????
+
+
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Mac/Makefile b/instrumentation/next-share/BaseLib/Player/Build/Mac/Makefile
new file mode 100644
index 0000000..cc99c6c
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/Build/Mac/Makefile
@@ -0,0 +1,116 @@
+# Building on Mac OS/X requires:
+# * Python 2.5
+# * wxPython 2.8-unicode
+# * py2app 0.3.6
+# * swig, subversion (available through MacPorts)
+# * XCode 2.4+
+#
+# Use lower versions at your own risk.
+
+APPNAME=SwarmPlayer
+PYTHON_VER=2.5
+PWD:=${shell pwd}
+ARCH:=${shell arch}
+
+PYTHON=python${PYTHON_VER}
+
+all:	clean SwarmPlayer-${ARCH}.dmg
+
+clean:
+	rm -rf build/imagecontents/ ${APPNAME}-${ARCH}.dmg
+
+.PHONY: 		all clean dirs
+
+# ----- SwarmPlayer 
+
+APPRES=build/imagecontents/${APPNAME}.app/Contents/Resources
+
+SRCDIR=../../../..
+
+build/imagecontents/:
+	rm -rf $@
+	mkdir -p $@
+
+	cd ${SRCDIR} && DYLD_LIBRARY_PATH=macbinaries PYTHONPATH=.:macbinaries ${PYTHON} -OO - < ${PWD}/setuptriblermac.py py2app
+	mv ${SRCDIR}/dist/* $@
+
+	# Thin everything for this architecture. Some things ship Universal (Python, wxPython, ...) and
+	# others get a stub for the other architecture (things built by Universal Python)
+	for i in `find build/imagecontents`; do ./smart_lipo_thin $$i; done
+
+        # Replace any rogue references to local ones. For instance, some libraries are accidently
+        # linked against /usr/local/lib/* or /opt/local/lib. Py2app puts them in the Frameworks dir,
+        # but fails to correct the references in the binaries.
+	#./process_libs build/imagecontents | bash -
+
+	# Background
+	mkdir -p $@/.background
+	cp background.png $@/.background
+
+	# Volume Icon
+	cp VolumeIcon.icns $@/.VolumeIcon.icns
+
+	# Shortcut to /Applications
+	ln -s /Applications $@/Applications
+
+	touch $@
+
+${APPNAME}-${ARCH}.dmg:		build/imagecontents/ SLAResources.rsrc
+	rm -f $@
+	mkdir -p build/temp
+
+	# create image
+	hdiutil create -srcfolder $< -format UDRW -scrub -volname ${APPNAME} $@
+
+	# open it
+	hdiutil attach -readwrite -noverify -noautoopen $@ -mountpoint build/temp/mnt
+
+	# make sure root folder is opened when image is
+	bless --folder build/temp/mnt --openfolder build/temp/mnt
+	# hack: wait for completion
+	sleep 1
+
+	# position items
+	# oddly enough, 'set f .. as alias' can fail, but a reboot fixes that
+	osascript -e "tell application \"Finder\"" \
+	-e "   set f to POSIX file (\"${PWD}/build/temp/mnt\" as string) as alias" \
+	-e "   tell folder f" \
+	-e "       open" \
+	-e "       tell container window" \
+	-e "          set toolbar visible to false" \
+	-e "          set statusbar visible to false" \
+	-e "          set current view to icon view" \
+	-e "          delay 1 -- Sync" \
+	-e "          set the bounds to {50, 100, 1000, 1000} -- Big size so the finder won't do silly things" \
+	-e "       end tell" \
+	-e "       delay 1 -- Sync" \
+	-e "       set icon size of the icon view options of container window to 128" \
+	-e "       set arrangement of the icon view options of container window to not arranged" \
+	-e "       set background picture of the icon view options of container window to file \".background:background.png\"" \
+	-e "       set position of item \"${APPNAME}.app\" to {150, 140}" \
+	-e "       set position of item \"Applications\" to {410, 140}" \
+	-e "       set the bounds of the container window to {50, 100, 600, 400}" \
+	-e "       update without registering applications" \
+	-e "       delay 5 -- Sync" \
+	-e "       close" \
+	-e "   end tell" \
+	-e "   -- Sync" \
+	-e "   delay 5" \
+	-e "end tell" || true
+
+	# turn on custom volume icon
+	/Developer/Tools/SetFile -a C build/temp/mnt || true
+
+	# close
+	hdiutil detach build/temp/mnt || true
+
+	# make read-only
+	mv $@ build/temp/rw.dmg
+	hdiutil convert build/temp/rw.dmg -format UDZO -imagekey zlib-level=9 -o $@
+	rm -f build/temp/rw.dmg
+
+	# add EULA
+	hdiutil unflatten $@
+	/Developer/Tools/DeRez -useDF SLAResources.rsrc > build/temp/sla.r
+	/Developer/Tools/Rez -a build/temp/sla.r -o $@
+	hdiutil flatten $@
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Mac/SLAResources.rsrc b/instrumentation/next-share/BaseLib/Player/Build/Mac/SLAResources.rsrc
new file mode 100644
index 0000000..162a889
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Player/Build/Mac/SLAResources.rsrc differ
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Mac/TriblerDoc.icns b/instrumentation/next-share/BaseLib/Player/Build/Mac/TriblerDoc.icns
new file mode 100644
index 0000000..8f9a1c2
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Player/Build/Mac/TriblerDoc.icns differ
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Mac/VolumeIcon.icns b/instrumentation/next-share/BaseLib/Player/Build/Mac/VolumeIcon.icns
new file mode 100644
index 0000000..8a9d383
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Player/Build/Mac/VolumeIcon.icns differ
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Mac/background.png b/instrumentation/next-share/BaseLib/Player/Build/Mac/background.png
new file mode 100644
index 0000000..fcd940a
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Player/Build/Mac/background.png differ
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Mac/icon_sources/appicon.png b/instrumentation/next-share/BaseLib/Player/Build/Mac/icon_sources/appicon.png
new file mode 100644
index 0000000..586b2d3
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Player/Build/Mac/icon_sources/appicon.png differ
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Mac/icon_sources/appicon.psd b/instrumentation/next-share/BaseLib/Player/Build/Mac/icon_sources/appicon.psd
new file mode 100644
index 0000000..d8604dd
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Player/Build/Mac/icon_sources/appicon.psd differ
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Mac/icon_sources/default_document.png b/instrumentation/next-share/BaseLib/Player/Build/Mac/icon_sources/default_document.png
new file mode 100644
index 0000000..7a14bb3
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Player/Build/Mac/icon_sources/default_document.png differ
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Mac/icon_sources/default_volumeicon.png b/instrumentation/next-share/BaseLib/Player/Build/Mac/icon_sources/default_volumeicon.png
new file mode 100644
index 0000000..5ca6d53
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Player/Build/Mac/icon_sources/default_volumeicon.png differ
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Mac/icon_sources/dmgicon.png b/instrumentation/next-share/BaseLib/Player/Build/Mac/icon_sources/dmgicon.png
new file mode 100644
index 0000000..24b2b22
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Player/Build/Mac/icon_sources/dmgicon.png differ
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Mac/icon_sources/dmgicon.psd b/instrumentation/next-share/BaseLib/Player/Build/Mac/icon_sources/dmgicon.psd
new file mode 100644
index 0000000..d3fcc79
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Player/Build/Mac/icon_sources/dmgicon.psd differ
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Mac/icon_sources/docicon.png b/instrumentation/next-share/BaseLib/Player/Build/Mac/icon_sources/docicon.png
new file mode 100644
index 0000000..e5c1ac6
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Player/Build/Mac/icon_sources/docicon.png differ
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Mac/icon_sources/docicon.psd b/instrumentation/next-share/BaseLib/Player/Build/Mac/icon_sources/docicon.psd
new file mode 100644
index 0000000..b343238
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Player/Build/Mac/icon_sources/docicon.psd differ
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Mac/mkinstalldirs b/instrumentation/next-share/BaseLib/Player/Build/Mac/mkinstalldirs
new file mode 100755
index 0000000..d2d5f21
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/Build/Mac/mkinstalldirs
@@ -0,0 +1,111 @@
+#! /bin/sh
+# mkinstalldirs --- make directory hierarchy
+# Author: Noah Friedman 
+# Created: 1993-05-16
+# Public domain
+
+errstatus=0
+dirmode=""
+
+usage="\
+Usage: mkinstalldirs [-h] [--help] [-m mode] dir ..."
+
+# process command line arguments
+while test $# -gt 0 ; do
+  case $1 in
+    -h | --help | --h*)         # -h for help
+      echo "$usage" 1>&2
+      exit 0
+      ;;
+    -m)                         # -m PERM arg
+      shift
+      test $# -eq 0 && { echo "$usage" 1>&2; exit 1; }
+      dirmode=$1
+      shift
+      ;;
+    --)                         # stop option processing
+      shift
+      break
+      ;;
+    -*)                         # unknown option
+      echo "$usage" 1>&2
+      exit 1
+      ;;
+    *)                          # first non-opt arg
+      break
+      ;;
+  esac
+done
+
+for file
+do
+  if test -d "$file"; then
+    shift
+  else
+    break
+  fi
+done
+
+case $# in
+  0) exit 0 ;;
+esac
+
+case $dirmode in
+  '')
+    if mkdir -p -- . 2>/dev/null; then
+      echo "mkdir -p -- $*"
+      exec mkdir -p -- "$@"
+    fi
+    ;;
+  *)
+    if mkdir -m "$dirmode" -p -- . 2>/dev/null; then
+      echo "mkdir -m $dirmode -p -- $*"
+      exec mkdir -m "$dirmode" -p -- "$@"
+    fi
+    ;;
+esac
+
+for file
+do
+  set fnord `echo ":$file" | sed -ne 's/^:\//#/;s/^://;s/\// /g;s/^#/\//;p'`
+  shift
+
+  pathcomp=
+  for d
+  do
+    pathcomp="$pathcomp$d"
+    case $pathcomp in
+      -*) pathcomp=./$pathcomp ;;
+    esac
+
+    if test ! -d "$pathcomp"; then
+      echo "mkdir $pathcomp"
+
+      mkdir "$pathcomp" || lasterr=$?
+
+      if test ! -d "$pathcomp"; then
+  	errstatus=$lasterr
+      else
+  	if test ! -z "$dirmode"; then
+	  echo "chmod $dirmode $pathcomp"
+    	  lasterr=""
+  	  chmod "$dirmode" "$pathcomp" || lasterr=$?
+
+  	  if test ! -z "$lasterr"; then
+  	    errstatus=$lasterr
+  	  fi
+  	fi
+      fi
+    fi
+
+    pathcomp="$pathcomp/"
+  done
+done
+
+exit $errstatus
+
+# Local Variables:
+# mode: shell-script
+# sh-indentation: 2
+# End:
+# mkinstalldirs ends here
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Mac/process_libs b/instrumentation/next-share/BaseLib/Player/Build/Mac/process_libs
new file mode 100755
index 0000000..d7a99a1
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/Build/Mac/process_libs
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+TARGETDIR=$1
+
+# process dependencies and their exact locations of all libs
+
+cd $TARGETDIR
+
+for i in `find . -name "*.dylib" -or -name "*.so"`
+do
+  otool -L $i | perl -ne '
+    if(m#/'`basename $i`' #) {
+      # skip references to self
+
+      next;
+    }
+
+    if(m#(/usr/local/lib/([^ /]+))#) {
+      # make reference to /usr/local/lib/* local
+
+      print "# Reference to $1 found in '$i'\n";
+      print "chmod a+w '$i'\n";
+      print "install_name_tool -change $1 \@executable_path/../Frameworks/$2 '$i'\n";
+    }
+
+    if(m#(/opt/local/lib/([^ /]+))#) {
+      # make reference to /opt/local/lib/* local
+
+      print "# Reference to $1 found in '$i'\n";
+      print "chmod a+w '$i'\n";
+      print "install_name_tool -change $1 \@executable_path/../Frameworks/$2 '$i'\n";
+    }
+  '
+done
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Mac/setuptriblermac.py b/instrumentation/next-share/BaseLib/Player/Build/Mac/setuptriblermac.py
new file mode 100644
index 0000000..0cf2143
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/Build/Mac/setuptriblermac.py
@@ -0,0 +1,120 @@
+# ---------------
+# This script builds build/SwarmPlayer.app
+#
+# Meant to be called from Tribler/Player/Build/Mac/Makefile
+# ---------------
+
+import py2app
+from distutils.util import get_platform
+import sys,os,platform,shutil
+from setuptools import setup
+
+from BaseLib.__init__ import LIBRARYNAME
+
+# modules to include into bundle
+includeModules=["encodings.hex_codec","encodings.utf_8","encodings.latin_1","xml.sax", "email.iterators"]
+
+# ----- some basic checks
+
+if __debug__:
+    print "WARNING: Non optimised python bytecode (.pyc) will be produced. Run with -OO instead to produce and bundle .pyo files."
+
+if sys.platform != "darwin":
+    print "WARNING: You do not seem to be running Mac OS/X." 
+
+# ----- import and verify wxPython
+
+import wxversion
+
+wxversion.select('2.8-unicode')
+
+import wx
+
+v = wx.__version__
+
+if v < "2.6":
+    print "WARNING: You need wxPython 2.6 or higher but are using %s." % v
+
+if v < "2.8.4.2":
+    print "WARNING: wxPython before 2.8.4.2 could crash when loading non-present fonts. You are using %s." % v
+
+# ----- import and verify M2Crypto
+
+import M2Crypto
+import M2Crypto.m2
+if "ec_init" not in M2Crypto.m2.__dict__:
+    print "WARNING: Could not import specialistic M2Crypto (imported %s)" % M2Crypto.__file__
+
+# ----- import VLC
+
+#import vlc
+
+#vlc = vlc.MediaControl(["--plugin-path",os.getcwd()+"/macbinaries/vlc_plugins"])
+
+# =================
+# build SwarmPlayer.app
+# =================
+
+from plistlib import Plist
+
+def includedir( srcpath, dstpath = None ):
+    """ Recursive directory listing, filtering out svn files. """
+
+    total = []
+
+    cwd = os.getcwd()
+    os.chdir( srcpath )
+
+    if dstpath is None:
+        dstpath = srcpath
+
+    for root,dirs,files in os.walk( "." ):
+        if '.svn' in dirs:
+            dirs.remove('.svn')
+
+        for f in files:
+            total.append( (root,f) )
+
+    os.chdir( cwd )
+
+    # format: (targetdir,[file])
+    # so for us, (dstpath/filedir,[srcpath/filedir/filename])
+    return [("%s/%s" % (dstpath,root),["%s/%s/%s" % (srcpath,root,f)]) for root,f in total]
+
+def filterincludes( l, f ):
+    """ Return includes which pass filter f. """
+
+    return [(x,y) for (x,y) in l if f(y[0])]
+
+# ----- build the app bundle
+mainfile = os.path.join(LIBRARYNAME,'Player','swarmplayer.py')
+
+setup(
+    setup_requires=['py2app'],
+    name='SwarmPlayer',
+    app=[mainfile],
+    options={ 'py2app': {
+        'argv_emulation': True,
+        'includes': includeModules,
+        'excludes': ["Tkinter","Tkconstants","tcl"],
+        'iconfile': LIBRARYNAME+'/Player/Build/Mac/tribler.icns',
+        'plist': Plist.fromFile(LIBRARYNAME+'/Player/Build/Mac/Info.plist'),
+        'optimize': 2*int(not __debug__),
+        'resources':
+            [(LIBRARYNAME+"/Lang", [LIBRARYNAME+"/Lang/english.lang"]),
+             LIBRARYNAME+"/binary-LICENSE.txt", 
+             LIBRARYNAME+"/readme.txt",
+             LIBRARYNAME+"/Images/SwarmPlayerIcon.ico",
+             LIBRARYNAME+"/Player/Build/Mac/TriblerDoc.icns",
+           ]
+           # add images
+           + includedir( LIBRARYNAME+"/Images" )
+
+           # add VLC plugins
+           + includedir( "macbinaries/vlc_plugins" )
+
+           # add ffmpeg binary
+           + ["macbinaries/ffmpeg"]
+            ,
+    } }
+)
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Mac/smart_lipo_merge b/instrumentation/next-share/BaseLib/Player/Build/Mac/smart_lipo_merge
new file mode 100755
index 0000000..3097e61
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/Build/Mac/smart_lipo_merge
@@ -0,0 +1,46 @@
+#!/bin/bash
+#
+# syntax: smart_lipo_merge filenative fileforeign fileout
+#
+# merges two binaries, taking the respective architecture part in case the input is fat
+#
+
+NATIVE=$1
+FOREIGN=$2
+FILEOUT=$3
+
+ARCH1=i386
+ARCH2=ppc
+ARCH=`arch`
+if [ $ARCH = $ARCH1 ]
+then
+    FOREIGNARCH=$ARCH2
+else
+    FOREIGNARCH=$ARCH1
+fi
+
+if [ `lipo -info $NATIVE | cut -d\  -f1` != "Non-fat" ]
+then
+  echo native file is fat -- extracting $ARCH
+  lipo -thin $ARCH $NATIVE -output $NATIVE.$ARCH
+else
+  echo native file is thin -- using as is
+  cp $NATIVE $NATIVE.$ARCH
+fi
+
+if [ `lipo -info $FOREIGN | cut -d\  -f1` != "Non-fat" ]
+then
+  echo foreign file is fat -- extracting $FOREIGNARCH
+  lipo -thin $FOREIGNARCH $FOREIGN -output $FOREIGN.$FOREIGNARCH
+else
+  echo foreign file is thin -- using as is
+  cp $FOREIGN $FOREIGN.$FOREIGNARCH
+fi
+
+echo merging... 
+lipo -create $NATIVE.$ARCH $FOREIGN.$FOREIGNARCH -output $FILEOUT
+echo cleanup..
+rm $NATIVE.$ARCH
+rm $FOREIGN.$FOREIGNARCH
+
+
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Mac/smart_lipo_thin b/instrumentation/next-share/BaseLib/Player/Build/Mac/smart_lipo_thin
new file mode 100755
index 0000000..b6fd13d
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/Build/Mac/smart_lipo_thin
@@ -0,0 +1,19 @@
+#!/bin/bash
+#
+# syntax: smart_lipo_thin file
+#
+# extracts the native architecture part of the fat input file, or does nothing if input is thin
+#
+
+INPUT=$1
+ARCH=`arch`
+
+REPORT=`lipo -info $INPUT 2>&1 | cut -d\  -f1-5`
+if [ "$REPORT" == "Architectures in the fat file:" ]
+then
+  echo thinning `basename $INPUT`
+  lipo -thin $ARCH $INPUT -output $INPUT.tmp
+  rm -f $INPUT
+  mv $INPUT.tmp $INPUT
+fi
+
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Mac/tribler.icns b/instrumentation/next-share/BaseLib/Player/Build/Mac/tribler.icns
new file mode 100644
index 0000000..8fd54eb
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Player/Build/Mac/tribler.icns differ
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Mac/vlc-macosx-compile.patch b/instrumentation/next-share/BaseLib/Player/Build/Mac/vlc-macosx-compile.patch
new file mode 100644
index 0000000..a09a3a6
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/Build/Mac/vlc-macosx-compile.patch
@@ -0,0 +1,509 @@
+Index: modules/gui/macosx/voutqt.m
+===================================================================
+--- modules/gui/macosx/voutqt.m	(revision 20403)
++++ modules/gui/macosx/voutqt.m	(working copy)
+@@ -39,6 +39,7 @@
+ 
+ #include "intf.h"
+ #include "vout.h"
++#include 
+ 
+ #define QT_MAX_DIRECTBUFFERS 10
+ #define VL_MAX_DISPLAYS 16
+@@ -138,13 +139,22 @@
+     p_vout->pf_display = DisplayVideo;
+     p_vout->pf_control = ControlVideo;
+ 
+-    /* Are we embedded?  If so, the drawable value will be a pointer to a
++    /* Are we embedded?  If so, the drawable value should be a pointer to a
+      * CGrafPtr that we're expected to use */
+     var_Get( p_vout->p_libvlc, "drawable", &value_drawable );
+-    if( value_drawable.i_int != 0 )
++    if( value_drawable.i_int != 0 ) {
++        vlc_value_t value_drawable_type;
++
++        var_Get( p_vout->p_libvlc, "macosx-drawable-type", &value_drawable_type );
++        if( value_drawable_type.i_int != VLCDrawableCGrafPtr ) {
++            msg_Err( p_vout, "QT interface requires a CGrafPtr when embedded" );
++            return( 1 );
++        }
++
+         p_vout->p_sys->b_embedded = VLC_TRUE;
+-    else
++    } else {
+         p_vout->p_sys->b_embedded = VLC_FALSE;
++    }
+ 
+     p_vout->p_sys->b_cpu_has_simd =
+         vlc_CPU() & (CPU_CAPABILITY_ALTIVEC|CPU_CAPABILITY_MMXEXT);
+Index: modules/gui/macosx/voutgl.m
+===================================================================
+--- modules/gui/macosx/voutgl.m	(revision 20403)
++++ modules/gui/macosx/voutgl.m	(working copy)
+@@ -35,6 +35,7 @@
+ #include                                             /* strerror() */
+ 
+ #include 
++#include 
+ 
+ #include "intf.h"
+ #include "vout.h"
+@@ -43,6 +44,7 @@
+ #include 
+ 
+ #include 
++#include 
+ 
+ /*****************************************************************************
+  * VLCGLView interface
+@@ -67,13 +69,18 @@
+     /* Mozilla plugin-related variables */
+     vlc_bool_t          b_embedded;
+     AGLContext          agl_ctx;
+-    AGLDrawable         agl_drawable;
+     int                 i_offx, i_offy;
+     int                 i_width, i_height;
+     WindowRef           theWindow;
+     WindowGroupRef      winGroup;
+     vlc_bool_t          b_clipped_out;
+-    Rect                clipBounds, viewBounds;             
++    Rect                clipBounds, viewBounds;
++
++    libvlc_macosx_drawable_type_t drawable_type;
++    union {
++        CGrafPtr        CGrafPtr;
++        ControlRef      ControlRef;
++    } drawable;
+ };
+ 
+ /*****************************************************************************
+@@ -462,17 +469,90 @@
+ static void aglReshape( vout_thread_t * p_vout );
+ static OSStatus WindowEventHandler(EventHandlerCallRef nextHandler, EventRef event, void *userData);
+ 
+-static int aglInit( vout_thread_t * p_vout )
++/* returns the bounds of the drawable control/window */
++static Rect aglGetBounds( vout_thread_t * p_vout )
+ {
++        WindowRef win;
++        Rect rect;
++
++        switch( p_vout->p_sys->drawable_type ) {
++            case VLCDrawableCGrafPtr:
++                win = GetWindowFromPort( p_vout->p_sys->drawable.CGrafPtr );
++                GetWindowPortBounds( win, &rect );
++                break;
++
++            case VLCDrawableControlRef:
++                win = GetControlOwner( p_vout->p_sys->drawable.ControlRef );
++                GetControlBounds( p_vout->p_sys->drawable.ControlRef, &rect );
++                break;
++        }
++
++        return rect;
++}
++
++/* returns the window containing the drawable area */
++static WindowRef aglGetWindow( vout_thread_t * p_vout )
++{
++        WindowRef window;
++
++        switch( p_vout->p_sys->drawable_type ) {
++            case VLCDrawableCGrafPtr:
++                window = GetWindowFromPort( p_vout->p_sys->drawable.CGrafPtr );
++                break;
++
++            case VLCDrawableControlRef:
++                window = GetControlOwner( p_vout->p_sys->drawable.ControlRef );
++                break;
++        }
++
++        return window;
++}
++
++/* gets the graphics port associated with our drawing area */
++static CGrafPtr aglGetPort( vout_thread_t * p_vout )
++{
++        CGrafPtr port;
++
++        switch( p_vout->p_sys->drawable_type ) {
++            case VLCDrawableCGrafPtr:
++                port = p_vout->p_sys->drawable.CGrafPtr;
++                break;
++
++            case VLCDrawableControlRef:
++                port = GetWindowPort( GetControlOwner(
++                           p_vout->p_sys->drawable.ControlRef
++                       ) );
++                break;
++        }
++
++        return port;
++}
++
++/* (re)process "drawable-*" and "macosx-drawable-type" variables. `drawable' is a
++   parameter to allow it to be overridden (REPARENT) */
++static int aglProcessDrawable( vout_thread_t * p_vout, libvlc_drawable_t drawable )
++{
+     vlc_value_t val;
++    vlc_value_t val_type;
++    AGLDrawable agl_drawable;
++    Rect clipBounds,viewBounds;
+ 
+-    Rect viewBounds;    
+-    Rect clipBounds;
+-    
+-    var_Get( p_vout->p_libvlc, "drawable", &val );
+-    p_vout->p_sys->agl_drawable = (AGLDrawable)val.i_int;
+-    aglSetDrawable(p_vout->p_sys->agl_ctx, p_vout->p_sys->agl_drawable);
++    var_Get( p_vout->p_libvlc, "macosx-drawable-type", &val_type );
+ 
++    p_vout->p_sys->drawable_type = val_type.i_int;
++    switch( val_type.i_int ) {
++        case VLCDrawableCGrafPtr:
++            p_vout->p_sys->drawable.CGrafPtr = (CGrafPtr)drawable;
++            break;
++
++        case VLCDrawableControlRef:
++            p_vout->p_sys->drawable.ControlRef = (ControlRef)drawable;
++            break;
++    }
++
++    agl_drawable = (AGLDrawable)aglGetPort( p_vout );
++    aglSetDrawable(p_vout->p_sys->agl_ctx, agl_drawable);
++
+     var_Get( p_vout->p_libvlc, "drawable-view-top", &val );
+     viewBounds.top = val.i_int;
+     var_Get( p_vout->p_libvlc, "drawable-view-left", &val );
+@@ -481,15 +561,21 @@
+     viewBounds.bottom = val.i_int;
+     var_Get( p_vout->p_libvlc, "drawable-view-right", &val );
+     viewBounds.right = val.i_int;
+-    var_Get( p_vout->p_libvlc, "drawable-clip-top", &val );
+-    clipBounds.top = val.i_int;
+-    var_Get( p_vout->p_libvlc, "drawable-clip-left", &val );
+-    clipBounds.left = val.i_int;
+-    var_Get( p_vout->p_libvlc, "drawable-clip-bottom", &val );
+-    clipBounds.bottom = val.i_int;
+-    var_Get( p_vout->p_libvlc, "drawable-clip-right", &val );
+-    clipBounds.right = val.i_int;
+ 
++    if( !viewBounds.top && !viewBounds.left && !viewBounds.right && !viewBounds.bottom ) {
++        /* view bounds not set, use control/window bounds */
++        clipBounds = viewBounds = aglGetBounds( p_vout );
++    } else {
++        var_Get( p_vout->p_libvlc, "drawable-clip-top", &val );
++        clipBounds.top = val.i_int;
++        var_Get( p_vout->p_libvlc, "drawable-clip-left", &val );
++        clipBounds.left = val.i_int;
++        var_Get( p_vout->p_libvlc, "drawable-clip-bottom", &val );
++        clipBounds.bottom = val.i_int;
++        var_Get( p_vout->p_libvlc, "drawable-clip-right", &val );
++        clipBounds.right = val.i_int;
++    }
++
+     p_vout->p_sys->b_clipped_out = (clipBounds.top == clipBounds.bottom)
+                                  || (clipBounds.left == clipBounds.right);
+     if( ! p_vout->p_sys->b_clipped_out )
+@@ -501,7 +587,15 @@
+     }
+     p_vout->p_sys->clipBounds = clipBounds;
+     p_vout->p_sys->viewBounds = viewBounds;
++}
+ 
++static int aglInit( vout_thread_t * p_vout )
++{
++    vlc_value_t val;
++
++    var_Get( p_vout->p_libvlc, "drawable", &val );
++    aglProcessDrawable( p_vout, val.i_int );
++
+     return VLC_SUCCESS;
+ }
+ 
+@@ -564,6 +658,26 @@
+ 
+ static int aglManage( vout_thread_t * p_vout )
+ {
++    if( p_vout->p_sys->drawable_type == VLCDrawableControlRef ) {
++        /* auto-detect size changes in the control by polling */
++        Rect clipBounds, viewBounds;
++
++        clipBounds = viewBounds = aglGetBounds( p_vout );
++
++        if( memcmp(&clipBounds, &(p_vout->p_sys->clipBounds), sizeof(clipBounds) )
++         && memcmp(&viewBounds, &(p_vout->p_sys->viewBounds), sizeof(viewBounds)) )
++        {
++            /* size has changed since last poll */
++
++            p_vout->p_sys->clipBounds = clipBounds;
++            p_vout->p_sys->viewBounds = viewBounds;
++            aglLock( p_vout );
++            aglSetViewport(p_vout, viewBounds, clipBounds);
++            aglReshape( p_vout );
++            aglUnlock( p_vout );
++        }
++    }
++
+     if( p_vout->i_changes & VOUT_ASPECT_CHANGE )
+     {
+         aglLock( p_vout );
+@@ -586,42 +700,28 @@
+         {
+             /* Close the fullscreen window and resume normal drawing */
+             vlc_value_t val;
+-            Rect viewBounds;    
+-            Rect clipBounds;
+ 
+             var_Get( p_vout->p_libvlc, "drawable", &val );
+-            p_vout->p_sys->agl_drawable = (AGLDrawable)val.i_int;
+-            aglSetDrawable(p_vout->p_sys->agl_ctx, p_vout->p_sys->agl_drawable);
++            aglProcessDrawable( p_vout, val.i_int );
+ 
+-            var_Get( p_vout->p_libvlc, "drawable-view-top", &val );
+-            viewBounds.top = val.i_int;
+-            var_Get( p_vout->p_libvlc, "drawable-view-left", &val );
+-            viewBounds.left = val.i_int;
+-            var_Get( p_vout->p_libvlc, "drawable-view-bottom", &val );
+-            viewBounds.bottom = val.i_int;
+-            var_Get( p_vout->p_libvlc, "drawable-view-right", &val );
+-            viewBounds.right = val.i_int;
+-            var_Get( p_vout->p_libvlc, "drawable-clip-top", &val );
+-            clipBounds.top = val.i_int;
+-            var_Get( p_vout->p_libvlc, "drawable-clip-left", &val );
+-            clipBounds.left = val.i_int;
+-            var_Get( p_vout->p_libvlc, "drawable-clip-bottom", &val );
+-            clipBounds.bottom = val.i_int;
+-            var_Get( p_vout->p_libvlc, "drawable-clip-right", &val );
+-            clipBounds.right = val.i_int;
++            /*the following was here, superfluous due to the same in aglLock?
++              aglSetCurrentContext(p_vout->p_sys->agl_ctx);*/
+ 
+-            aglSetCurrentContext(p_vout->p_sys->agl_ctx);
+-            aglSetViewport(p_vout, viewBounds, clipBounds);
+-
+             /* Most Carbon APIs are not thread-safe, therefore delagate some GUI visibilty update to the main thread */
+             sendEventToMainThread(GetWindowEventTarget(p_vout->p_sys->theWindow), kEventClassVLCPlugin, kEventVLCPluginHideFullscreen);
+         }
+         else
+         {
++            CGDirectDisplayID displayID;
++            CGRect displayBounds;
+             Rect deviceRect;
+             
+-            GDHandle deviceHdl = GetMainDevice();
+-            deviceRect = (*deviceHdl)->gdRect;
++            /* the main display has its origin at (0,0) */
++            displayBounds = CGDisplayBounds( CGMainDisplayID() );
++            deviceRect.left = 0;
++            deviceRect.top = 0;
++            deviceRect.right = displayBounds.size.width;
++            deviceRect.bottom = displayBounds.size.height;
+             
+             if( !p_vout->p_sys->theWindow )
+             {
+@@ -669,8 +769,9 @@
+                 SetWindowBounds(p_vout->p_sys->theWindow, kWindowContentRgn, &deviceRect);
+             }
+             glClear( GL_COLOR_BUFFER_BIT );
+-            p_vout->p_sys->agl_drawable = (AGLDrawable)GetWindowPort(p_vout->p_sys->theWindow);
+-            aglSetDrawable(p_vout->p_sys->agl_ctx, p_vout->p_sys->agl_drawable);
++            p_vout->p_sys->drawable_type = VLCDrawableCGrafPtr;
++            p_vout->p_sys->drawable.CGrafPtr = GetWindowPort(p_vout->p_sys->theWindow);
++            aglSetDrawable(p_vout->p_sys->agl_ctx, p_vout->p_sys->drawable.CGrafPtr);
+             aglSetCurrentContext(p_vout->p_sys->agl_ctx);
+             aglSetViewport(p_vout, deviceRect, deviceRect);
+             //aglSetFullScreen(p_vout->p_sys->agl_ctx, device_width, device_height, 0, 0);
+@@ -753,11 +854,10 @@
+ 
+         case VOUT_REPARENT:
+         {
+-            AGLDrawable drawable = (AGLDrawable)va_arg( args, int);
+-            if( !p_vout->b_fullscreen && drawable != p_vout->p_sys->agl_drawable )
++            libvlc_drawable_t drawable = (libvlc_drawable_t)va_arg( args, int);
++            if( !p_vout->b_fullscreen )
+             {
+-                p_vout->p_sys->agl_drawable = drawable;
+-                aglSetDrawable(p_vout->p_sys->agl_ctx, drawable);
++                aglProcessDrawable( p_vout, drawable );
+             }
+             return VLC_SUCCESS;
+         }
+@@ -771,8 +871,16 @@
+ {
+     if( ! p_vout->p_sys->b_clipped_out )
+     {
++        WindowRef win;
++        Rect rect;
++
+         p_vout->p_sys->b_got_frame = VLC_TRUE;
+         aglSwapBuffers(p_vout->p_sys->agl_ctx);
++
++        win = aglGetWindow( p_vout );
++        rect = aglGetBounds( p_vout );
++
++        InvalWindowRect( win, &rect );
+     }
+     else
+     {
+@@ -788,12 +896,14 @@
+     // however AGL coordinates are based on window structure region
+     // and are vertically flipped
+     GLint rect[4];
+-    CGrafPtr port = (CGrafPtr)p_vout->p_sys->agl_drawable;
++    WindowRef window;
+     Rect winBounds, clientBounds;
+ 
+-    GetWindowBounds(GetWindowFromPort(port),
++    window = aglGetWindow( p_vout );
++
++    GetWindowBounds(window,
+         kWindowStructureRgn, &winBounds);
+-    GetWindowBounds(GetWindowFromPort(port),
++    GetWindowBounds(window,
+         kWindowContentRgn, &clientBounds);
+ 
+     /* update video clipping bounds in drawable */
+Index: bindings/python/vlc_instance.c
+===================================================================
+--- bindings/python/vlc_instance.c	(revision 20403)
++++ bindings/python/vlc_instance.c	(working copy)
+@@ -349,6 +349,30 @@
+ }
+ 
+ static PyObject *
++vlcInstance_video_set_macosx_parent_type( PyObject *self, PyObject *args )
++{
++    libvlc_exception_t ex;
++    int i_drawable_type;
++
++    if( !PyArg_ParseTuple( args, "i", &i_drawable_type ) )
++        return NULL;
++
++    if( i_drawable_type != VLCDrawableCGrafPtr
++     && i_drawable_type != VLCDrawableControlRef )
++    {
++        PyErr_SetString( vlcInstance_Exception, "Invalid drawable type." );
++        return NULL;
++    }
++
++    LIBVLC_TRY;
++    libvlc_video_set_macosx_parent_type( LIBVLC_INSTANCE->p_instance, (libvlc_macosx_drawable_type_t) i_drawable_type, &ex );
++    LIBVLC_EXCEPT;
++
++    Py_INCREF( Py_None );
++    return Py_None;
++}
++
++static PyObject *
+ vlcInstance_video_set_size( PyObject *self, PyObject *args )
+ {
+     libvlc_exception_t ex;
+@@ -733,6 +757,8 @@
+       "playlist_get_input() -> object   Return the current input"},
+     { "video_set_parent", vlcInstance_video_set_parent, METH_VARARGS,
+       "video_set_parent(xid=int)       Set the parent xid or HWND"},
++    { "video_set_macosx_parent_type", vlcInstance_video_set_macosx_parent_type, METH_VARARGS,
++      "video_set_macosx_parent_type(drawabletype=int)       Set the type of parent used on Mac OS/X (see the Drawable* constants)"},
+     { "video_set_size", vlcInstance_video_set_size, METH_VARARGS,
+       "video_set_size(width=int, height=int)    Set the video width and height"},
+     { "audio_toggle_mute", vlcInstance_audio_toggle_mute, METH_VARARGS,
+Index: bindings/python/vlc_module.c
+===================================================================
+--- bindings/python/vlc_module.c	(revision 20403)
++++ bindings/python/vlc_module.c	(working copy)
+@@ -147,6 +147,10 @@
+                              mediacontrol_EndStatus );
+     PyModule_AddIntConstant( p_module, "UndefinedStatus",
+                              mediacontrol_UndefinedStatus );
++    PyModule_AddIntConstant( p_module, "DrawableCGrafPtr",
++                             VLCDrawableCGrafPtr );
++    PyModule_AddIntConstant( p_module, "DrawableControlRef",
++                             VLCDrawableControlRef );
+ }
+ 
+ 
+Index: src/control/video.c
+===================================================================
+--- src/control/video.c	(revision 20403)
++++ src/control/video.c	(working copy)
+@@ -277,6 +277,21 @@
+ 
+ /* global video settings */
+ 
++void libvlc_video_set_macosx_parent_type( libvlc_instance_t *p_instance, libvlc_macosx_drawable_type_t t,
++                              libvlc_exception_t *p_e )
++{
++    var_SetInteger(p_instance->p_libvlc_int, "macosx-drawable-type", (int)t);
++}
++
++libvlc_macosx_drawable_type_t libvlc_video_get_macosx_parent_type( libvlc_instance_t *p_instance, libvlc_exception_t *p_e )
++{
++    libvlc_macosx_drawable_type_t result;
++    
++    result = var_GetInteger( p_instance->p_libvlc_int, "macosx-drawable-type" );
++    
++    return result;
++}
++
+ void libvlc_video_set_parent( libvlc_instance_t *p_instance, libvlc_drawable_t d,
+                               libvlc_exception_t *p_e )
+ {
+Index: src/libvlc-common.c
+===================================================================
+--- src/libvlc-common.c	(revision 20403)
++++ src/libvlc-common.c	(working copy)
+@@ -941,6 +941,10 @@
+     var_Create( p_libvlc, "drawable-clip-bottom", VLC_VAR_INTEGER );
+     var_Create( p_libvlc, "drawable-clip-right", VLC_VAR_INTEGER );
+ 
++#ifdef __APPLE__
++    var_Create( p_libvlc, "macosx-drawable-type", VLC_VAR_INTEGER );
++#endif
++
+     /* Create volume callback system. */
+     var_Create( p_libvlc, "volume-change", VLC_VAR_BOOL );
+ 
+Index: include/vlc/libvlc.h
+===================================================================
+--- include/vlc/libvlc.h	(revision 20403)
++++ include/vlc/libvlc.h	(working copy)
+@@ -424,6 +424,10 @@
+  */
+ VLC_PUBLIC_API void libvlc_video_redraw_rectangle( libvlc_input_t *, const libvlc_rectangle_t *, libvlc_exception_t * );
+ 
++VLC_PUBLIC_API void libvlc_video_set_macosx_parent_type( libvlc_instance_t *, libvlc_macosx_drawable_type_t, libvlc_exception_t * );
++
++VLC_PUBLIC_API libvlc_macosx_drawable_type_t libvlc_video_get_macosx_parent_type( libvlc_instance_t *, libvlc_exception_t * );
++
+ /**
+  * Set the default video output parent
+  *  this settings will be used as default for all video outputs
+Index: include/vlc/libvlc_structures.h
+===================================================================
+--- include/vlc/libvlc_structures.h	(revision 20403)
++++ include/vlc/libvlc_structures.h	(working copy)
+@@ -83,12 +83,22 @@
+ /**
+ * Downcast to this general type as placeholder for a platform specific one, such as:
+ *  Drawable on X11,
+-*  CGrafPort on MacOSX,
++*  (libvlc_macosx_drawable_type_t) on MacOSX,
+ *  HWND on win32
+ */
+ typedef int libvlc_drawable_t;
+ 
+ /**
++* Type of libvlc_drawable_t on MaxOSX. Available types:
++* - VLCDrawableCGrafPtr
++* - VLCDrawableControlRef
++*/
++typedef enum {
++    VLCDrawableCGrafPtr = 0,
++    VLCDrawableControlRef,
++} libvlc_macosx_drawable_type_t;
++
++/**
+ * Rectangle type for video geometry
+ */
+ typedef struct
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/changelog b/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/changelog
new file mode 100644
index 0000000..dbf4e49
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/changelog
@@ -0,0 +1,5 @@
+swarmplayer (1.0.0-1ubuntu3) hardy; urgency=low
+
+  * First release
+
+ -- Tribler   Tue, 17 Jun 2008 11:22:05 +0200
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/compat b/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/compat
new file mode 100644
index 0000000..b8626c4
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/compat
@@ -0,0 +1 @@
+4
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/control b/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/control
new file mode 100644
index 0000000..614499d
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/control
@@ -0,0 +1,16 @@
+Source: swarmplayer 
+Section: net
+Priority: optional
+Maintainer: Arno Bakker 
+Standards-Version: 3.7.2
+Build-Depends: python, debhelper (>= 5.0.37.2), devscripts
+
+Package: swarmplayer 
+Architecture: all
+Depends: python, python-wxgtk2.8, python-m2crypto, python-apsw, vlc, ffmpeg
+Description: Python based Bittorrent/Internet TV viewer. 
+ It allows you to watch BitTorrent-hosted videos on demand and
+ plays live Tribler streams. It is based on the same core as the
+ Tribler TV application.
+ .
+ Homepage: http://www.tribler.org/
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/files b/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/files
new file mode 100644
index 0000000..06b52fd
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/files
@@ -0,0 +1 @@
+swarmplayer_1.0.0-1ubuntu3_all.deb net optional
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/prerm b/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/prerm
new file mode 100644
index 0000000..082c7fe
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/prerm
@@ -0,0 +1,47 @@
+#! /bin/sh
+# prerm script for #PACKAGE#
+#
+# see: dh_installdeb(1)
+
+set -e
+
+# summary of how this script can be called:
+#        *  `remove'
+#        *  `upgrade' 
+#        *  `failed-upgrade' 
+#        *  `remove' `in-favour'  
+#        *  `deconfigure' `in-favour'
+#            `removing'
+#           
+# for details, see http://www.debian.org/doc/debian-policy/ or
+# the debian-policy package
+
+PACKAGE="swarmplayer"
+
+dpkg --listfiles $PACKAGE |
+        awk '$0~/\.py$/ {print $0"c\n" $0"o"}' |
+        xargs rm -f >&2
+
+killall $PACKAGE || :
+
+
+case "$1" in
+    remove|upgrade|deconfigure)
+#       install-info --quiet --remove /usr/info/#PACKAGE#.info.gz
+        ;;
+    failed-upgrade)
+        ;;
+    *)
+        echo "prerm called with unknown argument \`$1'" >&2
+        exit 1
+    ;;
+esac
+
+# dh_installdeb will replace this with shell code automatically
+# generated by other debhelper scripts.
+
+#DEBHELPER#
+
+exit 0
+
+
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/pycompat b/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/pycompat
new file mode 100644
index 0000000..0cfbf08
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/pycompat
@@ -0,0 +1 @@
+2
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/rules b/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/rules
new file mode 100644
index 0000000..5832e45
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/rules
@@ -0,0 +1,85 @@
+#!/usr/bin/make -f
+# Sample debian/rules that uses debhelper.
+# GNU copyright 1997 to 1999 by Joey Hess.
+
+# Uncomment this to turn on verbose mode.
+#export DH_VERBOSE=1
+LIBRARYNAME=BaseLib
+
+configure: configure-stamp
+configure-stamp:
+	dh_testdir
+	# Add here commands to configure the package.
+
+	touch configure-stamp
+
+
+build: build-stamp
+
+build-stamp: configure-stamp 
+	dh_testdir
+
+	# Add here commands to compile the package.
+	#$(MAKE)
+	#/usr/bin/docbook-to-man debian/bittorrent.sgml > bittorrent.1
+
+	touch build-stamp
+
+clean:
+	dh_testdir
+	dh_testroot
+	rm -f build-stamp configure-stamp
+
+	# Add here commands to clean up after the build process.
+	#-$(MAKE) clean
+	find . -name '*.pyc' |xargs rm || :
+
+	dh_clean
+
+install: build
+	dh_testdir
+	dh_testroot
+	dh_clean -k
+	dh_installdirs
+
+# Build architecture-independent files here.
+binary-arch: build install
+# We have nothing to do by default.
+
+
+# Build architecture-dependent files here.
+binary-indep: build install
+	dh_testdir
+	dh_testroot
+	dh_installdocs
+	dh_installexamples
+	dh_installmenu
+	dh_installmime
+	dh_installman
+
+	mkdir -p debian/swarmplayer/usr/share/swarmplayer/
+	cp -rf `ls -1d ${LIBRARYNAME}` debian/swarmplayer/usr/share/swarmplayer/
+	rm -rf debian/swarmplayer/usr/share/swarmplayer/${LIBRARYNAME}/Test
+	# add other files
+	mkdir -p debian/swarmplayer/usr/bin
+	cp -f debian/swarmplayer.sh debian/swarmplayer/usr/bin/swarmplayer
+	cp -f ${LIBRARYNAME}/LICENSE.txt debian/copyright
+	# for the menu
+	mkdir -p debian/swarmplayer/usr/share/pixmaps
+	cp -f debian/swarmplayer.xpm debian/swarmplayer/usr/share/pixmaps/
+
+	dh_installchangelogs 
+	dh_installinit -r --no-start -- stop 20 0 6 .
+	dh_install --sourcedir=debian/tmp
+	dh_install debian/swarmplayer.desktop usr/share/applications
+	dh_link
+	dh_compress
+	dh_fixperms
+	dh_installdeb
+	dh_python
+	dh_gencontrol
+	dh_md5sums
+	dh_builddeb
+
+binary: binary-indep binary-arch
+.PHONY: build clean binary-indep binary-arch binary install configure
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/swarmplayer.1 b/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/swarmplayer.1
new file mode 100644
index 0000000..f909b4e
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/swarmplayer.1
@@ -0,0 +1,22 @@
+.\" SwarmPlayer: Python based Bittorrent/Internet TV viewer 
+.TH man 1 "12 June 2007" "1.0" "SwarmPlayer man page"
+.SH NAME
+swarmplayer \- Python based Bittorrent/Internet TV viewer
+.SH SYNOPSIS
+.B swarmplayer
+.SH DESCRIPTION
+.B SwarmPlayer 
+is a python-based Bittorrent/Internet TV viewer.
+It allows you to watch BitTorrent-hosted videos on demand and
+plays live Tribler streams. It is based on the same core as the
+Tribler TV application.
+
+Homepage: http://www.tribler.org
+.SH FILES
+.P 
+.I /usr/bin/swarmplayer
+.I /usr/share/swarmplayer
+.SH AUTHOR
+.nf
+Arno Bakker (arno@cs.vu.nl)
+.fi
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/swarmplayer.desktop b/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/swarmplayer.desktop
new file mode 100644
index 0000000..c3b1833
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/swarmplayer.desktop
@@ -0,0 +1,8 @@
+[Desktop Entry]
+Name=SwarmPlayer
+GenericName=Bittorrent Video-On-Demand / Live streaming client
+Exec=swarmplayer
+Icon=swarmplayer
+Terminal=false
+Type=Application
+Categories=Application;Network;P2P
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/swarmplayer.manpages b/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/swarmplayer.manpages
new file mode 100644
index 0000000..acc1bea
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/swarmplayer.manpages
@@ -0,0 +1 @@
+debian/swarmplayer.1
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/swarmplayer.menu b/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/swarmplayer.menu
new file mode 100644
index 0000000..1de95ce
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/swarmplayer.menu
@@ -0,0 +1,4 @@
+?package(swarmplayer):needs="x11" section="Apps/Net" \
+	 title="SwarmPlayer" \
+	 icon="/usr/share/pixmaps/swarmplayer.xpm" \
+	 command="swarmplayer"
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/swarmplayer.postinst.debhelper b/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/swarmplayer.postinst.debhelper
new file mode 100644
index 0000000..8637a4e
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/swarmplayer.postinst.debhelper
@@ -0,0 +1,5 @@
+# Automatically added by dh_installmenu
+if [ "$1" = "configure" ] && [ -x "`which update-menus 2>/dev/null`" ]; then
+	update-menus
+fi
+# End automatically added section
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/swarmplayer.postrm.debhelper b/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/swarmplayer.postrm.debhelper
new file mode 100644
index 0000000..2b4be4f
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/swarmplayer.postrm.debhelper
@@ -0,0 +1,3 @@
+# Automatically added by dh_installmenu
+if [ -x "`which update-menus 2>/dev/null`" ]; then update-menus ; fi
+# End automatically added section
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/swarmplayer.sh b/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/swarmplayer.sh
new file mode 100755
index 0000000..b5be675
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/swarmplayer.sh
@@ -0,0 +1,31 @@
+#!/bin/sh
+# Startup script for Ubuntu Linux
+
+# don't care about gtk/x11/whatever. Currently (>= 3.4.0) must be unicode
+WXPYTHONVER24=`ls -1d /usr/lib/python2.4/site-packages/wx-2.8* 2>/dev/null | grep -v ansi | sed -e 's/.*wx-//g' -e 's/-.*//g' | sort -nr | head -1`
+WXPYTHONVER25=`ls -1d /usr/lib/python2.5/site-packages/wx-2.8* 2>/dev/null | grep -v ansi | sed -e 's/.*wx-//g' -e 's/-.*//g' | sort -nr | head -1`
+
+if [ "$WXPYTHONVER24" = "" ] && [ "$WXPYTHONVER25" = "" ];
+then
+    echo "Hmmm... No wxPython unicode package found for python2.4 or 2.5, cannot run Tribler, sorry"
+    exit -1
+fi
+
+if [ "$WXPYTHONVER25" = "" ];
+then
+    PYTHON="python2.4"
+    WXPYTHONVER=$WXPYTHONVER24
+    echo "Using python2.4"
+else
+    PYTHON="python2.5"
+    WXPYTHONVER=$WXPYTHONVER25
+    echo "Using python2.5"
+fi
+
+WXPYTHON=`ls -1d /usr/lib/$PYTHON/site-packages/wx-$WXPYTHONVER* | grep -v ansi | head -1`
+
+PYTHONPATH=/usr/share/swarmplayer/:$WXPYTHON
+export PYTHONPATH
+
+cd /usr/share/swarmplayer
+exec $PYTHON /usr/share/swarmplayer/Tribler/Player/swarmplayer.py "$@" > /tmp/$USER-swarmplayer.log 2>&1
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/swarmplayer.xpm b/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/swarmplayer.xpm
new file mode 100644
index 0000000..579b53f
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/swarmplayer.xpm
@@ -0,0 +1,257 @@
+/* XPM */
+static char * swarmplayer_xpm[] = {
+"32 32 222 2",
+"  	c None",
+". 	c #8F8F90",
+"+ 	c #909191",
+"@ 	c #8F9090",
+"# 	c #8D8D8E",
+"$ 	c #8B8B8C",
+"% 	c #8A8A8B",
+"& 	c #8A8A8A",
+"* 	c #898A8A",
+"= 	c #89898A",
+"- 	c #898989",
+"; 	c #888989",
+"> 	c #888889",
+", 	c #888888",
+"' 	c #878888",
+") 	c #878788",
+"! 	c #878787",
+"~ 	c #868787",
+"{ 	c #868687",
+"] 	c #858686",
+"^ 	c #868686",
+"/ 	c #8B8B8B",
+"( 	c #C0C0C0",
+"_ 	c #EBECEC",
+": 	c #EEEFEF",
+"< 	c #EEEEEE",
+"[ 	c #EDEEEE",
+"} 	c #ECEDED",
+"| 	c #ECECEC",
+"1 	c #EAEBEB",
+"2 	c #C7C7C8",
+"3 	c #616161",
+"4 	c #C2E1C2",
+"5 	c #7BCA7A",
+"6 	c #E0EAE0",
+"7 	c #AEDCAE",
+"8 	c #CEE5CE",
+"9 	c #E6ECE7",
+"0 	c #EBEEEC",
+"a 	c #D0E5D0",
+"b 	c #3EB93B",
+"c 	c #83CD82",
+"d 	c #5AC058",
+"e 	c #5EC45C",
+"f 	c #37B234",
+"g 	c #34B631",
+"h 	c #99D298",
+"i 	c #AEDAAE",
+"j 	c #73C571",
+"k 	c #19AA15",
+"l 	c #1BB317",
+"m 	c #A7D8A7",
+"n 	c #46B543",
+"o 	c #51BF4F",
+"p 	c #A7D5A7",
+"q 	c #42B93F",
+"r 	c #5DBE5C",
+"s 	c #B9DDBA",
+"t 	c #DCE8DD",
+"u 	c #DAE9DA",
+"v 	c #32B530",
+"w 	c #61C45F",
+"x 	c #57C255",
+"y 	c #11B00D",
+"z 	c #1CAE19",
+"A 	c #2AB627",
+"B 	c #57BE54",
+"C 	c #7AC879",
+"D 	c #4BBD49",
+"E 	c #36B934",
+"F 	c #23B120",
+"G 	c #33B630",
+"H 	c #9CD49C",
+"I 	c #CCE4CC",
+"J 	c #56C254",
+"K 	c #52C450",
+"L 	c #3FBB3C",
+"M 	c #38B736",
+"N 	c #0AAD06",
+"O 	c #15AD11",
+"P 	c #0FA90B",
+"Q 	c #1CAF19",
+"R 	c #27B825",
+"S 	c #55BC53",
+"T 	c #1EAF1B",
+"U 	c #25B421",
+"V 	c #38BA35",
+"W 	c #2EB42A",
+"X 	c #40BC3E",
+"Y 	c #5EC15C",
+"Z 	c #84CE83",
+"` 	c #9BD59B",
+" .	c #EDEFEE",
+"..	c #616262",
+"+.	c #9AD499",
+"@.	c #35B831",
+"#.	c #1EB11B",
+"$.	c #25B322",
+"%.	c #27B424",
+"&.	c #92D192",
+"*.	c #84CB83",
+"=.	c #ABDAAB",
+"-.	c #D7E7D8",
+";.	c #C5E1C5",
+">.	c #BCDDBC",
+",.	c #DDE9DE",
+"'.	c #56C255",
+").	c #7DCB7C",
+"!.	c #83CE82",
+"~.	c #95D595",
+"{.	c #C7E3C7",
+"].	c #626262",
+"^.	c #75CC74",
+"/.	c #15AF12",
+"(.	c #33B72F",
+"_.	c #1BB118",
+":.	c #36B532",
+"<.	c #6EC16D",
+"[.	c #DAE8DA",
+"}.	c #42BA3F",
+"|.	c #72CA70",
+"1.	c #0EAF0A",
+"2.	c #2BB428",
+"3.	c #B6DBB6",
+"4.	c #DEE8DF",
+"5.	c #C7E2C7",
+"6.	c #1FB51C",
+"7.	c #7AC97A",
+"8.	c #1AB116",
+"9.	c #B7DCB7",
+"0.	c #E0E9E1",
+"a.	c #31B82E",
+"b.	c #4ABD48",
+"c.	c #4EBE4C",
+"d.	c #6EC86C",
+"e.	c #E3E4E4",
+"f.	c #D6D7D7",
+"g.	c #DBDCDC",
+"h.	c #D7D8D8",
+"i.	c #DDDEDE",
+"j.	c #E7ECE8",
+"k.	c #87CD86",
+"l.	c #34B732",
+"m.	c #1AAC17",
+"n.	c #ADDAAD",
+"o.	c #9E9F9F",
+"p.	c #343434",
+"q.	c #252525",
+"r.	c #2A2A2A",
+"s.	c #525253",
+"t.	c #2B2B2B",
+"u.	c #6B6B6B",
+"v.	c #E9EAEA",
+"w.	c #87D086",
+"x.	c #5BC259",
+"y.	c #5ABF59",
+"z.	c #AAD8A9",
+"A.	c #EAEAEA",
+"B.	c #393939",
+"C.	c #515151",
+"D.	c #CFD0D0",
+"E.	c #C7C8C8",
+"F.	c #949494",
+"G.	c #DEDEDE",
+"H.	c #808080",
+"I.	c #D9DADA",
+"J.	c #8D8D8D",
+"K.	c #B4B5B5",
+"L.	c #61C460",
+"M.	c #46BC44",
+"N.	c #B4DDB4",
+"O.	c #3F4040",
+"P.	c #3B3B3B",
+"Q.	c #969697",
+"R.	c #BCBCBD",
+"S.	c #DEDFDF",
+"T.	c #ECEDEE",
+"U.	c #9C9C9C",
+"V.	c #B1B1B1",
+"W.	c #A5DAA4",
+"X.	c #4ABD47",
+"Y.	c #6AC768",
+"Z.	c #E9EDEA",
+"`.	c #BCBCBC",
+" +	c #565656",
+".+	c #313131",
+"++	c #262626",
+"@+	c #484848",
+"#+	c #DFE0E0",
+"$+	c #262727",
+"%+	c #282828",
+"&+	c #575858",
+"*+	c #E5E6E6",
+"=+	c #E6EDE7",
+"-+	c #82CE81",
+";+	c #2EB52B",
+">+	c #D4E7D5",
+",+	c #E6E8E8",
+"'+	c #919292",
+")+	c #A9AAAA",
+"!+	c #787878",
+"~+	c #C5C6C6",
+"{+	c #CECFCF",
+"]+	c #4EC04C",
+"^+	c #9ED79D",
+"/+	c #6A6B6B",
+"(+	c #505050",
+"_+	c #CBCCCC",
+":+	c #8D8E8E",
+"<+	c #C5E3C6",
+"[+	c #AAAAAA",
+"}+	c #787979",
+"|+	c #646464",
+"1+	c #6F7070",
+"2+	c #B7B7B7",
+"3+	c #AAABAB",
+"4+	c #C8C8C8",
+"5+	c #D3D4D4",
+"6+	c #848484",
+"7+	c #838484",
+"8+	c #838383",
+"9+	c #828282",
+"                                                                ",
+"      . + @ # $ % & * = - ; > , ' ' ) ! ~ { ] ^ { , * / =       ",
+"    ( _ : : : : : : : : : : : : : < [ [ [ } } | _ _ _ _ 1 2     ",
+"    } : : : : : : : : : : : : : : : : : : : : : : : : : : [ 3   ",
+"    : : : : : : : : : : : : : : : : : : : : : : : : : : : : 3   ",
+"    : : : : : : : : : : : : : : : : : : : : : : : : : : : : 3   ",
+"    : : : : : : : : 4 5 6 7 8 9 : : : : : : : : : : : : : : 3   ",
+"    : : : : : : 0 a b c d e f g h : : : : : : : : : : : : : 3   ",
+"    : : : : : i j k l m n o p q r s t : : : : : : : : : : : 3   ",
+"    : : : : u v w x y z A B C D E F G H I 0 : : : : : : : : 3   ",
+"    : : : : J K L M N O P Q R S T U V W X Y Z `  .: : : : : ..  ",
+"    [ : : : +.@.#.$.%.&.*.=.-.;.>.,.'.e ).!.~.{. .: : : : : ].  ",
+"    [ : : : ^./.(._.:.<.: : : : : : : : : : : : : : : : : : ].  ",
+"    } : : [.}.|.1.2.3.4.: : : : : : : : : : : : : : : : : : ].  ",
+"    _ : : 5.6.7.8.9.: : : : : : : : : : : : : : : : : : : : ].  ",
+"    _ : : 0.a.b.c.d.: : : : : e.f.g.1 : } h.h.h.i.[ : : : : ].  ",
+"    _ : : j.k.l.m.n.: : : : o.p.q.r.s.1 i.q.q.q.t.u.v.: : : ].  ",
+"    | : : : w.x.y.z.: : : A.B.C.D.E.F.v.G.q.H.I.J.q.K.: : : ].  ",
+"    } : : : -.L.M.N.: : : 1 O.P.Q.R.v.: S.q.* T.U.q.V.: : : ].  ",
+"    [ : : : : W.X.Y.Z.: : : `. +.+++@+#+S.q.++$+%+&+*+: : : ].  ",
+"    [ : : : : =+-+;+>+: : [ ,+: e.'+q.)+S.q.!+~+{+} : : : : ].  ",
+"    [ : : : : : =+]+^+: : 1 @+/+H.(+%+_+S.q.:+: : : : : : : ].  ",
+"    : : : : : : : <+e 0 : : [+}+|+1+2+: e.|+3+: : : : : : : ].  ",
+"    : : : : : : : : : : : : : : : : : : : : : : : : : : : : ].  ",
+"    : : : : : : : : : : : : : : : : : : : : : : : : : : : : ].  ",
+"    : : : : : : : : : : : : : : : : : : : : : : : : : : : : ..  ",
+"    : : : : : : : : : : : : : : : : : : : : : : : : : : : : ..  ",
+"    : : : : : : : : : : : : : : : : : : : : : : : : : : : : ].  ",
+"    [ : : : : : : : : : : : : : : : : : : : : : : : : : : :     ",
+"    4+: : : : : : : : : : : : : : : : : : : : : : : : : : 5+    ",
+"              / ^ 6+7+7+7+8+9+9+9+9+9+9+9+9+9+6+,               ",
+"                                                                "};
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/swarmplayer_big.xpm b/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/swarmplayer_big.xpm
new file mode 100644
index 0000000..80af874
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/Build/Ubuntu/swarmplayer_big.xpm
@@ -0,0 +1,563 @@
+/* XPM */
+static char * swarmplayer_big_xpm[] = {
+"48 48 512 2",
+"  	c None",
+". 	c #8F8F8F",
+"+ 	c #A6A6A6",
+"@ 	c #A4A5A5",
+"# 	c #A4A4A4",
+"$ 	c #A3A4A4",
+"% 	c #A2A3A4",
+"& 	c #A2A2A2",
+"* 	c #A1A1A2",
+"= 	c #A0A1A1",
+"- 	c #A0A0A1",
+"; 	c #A0A0A0",
+"> 	c #9FA0A0",
+", 	c #9F9FA0",
+"' 	c #9E9F9F",
+") 	c #9E9E9F",
+"! 	c #9E9E9E",
+"~ 	c #9D9E9E",
+"{ 	c #9D9D9E",
+"] 	c #9C9D9D",
+"^ 	c #9C9C9D",
+"/ 	c #9B9C9C",
+"( 	c #9B9B9C",
+"_ 	c #9B9B9B",
+": 	c #9C9D9E",
+"< 	c #8E8E8F",
+"[ 	c #DDDEDE",
+"} 	c #F0F0F0",
+"| 	c #F1F2F2",
+"1 	c #F0F1F1",
+"2 	c #EFF0F0",
+"3 	c #EEEFEF",
+"4 	c #EDEEEE",
+"5 	c #ECEDEE",
+"6 	c #ECEDED",
+"7 	c #EBECEC",
+"8 	c #EAEBEB",
+"9 	c #EAEAEB",
+"0 	c #E9EAEA",
+"a 	c #E8E9EA",
+"b 	c #E8E9E9",
+"c 	c #E7E8E9",
+"d 	c #E6E7E8",
+"e 	c #E6E6E7",
+"f 	c #E5E6E7",
+"g 	c #E4E5E6",
+"h 	c #E3E4E5",
+"i 	c #E3E3E4",
+"j 	c #E2E3E4",
+"k 	c #E2E3E3",
+"l 	c #D9DADA",
+"m 	c #A9AAAA",
+"n 	c #F5F6F6",
+"o 	c #F4F5F5",
+"p 	c #F3F4F4",
+"q 	c #F2F3F3",
+"r 	c #F1F2F3",
+"s 	c #F0F1F2",
+"t 	c #EDEEEF",
+"u 	c #EBECED",
+"v 	c #EAEBEC",
+"w 	c #E9EAEB",
+"x 	c #DEDFDF",
+"y 	c #616161",
+"z 	c #5A5A5A",
+"A 	c #EFF0F1",
+"B 	c #EEEFF0",
+"C 	c #757575",
+"D 	c #5B5B5B",
+"E 	c #F1F1F2",
+"F 	c #F3F4F5",
+"G 	c #F2F3F4",
+"H 	c #E3ECE4",
+"I 	c #C1E1C2",
+"J 	c #97D397",
+"K 	c #EAECEC",
+"L 	c #DFE9E1",
+"M 	c #D1E5D2",
+"N 	c #E1E9E2",
+"O 	c #E7E8E8",
+"P 	c #E6E7E7",
+"Q 	c #747474",
+"R 	c #77CA75",
+"S 	c #35B532",
+"T 	c #49BA47",
+"U 	c #C0E0C1",
+"V 	c #57C156",
+"W 	c #38BC35",
+"X 	c #5CC15A",
+"Y 	c #6FC66E",
+"Z 	c #ACD8AC",
+"` 	c #DCE6DD",
+" .	c #E5E7E7",
+"..	c #737373",
+"+.	c #ECEEED",
+"@.	c #DEE9DF",
+"#.	c #B2DAB2",
+"$.	c #57C155",
+"%.	c #31B62E",
+"&.	c #C2E0C2",
+"*.	c #58BE56",
+"=.	c #48BB45",
+"-.	c #8CD18C",
+";.	c #2AAC27",
+">.	c #4FBB4D",
+",.	c #12AF0F",
+"'.	c #55BD52",
+").	c #D0E0D1",
+"!.	c #737374",
+"~.	c #EAEEEB",
+"{.	c #C7E2C7",
+"].	c #9BD39A",
+"^.	c #37B133",
+"/.	c #15A811",
+"(.	c #10B00B",
+"_.	c #44BD42",
+":.	c #CDE2CE",
+"<.	c #60BE5F",
+"[.	c #2FB12C",
+"}.	c #36B933",
+"|.	c #7EC97D",
+"1.	c #83CB82",
+"2.	c #57BF55",
+"3.	c #51BA4F",
+"4.	c #A7D4A8",
+"5.	c #DDE4DF",
+"6.	c #E3E5E5",
+"7.	c #737474",
+"8.	c #C7E1C7",
+"9.	c #31B02F",
+"0.	c #86CC86",
+"a.	c #5CBE5A",
+"b.	c #0EAF0A",
+"c.	c #13B20F",
+"d.	c #44BD41",
+"e.	c #72C871",
+"f.	c #22AE1F",
+"g.	c #7AC679",
+"h.	c #71C56F",
+"i.	c #C4DDC4",
+"j.	c #83CC82",
+"k.	c #2BB528",
+"l.	c #40B63E",
+"m.	c #4BBC49",
+"n.	c #61C05F",
+"o.	c #7ECA7E",
+"p.	c #BBDABB",
+"q.	c #E1EAE2",
+"r.	c #7ACF78",
+"s.	c #15AD13",
+"t.	c #2AB426",
+"u.	c #A0D69F",
+"v.	c #39BA36",
+"w.	c #0CAF08",
+"x.	c #13B00F",
+"y.	c #14A810",
+"z.	c #0FB00C",
+"A.	c #60C55E",
+"B.	c #32B32F",
+"C.	c #3CB73A",
+"D.	c #93D093",
+"E.	c #2DB62A",
+"F.	c #2DB72B",
+"G.	c #2CB529",
+"H.	c #18AF15",
+"I.	c #20B21D",
+"J.	c #3AB838",
+"K.	c #8ACD89",
+"L.	c #B4DBB4",
+"M.	c #CFE2D1",
+"N.	c #E6E8E8",
+"O.	c #67C666",
+"P.	c #14B510",
+"Q.	c #64C863",
+"R.	c #23B320",
+"S.	c #7DCA7D",
+"T.	c #1FB01C",
+"U.	c #09AD05",
+"V.	c #0BAB08",
+"W.	c #0CAB08",
+"X.	c #0DA809",
+"Y.	c #14AF10",
+"Z.	c #19AE16",
+"`.	c #12B20F",
+" +	c #6DC56D",
+".+	c #58BB56",
+"++	c #1FAE1B",
+"@+	c #1DB11A",
+"#+	c #1AB117",
+"$+	c #3CB839",
+"%+	c #4ABE47",
+"&+	c #22B01E",
+"*+	c #30B72D",
+"=+	c #38B835",
+"-+	c #77C875",
+";+	c #99D398",
+">+	c #C0DFC1",
+",+	c #BADDBA",
+"'+	c #41BF3E",
+")+	c #79CD78",
+"!+	c #3CBA3A",
+"~+	c #27B425",
+"{+	c #1BAF17",
+"]+	c #0EB10A",
+"^+	c #36B833",
+"/+	c #44B942",
+"(+	c #1AAA16",
+"_+	c #20B31D",
+":+	c #5CBF5A",
+"<+	c #6ECC6D",
+"[+	c #70C76F",
+"}+	c #46BC43",
+"|+	c #3FB83D",
+"1+	c #6DC36B",
+"2+	c #7BCD7A",
+"3+	c #1DB419",
+"4+	c #34B731",
+"5+	c #35B732",
+"6+	c #4BBE49",
+"7+	c #7BCC7A",
+"8+	c #46BB44",
+"9+	c #41BD3F",
+"0+	c #4CBE4A",
+"a+	c #88CF88",
+"b+	c #E7EBE9",
+"c+	c #9BD39B",
+"d+	c #34BA31",
+"e+	c #25B121",
+"f+	c #1FB11C",
+"g+	c #23B21F",
+"h+	c #25B421",
+"i+	c #39B836",
+"j+	c #44BC41",
+"k+	c #BDDBBE",
+"l+	c #9FD29F",
+"m+	c #BADBBB",
+"n+	c #D3E2D5",
+"o+	c #E3E6E5",
+"p+	c #E0E5E2",
+"q+	c #DAE4DC",
+"r+	c #E1E6E3",
+"s+	c #E2E7E4",
+"t+	c #84CD83",
+"u+	c #53C351",
+"v+	c #76CB75",
+"w+	c #90CF8F",
+"x+	c #99D399",
+"y+	c #92D291",
+"z+	c #BBDFBC",
+"A+	c #CEE4CF",
+"B+	c #E5EAE6",
+"C+	c #A7D7A7",
+"D+	c #19B216",
+"E+	c #1CAE19",
+"F+	c #34B831",
+"G+	c #27B323",
+"H+	c #15AF11",
+"I+	c #42BB40",
+"J+	c #34B032",
+"K+	c #8ECA8D",
+"L+	c #81CD80",
+"M+	c #18BA15",
+"N+	c #1DB319",
+"O+	c #1EB31A",
+"P+	c #38BA35",
+"Q+	c #10B20C",
+"R+	c #4EBD4C",
+"S+	c #57BA54",
+"T+	c #B0D5B1",
+"U+	c #BDDCBE",
+"V+	c #48B945",
+"W+	c #65C464",
+"X+	c #8BCE8A",
+"Y+	c #08AE04",
+"Z+	c #16AD13",
+"`+	c #3BB739",
+" @	c #C9DECB",
+".@	c #DEE4E0",
+"+@	c #A2D4A1",
+"@@	c #25B522",
+"#@	c #37BD36",
+"$@	c #A6D5A6",
+"%@	c #16B113",
+"&@	c #5AC258",
+"*@	c #DBE3DC",
+"=@	c #D1E1D3",
+"-@	c #2BB929",
+";@	c #2BB429",
+">@	c #56BA54",
+",@	c #27B424",
+"'@	c #37BB34",
+")@	c #91D291",
+"!@	c #E9EBEB",
+"~@	c #EEEEEF",
+"{@	c #747575",
+"]@	c #D2E0D3",
+"^@	c #34B832",
+"/@	c #29B626",
+"(@	c #61C75F",
+"_@	c #6BC56A",
+":@	c #2DB429",
+"<@	c #A3D6A3",
+"[@	c #D4D5D6",
+"}@	c #C9C9CA",
+"|@	c #C9CACA",
+"1@	c #E4E5E5",
+"2@	c #D8D9D9",
+"3@	c #CCCCCD",
+"4@	c #CCCDCD",
+"5@	c #CDCDCE",
+"6@	c #CFD0D0",
+"7@	c #D9E3DB",
+"8@	c #71C770",
+"9@	c #33B630",
+"0@	c #33B830",
+"a@	c #14A911",
+"b@	c #36B733",
+"c@	c #CFE1D0",
+"d@	c #D3D4D5",
+"e@	c #676767",
+"f@	c #333333",
+"g@	c #252525",
+"h@	c #303030",
+"i@	c #474747",
+"j@	c #B6B7B7",
+"k@	c #707070",
+"l@	c #272727",
+"m@	c #A3A3A3",
+"n@	c #E4E6E6",
+"o@	c #C0DEC1",
+"p@	c #80CC80",
+"q@	c #58BF56",
+"r@	c #51BE4E",
+"s@	c #47BB44",
+"t@	c #D1E3D2",
+"u@	c #EBEBEC",
+"v@	c #686969",
+"w@	c #252626",
+"x@	c #2E2E2E",
+"y@	c #646464",
+"z@	c #4F4F4F",
+"A@	c #2D2D2D",
+"B@	c #717171",
+"C@	c #363636",
+"D@	c #656666",
+"E@	c #666666",
+"F@	c #454545",
+"G@	c #262626",
+"H@	c #C4C5C5",
+"I@	c #E5E6E6",
+"J@	c #ADD9AD",
+"K@	c #47BC44",
+"L@	c #4DBA4B",
+"M@	c #77C876",
+"N@	c #BADCBB",
+"O@	c #E1E2E3",
+"P@	c #3E3E3E",
+"Q@	c #7E7E7F",
+"R@	c #C0C1C1",
+"S@	c #C7C8C8",
+"T@	c #717172",
+"U@	c #5B5B5C",
+"V@	c #D7D7D8",
+"W@	c #3C3C3C",
+"X@	c #8E8F8F",
+"Y@	c #DFE6E0",
+"Z@	c #85CE84",
+"`@	c #6AC669",
+" #	c #52BC50",
+".#	c #4AC048",
+"+#	c #DAE7DB",
+"@#	c #404041",
+"##	c #565656",
+"$#	c #B9BABA",
+"%#	c #DCDDDE",
+"&#	c #717272",
+"*#	c #E0E1E1",
+"=#	c #424242",
+"-#	c #8C8D8D",
+";#	c #C2E0C3",
+">#	c #5DC65B",
+",#	c #3EBC3B",
+"'#	c #4EBE4C",
+")#	c #A8D8A8",
+"!#	c #878788",
+"~#	c #282828",
+"{#	c #2C2C2C",
+"]#	c #535353",
+"^#	c #949494",
+"/#	c #727272",
+"(#	c #838484",
+"_#	c #848585",
+":#	c #575757",
+"<#	c #292929",
+"[#	c #E8EAEA",
+"}#	c #8AD289",
+"|#	c #3EB93B",
+"1#	c #8CD08C",
+"2#	c #4DBF4B",
+"3#	c #E0EAE1",
+"4#	c #A5A5A5",
+"5#	c #5D5D5D",
+"6#	c #3A3A3A",
+"7#	c #2B2B2B",
+"8#	c #787878",
+"9#	c #888989",
+"0#	c #D9E7DA",
+"a#	c #65C563",
+"b#	c #3BB838",
+"c#	c #35B832",
+"d#	c #C9E4CA",
+"e#	c #606060",
+"f#	c #727373",
+"g#	c #4A4A4A",
+"h#	c #B3B4B4",
+"i#	c #B4B5B5",
+"j#	c #B8B9B9",
+"k#	c #EBEDED",
+"l#	c #E7EBE8",
+"m#	c #7CCC7A",
+"n#	c #28B324",
+"o#	c #ADDBAD",
+"p#	c #959696",
+"q#	c #E1E2E2",
+"r#	c #E9E9EA",
+"s#	c #838383",
+"t#	c #DFE0E0",
+"u#	c #AFDCAF",
+"v#	c #45BE43",
+"w#	c #67C865",
+"x#	c #E8EEE9",
+"y#	c #414242",
+"z#	c #4E4E4E",
+"A#	c #494949",
+"B#	c #2A2A2A",
+"C#	c #E7E7E8",
+"D#	c #98D597",
+"E#	c #3CBA39",
+"F#	c #C2E2C2",
+"G#	c #9A9B9B",
+"H#	c #414141",
+"I#	c #313232",
+"J#	c #E3E4E4",
+"K#	c #7A7A7A",
+"L#	c #616262",
+"M#	c #ECEFEE",
+"N#	c #A7DBA7",
+"O#	c #C9E5C9",
+"P#	c #EEF0F0",
+"Q#	c #F2F4F4",
+"R#	c #D1D1D1",
+"S#	c #D0D1D1",
+"T#	c #EEEEEE",
+"U#	c #DCDDDD",
+"V#	c #DBDBDB",
+"W#	c #DBDCDC",
+"X#	c #DADBDB",
+"Y#	c #CECFCF",
+"Z#	c #929292",
+"`#	c #B3B3B3",
+" $	c #888888",
+".$	c #A5A6A6",
+"+$	c #9C9C9C",
+"@$	c #989999",
+"#$	c #646565",
+"$$	c #818181",
+"%$	c #686868",
+"&$	c #ACACAC",
+"*$	c #B5B6B6",
+"=$	c #8B8B8B",
+"-$	c #C9C9C9",
+";$	c #A7A7A7",
+">$	c #AEAFAF",
+",$	c #8B8C8C",
+"'$	c #939393",
+")$	c #B0B1B1",
+"!$	c #868686",
+"~$	c #707171",
+"{$	c #949595",
+"]$	c #7C7D7D",
+"^$	c #9F9F9F",
+"/$	c #999A9A",
+"($	c #CACBCB",
+"_$	c #909191",
+":$	c #B2B2B2",
+"<$	c #6B6B6B",
+"[$	c #CACACA",
+"}$	c #858585",
+"|$	c #979898",
+"1$	c #919292",
+"2$	c #BCBDBD",
+"3$	c #939494",
+"4$	c #C1C2C2",
+"5$	c #B2B3B3",
+"6$	c #BFC0C0",
+"7$	c #C2C3C3",
+"8$	c #A7A8A8",
+"9$	c #C4C4C4",
+"0$	c #ABACAC",
+"a$	c #C3C4C4",
+"b$	c #B1B2B2",
+"c$	c #CDCECE",
+"d$	c #BDBEBE",
+"e$	c #585858",
+"f$	c #797A7A",
+"g$	c #979797",
+"h$	c #AFAFAF",
+"i$	c #9D9D9D",
+"j$	c #989898",
+"k$	c #969696",
+"l$	c #959595",
+"m$	c #9A9A9A",
+"                                                                                                ",
+"                                                                                                ",
+"        . + @ # $ % & * = - - ; ; > , , ' ' ) ! ~ ~ { { ] ] ^ ^ / / ( ( _ ( / ] : ~ ) <         ",
+"      ! [ } | | | 1 1 2 2 3 3 4 4 5 6 7 7 8 9 0 0 a b c d d e f g g h h i j k j i h h l m       ",
+"      [ n n n o o p p q r | s 1 2 2 3 t 4 5 6 u 7 v 8 w a a c c d d f f g g f f d d c c x y     ",
+"    z 1 n o o p p q q | | 1 A 2 B 3 t 4 6 u 7 v 8 w 0 a c c d d f f g g f f d d c c b a 0 C     ",
+"    D E o F p G q | | 1 1 2 B H I J K L M N v w w a b c O d f f g g f f P d O c c a a 0 w Q     ",
+"    z 1 p p q r | s 1 2 2 3 t R S T U V W X Y Z ` c c d e f g g f f  .d d c c a a 0 w 8 v ..    ",
+"    z 2 q r | s 1 A 2 3 +.@.#.$.%.&.*.=.-.;.>.,.'.).d f f g g f f d d c c b a 0 w 8 v v 7 !.    ",
+"    z 3 | | 1 A 2 B ~.{.].^./.(._.:.<.[.}.|.1.2.3.4.5.6.g f f d d O c b a a w w 8 v 7 u 6 7.    ",
+"    z 4 1 1 2 B 3 t 8.9.0.a.b.c.d.e.f.g.h.i.j.k.l.m.n.o.p.f d d c c a a 0 w 8 v 7 u u 6 5 Q     ",
+"    z 5 2 2 3 t 4 q.r.s.t.u.v.w.x.y.z.A.B.C.D.E.F.G.H.I.J.K.L.M.N.a 0 w 8 v 7 7 u 6 5 4 t Q     ",
+"    z u B 3 t 4 6 O.P.Q.R.S.T.U.V.W.X.Y.Z.`. +.+++@+#+$+%+&+*+=+-+;+Z >+8 7 u 6 5 4 4 t 3 Q     ",
+"    z v t 4 5 6 u ,+'+)+!+~+{+]+^+/+(+_+:+<+[+}+|+1+2+3+4+5+6+7+8+9+0+a+b+6 5 5 4 t 3 B 2 Q     ",
+"    z 9 5 6 u 7 v c+d+e+f+g+h+i+j+k+l+m+n+o+o+p+q+r+s+t+u+v+w+x+y+z+A+B+6 5 4 t 3 B 2 2 1 Q     ",
+"    z 0 u 7 v 8 w C+D+E+F+G+H+I+J+K+g f f d d c c b a 0 w 8 v v 7 u 6 5 4 t 3 3 B 2 A 1 s Q     ",
+"    z b v 8 w 0 a L+M+N+O+P+Q+R+S+T+f d d c c b a a w w 8 v 7 u 6 5 4 4 t 3 B 2 A 1 1 | | Q     ",
+"    z c w 0 a b U+V+W+X+Y+Z+`+ @.@d d O c c a a w w 8 v 7 u 6 5 5 4 t 3 B 2 A 1 1 | | r q Q     ",
+"    z d a a c c +@@@#@$@%@&@*@f d d c c a a 0 w 8 v 7 u u 6 5 4 t 3 B 2 2 A 1 s | r q q p Q     ",
+"    z f c c d d =@-@;@>@,@'@)@d c c b a 0 w !@v v 7 u 6 5 4 t ~@3 B 2 A 1 s | | q q p p F {@    ",
+"    z g d d f f ]@^@/@(@_@:@<@c b a a w w 8 a [@}@|@[@1@4 t 3 2@3@4@4@5@6@O q q G p p o o C     ",
+"    z h f f g g 7@8@9@0@a@b@c@a a w w 8 v d@e@f@g@g@h@i@j@B 2 k@g@g@g@g@l@i@m@p p o o n n C     ",
+"    z i n@g f f f o@p@q@r@s@t@0 w 8 v u@7 v@w@x@D y@z@A@$ 2 A B@g@C@D@E@F@G@A@H@o o n n n C     ",
+"    z k I@f f d d J@K@v.L@M@N@w v v 7 u O@P@g@Q@8 3 g R@S@1 s T@g@U@s p V@W@g@X@n n n n o {@    ",
+"    z i f d d O c Y@Z@`@ #.#+#v 7 u 6 5 h @#g@##$#%#O A 1 | | &#g@D | p *#=#g@-#n n n o o {@    ",
+"    z h d d c c a a ;#>#,#'#)#u u 6 5 4 t !#~#g@{#C@]#^#I@r q /#g@P@(#_#:#l@<#$#n o o o o {@    ",
+"    z 1@c c b a 0 w [#}#|#1#2#3#5 4 t 3 B O 4#5#6#7#g@l@8#2 p /#g@g@g@g@G@C@9#q o o o o o {@    ",
+"    z I@b a a w w 8 v 0#a#b#c#d#4 3 3 B 2 A 1 s P S@e#g@6#[ p f#g@g#h#i#j#*#o o o o o o p Q     ",
+"    z f a w w 8 v 7 u k#l#m#n#o#3 B 2 A a p#R@q#6 r#s#g@W@t#o f#g@D p n o o o o o o p p p Q     ",
+"    z P w 8 v 7 u u 6 5 4 u#v#w#x#2 A 1 8 y#{#=#z#A#B#g@k@q n ..g@D q o o o o o o p p p p Q     ",
+"    z C#v v 7 u 6 5 4 t 3 ~.D#E#F#1 s | A G#y H#I#I#F@s#J#n n K#I#L#q o o o o p p p p p p Q     ",
+"    z c 7 u 6 5 4 4 ~@3 B 2 M#N#O#| | r q G 4 I@*#q#O o n n o b *#I@o o o p p p p p p p q Q     ",
+"    z a 6 5 5 4 t 3 B 2 2 1 1 s | r q G p p o o n n n n n o o o o o o p p p p p p p q q q Q     ",
+"    z 0 5 4 t 3 B 2 2 1 1 s | | q q p p o o o n n n n o o o o o o o p p p p p p p q q q q Q     ",
+"    z 8 t 3 3 P#2 A 1 s | | q q Q#p F o o n n n n o o o o o o o p p p p p p p q q q q q q Q     ",
+"    z v 3 B 2 A 1 1 | | r q G p p o o n n n n o o o o o o o p p p p p p p q q q q q q q | Q     ",
+"    z 7 2 2 1 1 | | r q q p p o o n n n n n o o o o o o o p p p p p p q q q q q q q | | | Q     ",
+"    z 6 1 1 s | | q q J#V@h p 8 0 R#0 S#b 1@T#U#S#b 0 p t#V#J#4 W#6@X#Y#J#q q q | | | | | Q     ",
+"    z 5 s | | q q G 1 Z#$ -#|@`# $.$8#+$@$#$$$%$&$ $*$3 =$_#^#-$%$;$>$,$'$2 | | | | | | | Q     ",
+"    z 4 | r q G p p p )$!$= ~$'$(#{$%$]$^$/$($%$_$; :$O /#<$[$}$|$1$2$Q 3$1 | | | | | 1 1 Q     ",
+"    z 3 q q p p o o o 4$.$b 5$6$7$7 8$l 4$9$*#;$7 t#Z#0$4$a$8 4$a$|$b$c$d$1 | | | | 1 1 1 Q     ",
+"    D 3 p p F o o n n n n o o o o o o o p p p p p p q q q q q q q | | | | | | | 1 1 1 1 1 C     ",
+"    e$2 p o o n n n n o o o o o o o p p p p p p p q q q q q q q | | | | | | 1 1 1 1 1 1 1 f$    ",
+"      I@o n n n n n o o o o o o p p p p p p p q q q q q q q | | | | | | 1 1 1 1 1 1 1 2 8       ",
+"      g$6 n n n o o o o o o o p p p p p p q q q q q q q | | | | | | | 1 1 1 1 1 1 2 2 7 h$      ",
+"          i$; ^$^$! i$_ j$k$p#p#l$l$l$l$l$l$l$l$l$l$l${${${${${$l$l$l$g$m$+$i$! ~ ! i$          ",
+"                                                                                                ",
+"                                                                                                "};
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Win32/heading.bmp b/instrumentation/next-share/BaseLib/Player/Build/Win32/heading.bmp
new file mode 100644
index 0000000..7bdbfcd
Binary files /dev/null and b/instrumentation/next-share/BaseLib/Player/Build/Win32/heading.bmp differ
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Win32/setuptriblerplay.py b/instrumentation/next-share/BaseLib/Player/Build/Win32/setuptriblerplay.py
new file mode 100644
index 0000000..068ba57
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/Build/Win32/setuptriblerplay.py
@@ -0,0 +1,52 @@
+# setup.py
+import sys
+import os
+
+try:
+    import py2exe.mf as modulefinder
+    import win32com
+    for p in win32com.__path__[1:]:
+        modulefinder.AddPackagePath("win32com", p)
+    for extra in ["win32com.shell"]:
+        __import__(extra)
+        m = sys.modules[extra]
+        for p in m.__path__[1:]:
+            modulefinder.AddPackagePath(extra, p)
+except ImportError:
+    pass
+
+from distutils.core import setup
+import py2exe
+
+from BaseLib.__init__ import LIBRARYNAME
+
+################################################################
+#
+# Setup script used for py2exe
+#
+# *** Important note: ***
+# Setting Python's optimize flag when building disables
+# "assert" statments, which are used throughout the
+# BitTornado core for error-handling.
+#
+################################################################
+
+mainfile = os.path.join(LIBRARYNAME,'Player','swarmplayer.py')
+progicofile = os.path.join(LIBRARYNAME,'Images','SwarmPlayerIcon.ico')
+
+target_player = {
+    "script": mainfile,
+    "icon_resources": [(1, progicofile)],
+}
+
+
+setup(
+#    (Disabling bundle_files for now -- apparently causes some issues with Win98)
+#    options = {"py2exe": {"bundle_files": 1}},
+#    zipfile = None,
+    options = {"py2exe": {"packages": [LIBRARYNAME+".Core","encodings"],"optimize": 2}},
+    data_files = [("installdir",[])], 
+    windows = [target_player],
+)
+
+#data_files = [("installdir", [manifest, nsifile, progicofile, toricofile, "binary-LICENSE.txt", "readme.txt"])],
\ No newline at end of file
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Win32/swarmplayer.exe.manifest b/instrumentation/next-share/BaseLib/Player/Build/Win32/swarmplayer.exe.manifest
new file mode 100644
index 0000000..78c537e
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/Build/Win32/swarmplayer.exe.manifest
@@ -0,0 +1,23 @@
+
+
+	
+	My Manifest Testing application
+	
+	   
+	     
+	   
+	
+
+
diff --git a/instrumentation/next-share/BaseLib/Player/Build/Win32/triblerplay.nsi b/instrumentation/next-share/BaseLib/Player/Build/Win32/triblerplay.nsi
new file mode 100644
index 0000000..e10c485
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/Build/Win32/triblerplay.nsi
@@ -0,0 +1,218 @@
+!define PRODUCT "SwarmPlayer"
+!define VERSION "1.1.0"
+!define LIBRARYNAME "BaseLib"
+
+
+!include "MUI.nsh"
+
+;--------------------------------
+;Configuration
+
+;General
+ Name "${PRODUCT} ${VERSION}"
+OutFile "${PRODUCT}_${VERSION}.exe"
+
+;Folder selection page
+InstallDir "$PROGRAMFILES\${PRODUCT}"
+ 
+;Remember install folder
+InstallDirRegKey HKCU "Software\${PRODUCT}" ""
+
+;
+; Uncomment for smaller file size
+;
+SetCompressor "lzma"
+;
+; Uncomment for quick built time
+;
+;SetCompress "off"
+
+CompletedText "Installation completed. Thank you for choosing ${PRODUCT}"
+
+BrandingText "${PRODUCT}"
+
+;--------------------------------
+;Modern UI Configuration
+
+!define MUI_ABORTWARNING
+!define MUI_HEADERIMAGE
+!define MUI_HEADERIMAGE_BITMAP "heading.bmp"
+
+;--------------------------------
+;Pages
+
+!define MUI_LICENSEPAGE_RADIOBUTTONS
+!define MUI_LICENSEPAGE_RADIOBUTTONS_TEXT_ACCEPT "I accept"
+!define MUI_LICENSEPAGE_RADIOBUTTONS_TEXT_DECLINE "I decline"
+;   !define MUI_FINISHPAGE_RUN "$INSTDIR\swarmplayer.exe"
+
+!insertmacro MUI_PAGE_LICENSE "binary-LICENSE.txt"
+!insertmacro MUI_PAGE_COMPONENTS
+!insertmacro MUI_PAGE_DIRECTORY
+!insertmacro MUI_PAGE_INSTFILES
+!insertmacro MUI_PAGE_FINISH
+
+!insertmacro MUI_UNPAGE_CONFIRM
+!insertmacro MUI_UNPAGE_INSTFILES
+
+;!insertmacro MUI_DEFAULT UMUI_HEADERIMAGE_BMP heading.bmp"
+
+;--------------------------------
+;Languages
+
+!insertmacro MUI_LANGUAGE "English"
+ 
+;--------------------------------
+;Language Strings
+
+;Description
+LangString DESC_SecMain ${LANG_ENGLISH} "Install ${PRODUCT}"
+LangString DESC_SecDesk ${LANG_ENGLISH} "Create Desktop Shortcuts"
+LangString DESC_SecStart ${LANG_ENGLISH} "Create Start Menu Shortcuts"
+LangString DESC_SecDefaultTStream ${LANG_ENGLISH} "Associate .tstream files with ${PRODUCT}"
+LangString DESC_SecDefaultTorrent ${LANG_ENGLISH} "Associate .torrent files with ${PRODUCT}"
+
+;--------------------------------
+;Installer Sections
+
+Section "!Main EXE" SecMain
+ SectionIn RO
+ SetOutPath "$INSTDIR"
+ File *.txt
+ File swarmplayer.exe.manifest
+ File swarmplayer.exe
+ File ffmpeg.exe
+ File /r vlc
+ File *.bat
+ Delete "$INSTDIR\*.pyd"
+ File *.pyd
+ Delete "$INSTDIR\python*.dll"
+ Delete "$INSTDIR\wx*.dll"
+ File *.dll
+ Delete "$INSTDIR\*.zip"
+ File *.zip
+ CreateDirectory "$INSTDIR\${LIBRARYNAME}"
+ CreateDirectory "$INSTDIR\${LIBRARYNAME}\Core"
+ SetOutPath "$INSTDIR\${LIBRARYNAME}\Core"
+ File ${LIBRARYNAME}\Core\*.txt
+ CreateDirectory "$INSTDIR\${LIBRARYNAME}\Core\Statistics"
+ SetOutPath "$INSTDIR\${LIBRARYNAME}\Core\Statistics"
+ File ${LIBRARYNAME}\Core\Statistics\*.txt
+ File ${LIBRARYNAME}\Core\Statistics\*.sql
+ CreateDirectory "$INSTDIR\${LIBRARYNAME}\Images"
+ SetOutPath "$INSTDIR\${LIBRARYNAME}\Images"
+ File ${LIBRARYNAME}\Images\*.*
+ CreateDirectory "$INSTDIR\${LIBRARYNAME}\Video"
+ CreateDirectory "$INSTDIR\${LIBRARYNAME}\Video\Images"
+ SetOutPath "$INSTDIR\${LIBRARYNAME}\Video\Images"
+ File ${LIBRARYNAME}\Video\Images\*.*
+ CreateDirectory "$INSTDIR\${LIBRARYNAME}\Lang"
+ SetOutPath "$INSTDIR\${LIBRARYNAME}\Lang"
+ IfFileExists user.lang userlang
+ File ${LIBRARYNAME}\Lang\*.*
+ userlang:
+ File /x user.lang ${LIBRARYNAME}\Lang\*.*
+ SetOutPath "$INSTDIR"
+ WriteRegStr HKEY_LOCAL_MACHINE "Software\Microsoft\Windows\CurrentVersion\Uninstall\${PRODUCT}" "DisplayName" "${PRODUCT} (remove only)"
+ WriteRegStr HKEY_LOCAL_MACHINE "Software\Microsoft\Windows\CurrentVersion\Uninstall\${PRODUCT}" "UninstallString" "$INSTDIR\Uninstall.exe"
+
+; Now writing to KHEY_LOCAL_MACHINE only -- remove references to uninstall from current user
+ DeleteRegKey HKEY_CURRENT_USER "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\${PRODUCT}"
+; Remove old error log if present
+ Delete "$INSTDIR\swarmplayer.exe.log"
+
+ WriteUninstaller "$INSTDIR\Uninstall.exe"
+
+ ; Add an application to the firewall exception list - All Networks - All IP Version - Enabled
+ SimpleFC::AddApplication "Tribler" "$INSTDIR\${PRODUCT}.exe" 0 2 "" 1
+ ; Pop $0 ; return error(1)/success(0)
+
+SectionEnd
+
+
+Section "Desktop Icons" SecDesk
+   CreateShortCut "$DESKTOP\${PRODUCT}.lnk" "$INSTDIR\${PRODUCT}.exe" ""
+SectionEnd
+
+
+Section "Startmenu Icons" SecStart
+   CreateDirectory "$SMPROGRAMS\${PRODUCT}"
+   CreateShortCut "$SMPROGRAMS\${PRODUCT}\Uninstall.lnk" "$INSTDIR\Uninstall.exe" "" "$INSTDIR\Uninstall.exe" 0
+   CreateShortCut "$SMPROGRAMS\${PRODUCT}\${PRODUCT}.lnk" "$INSTDIR\${PRODUCT}.exe" "" "$INSTDIR\${PRODUCT}.exe" 0
+SectionEnd
+
+
+Section "Make Default For .tstream" SecDefaultTStream
+   WriteRegStr HKCR .tstream "" tstream
+   WriteRegStr HKCR .tstream "Content Type" application/x-tribler-stream
+   WriteRegStr HKCR "MIME\Database\Content Type\application/x-tribler-stream" Extension .tstream
+   WriteRegStr HKCR tstream "" "TSTREAM File"
+   WriteRegBin HKCR tstream EditFlags 00000100
+   WriteRegStr HKCR "tstream\shell" "" open
+   WriteRegStr HKCR "tstream\shell\open\command" "" '"$INSTDIR\${PRODUCT}.exe" "%1"'
+   WriteRegStr HKCR "tstream\DefaultIcon" "" "$INSTDIR\${LIBRARYNAME}\Images\SwarmPlayerIcon.ico"
+SectionEnd
+
+
+Section /o "Make Default For .torrent" SecDefaultTorrent
+   ; Delete ddeexec key if it exists
+   DeleteRegKey HKCR "bittorrent\shell\open\ddeexec"
+   WriteRegStr HKCR .torrent "" bittorrent
+   WriteRegStr HKCR .torrent "Content Type" application/x-bittorrent
+   WriteRegStr HKCR "MIME\Database\Content Type\application/x-bittorrent" Extension .torrent
+   WriteRegStr HKCR bittorrent "" "TORRENT File"
+   WriteRegBin HKCR bittorrent EditFlags 00000100
+   WriteRegStr HKCR "bittorrent\shell" "" open
+   WriteRegStr HKCR "bittorrent\shell\open\command" "" '"$INSTDIR\${PRODUCT}.exe" "%1"'
+   WriteRegStr HKCR "bittorrent\DefaultIcon" "" "$INSTDIR\${LIBRARYNAME}\Images\torrenticon.ico"
+SectionEnd
+
+
+
+;--------------------------------
+;Descriptions
+
+!insertmacro MUI_FUNCTION_DESCRIPTION_BEGIN
+!insertmacro MUI_DESCRIPTION_TEXT ${SecMain} $(DESC_SecMain)
+!insertmacro MUI_DESCRIPTION_TEXT ${SecDesk} $(DESC_SecDesk)
+!insertmacro MUI_DESCRIPTION_TEXT ${SecStart} $(DESC_SecStart)
+;!insertmacro MUI_DESCRIPTION_TEXT ${SecLang} $(DESC_SecLang)
+!insertmacro MUI_DESCRIPTION_TEXT ${SecDefaultTStream} $(DESC_SecDefaultTStream)
+!insertmacro MUI_DESCRIPTION_TEXT ${SecDefaultTorrent} $(DESC_SecDefaultTorrent)
+!insertmacro MUI_FUNCTION_DESCRIPTION_END
+
+;--------------------------------
+;Uninstaller Section
+
+Section "Uninstall"
+
+ RMDir /r "$INSTDIR"
+
+ Delete "$DESKTOP\${PRODUCT}.lnk"
+ Delete "$SMPROGRAMS\${PRODUCT}\*.*"
+ RmDir  "$SMPROGRAMS\${PRODUCT}"
+
+ DeleteRegKey HKEY_LOCAL_MACHINE "SOFTWARE\${PRODUCT}"
+ DeleteRegKey HKEY_LOCAL_MACHINE "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\${PRODUCT}"
+
+ ; Remove an application from the firewall exception list
+ SimpleFC::RemoveApplication "$INSTDIR\${PRODUCT}.exe"
+ ; Pop $0 ; return error(1)/success(0)
+
+SectionEnd
+
+
+;--------------------------------
+;Functions Section
+
+Function .onInit
+  System::Call 'kernel32::CreateMutexA(i 0, i 0, t "SwarmPlayer") i .r1 ?e' 
+
+  Pop $R0 
+
+  StrCmp $R0 0 +3 
+
+  MessageBox MB_OK "The installer is already running."
+
+  Abort 
+FunctionEnd
diff --git a/instrumentation/next-share/BaseLib/Player/EmbeddedPlayer4Frame.py b/instrumentation/next-share/BaseLib/Player/EmbeddedPlayer4Frame.py
new file mode 100644
index 0000000..5077179
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/EmbeddedPlayer4Frame.py
@@ -0,0 +1,494 @@
+# Written by Fabian van der Werf and Arno Bakker
+# see LICENSE.txt for license information
+#
+# EmbeddedPlayerPanel is the panel used in Tribler 5.0
+# EmbeddedPlayer4FramePanel is the panel used in the SwarmPlayer / 4.5
+# 
+
+import wx
+import sys
+
+import os, shutil
+import time
+import random
+from time import sleep
+from tempfile import mkstemp
+from threading import currentThread,Event, Thread
+from traceback import print_stack,print_exc
+from textwrap import wrap
+
+from BaseLib.__init__ import LIBRARYNAME
+from BaseLib.Video.defs import *
+from BaseLib.Video.Progress import ProgressSlider, VolumeSlider
+from BaseLib.Video.Buttons import PlayerSwitchButton, PlayerButton
+from BaseLib.Video.VideoFrame import DelayTimer
+
+DEBUG = False
+
+class EmbeddedPlayer4FramePanel(wx.Panel):
+    """
+    The Embedded Player consists of a VLCLogoWindow and the media controls such 
+    as Play/Pause buttons and Volume Control.
+    """
+
+    def __init__(self, parent, utility, vlcwrap, logopath):
+        wx.Panel.__init__(self, parent, -1)
+        self.utility = utility
+
+        self.estduration = None
+
+        #self.SetBackgroundColour(wx.WHITE)
+        self.SetBackgroundColour(wx.BLACK)
+        mainbox = wx.BoxSizer(wx.VERTICAL)
+
+
+        if vlcwrap is None:
+            size = (320,64)
+        else:
+            size = (320,240) 
+        
+        self.vlcwin = VLCLogoWindow(self,size,vlcwrap,logopath, animate = False)
+        self.vlcwrap = vlcwrap
+
+        # Arno: until we figure out how to show in-playback prebuffering info
+        self.statuslabel = wx.StaticText(self, -1, 'Loading player...' )
+        self.statuslabel.SetForegroundColour(wx.WHITE)
+
+        if vlcwrap is not None:
+            ctrlsizer = wx.BoxSizer(wx.HORIZONTAL)        
+            #self.slider = wx.Slider(self, -1)
+            self.slider = ProgressSlider(self, self.utility, imgprefix='4frame')
+            self.slider.SetRange(0,1)
+            self.slider.SetValue(0)
+            self.oldvolume = None
+            
+                            
+            self.ppbtn = PlayerSwitchButton(self, os.path.join(self.utility.getPath(), LIBRARYNAME, 'Images'), 'pause', 'play')
+            self.ppbtn.Bind(wx.EVT_LEFT_UP, self.PlayPause)
+    
+            self.volumebox = wx.BoxSizer(wx.HORIZONTAL)
+            self.volumeicon = PlayerSwitchButton(self, os.path.join(self.utility.getPath(), LIBRARYNAME, 'Images'), 'volume', 'mute')   
+            self.volumeicon.Bind(wx.EVT_LEFT_UP, self.Mute)
+            self.volume = VolumeSlider(self, self.utility, imgprefix='4frame')
+            self.volume.SetRange(0, 100)
+            self.volumebox.Add(self.volumeicon, 0, wx.ALIGN_CENTER_VERTICAL)
+            self.volumebox.Add(self.volume, 0, wx.ALIGN_CENTER_VERTICAL, 0)
+    
+            self.fsbtn = PlayerButton(self, os.path.join(self.utility.getPath(), LIBRARYNAME, 'Images'), 'fullScreen')
+            self.fsbtn.Bind(wx.EVT_LEFT_UP, self.FullScreen)
+    
+            self.save_button = PlayerSwitchButton(self, os.path.join(self.utility.getPath(), LIBRARYNAME, 'Images'), 'saveDisabled', 'save')   
+            self.save_button.Bind(wx.EVT_LEFT_UP, self.Save)
+            self.save_callback = lambda:None
+            
+            ctrlsizer.Add(self.ppbtn, 0, wx.ALIGN_CENTER_VERTICAL)
+            ctrlsizer.Add(self.slider, 1, wx.ALIGN_CENTER_VERTICAL|wx.EXPAND)
+            ctrlsizer.Add(self.volumebox, 0, wx.ALIGN_CENTER_VERTICAL|wx.EXPAND)
+            ctrlsizer.Add(self.fsbtn, 0, wx.ALIGN_CENTER_VERTICAL)
+            ctrlsizer.Add(self.save_button, 0, wx.ALIGN_CENTER_VERTICAL)
+        
+        mainbox.Add(self.vlcwin, 1, wx.EXPAND, 1)
+        mainbox.Add(self.statuslabel, 0, wx.EXPAND|wx.LEFT|wx.RIGHT, 30)
+        if vlcwrap is not None:
+            mainbox.Add(ctrlsizer, 0, wx.ALIGN_BOTTOM|wx.EXPAND, 1)
+        self.SetSizerAndFit(mainbox)
+        
+        self.playtimer = None
+        self.update = False
+        self.timer = None
+        
+    def Load(self,url,streaminfo = None):
+        if DEBUG:
+            print >>sys.stderr,"embedplay: Load:",url,streaminfo,currentThread().getName()
+        # Arno: hack: disable dragging when not playing from file.
+        #if url is None or url.startswith('http:'):
+        #if url is not None and url.startswith('http:'):
+        #   self.slider.DisableDragging()
+        #else:
+        self.slider.EnableDragging()
+        self.SetPlayerStatus('')
+        if streaminfo is not None:
+            self.estduration = streaminfo.get('estduration',None)
+
+        # Arno, 2008-10-17: If we don't do this VLC gets the wrong playlist somehow
+        self.vlcwrap.stop()
+        self.vlcwrap.playlist_clear()
+             
+        self.vlcwrap.load(url,streaminfo=streaminfo)
+        
+        # Enable update of progress slider
+        self.update = True
+        wx.CallAfter(self.slider.SetValue,0)
+        if self.timer is None:
+            self.timer = wx.Timer(self)
+            self.Bind(wx.EVT_TIMER, self.UpdateSlider)
+        self.timer.Start(200)
+        
+    def StartPlay(self):
+        """ Start playing the new item after VLC has stopped playing the old
+        one
+        """
+        if DEBUG:
+            print >>sys.stderr,"embedplay: PlayWhenStopped"
+        self.playtimer = DelayTimer(self)
+
+    def Play(self, evt=None):
+        if DEBUG:
+            print >>sys.stderr,"embedplay: Play pressed"
+
+        if self.GetState() != MEDIASTATE_PLAYING:
+            self.ppbtn.setToggled(False)
+            self.vlcwrap.start()
+
+    def Pause(self, evt=None):
+        """ Toggle between playing and pausing of current item """
+        if DEBUG:
+            print >>sys.stderr,"embedplay: Pause pressed"
+        
+        if self.GetState() == MEDIASTATE_PLAYING:
+            self.ppbtn.setToggled(True)
+            self.vlcwrap.pause()
+
+
+    def PlayPause(self, evt=None):
+        """ Toggle between playing and pausing of current item """
+        if DEBUG:
+            print >>sys.stderr,"embedplay: PlayPause pressed"
+        
+        if self.GetState() == MEDIASTATE_PLAYING:
+            self.ppbtn.setToggled(True)
+            self.vlcwrap.pause()
+
+        else:
+            self.ppbtn.setToggled(False)
+            self.vlcwrap.resume()
+
+
+    def Seek(self, evt=None):
+        if DEBUG:
+            print >>sys.stderr,"embedplay: Seek"
+        
+        oldsliderpos = self.slider.GetValue()
+        print >>sys.stderr, 'embedplay: Seek: GetValue returned,',oldsliderpos
+        pos = int(oldsliderpos * 1000.0)
+        print >>sys.stderr, 'embedplay: Seek: newpos',pos
+        
+        try:
+            if self.GetState() == MEDIASTATE_STOPPED:
+                self.vlcwrap.start(pos)
+            else:
+                self.vlcwrap.set_media_position(pos)
+        except:
+            print_exc()
+            if DEBUG:
+                print >> sys.stderr, 'embedplay: could not seek'
+            self.slider.SetValue(oldsliderpos)
+        self.update = True
+        
+
+    def FullScreen(self,evt=None):
+        self.vlcwrap.set_fullscreen(True)
+
+    def Mute(self, evt = None):
+        if self.volumeicon.isToggled():
+            if self.oldvolume is not None:
+                self.vlcwrap.sound_set_volume(self.oldvolume)
+            self.volumeicon.setToggled(False)
+        else:
+            self.oldvolume = self.vlcwrap.sound_get_volume()
+            self.vlcwrap.sound_set_volume(0.0) # mute sound
+            self.volumeicon.setToggled(True)
+        
+    def Save(self, evt = None):
+        # save media content in different directory
+        if self.save_button.isToggled():
+            self.save_callback()
+            
+    
+    def SetVolume(self, evt = None):
+        if DEBUG:
+            print >> sys.stderr, "embedplay: SetVolume:",self.volume.GetValue()
+        self.vlcwrap.sound_set_volume(float(self.volume.GetValue()) / 100)
+        # reset mute
+        if self.volumeicon.isToggled():
+            self.volumeicon.setToggled(False)
+
+    def Stop(self):
+        if DEBUG:
+            print >> sys.stderr, "embedplay: Stop"
+        self.vlcwrap.stop()
+        self.ppbtn.SetLabel(self.utility.lang.get('playprompt'))
+        self.slider.SetValue(0)
+        if self.timer is not None:
+            self.timer.Stop()
+
+    def GetState(self):
+        """ Returns the state of VLC as summarized by Fabian: 
+        MEDIASTATE_PLAYING, MEDIASTATE_PAUSED, MEDIASTATE_STOPPED """
+        if DEBUG:
+            print >>sys.stderr,"embedplay: GetState"
+            
+        status = self.vlcwrap.get_stream_information_status()
+        
+        import vlc
+        if status == vlc.PlayingStatus:
+            return MEDIASTATE_PLAYING
+        elif status == vlc.PauseStatus:
+            return MEDIASTATE_PAUSED
+        else:
+            return MEDIASTATE_STOPPED
+
+
+    def EnableSaveButton(self, b, callback):
+        self.save_button.setToggled(b)
+        if b:
+            self.save_callback = callback
+        else:
+            self.save_callback = lambda:None
+
+    def Reset(self):
+        self.DisableInput()
+        self.Stop()
+        self.UpdateProgressSlider([False])
+
+    #
+    # Control on-screen information
+    #
+    def UpdateStatus(self,playerstatus,pieces_complete):
+        self.SetPlayerStatus(playerstatus)
+        if self.vlcwrap is not None:
+            self.UpdateProgressSlider(pieces_complete)
+    
+    def SetPlayerStatus(self,s):
+        self.statuslabel.SetLabel(s)
+
+    def SetContentName(self,s):
+        self.vlcwin.set_content_name(s)
+
+    def SetContentImage(self,wximg):
+        self.vlcwin.set_content_image(wximg)
+
+
+    #
+    # Internal methods
+    #
+    def EnableInput(self):
+        self.ppbtn.Enable(True)
+        self.slider.Enable(True)
+        self.fsbtn.Enable(True)
+
+    def UpdateProgressSlider(self, pieces_complete):
+        self.slider.setBufferFromPieces(pieces_complete)
+        self.slider.Refresh()
+        
+    def DisableInput(self):
+        return # Not currently used
+        
+        self.ppbtn.Disable()
+        self.slider.Disable()
+        self.fsbtn.Disable()
+
+    def UpdateSlider(self, evt):
+        if not self.volumeicon.isToggled():
+            self.volume.SetValue(int(self.vlcwrap.sound_get_volume() * 100))
+
+        if self.update and self.GetState() != MEDIASTATE_STOPPED:
+            len = self.vlcwrap.get_stream_information_length()
+            if len == -1 or len == 0:
+                if self.estduration is None:
+                    return
+                else:
+                    len = int(self.estduration)
+            else:
+                len /= 1000
+
+            cur = self.vlcwrap.get_media_position() / 1000
+
+            self.slider.SetRange(0, len)
+            self.slider.SetValue(cur)
+            self.slider.SetTimePosition(float(cur), len)
+
+    def StopSliderUpdate(self, evt):
+        self.update = False
+
+
+    def TellLVCWrapWindow4Playback(self):
+        if self.vlcwrap is not None:
+            self.vlcwin.tell_vclwrap_window_for_playback()
+
+    def ShowLoading(self):
+        pass
+    
+    
+
+
+class VLCLogoWindow(wx.Panel):
+    """ A wx.Window to be passed to the vlc.MediaControl to draw the video
+    in (normally). In addition, the class can display a logo, a thumbnail and a 
+    "Loading: bla.video" message when VLC is not playing.
+    """
+    
+    def __init__(self, parent, size, vlcwrap, logopath, fg=wx.WHITE, bg=wx.BLACK, animate = False, position = (300,300)):
+        wx.Panel.__init__(self, parent, -1, size=size)
+        self.parent = parent ##
+    
+        self.SetMinSize(size)
+        self.SetBackgroundColour(bg)
+        self.bg = bg
+        self.vlcwrap = vlcwrap
+        self.animation_running = False
+       
+        self.Bind(wx.EVT_KEY_UP, self.keyDown)
+
+        print >>sys.stderr,"VLCLogoWindow: logopath is",logopath
+
+        if logopath is not None and not animate:
+            self.logo = wx.BitmapFromImage(wx.Image(logopath),-1)
+        else:
+            self.logo = None
+        self.contentname = None
+        self.contentbm = None
+        self.Bind(wx.EVT_PAINT, self.OnPaint)
+        if sys.platform == 'darwin':
+            self.hsizermain = wx.BoxSizer(wx.HORIZONTAL)
+            self.vsizer = wx.BoxSizer(wx.VERTICAL)
+            self.vsizer.Add((0,70),0,0,0)
+        if animate:
+            if sys.platform == 'darwin':
+                self.agVideo = wx.animate.GIFAnimationCtrl(self, 1, logopath)
+            else:
+                self.agVideo = wx.animate.GIFAnimationCtrl(self, 1, logopath, pos = (110,70))
+            self.agVideo.Hide()
+            if sys.platform == 'darwin':
+                self.vsizer.Add(self.agVideo,0,wx.ALIGN_CENTRE_HORIZONTAL,0)
+                self.vsizer.Add((0,10),0,0,0)
+        else:
+            self.agVideo = None
+
+        #self.playbackText = wx.StaticText(self,-1,"Leave Tribler running\n for faster playback",wx.Point(30,140))
+        #self.playbackText.SetFont(wx.Font(8, wx.SWISS, wx.NORMAL, wx.BOLD, 0, "UTF-8"))
+        #self.playbackText.SetForegroundColour(wx.Colour(255,51,00))
+        if sys.platform == 'darwin':
+            self.loadingtext = wx.StaticText(self,-1,'')
+        else:
+            self.loadingtext = wx.StaticText(self,-1,'',wx.Point(0,200),wx.Size(320,30),style=wx.ALIGN_CENTRE)
+        if sys.platform == 'darwin':
+            self.loadingtext.SetFont(wx.Font(12, wx.SWISS, wx.NORMAL, wx.BOLD, 0, "UTF-8"))
+        else:
+            self.loadingtext.SetFont(wx.Font(10, wx.SWISS, wx.NORMAL, wx.BOLD, 0, "UTF-8"))
+        self.loadingtext.SetForegroundColour(wx.WHITE)
+
+        if sys.platform == 'darwin':
+            self.vsizer.Add(self.loadingtext,1,wx.ALIGN_CENTRE_HORIZONTAL,0)
+            self.hsizermain.Add(self.vsizer,1,wx.ALIGN_CENTRE_HORIZONTAL,0)
+            self.SetSizer(self.hsizermain)
+            self.SetAutoLayout(1)
+            self.Layout()
+            self.Refresh()
+        if self.vlcwrap is not None:
+            wx.CallAfter(self.tell_vclwrap_window_for_playback)
+        
+    def tell_vclwrap_window_for_playback(self):
+        """ This method must be called after the VLCLogoWindow has been
+        realized, otherwise the self.GetHandle() call that vlcwrap.set_window()
+        does, doesn't return a correct XID.
+        """
+        self.vlcwrap.set_window(self)
+
+    def get_vlcwrap(self):
+        return self.vlcwrap
+
+    def set_content_name(self,s):
+        if DEBUG:
+            print >>sys.stderr,"VLCWin: set_content_name"
+        self.contentname = s
+        self.Refresh()
+    
+    def set_content_image(self,wximg):
+        if DEBUG:
+            print >>sys.stderr,"VLCWin: set_content_image"
+        if wximg is not None:
+            self.contentbm = wx.BitmapFromImage(wximg,-1)
+        else:
+            self.contentbm = None
+
+    def is_animation_running(self):
+        return self.animation_running
+
+    def setloadingtext(self, text):
+        self.loadingtext.SetLabel(text)
+        self.Refresh()
+
+    def show_loading(self):
+        if self.agVideo:
+            self.agVideo.Show()
+            self.agVideo.Play()
+            self.animation_running = True
+            self.Refresh()
+
+
+        
+        
+    def stop_animation(self):
+        if self.agVideo:
+            self.agVideo.Stop()
+            self.agVideo.Hide()
+            self.animation_running = False
+            self.Refresh()
+
+    def OnPaint(self,evt):
+        dc = wx.PaintDC(self)
+        dc.Clear()
+        dc.BeginDrawing()        
+
+        x,y,maxw,maxh = self.GetClientRect()
+        halfx = (maxw-x)/2
+        halfy = (maxh-y)/2
+        if self.logo is None:
+            halfx = 10
+            halfy = 10
+            lheight = 20
+        else:
+            halfx -= self.logo.GetWidth()/2
+            halfy -= self.logo.GetHeight()/2
+            lheight = self.logo.GetHeight()
+
+        dc.SetPen(wx.Pen(self.bg,0))
+        dc.SetBrush(wx.Brush(self.bg))
+        if sys.platform == 'linux2':
+            dc.DrawRectangle(x,y,maxw,maxh)
+        if self.logo is not None:
+            dc.DrawBitmap(self.logo,halfx,halfy,True)
+        #logox = max(0,maxw-self.logo.GetWidth()-30)
+        #dc.DrawBitmap(self.logo,logox,20,True)
+
+        dc.SetTextForeground(wx.WHITE)
+        dc.SetTextBackground(wx.BLACK)
+        
+        lineoffset = 120
+        txty = halfy+lheight+lineoffset
+        if txty > maxh:
+            txty = 0
+        if self.contentname is not None:
+            txt = self.contentname
+            dc.DrawText(txt,30,txty)
+            lineoffset += 30
+
+        #txt = self.getStatus()
+        #dc.DrawText(txt,30,halfy+self.logo.GetHeight()+lineoffset)
+        
+        if self.contentbm is not None:
+            bmy = max(20,txty-20-self.contentbm.GetHeight())
+            dc.DrawBitmap(self.contentbm,30,bmy,True)
+        
+        dc.EndDrawing()
+        if evt is not None:
+            evt.Skip(True)
+
+
+    def keyDown(self, event):
+        Level = event.StopPropagation()
+        event.ResumePropagation(10)
+
+        event.Skip()
+
diff --git a/instrumentation/next-share/BaseLib/Player/PlayerVideoFrame.py b/instrumentation/next-share/BaseLib/Player/PlayerVideoFrame.py
new file mode 100644
index 0000000..3ecc453
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/PlayerVideoFrame.py
@@ -0,0 +1,96 @@
+# Written by Fabian van der Werf and Arno Bakker
+# see LICENSE.txt for license information
+
+import wx
+import sys
+
+from BaseLib.Video.VideoFrame import VideoBaseFrame
+from BaseLib.Player.EmbeddedPlayer4Frame import EmbeddedPlayer4FramePanel
+
+DEBUG = False
+
+
+
+
+class VideoFrame(wx.Frame,VideoBaseFrame):
+    """ Provides a wx.Frame around an EmbeddedPlayerPanel so the embedded player
+    is shown as a separate window. The Embedded Player consists of a VLCLogoWindow
+    and the media controls such as Play/Pause buttons and Volume Control.
+    """
+    
+    def __init__(self,parent,utility,title,iconpath,vlcwrap,logopath): ## rm utility
+        self.parent = parent    
+        self.utility = utility ## parent.utility
+        if title is None:
+            title = self.utility.lang.get('tb_video_short')
+        
+        if vlcwrap is None:
+            size = (800,150)
+        else:
+            if sys.platform == 'darwin':
+                size = (800,520)
+            else:
+                size = (800,520) # Use 16:9 aspect ratio: 500 = (800/16) * 9 + 50 for controls
+        wx.Frame.__init__(self, None, -1, title, size=size) 
+        self.Centre()
+        
+        self.create_videopanel(vlcwrap,logopath)
+
+        # Set icons for Frame
+        self.icons = wx.IconBundle()
+        self.icons.AddIconFromFile(iconpath,wx.BITMAP_TYPE_ICO)
+        self.SetIcons(self.icons)
+
+        self.Bind(wx.EVT_CLOSE, self.OnCloseWindow)
+
+    def create_videopanel(self,vlcwrap, logopath):
+        self.showingframe = False
+        self.videopanel = EmbeddedPlayer4FramePanel(self, self.utility, vlcwrap, logopath)
+        self.Hide()
+
+    def show_videoframe(self):
+        if DEBUG:
+            print >>sys.stderr,"videoframe: Swap IN videopanel"
+            
+        if self.videopanel is not None:
+            if not self.showingframe:
+                self.showingframe = True
+                self.Show()
+                
+            self.Raise()
+            self.SetFocus()
+                       
+            # H4x0r: We need to tell the VLC wrapper a XID of a
+            # window to paint in. Apparently on win32 the XID is only
+            # known when the window is shown. We give it the command
+            # to show here, so shortly after it should be shown.
+            #
+            wx.CallAfter(self.videopanel.TellLVCWrapWindow4Playback)
+
+    
+    def hide_videoframe(self):
+        if DEBUG:
+            print >>sys.stderr,"videoframe: Swap OUT videopanel"
+        if self.videopanel is not None:
+            self.videopanel.Reset()
+            if self.showingframe:
+                self.showingframe = False
+                self.Hide()
+
+    def get_videopanel(self):
+        return self.videopanel
+
+    def delete_videopanel(self):
+        self.videopanel = None
+
+    def get_window(self):
+        return self
+
+
+    def OnCloseWindow(self, event = None):
+        if sys.platform == 'darwin':
+            #self.videopanel.Stop()
+            self.videopanel.Reset()
+                
+        self.hide_videoframe()
+
diff --git a/instrumentation/next-share/BaseLib/Player/Reporter.py b/instrumentation/next-share/BaseLib/Player/Reporter.py
new file mode 100644
index 0000000..39cbef9
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/Reporter.py
@@ -0,0 +1,169 @@
+# Written by Jan David Mol
+# see LICENSE.txt for license information
+
+# Collects statistics about a download/VOD session, and sends it
+# home on a regular interval.
+
+import sys,urllib,zlib,pickle
+import thread
+import threading
+from random import shuffle
+from time import time
+from traceback import print_exc
+from BaseLib.Core.Session import Session
+
+PHONEHOME = True
+DEBUG = False
+
+class Reporter:
+    """ Old Reporter class used for July 2008 trial. See below for new """
+    
+    def __init__( self, sconfig ):
+        self.sconfig = sconfig
+
+        # time of initialisation
+        self.epoch = time()
+
+        # mapping from peer ids to (shorter) numbers
+        self.peernr = {}
+
+        # remember static peer information, such as IP
+        # self.peerinfo[id] = info string
+        self.peerinfo = {}
+
+        # remember which peers we were connected to in the last report
+        # self.connected[id] = timestamp when last seen
+        self.connected = {}
+
+        # collected reports
+        self.buffered_reports = []
+
+        # whether to phone home to send collected data
+        self.do_reporting = True
+
+        # send data at this interval (seconds)
+        self.report_interval = 30
+
+        # send first report immediately
+        self.last_report_ts = 0
+
+        # record when we started (used as a session id)
+        self.epoch = time()
+
+    def phone_home( self, report ):
+        """ Report status to a centralised server. """
+
+        #if DEBUG: print >>sys.stderr,"\nreport: ".join(reports)
+
+        # do not actually send if reporting is disabled
+        if not self.do_reporting or not PHONEHOME:
+            return
+
+        # add reports to buffer
+        self.buffered_reports.append( report )
+
+        # only process at regular intervals
+        now = time()
+        if now - self.last_report_ts < self.report_interval:
+            return
+        self.last_report_ts = now
+
+        # send complete buffer
+        s = pickle.dumps( self.buffered_reports )
+        self.buffered_reports = []
+
+        if DEBUG: print >>sys.stderr,"\nreport: phoning home."
+        try:
+            data = zlib.compress( s, 9 ).encode("base64")
+            sock = urllib.urlopen("http://swpreporter.tribler.org/reporting/report.cgi",data)
+            result = sock.read()
+            sock.close()
+
+            result = int(result)
+
+            if result == 0:
+                # remote server is not recording, so don't bother sending info
+                self.do_reporting = False
+            else:
+                self.report_interval = result
+        except IOError, e:
+            # error contacting server
+            print_exc(file=sys.stderr)
+            self.do_reporting = False
+        except ValueError, e:
+            # page did not obtain an integer
+            print >>sys.stderr,"report: got %s" % (result,)
+            print_exc(file=sys.stderr)
+            self.do_reporting = False
+        except:
+            # any other error
+            print_exc(file=sys.stderr)
+            self.do_reporting = False
+        if DEBUG: print >>sys.stderr,"\nreport: succes. reported %s bytes, will report again (%s) in %s seconds" % (len(data),self.do_reporting,self.report_interval)
+
+    def report_stat( self, ds ):
+        chokestr = lambda b: ["c","C"][int(bool(b))]
+        intereststr = lambda b: ["i","I"][int(bool(b))]
+        optstr = lambda b: ["o","O"][int(bool(b))]
+        protstr = lambda b: ["bt","g2g"][int(bool(b))]
+            
+        now = time()
+        v = ds.get_vod_stats() or { "played": 0, "stall": 0, "late": 0, "dropped": 0, "prebuf": -1, "pieces": {} }
+        # Arno, 2009-09-09: method removed, was unclean
+        vi = ds.get_videoinfo() or { "live": False, "inpath": "(none)", "status": None }
+        vs = vi["status"]
+
+        scfg = self.sconfig
+
+        down_total, down_rate, up_total, up_rate = 0, 0.0, 0, 0.0
+        peerinfo = {}
+
+        for p in ds.get_peerlist():
+            down_total += p["dtotal"]/1024
+            down_rate  += p["downrate"]/1024.0
+            up_total   += p["utotal"]/1024
+            up_rate    += p["uprate"]/1024.0
+
+            id = p["id"]
+            peerinfo[id] = {
+                "g2g": protstr(p["g2g"]),
+                "addr": "%s:%s:%s" % (p["ip"],p["port"],p["direction"]),
+                "id": id,
+                "g2g_score": "%s,%s" % (p["g2g_score"][0],p["g2g_score"][1]),
+                "down_str": "%s%s" % (chokestr(p["dchoked"]),intereststr(p["dinterested"])),
+                "down_total": p["dtotal"]/1024,
+                "down_rate": p["downrate"]/1024.0,
+                "up_str": "%s%s%s" % (chokestr(p["uchoked"]),intereststr(p["uinterested"]),optstr(p["optimistic"])),
+                "up_total": p["utotal"]/1024,
+                "up_rate": p["uprate"]/1024.0,
+            }
+
+        if vs:
+            valid_range = vs.download_range()
+        else:
+            valid_range = ""
+
+        stats = {
+            "timestamp":  time(),
+            "epoch":      self.epoch,
+            "listenport": scfg.get_listen_port(),
+            "infohash":   `ds.get_download().get_def().get_infohash()`,
+            "filename":   vi["inpath"],
+            "peerid":     `ds.get_peerid()`,  # Arno, 2009-09-09: method removed, should be Download method
+            "live":       vi["live"],
+            "progress":   100.00*ds.get_progress(),
+            "down_total": down_total,
+            "down_rate":  down_rate,
+            "up_total":   up_total,
+            "up_rate":    up_rate,
+            "p_played":   v["played"],
+            "t_stall":    v["stall"],
+            "p_late":     v["late"],
+            "p_dropped":  v["dropped"],
+            "t_prebuf":   v["prebuf"],
+            "peers":      peerinfo.values(),
+            "pieces":     v["pieces"],
+            "validrange": valid_range,
+        }
+
+        self.phone_home( stats )
diff --git a/instrumentation/next-share/BaseLib/Player/SvcTest.py b/instrumentation/next-share/BaseLib/Player/SvcTest.py
new file mode 100644
index 0000000..1183e17
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/SvcTest.py
@@ -0,0 +1,182 @@
+import wx
+import sys
+import time
+from traceback import print_exc
+from BaseLib.Video.utils import svcextdefaults, videoextdefaults
+from BaseLib.Core.API import *
+
+DEBUG = True
+# used to set different download speeds
+DOWNLOADSPEED = 200
+
+def svc_event_callback(d,event,params):
+    if event == VODEVENT_START:
+    
+        stream = params["stream"]
+        length   = params["length"]
+        mimetype = params["mimetype"]
+        
+        # save stream on a temp file for verification
+        f = open("stream","wb")
+
+        while True:
+            # Read data from the resulting stream.
+            # Every stream.read() call will give back the available layers for the
+            # following time slot.
+            # The first 6 Bytes tell us the piece size. Therefore depending on the 
+            # size of the stream, knowing the piece size, we can see how many layers 
+            # are given back for that specific time slot.
+            data = stream.read()
+            print >>sys.stderr,"main: VOD ready callback: reading",type(data)
+            print >>sys.stderr,"main: VOD ready callback: reading",len(data)
+            if len(data) == 0:
+                break
+            f.write(data)
+            time.sleep(2)
+        
+        # Stop the download
+        if STOP_AFTER:
+            d.stop()
+            
+        f.close()
+    
+        stream.close()
+        
+
+def state_callback(ds):
+    try:
+        d = ds.get_download()
+        p = "%.0f %%" % (100.0*ds.get_progress())
+        dl = "dl %.0f" % (ds.get_current_speed(DOWNLOAD))
+        ul = "ul %.0f" % (ds.get_current_speed(UPLOAD))
+        print >>sys.stderr,dlstatus_strings[ds.get_status() ],p,dl,ul,"====="
+    except:
+        print_exc()
+
+    return (1.0,False)
+
+
+def select_torrent_from_disk(self):
+        dlg = wx.FileDialog(None, 
+                            self.appname+': Select torrent to play', 
+                            '', # default dir
+                            '', # default file
+                            'TSTREAM and TORRENT files (*.tstream;*.torrent)|*.tstream;*.torrent', 
+                            wx.OPEN|wx.FD_FILE_MUST_EXIST)
+        if dlg.ShowModal() == wx.ID_OK:
+            filename = dlg.GetPath()
+        else:
+            filename = None
+        dlg.Destroy()
+        return filename
+
+
+def select_file_start_download(self,torrentfilename):
+    
+    if torrentfilename.startswith("http") or torrentfilename.startswith(P2PURL_SCHEME):
+        tdef = TorrentDef.load_from_url(torrentfilename)
+    else: 
+        tdef = TorrentDef.load(torrentfilename)
+    print >>sys.stderr,"main: Starting download, infohash is",`tdef.get_infohash()`
+    
+    # Select which video to play (if multiple)
+    videofiles = tdef.get_files(exts=videoextdefaults)
+    print >>sys.stderr,"main: Found video files",videofiles
+    
+    if len(videofiles) == 0:
+        print >>sys.stderr,"main: No video files found! Let user select"
+        # Let user choose any file
+        videofiles = tdef.get_files(exts=None)
+        
+    if len(videofiles) > 1:
+        selectedvideofile = self.ask_user_which_video_from_torrent(videofiles)
+        if selectedvideofile is None:
+            print >>sys.stderr,"main: User selected no video"
+            return False
+        dlfile = selectedvideofile
+    else:
+        dlfile = videofiles[0]
+
+# Ric: check if it as an SVC download. If it is add the enhancement layers to the dlfiles
+def is_svc(dlfile, tdef):
+    svcfiles = None
+    
+    if tdef.is_multifile_torrent():
+        enhancement =  tdef.get_files(exts=svcextdefaults)
+        # Ric: order the enhancement layer in the svcfiles list
+        enhancement.sort()
+        if tdef.get_length(enhancement[0]) == tdef.get_length(dlfile):
+            svcfiles = [dlfile]
+            svcfiles.extend(enhancement)
+            
+    return svcfiles
+
+def run_test(params = None):
+    
+    if params is None:
+        params = [""]
+    
+    if len(sys.argv) > 1:
+        params = sys.argv[1:]
+        torrentfilename = params[0]
+    else:
+        torrentfilename = self.select_torrent_from_disk()
+        if torrentfilename is None:
+            print >>sys.stderr,"main: User selected no file"
+            self.OnExit()
+            return False
+    
+    scfg = SessionStartupConfig()
+    scfg.set_megacache( False )
+    scfg.set_overlay( False )
+
+    s = Session( scfg )
+
+    tdef = TorrentDef.load(torrentfilename)
+    dcfg = DownloadStartupConfig()
+
+    
+    # Select which video to play (if multiple)
+    videofiles = tdef.get_files(exts=videoextdefaults)
+    print >>sys.stderr,"main: Found video files",videofiles
+    
+    if len(videofiles) == 0:
+        print >>sys.stderr,"main: No video files found! Let user select"
+        # Let user choose any file
+        
+    if len(videofiles) > 1:
+        print >>sys.stderr,"main: More then one video file found!!"
+    else:
+        videofile = videofiles[0]
+
+    # Ric: check for svc
+    if tdef.is_multifile_torrent():
+
+        dlfiles = is_svc(videofile, tdef)
+
+        if dlfiles is not None:
+            print >>sys.stderr,"main: Found SVC video!!"
+            dcfg.set_video_event_callback(svc_event_callback, svc=True)
+            dcfg.set_selected_files(dlfiles)
+    else:
+        dcfg.set_video_event_callback(svc_event_callback)
+        dcfg.set_selected_files([dlfile])        
+
+    
+    # Ric: Set here the desired download speed
+    dcfg.set_max_speed(DOWNLOAD,DOWNLOADSPEED)
+
+    d = s.start_download( tdef, dcfg )
+
+    d.set_state_callback(state_callback,getpeerlist=False)
+    print >>sys.stderr,"main: Saving content to", d.get_dest_files()
+
+    while True:
+      time.sleep(360)
+    print >>sys.stderr,"Sleeping seconds to let other threads finish"
+    time.sleep(2)
+
+
+
+if __name__ == '__main__':
+    run_test()
diff --git a/instrumentation/next-share/BaseLib/Player/UtilityStub.py b/instrumentation/next-share/BaseLib/Player/UtilityStub.py
new file mode 100644
index 0000000..8823655
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/UtilityStub.py
@@ -0,0 +1,37 @@
+# Written by ABC authors and Arno Bakker 
+# see LICENSE.txt for license information
+import sys
+import os
+
+from BaseLib.__init__ import LIBRARYNAME
+from BaseLib.Lang.lang import Lang
+
+################################################################
+#
+# Class: UtilityStub
+#
+################################################################
+class UtilityStub:
+    def __init__(self,installdir,statedir):
+        self.installdir = installdir
+        self.statedir = statedir
+
+        self.config = self
+
+        # Setup language files
+        self.lang = Lang(self)
+
+        
+        
+    def getConfigPath(self):
+        return self.statedir
+    
+    def getPath(self):
+        return self.installdir.decode(sys.getfilesystemencoding())
+
+    def Read(self,key):
+        if key == 'language_file':
+            return os.path.join(self.installdir,LIBRARYNAME,'Lang','english.lang')
+        elif key == 'videoplayerpath':
+            return 'vlc'
+        return None
diff --git a/instrumentation/next-share/BaseLib/Player/__init__.py b/instrumentation/next-share/BaseLib/Player/__init__.py
new file mode 100644
index 0000000..395f8fb
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/__init__.py
@@ -0,0 +1,2 @@
+# Written by Arno Bakker
+# see LICENSE.txt for license information
diff --git a/instrumentation/next-share/BaseLib/Player/swarmplayer-njaal.py b/instrumentation/next-share/BaseLib/Player/swarmplayer-njaal.py
new file mode 100644
index 0000000..e69de29
diff --git a/instrumentation/next-share/BaseLib/Player/swarmplayer.py b/instrumentation/next-share/BaseLib/Player/swarmplayer.py
new file mode 100644
index 0000000..9198c73
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/swarmplayer.py
@@ -0,0 +1,688 @@
+# Written by Arno Bakker, Choopan RATTANAPOKA, Jie Yang
+# see LICENSE.txt for license information
+#
+# This is the main file for the SwarmPlayer V1, which is a standalone P2P-based 
+# video player. SwarmPlayer V2, the transport protocol for use with HTML5 can be
+# found in Transport/SwarmEngine.py (Sharing code with SwarmPlugin and v1,
+# confusing the code a bit).
+#
+#
+# TODO: 
+# * set 'download_slice_size' to 32K, such that pieces are no longer
+#   downloaded in 2 chunks. This particularly avoids a bad case where you
+#   kick the source: you download chunk 1 of piece X
+#   from lagging peer and download chunk 2 of piece X from source. With the piece
+#   now complete you check the sig. As the first part of the piece is old, this
+#   fails and we kick the peer that gave us the completing chunk, which is the 
+#   source.
+#
+#   Note that the BT spec says: 
+#   "All current implementations use 2 15 , and close connections which request 
+#   an amount greater than 2 17." http://www.bittorrent.org/beps/bep_0003.html
+#
+#   So it should be 32KB already. However, the BitTorrent (3.4.1, 5.0.9), 
+#   BitTornado and Azureus all use 2 ** 14 = 16KB chunks.
+#
+# - See if we can use stream.seek() to optimize SwarmPlayer as well (see SwarmPlugin)
+
+# modify the sys.stderr and sys.stdout for safe output
+import BaseLib.Debug.console
+
+import os
+import sys
+import time
+import tempfile
+import shutil
+from traceback import print_exc
+from cStringIO import StringIO
+from threading import Thread
+from base64 import encodestring
+
+if sys.platform == "darwin":
+    # on Mac, we can only load VLC/OpenSSL libraries
+    # relative to the location of tribler.py
+    os.chdir(os.path.abspath(os.path.dirname(sys.argv[0])))
+try:
+    import wxversion
+    wxversion.select('2.8')
+except:
+    pass
+import wx
+
+from BaseLib.__init__ import LIBRARYNAME
+from BaseLib.Core.API import *
+from BaseLib.Core.Utilities.unicode import bin2unicode
+from BaseLib.Core.Utilities.timeouturlopen import urlOpenTimeout
+
+from BaseLib.Video.defs import * 
+from BaseLib.Video.VideoPlayer import VideoPlayer, VideoChooser
+from BaseLib.Video.utils import videoextdefaults
+from BaseLib.Utilities.LinuxSingleInstanceChecker import *
+from BaseLib.Utilities.Instance2Instance import Instance2InstanceClient
+
+from BaseLib.Player.PlayerVideoFrame import VideoFrame
+from BaseLib.Player.BaseApp import BaseApp
+
+from BaseLib.Core.Statistics.Status import *
+
+DEBUG = True
+ONSCREENDEBUG = False
+ALLOW_MULTIPLE = False
+
+PLAYER_VERSION = '1.1.0'
+
+I2I_LISTENPORT = 57894
+PLAYER_LISTENPORT = 8620
+VIDEOHTTP_LISTENPORT = 6879
+
+# Arno, 2010-03-08: declaration here gives warning, can't get rid of it.
+START_TIME = 0
+
+
+class PlayerApp(BaseApp):
+    def __init__(self, redirectstderrout, appname, appversion, params, single_instance_checker, installdir, i2iport, sport):
+        self.videoFrame = None
+        BaseApp.__init__(self, redirectstderrout, appname, appversion, params, single_instance_checker, installdir, i2iport, sport)
+
+        self.said_start_playback = False
+        self.decodeprogress = 0
+
+        
+    def OnInit(self):
+        try:
+            # If already running, and user starts a new instance without a URL 
+            # on the cmd line
+            if not ALLOW_MULTIPLE and self.single_instance_checker.IsAnotherRunning():
+                print >> sys.stderr,"main: Another instance running, no URL on CMD, asking user"
+                torrentfilename = self.select_torrent_from_disk()
+                if torrentfilename is not None:
+                    i2ic = Instance2InstanceClient(I2I_LISTENPORT,'START',torrentfilename)
+                    return False
+
+            # Do common initialization
+            BaseApp.OnInitBase(self)
+        
+            # Fire up the VideoPlayer, it abstracts away whether we're using
+            # an internal or external video player.
+            self.videoplayer = VideoPlayer.getInstance(httpport=VIDEOHTTP_LISTENPORT)
+            playbackmode = PLAYBACKMODE_INTERNAL
+            self.videoplayer.register(self.utility,preferredplaybackmode=playbackmode)
+            
+            # Open video window
+            self.start_video_frame()
+
+            # Load torrent
+            if self.params[0] != "":
+                torrentfilename = self.params[0]
+                
+                # TEST: just play video file
+                #self.videoplayer.play_url(torrentfilename)
+                #return True
+                
+            else:
+                torrentfilename = self.select_torrent_from_disk()
+                if torrentfilename is None:
+                    print >>sys.stderr,"main: User selected no file"
+                    self.OnExit()
+                    return False
+
+
+            # Start download
+            if not self.select_file_start_download(torrentfilename):
+                
+                self.OnExit()
+                return False
+
+            return True
+        
+        except Exception,e:
+            print_exc()
+            self.show_error(str(e))
+            self.OnExit()
+            return False
+
+
+    def start_video_frame(self):
+        self.videoFrame = PlayerFrame(self,self.appname)
+        self.Bind(wx.EVT_CLOSE, self.videoFrame.OnCloseWindow)
+        self.Bind(wx.EVT_QUERY_END_SESSION, self.videoFrame.OnCloseWindow)
+        self.Bind(wx.EVT_END_SESSION, self.videoFrame.OnCloseWindow)
+        self.videoFrame.show_videoframe()
+
+        if self.videoplayer is not None:
+            self.videoplayer.set_videoframe(self.videoFrame)
+        self.said_start_playback = False
+        
+        
+    def select_torrent_from_disk(self):
+        dlg = wx.FileDialog(None, 
+                            self.appname+': Select torrent to play', 
+                            '', # default dir
+                            '', # default file
+                            'TSTREAM and TORRENT files (*.tstream;*.torrent)|*.tstream;*.torrent', 
+                            wx.OPEN|wx.FD_FILE_MUST_EXIST)
+        if dlg.ShowModal() == wx.ID_OK:
+            filename = dlg.GetPath()
+        else:
+            filename = None
+        dlg.Destroy()
+        return filename
+
+
+    def select_file_start_download(self,torrentfilename):
+        
+        if torrentfilename.startswith("http") or torrentfilename.startswith(P2PURL_SCHEME):
+            tdef = TorrentDef.load_from_url(torrentfilename)
+        else: 
+            tdef = TorrentDef.load(torrentfilename)
+        print >>sys.stderr,"main: Starting download, infohash is",`tdef.get_infohash()`
+        poa = None
+        if tdef.get_cs_keys():
+            # This is a closed swarm, try to get a POA
+            poa = self._get_poa(tdef)
+        
+        # Select which video to play (if multiple)
+        videofiles = tdef.get_files(exts=videoextdefaults)
+        print >>sys.stderr,"main: Found video files",videofiles
+        
+        if len(videofiles) == 0:
+            print >>sys.stderr,"main: No video files found! Let user select"
+            # Let user choose any file
+            videofiles = tdef.get_files(exts=None)
+            
+        if len(videofiles) > 1:
+            selectedvideofile = self.ask_user_which_video_from_torrent(videofiles)
+            if selectedvideofile is None:
+                print >>sys.stderr,"main: User selected no video"
+                return False
+            dlfile = selectedvideofile
+        else:
+            dlfile = videofiles[0]
+
+
+        # Start video window if not open
+        if self.videoFrame is None:
+            self.start_video_frame()
+        else:
+            # Stop playing, reset stream progress info + sliders 
+            self.videoplayer.stop_playback(reset=True)
+            self.said_start_playback = False
+        self.decodeprogress = 0
+
+        # Display name and thumbnail
+        cname = tdef.get_name_as_unicode()
+        if len(videofiles) > 1:
+            cname += u' - '+bin2unicode(dlfile)
+        self.videoplayer.set_content_name(u'Loading: '+cname)
+        
+        try:
+            [mime,imgdata] = tdef.get_thumbnail()
+            if mime is not None:
+                f = StringIO(imgdata)
+                img = wx.EmptyImage(-1,-1)
+                img.LoadMimeStream(f,mime,-1)
+                self.videoplayer.set_content_image(img)
+            else:
+                self.videoplayer.set_content_image(None)
+        except:
+            print_exc()
+
+
+        # Start actual download
+        self.start_download(tdef,dlfile, poa)
+        return True
+
+
+
+    def ask_user_which_video_from_torrent(self,videofiles):
+        dlg = VideoChooser(self.videoFrame,self.utility,videofiles,title=self.appname,expl='Select which file to play')
+        result = dlg.ShowModal()
+        if result == wx.ID_OK:
+            index = dlg.getChosenIndex()
+            filename = videofiles[index]
+        else:
+            filename = None
+        dlg.Destroy()
+        return filename
+
+
+    # ARNOTODO: see how VideoPlayer manages stopping downloads
+    
+    def sesscb_vod_event_callback(self,d,event,params):
+        self.videoplayer.sesscb_vod_event_callback(d,event,params)
+        
+        
+    def get_supported_vod_events(self):
+        return self.videoplayer.get_supported_vod_events()
+
+
+    #
+    # Remote start of new torrents
+    #
+    def i2ithread_readlinecallback(self,ic,cmd):
+        """ Called by Instance2Instance thread """
+        
+        print >>sys.stderr,"main: Another instance called us with cmd",cmd
+        ic.close()
+        
+        if cmd.startswith('START '):
+            param = cmd[len('START '):]
+            torrentfilename = None
+            if param.startswith('http:'):
+                # Retrieve from web 
+                f = tempfile.NamedTemporaryFile()
+                n = urlOpenTimeout(param)
+                data = n.read()
+                f.write(data)
+                f.close()
+                n.close()
+                torrentfilename = f.name
+            else:
+                torrentfilename = param
+                
+            # Switch to GUI thread
+            wx.CallAfter(self.remote_start_download,torrentfilename)
+
+    def remote_start_download(self,torrentfilename):
+        """ Called by GUI thread """
+        self.videoplayer.stop_playback(reset=True)
+
+        self.remove_downloads_in_vodmode_if_not_complete()
+        self.select_file_start_download(torrentfilename)
+
+
+    #
+    # Display stats in videoframe
+    #
+    def gui_states_callback(self,dslist,haspeerlist):
+        """ Override BaseApp """
+        
+        (playing_dslist,totalhelping,totalspeed) = BaseApp.gui_states_callback(self,dslist,haspeerlist)
+        
+        # Don't display stats if there is no video frame to show them on.
+        if self.videoFrame is None:
+            return
+        elif len(playing_dslist) > 0:
+            ds = playing_dslist[0] # only single playing Download at the moment in swarmplayer 
+            self.display_stats_in_videoframe(ds,totalhelping,totalspeed)
+
+
+    def display_stats_in_videoframe(self,ds,totalhelping,totalspeed):
+        # Display stats for currently playing Download
+        
+        videoplayer_mediastate = self.videoplayer.get_state()
+        #print >>sys.stderr,"main: Stats: VideoPlayer state",videoplayer_mediastate
+        
+        [topmsg,msg,self.said_start_playback,self.decodeprogress] = get_status_msgs(ds,videoplayer_mediastate,self.appname,self.said_start_playback,self.decodeprogress,totalhelping,totalspeed)
+        # Display helping info on "content name" line.
+        self.videoplayer.set_content_name(topmsg)
+
+        # Update status msg and progress bar
+        self.videoplayer.set_player_status_and_progress(msg,ds.get_pieces_complete())
+        
+        # Toggle save button
+        self.videoplayer.set_save_button(ds.get_status() == DLSTATUS_SEEDING, self.save_video_copy)    
+            
+        if False: # Only works if the sesscb_states_callback() method returns (x,True)
+            peerlist = ds.get_peerlist()
+            print >>sys.stderr,"main: Connected to",len(peerlist),"peers"
+            for peer in peerlist:
+                print >>sys.stderr,"main: Connected to",peer['ip'],peer['uprate'],peer['downrate']
+
+
+    def videoserver_set_status_guicallback(self,status):
+        """ Override BaseApp """
+        if self.videoFrame is not None:
+            self.videoFrame.set_player_status(status)
+
+    #
+    # Save button logic
+    #
+    def save_video_copy(self):
+        # Save a copy of playing download to other location
+        for d2 in self.downloads_in_vodmode:
+            # only single playing Download at the moment in swarmplayer
+            d = d2
+        dest_files = d.get_dest_files()  
+        dest_file = dest_files[0] # only single file at the moment in swarmplayer
+        savethread_callback_lambda = lambda:self.savethread_callback(dest_file)
+        
+        t = Thread(target = savethread_callback_lambda)
+        t.setName( self.appname+"Save"+t.getName() )
+        t.setDaemon(True)
+        t.start()
+    
+    def savethread_callback(self,dest_file):
+        
+        # Save a copy of playing download to other location
+        # called by new thread from self.save_video_copy
+        try:
+            if sys.platform == 'win32':
+                # Jelle also goes win32, find location of "My Documents"
+                # see http://www.mvps.org/access/api/api0054.htm
+                from win32com.shell import shell
+                pidl = shell.SHGetSpecialFolderLocation(0,0x05)
+                defaultpath = shell.SHGetPathFromIDList(pidl)
+            else:
+                defaultpath = os.path.expandvars('$HOME')
+        except Exception, msg:
+            defaultpath = ''
+            print_exc()
+
+        dest_file_only = os.path.split(dest_file[1])[1]
+        
+        print >> sys.stderr, 'Defaultpath:', defaultpath, 'Dest:', dest_file
+        dlg = wx.FileDialog(self.videoFrame, 
+                            message = self.utility.lang.get('savemedia'), 
+                            defaultDir = defaultpath, 
+                            defaultFile = dest_file_only,
+                            wildcard = self.utility.lang.get('allfileswildcard') + ' (*.*)|*.*', 
+                            style = wx.SAVE)
+        dlg.Raise()
+        result = dlg.ShowModal()
+        dlg.Destroy()
+        
+        if result == wx.ID_OK:
+            path = dlg.GetPath()
+            print >> sys.stderr, 'Path:', path
+            print >> sys.stderr, 'Copy: %s to %s' % (dest_file[1], path)
+            if sys.platform == 'win32':
+                try:
+                    import win32file
+                    win32file.CopyFile(dest_file[1], path, 0) # do succeed on collision
+                except:
+                    shutil.copyfile(dest_file[1], path)
+            else:
+                shutil.copyfile(dest_file[1], path)
+
+    # On Exit
+
+    def clear_session_state(self):
+        """ Try to fix apps by doing hard reset. Called from systray menu """
+        try:
+            self.videoplayer.stop_playback()
+        except:
+            print_exc()
+        BaseApp.clear_session_state(self)
+
+def get_status_msgs(ds,videoplayer_mediastate,appname,said_start_playback,decodeprogress,totalhelping,totalspeed):
+
+    intime = "Not playing for quite some time."
+    ETA = ((60 * 15, "Playing in less than 15 minutes."),
+           (60 * 10, "Playing in less than 10 minutes."),
+           (60 * 5, "Playing in less than 5 minutes."),
+           (60, "Playing in less than a minute."))
+
+    topmsg = ''
+    msg = ''
+    
+    logmsgs = ds.get_log_messages()
+    logmsg = None
+    if len(logmsgs) > 0:
+        print >>sys.stderr,"main: Log",logmsgs[0]
+        logmsg = logmsgs[-1][1]
+        
+    preprogress = ds.get_vod_prebuffering_progress()
+    playable = ds.get_vod_playable()
+    t = ds.get_vod_playable_after()
+
+    # Instrumentation
+    # Status elements (reported periodically):
+    #   playable: True if playable
+    #   prebuffering: float of percentage full?
+    #
+    # Events:
+    #   failed_after: Failed to play after X seconds (since starting to play)
+    #   playable_in: Started playing after X seconds
+    status = Status.get_status_holder("LivingLab")
+
+    s_play = status.get_or_create_status_element("playable", False)
+    if playable:
+        if preprogress < 1.0:
+            if s_play.get_value() == True:
+                global START_TIME
+                status.create_and_add_event("failed_after", [time.time() - START_TIME])
+                START_TIME = time.time()
+                
+            s_play.set_value(False)
+
+        elif s_play.get_value() == False:
+            s_play.set_value(True)
+            global START_TIME
+            status.create_and_add_event("playable_in", [time.time() - START_TIME])
+            START_TIME = time.time()
+
+    elif preprogress < 1.0:
+        status.get_or_create_status_element("prebuffering").set_value(preprogress)
+    # /Instrumentation
+
+    intime = ETA[0][1]
+    for eta_time, eta_msg in ETA:
+        if t > eta_time:
+            break
+        intime = eta_msg
+    
+    #print >>sys.stderr,"main: playble",playable,"preprog",preprogress
+    #print >>sys.stderr,"main: ETA is",t,"secs"
+    # if t > float(2 ** 30):
+    #     intime = "inf"
+    # elif t == 0.0:
+    #     intime = "now"
+    # else:
+    #     h, t = divmod(t, 60.0*60.0)
+    #     m, s = divmod(t, 60.0)
+    #     if h == 0.0:
+    #         if m == 0.0:
+    #             intime = "%ds" % (s)
+    #         else:
+    #             intime = "%dm:%02ds" % (m,s)
+    #     else:
+    #         intime = "%dh:%02dm:%02ds" % (h,m,s)
+            
+    #print >>sys.stderr,"main: VODStats",preprogress,playable,"%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%"
+
+    if ds.get_status() == DLSTATUS_HASHCHECKING:
+        genprogress = ds.get_progress()
+        pstr = str(int(genprogress*100))
+        msg = "Checking already downloaded parts "+pstr+"% done"
+    elif ds.get_status() == DLSTATUS_STOPPED_ON_ERROR:
+        msg = 'Error playing: '+str(ds.get_error())
+    elif ds.get_progress() == 1.0:
+        msg = ''
+    elif playable:
+        if not said_start_playback:
+            msg = "Starting playback..."
+            
+        if videoplayer_mediastate == MEDIASTATE_STOPPED and said_start_playback:
+            if totalhelping == 0:
+                topmsg = u"Please leave the "+appname+" running, this will help other "+appname+" users to download faster."
+            else:
+                topmsg = u"Helping "+str(totalhelping)+" "+appname+" users to download. Please leave it running in the background."
+                
+            # Display this on status line
+            # TODO: Show balloon in systray when closing window to indicate things continue there
+            msg = ''
+            
+        elif videoplayer_mediastate == MEDIASTATE_PLAYING:
+            said_start_playback = True
+            # It may take a while for VLC to actually start displaying
+            # video, as it is trying to tune in to the stream (finding
+            # I-Frame). Display some info to show that:
+            #
+            cname = ds.get_download().get_def().get_name_as_unicode()
+            topmsg = u'Decoding: '+cname+' '+str(decodeprogress)+' s'
+            decodeprogress += 1
+            msg = ''
+        elif videoplayer_mediastate == MEDIASTATE_PAUSED:
+            # msg = "Buffering... " + str(int(100.0*preprogress))+"%" 
+            msg = "Buffering... " + str(int(100.0*preprogress))+"%. " + intime
+        else:
+            msg = ''
+            
+    elif preprogress != 1.0:
+        pstr = str(int(preprogress*100))
+        npeers = ds.get_num_peers()
+        npeerstr = str(npeers)
+        if npeers == 0 and logmsg is not None:
+            msg = logmsg
+        elif npeers == 1:
+            msg = "Prebuffering "+pstr+"% done (connected to 1 person). " + intime
+        else:
+            msg = "Prebuffering "+pstr+"% done (connected to "+npeerstr+" people). " + intime
+            
+        try:
+            d = ds.get_download()
+            tdef = d.get_def()
+            videofiles = d.get_selected_files()
+            if len(videofiles) >= 1:
+                videofile = videofiles[0]
+            else:
+                videofile = None
+            if tdef.get_bitrate(videofile) is None:
+                msg += ' This video may not play properly because its bitrate is unknown'
+        except:
+            print_exc()
+    else:
+        # msg = "Waiting for sufficient download speed... "+intime
+        msg = 'Waiting for sufficient download speed... ' + intime
+        
+    global ONSCREENDEBUG
+    if msg == '' and ONSCREENDEBUG:
+        uptxt = "up %.1f" % (totalspeed[UPLOAD])
+        downtxt = " down %.1f" % (totalspeed[DOWNLOAD])
+        peertxt = " peer %d" % (totalhelping)
+        msg = uptxt + downtxt + peertxt
+
+    return [topmsg,msg,said_start_playback,decodeprogress]
+
+
+
+class PlayerFrame(VideoFrame):
+    def __init__(self,parent,appname):
+        VideoFrame.__init__(self,parent,parent.utility,appname+' '+PLAYER_VERSION,parent.iconpath,parent.videoplayer.get_vlcwrap(),parent.logopath)
+        self.parent = parent
+        self.closed = False
+
+        dragdroplist = FileDropTarget(self.parent)
+        self.SetDropTarget(dragdroplist)
+        
+        self.Bind(wx.EVT_CLOSE, self.OnCloseWindow)
+    
+    def OnCloseWindow(self, event = None):
+        
+        print >>sys.stderr,"main: ON CLOSE WINDOW"
+
+        # TODO: first event.Skip does not close window, second apparently does
+        # Check how event differs
+
+        # This gets called multiple times somehow
+        if not self.closed:
+            self.closed = True
+            self.parent.videoFrame = None
+
+            self.parent.videoplayer.stop_playback()
+            self.parent.remove_downloads_in_vodmode_if_not_complete()
+            self.parent.restart_other_downloads()
+
+        if event is not None:
+            nr = event.GetEventType()
+            lookup = { wx.EVT_CLOSE.evtType[0]: "EVT_CLOSE", wx.EVT_QUERY_END_SESSION.evtType[0]: "EVT_QUERY_END_SESSION", wx.EVT_END_SESSION.evtType[0]: "EVT_END_SESSION" }
+            if nr in lookup: 
+                nr = lookup[nr]
+            print >>sys.stderr,"main: Closing due to event ",nr
+            event.Skip()
+        else:
+            print >>sys.stderr,"main: Closing untriggered by event"
+
+        print >>sys.stderr,"main: Closing done"
+        # TODO: Show balloon in systray when closing window to indicate things continue there
+
+    def set_player_status(self,s):
+        pass
+
+
+class FileDropTarget(wx.FileDropTarget):
+    """ To enable drag and drop of .tstream to window """
+ 
+    def __init__(self,app):
+        wx.FileDropTarget.__init__(self) 
+        self.app = app
+      
+    def OnDropFiles(self, x, y, filenames):
+        for filename in filenames:
+            self.app.remote_start_download(filename)
+        return True
+
+
+                
+        
+##############################################################
+#
+# Main Program Start Here
+#
+##############################################################
+def run_playerapp(appname,appversion,params = None):
+    global START_TIME
+    START_TIME = time.time()
+    
+    if params is None:
+        params = [""]
+    
+    if len(sys.argv) > 1:
+        params = sys.argv[1:]
+    
+    if 'debug' in params:
+        global ONSCREENDEBUG
+        ONSCREENDEBUG=True
+    if 'raw' in params:
+        BaseLib.Video.VideoPlayer.USE_VLC_RAW_INTERFACE = True
+    
+    # Create single instance semaphore
+    # Arno: On Linux and wxPython-2.8.1.1 the SingleInstanceChecker appears
+    # to mess up stderr, i.e., I get IOErrors when writing to it via print_exc()
+    #
+    siappname = appname.lower() # For backwards compatibility
+    if sys.platform != 'linux2':
+        single_instance_checker = wx.SingleInstanceChecker(siappname+"-"+ wx.GetUserId())
+    else:
+        single_instance_checker = LinuxSingleInstanceChecker(siappname)
+
+    #print "[StartUpDebug]---------------- 1", time()-start_time
+    if not ALLOW_MULTIPLE and single_instance_checker.IsAnotherRunning():
+        if params[0] != "":
+            torrentfilename = params[0]
+            i2ic = Instance2InstanceClient(I2I_LISTENPORT,'START',torrentfilename)
+            time.sleep(1)
+            return
+        
+    arg0 = sys.argv[0].lower()
+    if arg0.endswith('.exe'):
+        installdir = os.path.abspath(os.path.dirname(sys.argv[0]))
+    else:
+        installdir = os.getcwd()  
+
+    # Launch first single instance
+    app = PlayerApp(0, appname, appversion, params, single_instance_checker, installdir, I2I_LISTENPORT, PLAYER_LISTENPORT)
+
+    # Setup the statistic reporter while waiting for proper integration
+    status = Status.get_status_holder("LivingLab")
+    s = Session.get_instance()
+    id = encodestring(s.get_permid()).replace("\n","")
+    #reporter = LivingLabReporter.LivingLabPeriodicReporter("Living lab CS reporter", 300, id) # Report every 5 minutes
+    reporter = LivingLabReporter.LivingLabPeriodicReporter("Living lab CS reporter", 30, id) # Report every 30 seconds - ONLY FOR TESTING
+    status.add_reporter(reporter)
+
+    app.MainLoop()
+
+    reporter.stop()
+    
+    print >>sys.stderr,"Sleeping seconds to let other threads finish"
+    time.sleep(2)
+    
+    if not ALLOW_MULTIPLE:
+        del single_instance_checker
+
+
+if __name__ == '__main__':
+    run_playerapp("SwarmPlayer","1.1.0")
+
diff --git a/instrumentation/next-share/BaseLib/Player/systray.py b/instrumentation/next-share/BaseLib/Player/systray.py
new file mode 100644
index 0000000..3a67872
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Player/systray.py
@@ -0,0 +1,218 @@
+# Written by Arno Bakker
+# see LICENSE.txt for license information
+
+import sys
+import os
+import textwrap
+import time
+from traceback import print_exc
+import wx
+
+from BaseLib.Core.API import *
+from BaseLib.Plugin.defs import *
+
+class PlayerTaskBarIcon(wx.TaskBarIcon):
+    
+    def __init__(self,wxapp,iconfilename):
+        wx.TaskBarIcon.__init__(self)
+        self.wxapp = wxapp
+        
+        self.icons = wx.IconBundle()
+        self.icons.AddIconFromFile(iconfilename,wx.BITMAP_TYPE_ICO)
+        self.icon = self.icons.GetIcon(wx.Size(-1,-1))
+
+        self.Bind(wx.EVT_TASKBAR_LEFT_UP, self.OnLeftClicked)
+
+        if sys.platform != "darwin":
+            # Mac already has the right icon set at startup
+            self.SetIcon(self.icon,self.wxapp.appname)
+        else:
+            menuBar = wx.MenuBar()
+
+            # Setting up the file menu.
+            filemenu = wx.Menu()
+            item = filemenu.Append(-1,'E&xit','Terminate the program')
+            self.Bind(wx.EVT_MENU, self.OnExit, item)
+
+            wx.App.SetMacExitMenuItemId(item.GetId())
+
+    def OnExit(self,e):
+        self.wxapp.ExitMainLoop()
+        # Close the frame.
+
+    def CreatePopupMenu(self):        
+        menu = wx.Menu()
+        
+        mi = menu.Append(-1,"Options...")
+        self.Bind(wx.EVT_MENU, self.OnOptions, id=mi.GetId())
+        menu.AppendSeparator()
+        mi = menu.Append(-1,"Exit")
+        self.Bind(wx.EVT_MENU, self.OnExitClient, id=mi.GetId())
+        return menu
+        
+    def OnOptions(self,event=None):
+        #print >>sys.stderr,"PlayerTaskBarIcon: OnOptions"
+        dlg = PlayerOptionsDialog(self.wxapp,self.icons)
+        ret = dlg.ShowModal()
+        #print >>sys.stderr,"PlayerTaskBarIcon: Dialog returned",ret
+        dlg.Destroy()
+
+    def OnExitClient(self,event=None):
+        #print >>sys.stderr,"PlayerTaskBarIcon: OnExitClient"
+        self.wxapp.ExitMainLoop()
+    
+    
+    def set_icon_tooltip(self,txt):
+        if sys.platform == "darwin":
+            # no taskbar tooltip on OS/X
+            return
+
+        self.SetIcon(self.icon,txt)
+    
+    def OnLeftClicked(self,event=None):
+        import webbrowser
+        url = 'http://127.0.0.1:'+str(self.wxapp.httpport)+URLPATH_WEBIF_PREFIX
+        webbrowser.open_new_tab(url)
+            
+
+    
+class PlayerOptionsDialog(wx.Dialog):
+    
+    def __init__(self,wxapp,icons):
+        self.wxapp = wxapp
+        self.icons = icons
+        self.port = None
+        
+        style = wx.DEFAULT_DIALOG_STYLE | wx.RESIZE_BORDER
+        wx.Dialog.__init__(self, None, -1, self.wxapp.appname+' Options', size=(400,200), style=style)
+        self.SetIcons(self.icons)
+
+        mainbox = wx.BoxSizer(wx.VERTICAL)
+        
+        aboutbox = wx.BoxSizer(wx.VERTICAL)
+        aboutlabel1 = wx.StaticText(self, -1, self.wxapp.appname+' is a product of the P2P-Next project')
+        aboutlabel2 = wx.StaticText(self, -1, 'Visit us at www.p2p-next.org!')
+        aboutbox.Add(aboutlabel1, 1, wx.EXPAND|wx.LEFT|wx.RIGHT, 5)
+        aboutbox.Add(aboutlabel2, 1, wx.EXPAND|wx.LEFT|wx.RIGHT, 5)
+        
+        uploadrate = self.wxapp.get_playerconfig('total_max_upload_rate')
+        
+        uploadratebox = wx.BoxSizer(wx.HORIZONTAL)
+        label = wx.StaticText(self, -1, 'Max upload to others (KB/s)')
+        self.uploadratectrl = wx.TextCtrl(self, -1, str(uploadrate))
+        uploadratebox.Add(label, 1, wx.ALIGN_CENTER_VERTICAL)
+        uploadratebox.Add(self.uploadratectrl)
+
+
+        buttonbox2 = wx.BoxSizer(wx.HORIZONTAL)
+        advbtn = wx.Button(self, -1, 'Advanced...')
+        buttonbox2.Add(advbtn, 0, wx.ALL, 5)
+
+        
+        buttonbox = wx.BoxSizer(wx.HORIZONTAL)
+        okbtn = wx.Button(self, wx.ID_OK, 'OK')
+        buttonbox.Add(okbtn, 0, wx.ALL, 5)
+        cancelbtn = wx.Button(self, wx.ID_CANCEL, 'Cancel')
+        buttonbox.Add(cancelbtn, 0, wx.ALL, 5)
+        applybtn = wx.Button(self, -1, 'Apply')
+        buttonbox.Add(applybtn, 0, wx.ALL, 5)
+
+        mainbox.Add(aboutbox, 1, wx.ALL, 5)
+        mainbox.Add(uploadratebox, 1, wx.EXPAND|wx.ALL, 5)
+        mainbox.Add(buttonbox2, 1, wx.EXPAND, 1)
+        mainbox.Add(buttonbox, 1, wx.EXPAND, 1)
+        self.SetSizerAndFit(mainbox)
+
+        self.Bind(wx.EVT_BUTTON, self.OnAdvanced, advbtn)
+        self.Bind(wx.EVT_BUTTON, self.OnOK, okbtn)
+        #self.Bind(wx.EVT_BUTTON, self.OnCancel, cancelbtn)
+        self.Bind(wx.EVT_BUTTON, self.OnApply, applybtn)
+        #self.Bind(wx.EVT_CLOSE, self.OnCloseWindow)
+
+    def OnOK(self,event = None):
+        self.OnApply(event)
+        self.EndModal(wx.ID_OK)
+        
+    #def OnCancel(self,event = None):
+    #    self.EndModal(wx.ID_CANCEL)
+        
+    def OnApply(self,event = None):
+        print >>sys.stderr,"PlayerOptionsDialog: OnApply",self.port
+        
+        if self.port is not None:
+            session = self.wxapp.s
+            state_dir = session.get_state_dir()
+            cfgfilename = Session.get_default_config_filename(state_dir)
+            scfg = SessionStartupConfig.load(cfgfilename)
+            
+            scfg.set_listen_port(self.port)
+            print >>sys.stderr,"PlayerOptionsDialog: OnApply: Saving SessionStartupConfig to",cfgfilename
+            scfg.save(cfgfilename)
+        
+        uploadrate = int(self.uploadratectrl.GetValue())
+        # Updates value for global rate limiter too
+        self.wxapp.set_playerconfig('total_max_upload_rate',uploadrate)
+        self.wxapp.save_playerconfig()
+         
+        if self.port is not None and self.port != self.wxapp.s.get_listen_port():
+            dlg = wx.MessageDialog(None, "The SwarmPlugin will now exit to change the port. Reload the Web page to restart it", self.wxapp.appname+" Restart", wx.OK|wx.ICON_INFORMATION)
+            result = dlg.ShowModal()
+            dlg.Destroy()
+            self.wxapp.OnExit()
+            # F*cking wx won't exit. Die
+            os._exit(1)
+
+
+    def OnAdvanced(self,event = None):
+
+        if self.port is None:
+            self.port = self.wxapp.s.get_listen_port()
+        #destdir = self.wxapp.s.get_dest_dir()
+
+        dlg = PlayerAdvancedOptionsDialog(self.icons,self.port,self.wxapp)
+        ret = dlg.ShowModal()
+        if ret == wx.ID_OK:
+            self.port = dlg.get_port()
+        dlg.Destroy()
+
+
+class PlayerAdvancedOptionsDialog(wx.Dialog):
+    
+    def __init__(self,icons,port,wxapp):
+        style = wx.DEFAULT_DIALOG_STYLE | wx.RESIZE_BORDER  # TODO: Add OK+Cancel
+        wx.Dialog.__init__(self, None, -1, 'SwarmPlugin Advanced Options', size=(400,200), style=style)
+        self.wxapp = wxapp
+
+        self.SetIcons(icons)
+
+        mainbox = wx.BoxSizer(wx.VERTICAL)
+        
+        portbox = wx.BoxSizer(wx.HORIZONTAL)
+        label = wx.StaticText(self, -1, 'Port')
+        self.portctrl = wx.TextCtrl(self, -1, str(port))
+        portbox.Add(label, 1, wx.ALIGN_CENTER_VERTICAL) 
+        portbox.Add(self.portctrl)
+
+        button2box = wx.BoxSizer(wx.HORIZONTAL)
+        clearbtn = wx.Button(self, -1, 'Clear disk cache and exit')
+        button2box.Add(clearbtn, 0, wx.ALL, 5)
+        self.Bind(wx.EVT_BUTTON, self.OnClear, clearbtn)
+        
+        buttonbox = wx.BoxSizer(wx.HORIZONTAL)
+        okbtn = wx.Button(self, wx.ID_OK, 'OK')
+        buttonbox.Add(okbtn, 0, wx.ALL, 5)
+        cancelbtn = wx.Button(self, wx.ID_CANCEL, 'Cancel')
+        buttonbox.Add(cancelbtn, 0, wx.ALL, 5)
+
+        mainbox.Add(portbox, 1, wx.EXPAND|wx.ALL, 5)
+        mainbox.Add(button2box, 1, wx.EXPAND, 1)
+        mainbox.Add(buttonbox, 1, wx.EXPAND, 1)
+        self.SetSizerAndFit(mainbox)
+
+    def get_port(self):
+        return int(self.portctrl.GetValue())
+        
+    def OnClear(self,event=None):
+        self.wxapp.clear_session_state()
+        
+        
diff --git a/instrumentation/next-share/BaseLib/Plugin/AtomFeedParser.py b/instrumentation/next-share/BaseLib/Plugin/AtomFeedParser.py
new file mode 100644
index 0000000..50daef4
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Plugin/AtomFeedParser.py
@@ -0,0 +1,126 @@
+# Written by Arno Bakker
+# see LICENSE.txt for license information
+import sys
+import time
+import xml.etree.ElementTree as etree
+
+from BaseLib.Core.Search.KeywordSearch import KeywordSearch
+from BaseLib.Core.Utilities.timeouturlopen import urlOpenTimeout
+
+
+class MetaFeedParser:
+    """ Parse an Atom feed that has Atom feeds as entries """
+    
+    def __init__(self,metafeedurl):
+        self.metafeedurl = metafeedurl
+        self.tree = None
+        
+    def parse(self):
+        self.feedurls = []
+        stream = urlOpenTimeout(self.metafeedurl,10)
+        self.tree = etree.parse(stream)
+        entries = self.tree.findall('{http://www.w3.org/2005/Atom}entry')
+        for entry in entries:
+            titleelement = entry.find('{http://www.w3.org/2005/Atom}title')
+            linkelement = entry.find('{http://www.w3.org/2005/Atom}link')
+            if linkelement is not None:
+                if linkelement.attrib['type'] == 'application/atom+xml':
+                    # Got feed
+                    feedurl = linkelement.attrib['href']
+                    self.feedurls.append(feedurl)
+                
+    def get_feedurls(self):
+        return self.feedurls
+
+
+class FeedParser:
+    
+    def __init__(self,feedurl):
+        self.feedurl = feedurl
+        self.tree = None
+        
+    def parse(self):
+        self.title2entrymap = {}
+        print >>sys.stderr,"feedp: Parsing",self.feedurl
+        stream = urlOpenTimeout(self.feedurl,10)
+        self.tree = etree.parse(stream)
+        entries = self.tree.findall('{http://www.w3.org/2005/Atom}entry')
+        for entry in entries:
+            titleelement = entry.find('{http://www.w3.org/2005/Atom}title')
+            #print >> sys.stderr,"feedp: Got title",titleelement.text
+            self.title2entrymap[titleelement.text] = entry
+                
+    def search(self,searchstr):
+        """ Use Jelle's smart keyword search """
+        needles = searchstr.strip().split(' ')
+        
+        haystack = []
+        for title,entry in self.title2entrymap.iteritems():
+            record = {}
+            record['name'] = title
+            record['entry'] = entry
+            haystack.append(record)
+            
+        records = KeywordSearch().search(haystack,needles)
+        hits = []
+        for record in records:
+            hits.append(record['entry'])
+        return hits
+
+
+def feedhits2atomxml(feedhits,searchstr,urlpathprefix):
+    
+    new_feed = etree.Element('{http://www.w3.org/2005/Atom}feed',attrib={'xmlns:rdf':"http://www.w3.org/1999/02/22-rdf-syntax-ns#", 'xmlns:sy':"http://purl.org/rss/1.0/modules/syndication/", 'xmlns:dc':"http://purl.org/dc/elements/1.1/", 'xmlns:p2pnext':"urn:p2pnext:contentfeed:2009", 'xmlns:taxo':"http://purl.org/rss/1.0/modules/taxonomy/"})
+    
+    title = etree.SubElement(new_feed, 'title')
+    title.text = 'Hits for '+searchstr
+
+    link = etree.SubElement(new_feed, 'link',attrib={'rel':'self','href':urlpathprefix})
+    author = etree.SubElement(new_feed, 'author')
+    name = etree.SubElement(author, 'name')
+    name.text = 'NSSA'
+    id = etree.SubElement(new_feed, 'id')
+    id.text = 'urn:nssa'
+    updated = etree.SubElement(new_feed, 'updated')
+    updated.text = now2formatRFC3339()
+    #TODO image = etree.SubElement(new_feed,'p2pnext:image',attrib={'src':"http://p2pnextfeed1.rad0.net/images/bbc.png"})
+
+    for entry in feedhits:
+        new_feed.append(entry)
+    
+    atom = '\n'
+    atom += etree.tostring(new_feed)
+    # Parser anomaly / formally correct bla bla
+    atom = atom.replace(":ns0=","=")
+    atom = atom.replace("ns0:","") 
+    return atom
+
+def now2formatRFC3339():
+    formatstr = "%Y-%m-%dT%H:%M:%S"
+    s = time.strftime(formatstr, time.gmtime())
+    s += 'Z'
+    return s 
+
+
+
+if __name__ == '__main__':
+    searchstr = "Episode"
+    
+    metafp = MetaFeedParser('http://p2pnextfeed1.rad0.net/content/feed/bbc')
+    metafp.parse()
+    
+    allhits = []
+    for feedurl in metafp.get_feedurls():
+        feedp = FeedParser(feedurl)
+        feedp.parse()
+        hits = feedp.search(searchstr)
+        allhits.extend(hits)
+    
+    #print >>sys.stderr,"Got hits",`hits`
+    
+    for hitentry in allhits:
+        titleelement = hitentry.find('{http://www.w3.org/2005/Atom}title')
+        print >>sys.stderr,"Got hit",titleelement.text
+        
+    atomxml = feedhits2atomxml(allhits,searchstr,"http://localhost/bla")
+    print >>sys.stderr,"Result feed",atomxml
diff --git a/instrumentation/next-share/BaseLib/Plugin/BackgroundProcess.py b/instrumentation/next-share/BaseLib/Plugin/BackgroundProcess.py
new file mode 100644
index 0000000..bcacc60
--- /dev/null
+++ b/instrumentation/next-share/BaseLib/Plugin/BackgroundProcess.py
@@ -0,0 +1,959 @@
+# Written by Arno Bakker, Diego Rabioli
+# see LICENSE.txt for license information
+#
+# Implements the BackgroundProcess, i.e. SwarmEngine for SwarmPlugin and 
+# SwarmTransport=SwarmPlayer v2. See Plugin/SwarmEngine.py and Transport/SwarmEngine.py
+# for main startup.
+#
+# The BackgroundProcess shares a base class BaseApp with the SwarmPlayer v1,
+# which is a standalone P2P-based video player.
+#
+#
+# Notes: 
+# - Implement play while hashcheck?
+#        Not needed when proper shutdown & restart was done.
+# - load_checkpoint with DLSTATUS_DOWNLOADING for Plugin? 
+#        Nah, if we start BG when plugin started we have a video to play soon,
+#        so start others in STOPPED state (rather than switching them all
+#        to off and restart one in VOD mode just after)
+#
+
+# History:
+#
+# NSSA API 1.0.2
+#
+#  1.0.2    Added STOP message to tell plugin to stop playing the current item
+#           (needed to support new behaviour where control conn is not always
+#           shutdown anymore to support input.set_p2ptarget.
+#
+#           Added ERROR message to tell plugin NSSA won't be able to serve the
+#           content requested via START (for