instrumentation/next-share/BaseLib/Test/test_dlhelp.py

   1 # Written by Arno Bakker, George Milescu
   2 # see LICENSE.txt for license information
   3 #
   4 # Like test_secure_overlay, we start a new python interpreter for each test.
   5 # Although we don't have the singleton problem here, we do need to do this as the
   6 # HTTPServer that MyTracker uses won't relinquish the listen socket, causing
   7 # "address in use" errors in the next test. This is probably due to the fact that
   8 # MyTracker has a thread mixed in, as a listensocket.close() normally releases it
   9 # (according to lsof).
  10 #
  11
  12 import unittest
  13 import os
  14 import sys
  15 import time
  16 from traceback import print_exc
  17 import socket
  18 from types import ListType
  19 import tempfile
  20
  21 from BaseLib.Test.test_as_server import TestAsServer
  22 from btconn import BTConnection
  23 from olconn import OLConnection
  24 from BaseLib.Core.RequestPolicy import AllowAllRequestPolicy
  25 from BaseLib.Core.BitTornado.bencode import bencode,bdecode
  26 from BaseLib.Core.BitTornado.bitfield import Bitfield
  27 from BaseLib.Core.BitTornado.BT1.MessageID import *
  28 from BaseLib.Core.BitTornado.BT1.convert import toint
  29 from BaseLib.Core.CacheDB.CacheDBHandler import FriendDBHandler
  30 from BaseLib.Test.test_connect_overlay import MyTracker
  31
  32 DEBUG=True
  33
  34 class TestDownloadHelp(TestAsServer):
  35     """
  36     Testing download helping
  37     """
  38
  39     def setUp(self):
  40         """ override TestAsServer """
  41         TestAsServer.setUp(self)
  42         print >>sys.stderr,"test: Giving MyLaunchMany time to startup"
  43         time.sleep(5)
  44         print >>sys.stderr,"test: MyLaunchMany should have started up"
  45
  46     def setUpPreSession(self):
  47         """ override TestAsServer """
  48         TestAsServer.setUpPreSession(self)
  49
  50         self.setUpMyListenSockets()
  51
  52         # Must be changed in test/extend_hs_dir/dummydata.merkle.torrent as well
  53         self.mytrackerport = 4901
  54         self.myid = 'R410-----HgUyPu56789'
  55         self.mytracker = MyTracker(self.mytrackerport,self.myid,'127.0.0.1',self.mylistenport)
  56         self.mytracker.background_serve()
  57
  58         self.myid2 = 'R410-----56789HuGyx0'
  59
  60         # Arno, 2009-12-15: Make sure coop downloads have their own destdir
  61         destdir = tempfile.mkdtemp()
  62         self.config.set_download_help_dir(destdir)
  63
  64     def setUpMyListenSockets(self):
  65         # Start our server side, to with Tribler will try to connect
  66         self.mylistenport = 4810
  67         self.myss = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  68         self.myss.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
  69         self.myss.bind(('', self.mylistenport))
  70         self.myss.listen(1)
  71
  72         self.mylistenport2 = 3726
  73         self.myss2 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  74         self.myss2.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
  75         self.myss2.bind(('', self.mylistenport2))
  76         self.myss2.listen(1)
  77
  78
  79     def setUpPostSession(self):
  80         """ override TestAsServer """
  81         TestAsServer.setUpPostSession(self)
  82
  83         self.mypermid = str(self.my_keypair.pub().get_der())
  84         self.hispermid = str(self.his_keypair.pub().get_der())
  85
  86         # This is the infohash of the torrent in test/extend_hs_dir
  87         self.infohash = '\xccg\x07\xe2\x9e!]\x16\xae{\xb8\x10?\xf9\xa5\xf9\x07\xfdBk'
  88         self.torrentfile = os.path.join('extend_hs_dir','dummydata.merkle.torrent')
  89
  90         # Add us as friend, so he will accept the ASK_FOR_HELP
  91         if False:  # TEMP: friendsdb doesn't have an addFriend method
  92             # friendsdb = FriendDBHandler.getInstance()
  93             # friendsdb.addFriend(self.mypermid)
  94             pass
  95         else:
  96             self.session.set_overlay_request_policy(AllowAllRequestPolicy())
  97
  98         self.session.set_download_states_callback(self.states_callback)
  99
 100     def tearDown(self):
 101         """ override TestAsServer """
 102         print >> sys.stderr,"test: *** TEARDOWN"
 103         TestAsServer.tearDown(self)
 104         self.mytracker.shutdown()
 105         self.tearDownMyListenSockets()
 106
 107
 108     def tearDownMyListenSockets(self):
 109         self.myss.close()
 110         self.myss2.close()
 111
 112
 113     def states_callback(self,dslist):
 114         print >>sys.stderr,"stats: dslist",len(dslist)
 115         for ds in dslist:
 116             print >>sys.stderr,"stats: coordinator",`ds.get_coopdl_coordinator()`
 117             print >>sys.stderr,"stats: helpers",`ds.get_coopdl_helpers()`
 118         return (0.5,False)
 119
 120     #
 121     # Good 2fast
 122     #
 123     def singtest_good_2fast(self):
 124         genresdict = self.get_genresdict()
 125         print >>sys.stderr,"test: good ASK_FOR_HELP"
 126         self._test_2fast(genresdict)
 127
 128
 129     def get_genresdict(self):
 130         genresdict = {}
 131         genresdict[ASK_FOR_HELP] = (self.create_good_dlhelp,True)
 132         genresdict[METADATA] = (self.create_good_metadata,True)
 133         genresdict[PIECES_RESERVED] = (self.create_good_pieces_reserved,True)
 134         genresdict[STOP_DOWNLOAD_HELP] = (self.create_good_stop_dlhelp,True)
 135         return genresdict
 136
 137     #
 138     # Bad 2fast
 139     #
 140     def singtest_bad_2fast_dlhelp(self):
 141         genresdict = self.get_genresdict()
 142         genresdict[ASK_FOR_HELP] = (self.create_bad_dlhelp_not_infohash,False)
 143         print >>sys.stderr,"test: bad dlhelp"
 144         self._test_2fast(genresdict)
 145
 146     def singtest_bad_2fast_metadata_not_bdecodable(self):
 147         genresdict = self.get_genresdict()
 148         genresdict[METADATA] = (self.create_bad_metadata_not_bdecodable,False)
 149         print >>sys.stderr,"test: bad METADATA",genresdict[METADATA][0]
 150         self._test_2fast(genresdict)
 151
 152     def singtest_bad_2fast_metadata_not_dict1(self):
 153         genresdict = self.get_genresdict()
 154         genresdict[METADATA] = (self.create_bad_metadata_not_dict1,False)
 155         print >>sys.stderr,"test: bad METADATA",genresdict[METADATA][0]
 156         self._test_2fast(genresdict)
 157
 158     def singtest_bad_2fast_metadata_not_dict2(self):
 159         genresdict = self.get_genresdict()
 160         genresdict[METADATA] = (self.create_bad_metadata_not_dict2,False)
 161         print >>sys.stderr,"test: bad METADATA",genresdict[METADATA][0]
 162         self._test_2fast(genresdict)
 163
 164
 165     def singtest_bad_2fast_metadata_empty_dict(self):
 166         genresdict = self.get_genresdict()
 167         genresdict[METADATA] = (self.create_bad_metadata_empty_dict,False)
 168         print >>sys.stderr,"test: bad METADATA",genresdict[METADATA][0]
 169         self._test_2fast(genresdict)
 170
 171     def singtest_bad_2fast_metadata_wrong_dict_keys(self):
 172         genresdict = self.get_genresdict()
 173         genresdict[METADATA] = (self.create_bad_metadata_wrong_dict_keys,False)
 174         print >>sys.stderr,"test: bad METADATA",genresdict[METADATA][0]
 175         self._test_2fast(genresdict)
 176
 177     def singtest_bad_2fast_metadata_bad_torrent1(self):
 178         genresdict = self.get_genresdict()
 179         genresdict[METADATA] = (self.create_bad_metadata_bad_torrent1,False)
 180         print >>sys.stderr,"test: bad METADATA",genresdict[METADATA][0]
 181         self._test_2fast(genresdict)
 182
 183
 184     def singtest_bad_2fast_metadata_bad_torrent2(self):
 185         genresdict = self.get_genresdict()
 186         genresdict[METADATA] = (self.create_bad_metadata_bad_torrent2,False)
 187         print >>sys.stderr,"test: bad METADATA",genresdict[METADATA][0]
 188         self._test_2fast(genresdict)
 189
 190     def singtest_bad_2fast_metadata_bad_torrent3(self):
 191         genresdict = self.get_genresdict()
 192         genresdict[METADATA] = (self.create_bad_metadata_bad_torrent3,False)
 193         print >>sys.stderr,"test: bad METADATA",genresdict[METADATA][0]
 194         self._test_2fast(genresdict)
 195
 196
 197
 198     def _test_2fast(self,genresdict):
 199         """
 200             test ASK_FOR_HELP, METADATA, PIECES_RESERVED and STOP_DOWNLOAD_HELP sequence
 201         """
 202         # 1. Establish overlay connection to Tribler
 203         s = OLConnection(self.my_keypair,'localhost',self.hisport,mylistenport=self.mylistenport2)
 204
 205         (func,good) = genresdict[ASK_FOR_HELP]
 206         msg = func()
 207         s.send(msg)
 208         if good:
 209             resp = s.recv()
 210             self.assert_(resp[0] == GET_METADATA)
 211             self.check_get_metadata(resp[1:])
 212             print >>sys.stderr,"test: Got GET_METADATA for torrent, good"
 213         else:
 214             resp = s.recv()
 215             self.assert_(len(resp)==0)
 216             s.close()
 217             return
 218
 219         (func,good) = genresdict[METADATA]
 220         msg = func()
 221         s.send(msg)
 222
 223         if good:
 224             # 2. Accept the data connection Tribler wants to establish with us, the coordinator
 225             self.myss2.settimeout(10.0)
 226             conn, addr = self.myss2.accept()
 227             s3 = BTConnection('',0,conn,user_infohash=self.infohash,myid=self.myid2)
 228             s3.read_handshake_medium_rare()
 229
 230             msg = UNCHOKE
 231             s3.send(msg)
 232             print >>sys.stderr,"test: Got data connection to us, as coordinator, good"
 233         else:
 234             resp = s.recv()
 235             self.assert_(len(resp)==0)
 236             s.close()
 237             return
 238
 239         # 3. Our tracker says there is another peer (also us) on port 4810
 240         # Now accept a connection on that port and pretend we're a seeder
 241         self.myss.settimeout(10.0)
 242         conn, addr = self.myss.accept()
 243         options = '\x00\x00\x00\x00\x00\x00\x00\x00'
 244         s2 = BTConnection('',0,conn,user_option_pattern=options,user_infohash=self.infohash,myid=self.myid)
 245         s2.read_handshake_medium_rare()
 246
 247         numpieces = 10 # must correspond to the torrent in test/extend_hs_dir
 248         b = Bitfield(numpieces)
 249         for i in range(numpieces):
 250             b[i] = True
 251         self.assert_(b.complete())
 252         msg = BITFIELD+b.tostring()
 253         s2.send(msg)
 254         msg = UNCHOKE
 255         s2.send(msg)
 256         print >>sys.stderr,"test: Got BT connection to us, as fake seeder, good"
 257
 258         # 4. Await a RESERVE_PIECES message on the overlay connection
 259         resp = s.recv()
 260         self.assert_(resp[0] == RESERVE_PIECES)
 261         pieces = self.check_reserve_pieces(resp[1:])
 262         print >>sys.stderr,"test: Got RESERVE_PIECES, good"
 263
 264         (func,good) = genresdict[PIECES_RESERVED]
 265
 266         # 5. Reply with PIECES_RESERVED
 267         msg = func(pieces)
 268         s.send(msg)
 269
 270         if good:
 271             # 6. Await REQUEST on fake seeder
 272             try:
 273                 while True:
 274                     s2.s.settimeout(10.0)
 275                     resp = s2.recv()
 276                     self.assert_(len(resp) > 0)
 277                     print "test: Fake seeder got message",getMessageName(resp[0])
 278                     if resp[0] == REQUEST:
 279                         self.check_request(resp[1:],pieces)
 280                         print >>sys.stderr,"test: Fake seeder got REQUEST for reserved piece, good"
 281                         break
 282
 283             except socket.timeout:
 284                 print >> sys.stderr,"test: Timeout, bad, fake seeder didn't reply with message"
 285                 self.assert_(False)
 286         else:
 287             resp = s.recv()
 288             self.assert_(len(resp)==0)
 289             s.close()
 290             return
 291
 292         (func,good) = genresdict[STOP_DOWNLOAD_HELP]
 293         # 5. Reply with STOP_DOWNLOAD_HELP
 294         msg = func()
 295         s.send(msg)
 296
 297         # the other side should close the connection, whether the msg was good or bad
 298         resp = s.recv()
 299         self.assert_(len(resp)==0)
 300         s.close()
 301
 302
 303     def create_good_dlhelp(self):
 304         return ASK_FOR_HELP+self.infohash
 305
 306     def check_get_metadata(self,data):
 307         infohash = bdecode(data) # is bencoded for unknown reason, can't change it
 308         self.assert_(infohash == self.infohash)
 309
 310     def create_good_metadata(self):
 311         f = open(self.torrentfile,"rb")
 312         data = f.read()
 313         f.close()
 314
 315         d = self.create_good_metadata_dict(data)
 316         bd = bencode(d)
 317         return METADATA+bd
 318
 319     def create_good_metadata_dict(self,data):
 320         d = {}
 321         d['torrent_hash'] = self.infohash
 322         d['metadata'] = data
 323         d['leecher'] = 1
 324         d['seeder'] = 1
 325         d['last_check_time'] = int(time.time())
 326         d['status'] = 'good'
 327         return d
 328
 329     def check_reserve_pieces(self,data):
 330         # torrent_hash + 1-byte all_or_nothing + bencode([piece num,...])
 331         self.assert_(len(data) > 21)
 332         infohash = data[0:20]
 333         allflag = data[20]
 334         plist = bdecode(data[21:])
 335
 336         self.assert_(infohash == self.infohash)
 337         self.assert_(type(plist) == ListType)
 338         return plist
 339
 340     def create_good_pieces_reserved(self,pieces):
 341         payload = self.infohash + bencode(pieces)
 342         return PIECES_RESERVED + payload
 343
 344     def check_request(self,data,pieces):
 345         piece = toint(data[0:4])
 346         self.assert_(piece in pieces)
 347
 348     def create_good_stop_dlhelp(self):
 349         return STOP_DOWNLOAD_HELP+self.infohash
 350
 351
 352     #
 353     # Bad ASK_FOR_HELP
 354     #
 355
 356     def create_bad_dlhelp_not_infohash(self):
 357         return ASK_FOR_HELP+"481"
 358
 359     #
 360     # Bad METADATA
 361     #
 362
 363     def create_bad_metadata_not_bdecodable(self):
 364         return METADATA+"bla"
 365
 366     def create_bad_metadata_not_dict1(self):
 367         d  = 481
 368         return METADATA+bencode(d)
 369
 370     def create_bad_metadata_not_dict2(self):
 371         d  = []
 372         return METADATA+bencode(d)
 373
 374     def create_bad_metadata_empty_dict(self):
 375         d = {}
 376         return METADATA+bencode(d)
 377
 378     def create_bad_metadata_wrong_dict_keys(self):
 379         d = {}
 380         d['bla1'] = '\x00\x00\x00\x00\x00\x30\x00\x00'
 381         d['bla2'] = '\x00\x00\x00\x00\x00\x30\x00\x00'
 382         return METADATA+bencode(d)
 383
 384     def create_bad_metadata_bad_torrent1(self):
 385         d = self.create_good_metadata_dict(None)
 386         d['metadata'] = '\x12\x34' * 100 # random data
 387         bd = bencode(d)
 388         return METADATA+bd
 389
 390     def create_bad_metadata_bad_torrent2(self):
 391         torrent = {}
 392         data = bencode(torrent)
 393
 394         d = self.create_good_metadata_dict(data)
 395         d['metadata'] = data
 396         bd = bencode(d)
 397         return METADATA+bd
 398
 399
 400     def create_bad_metadata_bad_torrent3(self):
 401         torrent = {'info':481}
 402         data = bencode(torrent)
 403
 404         d = self.create_good_metadata_dict(data)
 405         d['metadata'] = data
 406         bd = bencode(d)
 407         return METADATA+bd
 408
 409
 410
 411 def test_suite():
 412     suite = unittest.TestSuite()
 413     # We should run the tests in a separate Python interpreter to prevent
 414     # problems with our singleton classes, e.g. PeerDB, etc.
 415     if len(sys.argv) != 2:
 416         print "Usage: python test_dl.py <method name>"
 417     else:
 418         suite.addTest(TestDownloadHelp(sys.argv[1]))
 419
 420     return suite
 421
 422 def main():
 423     unittest.main(defaultTest='test_suite',argv=[sys.argv[0]])
 424
 425 if __name__ == "__main__":
 426     main()