First step for using multiple recvs from mptp.
[swifty.git] / src / libswift / swift.cpp
1 /*
2  *  swift.cpp
3  *  swift the multiparty transport protocol
4  *
5  *  Created by Victor Grishchenko on 2/15/10.
6  *  Copyright 2009-2012 TECHNISCHE UNIVERSITEIT DELFT. All rights reserved.
7  *
8  */
9 #include <stdio.h>
10 #include <stdlib.h>
11 #include "compat.h"
12 #include "swift.h"
13 #include <cfloat>
14
15 using namespace swift;
16
17
18 // Local constants
19 #define RESCAN_DIR_INTERVAL     30 // seconds
20
21
22 // Local prototypes
23 #define quit(...) {fprintf(stderr,__VA_ARGS__); exit(1); }
24 int OpenSwiftFile(const TCHAR* filename, const Sha1Hash& hash, Address tracker, bool force_check_diskvshash, uint32_t chunk_size);
25 int OpenSwiftDirectory(const TCHAR* dirname, Address tracker, bool force_check_diskvshash, uint32_t chunk_size);
26
27 void ReportCallback(int fd, short event, void *arg);
28 void EndCallback(int fd, short event, void *arg);
29 void RescanDirCallback(int fd, short event, void *arg);
30
31
32 // Gateway stuff
33 bool InstallHTTPGateway(struct event_base *evbase,Address addr,uint32_t chunk_size, double *maxspeed);
34 bool InstallStatsGateway(struct event_base *evbase,Address addr);
35 bool InstallCmdGateway (struct event_base *evbase,Address cmdaddr,Address httpaddr);
36 bool HTTPIsSending();
37 bool StatsQuit();
38 void CmdGwUpdateDLStatesCallback();
39
40
41 // Global variables
42 struct event evreport, evrescan, evend;
43 int single_fd = -1;
44 bool file_enable_checkpoint = false;
45 bool file_checkpointed = false;
46 bool report_progress = false;
47 bool quiet=false;
48 bool exitoncomplete=false;
49 bool httpgw_enabled=false,cmdgw_enabled=false;
50 // Gertjan fix
51 bool do_nat_test = false;
52
53 char *scan_dirname = 0;
54 uint32_t chunk_size = SWIFT_DEFAULT_CHUNK_SIZE;
55 Address tracker;
56
57
58 int main (int argc, char** argv)
59 {
60     static struct option long_options[] =
61     {
62         {"hash",    required_argument, 0, 'h'},
63         {"file",    required_argument, 0, 'f'},
64         {"dir",     required_argument, 0, 'd'}, // SEEDDIR reuse
65         {"listen",  required_argument, 0, 'l'},
66         {"tracker", required_argument, 0, 't'},
67         {"debug",   no_argument, 0, 'D'},
68         {"progress",no_argument, 0, 'p'},
69         {"httpgw",  required_argument, 0, 'g'},
70         {"wait",    optional_argument, 0, 'w'},
71         {"nat-test",no_argument, 0, 'N'},
72         {"statsgw", required_argument, 0, 's'}, // SWIFTPROC
73         {"cmdgw",   required_argument, 0, 'c'}, // SWIFTPROC
74         {"destdir", required_argument, 0, 'o'}, // SWIFTPROC
75         {"uprate",  required_argument, 0, 'u'}, // RATELIMIT
76         {"downrate",required_argument, 0, 'y'}, // RATELIMIT
77         {"checkpoint",no_argument, 0, 'H'},
78         {"chunksize",required_argument, 0, 'z'}, // CHUNKSIZE
79         {"printurl",no_argument, 0, 'm'},
80         {0, 0, 0, 0}
81     };
82
83     Sha1Hash root_hash;
84     char* filename = 0;
85     const char *destdir = 0, *trackerargstr = 0; // UNICODE?
86     bool printurl=false;
87     Address bindaddr;
88     Address httpaddr;
89     Address statsaddr;
90     Address cmdaddr;
91     tint wait_time = 0;
92     double maxspeed[2] = {DBL_MAX,DBL_MAX};
93
94     LibraryInit();
95     Channel::evbase = event_base_new();
96
97     int c,n;
98     while ( -1 != (c = getopt_long (argc, argv, ":h:f:d:l:t:D:pg:s:c:o:u:y:z:wBNHm", long_options, 0)) ) {
99         switch (c) {
100             case 'h':
101                 if (strlen(optarg)!=40)
102                     quit("SHA1 hash must be 40 hex symbols\n");
103                 root_hash = Sha1Hash(true,optarg); // FIXME ambiguity
104                 if (root_hash==Sha1Hash::ZERO)
105                     quit("SHA1 hash must be 40 hex symbols\n");
106                 break;
107             case 'f':
108                 filename = strdup(optarg);
109                 break;
110             case 'd':
111                 scan_dirname = strdup(optarg);
112                 break;
113             case 'l':
114                 bindaddr = Address(optarg);
115                 if (bindaddr==Address())
116                     quit("address must be hostname:port, ip:port or just port\n");
117                 wait_time = TINT_NEVER;
118                 break;
119             case 't':
120                 tracker = Address(optarg);
121                 trackerargstr = strdup(optarg); // UNICODE
122                 if (tracker==Address())
123                     quit("address must be hostname:port, ip:port or just port\n");
124                 SetTracker(tracker);
125                 break;
126             case 'D':
127                 Channel::debug_file = optarg ? fopen(optarg,"a") : stderr;
128                 break;
129             // Arno hack: get opt diff Win32 doesn't allow -D without arg
130             case 'B':
131                 fprintf(stderr,"SETTING DEBUG TO STDOUT\n");
132                 Channel::debug_file = stderr;
133                 break;
134             case 'p':
135                 report_progress = true;
136                 break;
137             case 'g':
138                 httpgw_enabled = true;
139                 httpaddr = Address(optarg);
140                                 wait_time = TINT_NEVER; // seed
141                 break;
142             case 'w':
143                 if (optarg) {
144                     char unit = 'u';
145                     if (sscanf(optarg,"%lli%c",&wait_time,&unit)!=2)
146                         quit("time format: 1234[umsMHD], e.g. 1M = one minute\n");
147
148                     switch (unit) {
149                         case 'D': wait_time *= 24;
150                         case 'H': wait_time *= 60;
151                         case 'M': wait_time *= 60;
152                         case 's': wait_time *= 1000;
153                         case 'm': wait_time *= 1000;
154                         case 'u': break;
155                         default:  quit("time format: 1234[umsMHD], e.g. 1D = one day\n");
156                     }
157                 } else
158                     wait_time = TINT_NEVER;
159                 break;
160             case 'N': // Gertjan fix
161                 do_nat_test = true;
162                 break;
163             case 's': // SWIFTPROC
164                 statsaddr = Address(optarg);
165                 if (statsaddr==Address())
166                     quit("address must be hostname:port, ip:port or just port\n");
167                 break;
168             case 'c': // SWIFTPROC
169                 cmdgw_enabled = true;
170                 cmdaddr = Address(optarg);
171                 if (cmdaddr==Address())
172                     quit("address must be hostname:port, ip:port or just port\n");
173                 wait_time = TINT_NEVER; // seed
174                 break;
175             case 'o': // SWIFTPROC
176                 destdir = strdup(optarg); // UNICODE
177                 break;
178             case 'u': // RATELIMIT
179                 n = sscanf(optarg,"%lf",&maxspeed[DDIR_UPLOAD]);
180                 if (n != 1)
181                         quit("uprate must be KiB/s as float\n");
182                 maxspeed[DDIR_UPLOAD] *= 1024.0;
183                 break;
184             case 'y': // RATELIMIT
185                 n = sscanf(optarg,"%lf",&maxspeed[DDIR_DOWNLOAD]);
186                 if (n != 1)
187                         quit("downrate must be KiB/s as float\n");
188                 maxspeed[DDIR_DOWNLOAD] *= 1024.0;
189                 break;
190             case 'H': //CHECKPOINT
191                 file_enable_checkpoint = true;
192                 break;
193             case 'z': // CHUNKSIZE
194                 n = sscanf(optarg,"%i",&chunk_size);
195                 if (n != 1)
196                         quit("chunk size must be bytes as int\n");
197                 break;
198             case 'm': // printurl
199                 printurl = true;
200                 quiet = true;
201                 wait_time = 0;
202                 break;
203         }
204
205     }   // arguments parsed
206
207
208     if (httpgw_enabled)
209     {
210         // Change current directory to a temporary one
211 #ifdef _WIN32
212         if (destdir == 0) {
213                 std::string destdirstr = gettmpdir();
214                 !::SetCurrentDirectory(destdirstr.c_str());
215         }
216         else
217                 !::SetCurrentDirectory(destdir);
218         TCHAR szDirectory[MAX_PATH] = "";
219
220         !::GetCurrentDirectory(sizeof(szDirectory) - 1, szDirectory);
221         fprintf(stderr,"CWD %s\n",szDirectory);
222 #else
223         if (destdir == 0)
224                 chdir(gettmpdir().c_str());
225         else
226                 chdir(destdir);
227 #endif
228     }
229       
230     if (bindaddr!=Address()) { // seeding
231         if (Listen(bindaddr)<=0)
232             quit("cant listen to %s\n",bindaddr.str())
233     } else if (tracker!=Address() || httpgw_enabled || cmdgw_enabled) { // leeching
234         evutil_socket_t sock = INVALID_SOCKET;
235         for (int i=0; i<=10; i++) {
236             bindaddr = Address((uint32_t)INADDR_ANY,0);
237             sock = Listen(bindaddr);
238             if (sock>0)
239                 break;
240             if (i==10)
241                 quit("cant listen on %s\n",bindaddr.str());
242         }
243         if (!quiet)
244                 fprintf(stderr,"swift: My listen port is %d\n", BoundAddress(sock).port() );
245     }
246
247     if (tracker!=Address())
248         SetTracker(tracker);
249
250     if (httpgw_enabled)
251         InstallHTTPGateway(Channel::evbase,httpaddr,chunk_size,maxspeed);
252     if (cmdgw_enabled)
253                 InstallCmdGateway(Channel::evbase,cmdaddr,httpaddr);
254
255     // TRIALM36: Allow browser to retrieve stats via AJAX and as HTML page
256     if (statsaddr != Address())
257         InstallStatsGateway(Channel::evbase,statsaddr);
258
259
260     if (!cmdgw_enabled && !httpgw_enabled)
261     {
262                 int ret = -1;
263                 if (filename || root_hash != Sha1Hash::ZERO) {
264                         // Single file
265                         if (root_hash!=Sha1Hash::ZERO && !filename)
266                                 filename = strdup(root_hash.hex().c_str());
267
268                         single_fd = OpenSwiftFile(filename,root_hash,Address(),false,chunk_size);
269                         if (single_fd < 0)
270                                 quit("cannot open file %s",filename);
271                         if (printurl) {
272                                 if (trackerargstr == NULL)
273                                         trackerargstr = "";
274                                 printf("tswift://%s/%s$%i\n", trackerargstr, RootMerkleHash(single_fd).hex().c_str(), chunk_size);
275
276                                 // Arno, 2012-01-04: LivingLab: Create checkpoint such that content
277                                 // can be copied to scanned dir and quickly loaded
278                                 swift::Checkpoint(single_fd);
279                         }
280                         else
281                                 printf("Root hash: %s\n", RootMerkleHash(single_fd).hex().c_str());
282
283                         // RATELIMIT
284                         FileTransfer *ft = FileTransfer::file(single_fd);
285                         ft->SetMaxSpeed(DDIR_DOWNLOAD,maxspeed[DDIR_DOWNLOAD]);
286                         ft->SetMaxSpeed(DDIR_UPLOAD,maxspeed[DDIR_UPLOAD]);
287
288                         ret = single_fd;
289                 }
290                 else if (scan_dirname != NULL)
291                         ret = OpenSwiftDirectory(scan_dirname,Address(),false,chunk_size);
292                 else
293                         ret = -1;
294
295                 // No file/dir nor HTTP gateway nor CMD gateway, will never know what to swarm
296                 if (ret == -1) {
297                         fprintf(stderr,"Usage:\n");
298                         fprintf(stderr,"  -h, --hash\troot Merkle hash for the transmission\n");
299                         fprintf(stderr,"  -f, --file\tname of file to use (root hash by default)\n");
300                         fprintf(stderr,"  -l, --listen\t[ip:|host:]port to listen to (default: random)\n");
301                         fprintf(stderr,"  -t, --tracker\t[ip:|host:]port of the tracker (default: none)\n");
302                         fprintf(stderr,"  -D, --debug\tfile name for debugging logs (default: stdout)\n");
303                         fprintf(stderr,"  -B\tdebugging logs to stdout (win32 hack)\n");
304                         fprintf(stderr,"  -p, --progress\treport transfer progress\n");
305                         fprintf(stderr,"  -g, --httpgw\t[ip:|host:]port to bind HTTP content gateway to (no default)\n");
306                         fprintf(stderr,"  -s, --statsgw\t[ip:|host:]port to bind HTTP stats listen socket to (no default)\n");
307                         fprintf(stderr,"  -c, --cmdgw\t[ip:|host:]port to bind CMD listen socket to (no default)\n");
308                         fprintf(stderr,"  -o, --destdir\tdirectory for saving data (default: none)\n");
309                         fprintf(stderr,"  -u, --uprate\tupload rate limit in KiB/s (default: unlimited)\n");
310                         fprintf(stderr,"  -y, --downrate\tdownload rate limit in KiB/s (default: unlimited)\n");
311                         fprintf(stderr,"  -w, --wait\tlimit running time, e.g. 1[DHMs] (default: infinite with -l, -g)\n");
312                         fprintf(stderr,"  -H, --checkpoint\tcreate checkpoint of file when complete for fast restart\n");
313                         fprintf(stderr,"  -z, --chunksize\tchunk size in bytes (default: %d)\n", SWIFT_DEFAULT_CHUNK_SIZE);
314                         fprintf(stderr,"  -m, --printurl\tcompose URL from tracker, file and chunksize\n");
315                         return 1;
316                 }
317     }
318
319     // Arno, 2012-01-04: Allow download and quit mode
320     if (single_fd != -1 && root_hash != Sha1Hash::ZERO && wait_time == 0) {
321         wait_time = TINT_NEVER;
322         exitoncomplete = true;
323     }
324
325     // End after wait_time
326     if ((long)wait_time > 0) {
327         evtimer_assign(&evend, Channel::evbase, EndCallback, NULL);
328         evtimer_add(&evend, tint2tv(wait_time));
329     }
330
331     // Enter mainloop, if daemonizing
332     if (wait_time == TINT_NEVER || (long)wait_time > 0) {
333                 // Arno: always, for statsgw, rate control, etc.
334                 evtimer_assign(&evreport, Channel::evbase, ReportCallback, NULL);
335                 evtimer_add(&evreport, tint2tv(TINT_SEC));
336
337
338                 // Arno:
339                 if (scan_dirname != NULL) {
340                         evtimer_assign(&evrescan, Channel::evbase, RescanDirCallback, NULL);
341                         evtimer_add(&evrescan, tint2tv(RESCAN_DIR_INTERVAL*TINT_SEC));
342                 }
343
344
345                 fprintf(stderr,"swift: Mainloop\n");
346                 // Enter libevent mainloop
347                 event_base_dispatch(Channel::evbase);
348
349                 // event_base_loopexit() was called, shutting down
350     }
351
352     // Arno, 2012-01-03: Close all transfers
353         for (int i=0; i<FileTransfer::files.size(); i++) {
354                 if (FileTransfer::files[i] != NULL)
355             Close(FileTransfer::files[i]->fd());
356     }
357
358     if (Channel::debug_file)
359         fclose(Channel::debug_file);
360
361     swift::Shutdown();
362
363     return 0;
364 }
365
366
367
368 int OpenSwiftFile(const TCHAR* filename, const Sha1Hash& hash, Address tracker, bool force_check_diskvshash, uint32_t chunk_size)
369 {
370         std::string bfn;
371         bfn.assign(filename);
372         bfn.append(".mbinmap");
373         const char *binmap_filename = bfn.c_str();
374
375         // Arno, 2012-01-03: Hack to discover root hash of a file on disk, such that
376         // we don't load it twice while rescanning a dir of content.
377         HashTree *ht = new HashTree(true,binmap_filename);
378
379         //      fprintf(stderr,"swift: parsedir: File %s may have hash %s\n", filename, ht->root_hash().hex().c_str() );
380
381         int fd = swift::Find(ht->root_hash());
382         if (fd == -1) {
383                 if (!quiet)
384                         fprintf(stderr,"swift: parsedir: Opening %s\n", filename);
385
386                 fd = swift::Open(filename,hash,tracker,force_check_diskvshash,true,chunk_size);
387         }
388         else if (!quiet)
389                 fprintf(stderr,"swift: parsedir: Ignoring loaded %s\n", filename);
390         return fd;
391 }
392
393
394 int OpenSwiftDirectory(const TCHAR* dirname, Address tracker, bool force_check_diskvshash, uint32_t chunk_size)
395 {
396 #ifdef _WIN32
397         HANDLE hFind;
398         WIN32_FIND_DATA FindFileData;
399
400         TCHAR pathsearch[MAX_PATH];
401         strcpy(pathsearch,dirname);
402         strcat(pathsearch,"\\*.*");
403         hFind = FindFirstFile(pathsearch, &FindFileData);
404         if(hFind != INVALID_HANDLE_VALUE) {
405             do {
406                 //fprintf(stderr,"swift: parsedir: %s\n", FindFileData.cFileName);
407                 if ((FindFileData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0 && !strstr(FindFileData.cFileName,".mhash") && !strstr(FindFileData.cFileName,".mbinmap") ) {
408                                 TCHAR path[MAX_PATH];
409                                 strcpy(path,dirname);
410                                 strcat(path,"\\");
411                                 strcat(path,FindFileData.cFileName);
412
413                                 int fd = OpenSwiftFile(path,Sha1Hash::ZERO,tracker,force_check_diskvshash,chunk_size);
414                                 if (fd >= 0)
415                                         Checkpoint(fd);
416                 }
417             } while(FindNextFile(hFind, &FindFileData));
418
419             FindClose(hFind);
420             return 1;
421         }
422         else
423                 return -1;
424 #else
425         DIR *dirp = opendir(dirname);
426         if (dirp == NULL)
427                 return -1;
428
429         while(1)
430         {
431                 struct dirent *de = readdir(dirp);
432                 if (de == NULL)
433                         break;
434                 if ((de->d_type & DT_DIR) !=0 || strstr(de->d_name,".mhash") || strstr(de->d_name,".mbinmap"))
435                         continue;
436                 char path[PATH_MAX];
437                 strcpy(path,dirname);
438                 strcat(path,"/");
439                 strcat(path,de->d_name);
440
441                 int fd = OpenSwiftFile(path,Sha1Hash::ZERO,tracker,force_check_diskvshash,chunk_size);
442                 if (fd >= 0)
443                         Checkpoint(fd);
444         }
445         closedir(dirp);
446         return 1;
447 #endif
448 }
449
450
451
452 int CleanSwiftDirectory(const TCHAR* dirname)
453 {
454         std::set<int>   delset;
455         std::vector<FileTransfer*>::iterator iter;
456         for (iter=FileTransfer::files.begin(); iter!=FileTransfer::files.end(); iter++)
457         {
458                 FileTransfer *ft = *iter;
459                 if (ft != NULL) {
460                         std::string filename = ft->file().filename();
461 #ifdef WIN32
462                         struct _stat buf;
463 #else
464                         struct stat buf;
465 #endif
466                         fprintf(stderr,"swift: clean: Checking %s\n", filename.c_str() );
467                         int res = stat( filename.c_str(), &buf );
468                         if( res < 0 && errno == ENOENT) {
469                                 fprintf(stderr,"swift: clean: Missing %s\n", filename.c_str() );
470                                 delset.insert(ft->fd());
471                         }
472                 }
473         }
474
475         std::set<int>::iterator iiter;
476         for (iiter=delset.begin(); iiter!=delset.end(); iiter++)
477         {
478                 int fd = *iiter;
479                 fprintf(stderr,"swift: clean: Deleting transfer %d\n", fd );
480                 swift::Close(fd);
481         }
482
483         return 1;
484 }
485
486
487
488
489
490 void ReportCallback(int fd, short event, void *arg) {
491         // Called every second to print/calc some stats
492
493         if (single_fd  >= 0)
494         {
495                 if (report_progress) {
496                         fprintf(stderr,
497                                 "%s %lli of %lli (seq %lli) %lli dgram %lli bytes up, " \
498                                 "%lli dgram %lli bytes down\n",
499                                 IsComplete(single_fd ) ? "DONE" : "done",
500                                 Complete(single_fd), Size(single_fd), SeqComplete(single_fd),
501                                 Channel::global_dgrams_up, Channel::global_raw_bytes_up,
502                                 Channel::global_dgrams_down, Channel::global_raw_bytes_down );
503                 }
504
505         FileTransfer *ft = FileTransfer::file(single_fd);
506         if (report_progress) { // TODO: move up
507                 fprintf(stderr,"upload %lf\n",ft->GetCurrentSpeed(DDIR_UPLOAD));
508                 fprintf(stderr,"dwload %lf\n",ft->GetCurrentSpeed(DDIR_DOWNLOAD) );
509                 //fprintf(stderr,"npeers %d\n",ft->GetNumLeechers()+ft->GetNumSeeders() );
510         }
511         // Update speed measurements such that they decrease when DL/UL stops
512         // Always
513         ft->OnRecvData(0);
514         ft->OnSendData(0);
515
516         // CHECKPOINT
517         if (file_enable_checkpoint && !file_checkpointed && IsComplete(single_fd))
518         {
519                 std::string binmap_filename = ft->file().filename();
520                 binmap_filename.append(".mbinmap");
521                 fprintf(stderr,"swift: Complete, checkpointing %s\n", binmap_filename.c_str() );
522                 FILE *fp = fopen(binmap_filename.c_str(),"wb");
523                 if (!fp) {
524                         print_error("cannot open mbinmap for writing");
525                         return;
526                 }
527                 if (ft->file().serialize(fp) < 0)
528                         print_error("writing to mbinmap");
529                 else
530                         file_checkpointed = true;
531                 fclose(fp);
532         }
533
534
535         if (exitoncomplete && IsComplete(single_fd))
536                 // Download and stop mode
537             event_base_loopexit(Channel::evbase, NULL);
538
539         }
540     if (httpgw_enabled)
541     {
542         fprintf(stderr,".");
543
544         // ARNOSMPTODO: Restore fail behaviour when used in SwarmPlayer 3000.
545         if (!HTTPIsSending()) {
546                 // TODO
547                 //event_base_loopexit(Channel::evbase, NULL);
548             return;
549         }
550     }
551     if (StatsQuit())
552     {
553         // SwarmPlayer 3000: User click "Quit" button in webUI.
554         struct timeval tv;
555         tv.tv_sec = 1;
556         int ret = event_base_loopexit(Channel::evbase,&tv);
557     }
558         // SWIFTPROC
559         // ARNOSMPTODO: SCALE: perhaps less than once a second if many swarms
560         CmdGwUpdateDLStatesCallback();
561
562         // Gertjan fix
563         // Arno, 2011-10-04: Temp disable
564     //if (do_nat_test)
565     //     nat_test_update();
566
567         evtimer_add(&evreport, tint2tv(TINT_SEC));
568 }
569
570 void EndCallback(int fd, short event, void *arg) {
571         // Called when wait timer expires == fixed time daemon
572     event_base_loopexit(Channel::evbase, NULL);
573 }
574
575
576 void RescanDirCallback(int fd, short event, void *arg) {
577
578         // SEEDDIR
579         // Rescan dir: CAREFUL: this is blocking, better prepare .m* files first
580         // by running swift separately and then copy content + *.m* to scanned dir,
581         // such that a fast restore from checkpoint is done.
582         //
583         OpenSwiftDirectory(scan_dirname,tracker,false,chunk_size);
584
585         CleanSwiftDirectory(scan_dirname);
586
587         evtimer_add(&evrescan, tint2tv(RESCAN_DIR_INTERVAL*TINT_SEC));
588 }
589
590
591
592 #ifdef _WIN32
593 int WINAPI WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdLine, int nShowCmd)
594 {
595     return main(__argc,__argv);
596 }
597 #endif
598