operator uint64_t () const { return v; }
bool operator == (bin64_t& b) const { return v==b.v; }
+ static bin64_t none () { return NONE; }
+ static bin64_t all () { return ALL; }
+
uint64_t tail_bits () const {
return v ^ (v+1);
}
return (tail_bits()+1)>>1;
}
+ bin64_t sibling () const {
+ // if (v==ALL) return NONE;
+ return bin64_t(v^(tail_bit()<<1));
+ }
int layer () const {
int r = 0;
}
uint64_t base_offset () const {
- return v&~(tail_bits());
+ return (v&~(tail_bits()))>>1;
}
uint64_t offset () const {
return v >> (layer()+1);
}
+ bin64_t to (bool right) const {
+ if (!(v&1))
+ return NONE;
+ uint64_t tb = tail_bit()>>1;
+ if (right)
+ tb |= (tb<<1);
+ return bin64_t(v^tb);
+ }
+
bin64_t left () const {
- assert(layer());
- return bin64_t( v ^ (tail_bit()>>1) );
+ return to(false);
}
bin64_t right () const {
- assert(layer());
- uint64_t tb = tail_bit();
- return bin64_t( v ^ (tb|(tb>>1)) );
+ return to(true);
+ }
+
+ bool within (bin64_t maybe_asc) {
+ uint64_t short_tail = maybe_asc.tail_bits();
+ if (tail_bits()>short_tail)
+ return false;
+ return (v&~short_tail) == (maybe_asc.v&~short_tail) ;
+ }
+
+ bin64_t towards (bin64_t desc) const {
+ if (!desc.within(*this))
+ return NONE;
+ if (desc.within(left()))
+ return left();
+ else
+ return right();
}
bin64_t parent () const {
uint64_t tb = tail_bit();
return !(v&(tb<<1));
}
+ bool is_right() const { return !is_left(); }
/** The array must have 64 cells, as it is the max
number of peaks possible (and there are no reason
#endif
+
+/**
+ 00111
+ 0011 1011
+ 001 101 1001 1101
+0 10 100 110 1000 1010 1100 1110
+
+ 7
+ 3 11
+ 1 5 9 13
+0 2 4 6 8 10 12 14
+
+once we have peak hashes, this struture is more natural than bin-v1
+
+*/
*
*/
#include "bins.h"
+#include <string.h>
-uint16_t bins::SPLIT[256];
-uint8_t bins::JOIN[256];
-uint16_t bins::MASK1000[32];
-
-bin64_t bins::find (bin64_t range, int layer) {
- // fall to left border
- // do
- // if up && has 0 success
- // if lower && range-shift-trick
- // success
- // while (next && within range)
- // return fail
+// make it work piece by piece
+
+const uint8_t bins::SPLIT[16] =
+{0, 3, 12, 15, 48, 51, 60, 63, 192, 195, 204, 207, 240, 243, 252, 255};
+const uint8_t bins::JOIN[16] =
+{0, 1, 4, 5, 2, 3, 6, 7, 8, 9, 12, 13, 10, 11, 14, 15};
+const int bins::NOJOIN = 0x10000;
+
+
+void bins::extend () {
+ uint16_t nblocks = blocks_allocated ? (blocks_allocated<<1) : 1;
+ size_t had_bytes = blocks_allocated<<6;
+ size_t need_bytes = nblocks<<6;
+ cells = (uint32_t*) realloc(cells,need_bytes);
+ memset(((char*)cells)+had_bytes,0,need_bytes-had_bytes);
+ for(int b=blocks_allocated; b<nblocks; b++)
+ cells[(b<<4)|0xf] = 0x55555555; // cells are free
+ blocks_allocated = nblocks;
+}
+
+bins::bins() : height(4), blocks_allocated(0), cells(NULL),
+ ap(0), cells_allocated(0) {
+ extend();
+ assert(!alloc_cell());
+}
+
+void bins::dump (const char* note) {
+ printf("%s\t",note);
+ for(int i=0; i<(blocks_allocated<<5); i++) {
+ if ( (i&0x1f)>29 )
+ printf("|%x ",halves[i]);
+ else if (deep(i))
+ printf(">%i ",halves[i]);
+ else
+ printf("%x ",halves[i]);
+ if (i&1)
+ printf(" ");
+ }
+ printf("\n");
+}
+
+uint32_t bins::split16to32(uint16_t halfval) {
+ uint32_t nval = 0;
+ for(int i=0; i<4; i++) {
+ nval >>= 8;
+ nval |= (SPLIT[halfval&0xf])<<24;
+ halfval >>= 4;
+ }
+ return nval;
+}
+
+
+int bins::join32to16(uint32_t cval) {
+ union { uint32_t i; uint8_t a[4]; } uvar;
+ uvar.i = cval & (cval>>1) & 0x55555555;
+ if ( (uvar.i|(uvar.i<<1)) != cval )
+ return NOJOIN;
+ uvar.i = (uvar.i&0x05050505) | ((uvar.i&0x50505050U)>>3);
+ uint16_t res = 0;
+ for(int i=3; i>=0; i--) {
+ res <<= 4;
+ res |= JOIN[uvar.a[i]];
+ }
+ return res;
+}
+
+
+void bins::split (uint32_t half) {
+ if (deep(half))
+ return;
+ uint32_t cell = alloc_cell(), left=cell<<1, right=left+1;
+ mark(half); //cells[(half>>1)|0xf] |= 1<<(half&0x1f);
+ uint16_t halfval = halves[half];
+ uint32_t nval = split16to32(halfval);
+ halves[left] = nval&0xffff;
+ halves[right] = nval>>16;
+ halves[half] = cell;
+}
+
+
+bool bins::join (uint32_t half) {
+ uint32_t cellno = halves[half];
+ int left = cellno<<1, right=left+1;
+ if (deep(left) || deep(right))
+ return false;
+ int res = join32to16(cells[cellno]);
+ if (res>0xffff)
+ return false;
+ halves[half] = (uint16_t)res;
+ unmark(half);
+ free_cell(cellno);
+ //cells[(half>>1)|0xf] &= ~(1<<(half&0x1f));
+ //(*childdeepcell) &= 0xffff>>1; // clean the full bit
+ return true;
+}
+
+void bins::free_cell (uint16_t cell) {
+ cells[cell] = 0;
+ int left = cell<<1, right=left+1;
+ mark(left);
+ unmark(right);
+ if (ap==cell+1)
+ ap--;
+ cells_allocated--;
+}
+
+/** Get a free cell. */
+uint16_t bins::alloc_cell () {
+ uint16_t ap1 = ap;
+ cells_allocated++;
+ for(; ap<(blocks_allocated<<4); ap++) {
+ if ((ap&0xf)==0xf)
+ continue;
+ if (!cells[ap] && deep(ap<<1)) {
+ unmark(ap<<1);
+ return ap++;
+ }
+ }
+ if (ap1)
+ ap = 0;
+ else
+ extend();
+ cells_allocated--;
+ return alloc_cell();
+}
+
+
+bin64_t iterator::next (bool need_solid) {
+ assert(!deep());
+ while (pos.is_right())
+ parent();
+ //parent();
+ //if (need_solid ? !solid() : deep())
+ // right();
+ sibling();
+ while (need_solid ? !solid() : deep())
+ left();
+ return pos;
+}
+
+
+iterator::iterator(bins* host_, bin64_t start, bool split) {
+ host = host_;
+ half = 0;
+ for(int i=0; i<64; i++)
+ history[i] = 1;
+ pos = bin64_t(host->height,0);
+ while (!start.within(pos))
+ parent();
+ while (pos!=start && (deep() || split))
+ towards(start);
+}
+
+
+iterator::~iterator () {
+ while (half>1 && !deep())
+ parent();
+ // PROBLEM: may hang in the air if two iters
+ // exist simultaneously
+ // FIX: iterators are not exposed (protected)
+}
+
+
+void iterator::to (bool right) {
+ if (!deep())
+ host->split(half);
+ history[pos.layer()] = half; // FIXME
+ pos = pos.to(right);
+ half = (host->halves[half]<<1) + right;
+ //host->dump("/\\ ");
+}
+
+
+void bins::extend_range () {
+ assert(height<62);
+ height++;
+ uint16_t newroot = alloc_cell();
+ int left = newroot<<1, right = left+1;
+ cells[newroot] = cells[0];
+ halves[0] = newroot;
+ halves[1] = 0;
+ if (deep(0))
+ mark(left);
+ else
+ mark(0);
+ if (deep(1)) {
+ mark(right);
+ unmark(1);
+ }
+}
+
+void iterator::parent () {
+ if (!half) {
+ host->extend_range();
+ history[pos.layer()+1] = 0;
+ }
+ pos = pos.parent();
+ half = history[pos.layer()];
+ host->join(half);
+ //host->dump("| ");
+}
+
+
+bin64_t bins::find (const bin64_t range, const uint8_t layer) {
+ iterator i(this,range,true);
+ while (true) {
+ while ( i.bin().layer()>layer && (i.deep() || *i!=FILLED) )
+ i.left();
+ if (i.bin().layer()==layer && !i.deep() && *i==EMPTY)
+ return i.bin();
+ while (i.bin().is_right() && i.bin()!=range)
+ i.parent();
+ if (i.bin()==range)
+ break;
+ i.parent();
+ i.right();
+ }
+ return bin64_t::NONE;
+}
+
+
+uint16_t bins::get (bin64_t bin) {
+ iterator i(this,bin,true);
+ //while ( i.pos!=bin &&
+ // (i.deep() || (*i!=BIN_FULL && *i!=BIN_EMPTY)) )
+ // i.towards(bin);
+ //printf("at %i ",i.half);
+ //dump("get made");
+ return *i; // deep cell is never 0xffff or 0x0000
+}
+
+
+void bins::set (bin64_t bin, fill_t val) {
+ assert(val==FILLED || val==EMPTY);
+ iterator i(this,bin,false);
+ while (i.bin()!=bin && (i.deep() || *i!=val))
+ i.towards(bin);
+ if (!i.deep() && *i==val)
+ return;
+ while (i.deep())
+ i.left();
+ do {
+ *i = val;
+ i.next();
+ } while (i.bin().within(bin));
+ // dump("just set");
+}
+
+
+uint64_t* bins::get_stripes (int& count) {
+ int size = 32;
+ uint64_t *stripes = (uint64_t*) malloc(32*8);
+ count = 0;
+ uint16_t cur = bins::EMPTY;
+ stripes[count++] = 0;
+ iterator i(this,0,false);
+ while (!i.solid())
+ i.left();
+
+ while (!i.end()) {
+
+ if (cur!=*i) { // new stripe
+ cur = *i;
+ stripes[count++] = i.bin().base_offset();
+ if (count==size) {
+ size <<= 1;
+ stripes = (uint64_t*) realloc(stripes,size*8);
+ }
+ }
+
+ i.next(true);
+
+ }
+
+ if ( !(count&1) )
+ stripes[count++] = i.bin().base_offset();
+
+ return stripes;
}
* Copyright 2009 Delft Technical University. All rights reserved.
*
*/
-#ifndef SERP_SBIT_H
-#define SERP_SBIT_H
+#ifndef BINS_H
+#define BINS_H
#include "bin64.h"
+#include <gtest/gtest.h>
-class bins64 {
+/** A binmap covering 2^64 range. Complexity limit: 100+200LoC */
+class bins {
+
+public:
+ typedef enum { FILLED=0xffff, EMPTY=0x0000 } fill_t;
+ static const int NOJOIN;
+
+ bins();
+
+ uint16_t get (bin64_t bin);
+
+ void set (bin64_t bin, fill_t val=FILLED);
+
+ bin64_t find (const bin64_t range, const uint8_t layer) ;
+
+ void dump(const char* note);
- private:
+ uint64_t* get_stripes (int& count);
- uint32_t *bits;
- uint32_t alloc_block;
-
- protected:
-
- bool join(uint32_t half) {
- uint32_t cellno = bits[half]>>(half&1?16:0);
-
- if (deep(left) || deep(right)) // some half is deep
- return false;
- uint8_t b1=JOIN[cell&0xf],
- b2=JOIN[(cell>>8)&0xf],
- b3=JOIN[(cell>>16)&0xf],
- b4=JOIN[cell>>24];
- if (b1==0xff || b2==0xff || b3==0xff || b4==0xff)
- return false;
- bits[half] = b1 | (b2<<4) | (b3<<8) | (b4<<12) ;
- (*parentdeepcell) ^= 1<<(halfno&32);
- (*childdeepcell) &= 0xffff>>1; // clean the full bit
- }
-
- bool split(uint32_t half) {
- if (deep(half))
- return false;
- uint32_t cell = alloc_cell(), left=cell<<1, right=left+1;
- bits[half|0xf] |= 1<<(half&0xf);
- bits[left] = SPLIT[bits[half]>>8];
- bits[right] = SPLIT[bits[half&0xff]];
- bits[half] = cell;
- return true;
- }
-
- uint32_t alloc_cell () {
- do{
- for(int block=alloc_block; bits[block]&(1<<32); block+=32);
- for(int off=0; bits[block+off]==0 && off<31; off++);
- if (off==31)
- bits[block] |= 1<<32;
- if (block&(1<<31)) {
- bits = realloc(bits,block*2);
- memset();
- }
- } while (off==31);
- alloc_block = block;
- return block+off;
- }
-
- public:
-
- class iterator {
- bins64_t *host;
- uint32_t back[32];
- uint32_t half;
- bin64_t top;
- bin64_t focus;
- bin16_t mask;
- public:
- void left();
- void right();
- void parent();
- bin64_t next();
- bool defined();
- uint16_t& operator* ();
- };
- friend class iterator;
-
- bool get (uint64_t bin);
-
- void set (uint64_t bin);
-
- bin64_t find (bin64_t range, int layer);
+ uint32_t size() { return cells_allocated; }
+ // TODO: bitwise operators
+
+private:
+
+ /** Every 16th uint32 is a flag field denoting whether
+ previous 30 halves (in 15 cells) are deep or not.
+ The last bit is used as a fill-flag.
+ Free cells have a value of 0; neither deep nor flat
+ cell could have a value of 0 except for the root
+ cell in case the binmap is all-0. */
+ union {
+ uint32_t *cells;
+ uint16_t *halves;
+ };
+ uint32_t blocks_allocated;
+ uint32_t cells_allocated;
+ int height;
+ uint32_t ap;
+
+ void extend();
+
+ static const uint8_t SPLIT[16];
+ static const uint8_t JOIN[16];
+
+ bool deep(uint32_t half) const {
+ return cells[(half>>1)|0xf] & (1<<(half&0x1f));
+ }
+ void mark(uint32_t half) {
+ cells[(half>>1)|0xf] |= (1<<(half&0x1f));
+ }
+ void unmark(uint32_t half) {
+ cells[(half>>1)|0xf] &= ~(1<<(half&0x1f));
+ }
+
+ void extend_range();
+
+ uint16_t alloc_cell ();
+ void free_cell (uint16_t cell);
+
+ /** Join the cell this half is pointing to
+ (in other words, flatten the half). */
+ bool join(uint32_t half) ;
+
+ /** Make the half deep. */
+ void split(uint32_t half) ;
+
+ static uint32_t split16to32(uint16_t half);
+ static int join32to16(uint32_t cell);
+
+ friend class iterator;
+ FRIEND_TEST(BinsTest,Routines);
+};
- // TODO: bitwise operators
+/** Iterates over bins; for deep halves, bin==half.
+ For flat halves, bin is a range of bits in a half.
+ Iterator may split cells if needed.
+ Value is either undefined (deep cell, mixed cell)
+ or FILLED/EMPTY. */
+class iterator {
+public: // rm this
+ bins *host;
+ uint32_t history[64];
+ uint32_t half;
+ bin64_t pos; // TODO: half[] layer bin
+public:
+ iterator(bins* host, bin64_t start=0, bool split=false);
+ ~iterator();
+ bool deep () { return host->deep(half); }
+ bool solid () {
+ return !deep() && (host->halves[half]==bins::FILLED ||
+ host->halves[half]==bins::EMPTY);
+ }
+ void sibling () { half^=1; pos=pos.sibling(); }
+ bool end () { return half==1; }
+ void to (bool right);
+ void left() {to(0);}
+ void right() {to(1);}
+ /** Move to the next defined (non-deep, flat) cell.
+ If solid==true, move to a solid (0xffff/0x0) cell. */
+ bin64_t next (bool solid=false);
+ bin64_t bin() { return pos; }
+ void towards(bin64_t bin) {
+ bin64_t next = pos.towards(bin);
+ assert(next!=bin64_t::NONE);
+ to(next.is_right());
+ }
+ void parent() ;
+ bool defined() { return !host->deep(half); }
+ uint16_t& operator* () { return host->halves[half]; }
};
+
+#endif
+++ /dev/null
-/*
- * collisions.cpp
- * serp++
- *
- * Created by Victor Grishchenko on 3/15/09.
- * Copyright 2009 Delft Technical University. All rights reserved.
- *
- */
-#include "bin.h"
-#include "sbit.h"
-#include <deque>
-#include <cmath>
-
-using namespace std;
-
-#define NUMPEERS 40
-#define QUEUE_LEN 32
-#define WIDTH 20
-#define TOP (bin(WIDTH,0))
-
-sbit bigpic; // common knowledge
-
-struct Peer {
- deque<bin> packets; // packets in flight & unacknowledged
- feye map;
- int peerno;
- bin focus;
- Peer () : map(bigpic,bin(0,rand()%(1<<WIDTH))) {
- focus = map.focus;
- }
- void jump () {
- bin oldfoc = focus;
-
- map = feye(bigpic,focus);
- for(int i=0; i<packets.size(); i++)
- map|=packets[i];
-
- while (map.get(focus) && focus<TOP)
- focus = focus.parent();
- if (focus==TOP) {
- printf("DONE\n");
- packets.push_back(0);
- return;
- }
- //if (focus!=bin(WIDTH,0))
- // focus = focus.parent();
- while (focus.layer()) {
- bin left = focus.left(), right = focus.right();
- bool lfull = map.get(left);
- bool rfull = map.get(right);
- if (lfull)
- focus = right;
- else if (rfull)
- focus = left;
- else
- focus = rand()%2 ? left : right;
- }
- if (map.focus.commonParent(focus).layer()>6)
- map.refocus(focus);
- if (labs(map.focus-focus)>=129)
- printf("bred!\n");
- map |= focus;
- // sanity check
- if (bigpic.get(focus))
- printf("zhopa: peer %i redid %x after jumped %i\n",
- peerno,focus.offset(),oldfoc.commonParent(focus).layer());
- assert(focus.layer()==0 && focus<TOP);
- packets.push_back(focus);
- }
- void hint (bin newfoc) {
- printf("peer %i hinted at %x\n",
- peerno,newfoc.offset());
- focus = newfoc;
- //map.refocus(newfoc); // preserve recent sends if possible
- //map |= feye(bigpic,newfoc); // update with the big picture
- map = feye(bigpic,newfoc);
- }
- void ack (bin packet) {
- feye a(bigpic,packet);
- a.refocus(map.focus);
- map |= a;
- }
-};
-
-
-bin random_descend() {
- bin ret = bin(WIDTH,0);
- while (ret.layer()) {
- bin left = ret.left(), right = ret.right();
- bool lfull = bigpic.get(left);
- bool rfull = bigpic.get(right);
- if (lfull)
- ret = right;
- else if (rfull)
- ret = left;
- else
- ret = rand()%2 ? left : right;
- }
- return ret;
-}
-
-
-int main (int argc, char** args) {
- bin::init();
- sbit::init();
- freopen( "log", "w", stdout );
- Peer peers[NUMPEERS];
- int numpack = 0;
- int coll_layer[32] = { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
- 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 };
- for(int i=0; i<NUMPEERS; i++)
- peers[i].peerno = i;
- while ( ! bigpic.get(bin(WIDTH,0)) ) {
- for(int i=0; i<NUMPEERS; i++)
- peers[i].jump();
- for(int i=0; i<NUMPEERS; i++)
- if (peers[i].packets.size()>=QUEUE_LEN) { // round trip, acks reach senders
- bin packet = peers[i].packets.front();
- peers[i].packets.pop_front();
- if (packet==0) continue;
- bool collision = bigpic.get(packet);
- bin round = bigpic.set(packet);
- printf("peer %i arrived %x filled %i coll %i\n",
- i,packet.offset(),round.layer(),collision);
- for(int j=0; j<NUMPEERS; j++)
- peers[j].map |= round;
- peers[i].ack(packet);
- numpack++;
- if (collision) {
- peers[i].hint(random_descend());
- coll_layer[round.layer()]++;
- }
- /*{ // update with the big picture
- feye update(bigpic,packet); // current focus is unknown
- update.refocus(peers[i].focus);
- peers[i].map |= update;
- }*/
- }
- }
- printf("%i useful, %i total plus the tail\n",1<<WIDTH,numpack);
- for(int l=0;l<32; l++)
- printf("%i ",coll_layer[l]);
- printf("\n");
-}
-
+++ /dev/null
-/*
- * congctrl.cpp
- * p2tp
- *
- * Created by Victor Grishchenko on 4/14/09.
- * Copyright 2009 Delft Technical University. All rights reserved.
- *
- */
-#include "p2tp.h"
-#include <glog/logging.h>
-
-using namespace p2tp;
-
-CongestionControl::CongestionControl () {
- state_ = SLOW_START_STATE;
- cwnd_ = 1;
- cainc_ = 0;
- ssthresh_ = 100;
- rtt_avg_ = 0;
- dev_avg_ = 0;
- peer_cwnd_ = 0;
- data_ins_ = 0;
- last_arrival_ = 0;
- rate_ = TINT_1SEC/10;
-}
-
-
-void CongestionControl::RttSample (tint rtt) {
- if (rtt_avg_>0) {
- rtt_avg_ = (rtt_avg_*7 + rtt) >> 3; // affected by reordering
- dev_avg_ = (dev_avg_*7 + ::abs(rtt-rtt_avg_)) >> 3;
- } else {
- rtt_avg_ = rtt;
- dev_avg_ = rtt>>3;
- }
- DLOG(INFO)<<"sample "<<rtt<<" rtt "<<rtt_avg_<<" dev "<<dev_avg_;
-}
-
-
-void CongestionControl::OnCongestionEvent (CongCtrlEvents ev) {
- switch (ev) {
- case LOSS_EV:
- cwnd_ /= 2;
- state_ = CONG_AVOID_STATE;
- break;
- case ACK_EV:
- if (state_==SLOW_START_STATE) {
- cwnd_++;
- if (cwnd_>=ssthresh_)
- state_ = CONG_AVOID_STATE;
- } else if (state_==CONG_AVOID_STATE) {
- cainc_++;
- if (cainc_>=cwnd_) {
- cainc_ = 0;
- cwnd_++;
- }
- }
- break;
- case DATA_EV:
- tint interarrival = last_arrival_ ?
- Datagram::now - last_arrival_ :
- rtt_avg_; // starting est. corresp. cwnd==1
- last_arrival_ = Datagram::now;
- if (rate_)
- rate_ = ( rate_*3 + interarrival ) / 4;
- else
- rate_ = interarrival;
- break;
- }
- DLOG(INFO)<<"peer irr "<<rate_<<" pcwnd "<<peer_cwnd();
-}
-
-int CongestionControl::peer_cwnd () const {
- if (!rate_)
- return 0;
- int pc = rtt_avg_ / rate_;
- if ( rtt_avg_%rate_ > rate_/2 ) // too many /
- pc++;
- return pc;
-}
-
-int CongestionControl::peer_bps () const {
- return 1024 * TINT_1SEC / rate_;
-}
--- /dev/null
+/*
+ * ledbat_controller.cpp
+ * p2tp
+ *
+ * Created by Victor Grishchenko on 10/6/09.
+ * Copyright 2009 Delft Technical University. All rights reserved.
+ *
+ */
+
+#include "p2tp.h"
+
+using namespace p2tp;
+
+class LedbatController : public CongestionController {
+public:
+ /*tint rtt_avg;
+ tint dev_avg;
+ int cwnd;
+ int peer_cwnd;*/
+ virtual void OnTimeout () {
+ }
+
+ virtual void OnDataSent(bin64_t b) {
+ }
+
+ virtual void OnDataRecvd(bin64_t b) {
+ }
+
+ virtual void OnAckRcvd(bin64_t b, tint peer_stamp) {
+ }
+
+ virtual ~CongestionControl() {
+ }
+
+};
initial handshake packet also has the root hash
(a HASH message).
- DATA 01, bin, buffer
+ DATA 01, bin_32, buffer
1K of data.
- ACK 02, bin
+ ACK 02, bin_32
+ ACKTS 08, bin_32, timestamp_32
Confirms successfull delivery of data. Used for
congestion control, as well.
-
- HINT 03, bin
+
+ HINT 03, bin_32
Practical value of "hints" is to avoid overlap, mostly.
Hints might be lost in the network or ignored.
Peer might send out data without a hint.
As peers cant pick randomly kilobyte here and there,
they send out "long hints" for non-base bins.
- HASH 04, bin, sha1hash
+ HASH 04, bin_32, sha1hash
SHA1 hash tree hashes for data verification. The
connection to a fresh peer starts with bootstrapping
him with peak hashes. Later, before sending out
any data, a peer sends the necessary uncle hashes.
- PEX+/PEX- 05/06, ip, port
+ PEX+/PEX- 05/06, ipv4 addr, port
Peer exchange messages; reports all connected and
disconected peers. Might has special meaning (as
in the case with swarm supervisors).
- --8X---- maybe (give2get)
-
- CRED 07, scrchid
- Communicated the public channel id at the sender
- side. Any grandchildren's credits will go to this
- channel.
-
- CRED1/2 08/09, ip, port, scrchid
- Second-step and trird-step credits.
-
- ACK1/2 10/11, scrchid, packets
- Grandchildren's acknowledgements of data being
- received from a child; the public channel id
- is mentioned.
-
*/
#ifndef P2TP_H
#define P2TP_H
#include <stdint.h>
-#include <iostream>
#include <vector>
#include <deque>
-#include "bin.h"
-#include "sbit.h"
+#include "bin64.h"
+#include "bins.h"
#include "datagram.h"
#include "hashtree.h"
namespace p2tp {
- /* 64-bit time counter, microseconds since epoch
- typedef int64_t tint;
- static const tint TINT_1SEC = 1000000;
- static const tint TINT_1MSEC = 1000;
- static const tint TINT_INFINITY = 0x7fffffffffffffffULL;
- */
- struct tintbin {
- tint time;
- bin pos;
- tintbin(tint t, bin p) : time(t), pos(p) {}
- };
- typedef std::deque<tintbin> tbinvec;
+ typedef std::pair<tint,bin64_t> tintbin;
+
+ typedef std::deque<tintbin> tbqueue;
+ typedef std::deque<bin64_t> binqueue;
typedef enum {
P2TP_HANDSHAKE = 0,
P2TP_DATA = 1,
P2TP_ACK = 2,
+ P2TP_ACKTS = 8,
P2TP_HINT = 3,
P2TP_HASH = 4,
P2TP_PEX_ADD = 5,
P2TP_PEX_RM = 6,
P2TP_MESSAGE_COUNT = 7
} messageid_t;
+
+ class PiecePicker;
+ class CongestionController;
+ class PeerSelector;
- struct File {
-
- typedef enum {EMPTY,IN_PROGRESS,DONE} status_t;
-
- static std::vector<File*> files;
+ class FileTransfer {
- /** File descriptor. */
- int fd;
- /** Whether the file is completely downloaded. */
- status_t status_;
-
- // A map for all packets obtained and succesfully checked.
- bins ack_out;
- // Hinted packets.
- bins hint_out;
+ public:
- HashTree hashes;
- // History of bin retrieval.
- bin::vec history;
+ /** Offer a hash; returns true if it verified; false otherwise.
+ Once it cannot be verified (no sibling or parent), the hash
+ is remembered, while returning false. */
+ bool OfferHash (bin64_t pos, const Sha1Hash& hash);
+ /** Offer data; the behavior is the same as with a hash:
+ accept or remember or drop. */
+ bool OfferData (bin64_t bin, uint8_t* data);
+
+ const Sha1Hash& root_hash () const { return *root_hash; }
+
+ size_t size () const;
+ size_t packet_size () const;
+ size_t size_complete () const;
+ size_t packets_complete () const;
- // TBD
- uint64_t options;
+ static FileTransfer* find (const Sha1Hash& hash);
+ static FileTransfer* file (int fd) { return fd<files.size() ? files[fd] : NULL; }
+
+ friend int Open (const char* filename);
+ friend int Open (const Sha1Hash& hash, const char* filename);
+ friend void Close (int fdes);
+
+ private:
- /** Submit a fresh file. */
- File (int fd);
- /** Retrieve a file. */
- File (Sha1Hash hash, int fd);
- /** Placeholder. */
- File () : fd(-1), hashes(Sha1Hash::ZERO) {}
+ static std::vector<FileTransfer*> files;
+
+ /** Open/submit/retrieve a file. */
+ FileTransfer (Sha1Hash hash, char* file_name);
/** Close everything. */
- ~File();
+ ~FileTransfer();
- bool OfferHash (bin pos, const Sha1Hash& hash);
- const Sha1Hash& root_hash () const { return hashes.root; }
- size_t size () const { return hashes.data_size()<<10; }
- size_t packet_size () const { return hashes.data_size(); }
- status_t status () const { return status_; }
-
- static File* find (const Sha1Hash& hash);
- static File* file (int fd) { return fd<files.size() ? files[fd] : NULL; }
+ /** file descriptor. */
+ int fd;
+ /** File size, as derived from the hashes. */
+ size_t sizek;
+ /** Whether the file is completely downloaded. */
+ bool complete;
+ /** A map for all packets obtained and succesfully checked. */
+ bins ack_out;
+ /** History of bin retrieval. */
+ binqueue ack_history;
+ /** Piece picker strategy. */
+ PiecePicker* picker;
+ /** File for keeping the Merkle hash tree. */
+ int hashfd;
+ /** Merkle hash tree: root */
+ Sha1Hash* root_hash;
+ /** Merkle hash tree: peak hashes */
+ Sha1Hash* peak_hashes;
+ /** Merkle hash tree: the tree, as a bin64_t-indexed array */
+ Sha1Hash* hashes;
+ protected:
+ void Resize(size_t bytes);
- friend int Open (const char* filename);
- friend int Open (const Sha1Hash& hash, const char* filename);
- friend void Close (int fdes);
friend class Channel;
};
int Open (const char* filename) ;
int Open (const Sha1Hash& root_hash, const char* filename);
- size_t file_size (int fd);
void Close (int fid) ;
+ void Loop (tint till);
+ int Bind (int port);
+ void Shutdown (int fd);
+ void HeardOfPeer (const Sha1Hash& root, struct sockaddr_in address);
- struct CongestionControl {
- typedef enum {SLOW_START_STATE,CONG_AVOID_STATE} stdtcpstate_t;
- typedef enum { DATA_EV, ACK_EV, LOSS_EV } CongCtrlEvents;
-
- stdtcpstate_t state_;
- tint rtt_avg_, dev_avg_, last_arrival_, rate_;
- int cwnd_, cainc_, ssthresh_;
- int peer_cwnd_, data_ins_;
-
- CongestionControl();
- virtual ~CongestionControl() {}
- virtual void OnCongestionEvent (CongCtrlEvents ev);
- void RttSample (tint rtt);
-
- tint avg_rtt() const { return rtt_avg_; }
- tint safe_avg_rtt() const { return avg_rtt() + avg_rtt_dev()*8; }
- tint avg_rtt_dev() const { return dev_avg_; }
- int cwnd () const { return cwnd_; }
- int peer_cwnd () const;
- int peer_bps () const;
- tint data_in_rate () const { return rate_; }
+ class CongestionController {
+ public:
+ tint rtt_avg;
+ tint dev_avg;
+ int cwnd;
+ int peer_cwnd;
+ virtual void OnDataSent(bin64_t b) = 0;
+ virtual void OnDataRecvd(bin64_t b) = 0;
+ virtual void OnAckRcvd(bin64_t b, tint peer_stamp) = 0;
+ virtual ~CongestionControl() = 0;
};
-
+
+ class PiecePicker {
+ public:
+ virtual bin64_t Pick (bins& from, uint8_t layer) = 0;
+ virtual void Received (bin64_t b) = 0;
+ virtual void Snubbed (bin64_t b) = 0;
+ };
+
+ class PeerSelector {
+ public:
+ virtual void PeerKnown (const Sha1Hash& root, struct sockaddr_in& addr) = 0;
+ virtual sockaddr_in GetPeer (const Sha1Hash& for_root) = 0;
+ };
- /** P2TP "control block". */
+ /** P2TP channel's "control block"; channels loosely correspond to TCP
+ connections or FTP sessions; one channel is created for one file
+ being transferred between two peers. As we don't need buffers and
+ lots of other TCP stuff, sizeof(Channel+members) must be below 1K.
+ (There was a seductive idea to remove channels, just put the root
+ hash or a fragment of it into every datagram.) */
class Channel {
-
public:
Channel (int filedes, int socket, struct sockaddr_in peer,
uint32_t peer_channel, uint64_t supports=0);
static void Recv (int socket);
void Recv (Datagram& dgram);
- void Send ();
- void SendSomething ();
+ tint Send ();
+ tint SendSomething ();
void SendHandshake ();
- void Tick ();
- typedef enum {HS_REQ_OUT,HS_RES_OUT,HS_DONE} state_t;
- File& file () { return *File::files[fd]; }
-
-
+ FileTransfer& file () { return *FileTransfer::files[fd]; }
void OnAck (Datagram& dgram);
void OnData (Datagram& dgram);
void AddUncleHashes (Datagram& dgram, bin pos);
void AddPeakHashes (Datagram& dgram);
- bin SenderPiecePick () ;
- bin ReceiverPiecePick (int sizelim) ;
- void CleanStaleDataOut(bin ack_pos);
- void CleanStaleHintOut();
- void CleanStaleHintIn();
+ void CleanStaleHints();
state_t state () const;
- File::status_t peer_status() const { return peer_status_; }
- const CongestionControl& congestion_control() const { return cc_; }
static int DecodeID(int scrambled);
static int EncodeID(int unscrambled);
- static void Loop (tint time);
- static Channel* channel(int ind) {return ind<channels.size()?channels[ind]:NULL;}
- static int MAX_REORDERING;
- static tint TIMEOUT;
- static std::vector<Channel*> channels;
- static int* sockets_;
- static int sock_count_;
- static tint last_tick;
- //static int socket;
+ static Channel* channel(int i) {return i<channels.size()?channels[i]:NULL;}
- friend int Connect (int fd, int sock,
+ /*friend int Connect (int fd, int sock,
const struct sockaddr_in& addr, uint32_t peerch=0);
friend int Init (int portno);
- friend std::ostream& operator << (std::ostream& os, const Channel& s);
+ friend std::ostream& operator << (std::ostream& os, const Channel& s);*/
private:
- // index in the channel array
- int id;
- // Socket address of the peer.
+ /** Channel id: index in the channel array. */
+ uint32_t id;
+ /** Socket address of the peer. */
struct sockaddr_in peer;
- // The UDP socket fd.
- int socket_;
- File::status_t peer_status_;
- // Descriptor of the file in question
- int fd;
- // Zero if we are trying to open a channel.
+ /** The UDP socket fd. */
+ int socket;
+ /** Descriptor of the file in question. */
+ FileTransfer* file;
+ /** Peer channel id; zero if we are trying to open a channel. */
uint32_t peer_channel_id;
- // Temporarily, a pointer to a composed/parsed datagram.
- //Datagram* dgram;
-
- // Peer's retrieved pieces.
+ /** Peer's progress, based on acknowledgements. */
bins ack_in;
- // Progress of piece acknowledgement to the peer.
- bin data_in_;
+ /** Last data received; needs to be acked immediately. */
+ bin64_t data_in;
+ /** Index in the history array. */
int ack_out;
- // Transmit schedule: in most cases filled with the peer's hints
- tbinvec hint_in;
- // Hints sent (to detect and reschedule ignored hints).
- tbinvec hint_out;
- // Uncle hash send pos; in the case both data and hashes do not fit
- // into a single datagram, we set this.
- bin hash_out;
- // Send history: all cwnd pieces.
- tbinvec data_out;
-
- CongestionControl cc_;
- tint last_send_time;
+ /** Transmit schedule: in most cases filled with the peer's hints */
+ tbqueue hint_in;
+ /** Hints sent (to detect and reschedule ignored hints). */
+ tbqueue hint_out;
+ /** The congestion control strategy. */
+ CongestionController *cc;
+ /** For repeats. */
+ tint last_send_time, last_recv_time;
+
+ static PeerSelector* peer_selector;
+
+ static int MAX_REORDERING;
+ static tint TIMEOUT;
+ static std::vector<Channel*> channels;
+ static tint last_tick;
+
};
- //int connect (int fd, const struct sockaddr_in& addr, uint32_t peerch=0) ;
- void Loop (tint time=0) ;
- int Init (int portno) ;
- void Shutdown (int portno);
-
- uint32_t Width (const tbinvec& v);
+ //uint32_t Width (const tbinvec& v);
}
#define RETLOG(str) { LOG(WARNING)<<str; return; }
--- /dev/null
+/*
+ * seq_picker.cpp
+ * p2tp
+ *
+ * Created by Victor Grishchenko on 10/6/09.
+ * Copyright 2009 Delft Technical University. All rights reserved.
+ *
+ */
+
+#include "p2tp.h"
+
+using namespace p2tp;
+
+class SeqPiecePicker : public PiecePicker {
+
+ bins hint_out;
+ FileTransfer* file;
+
+public:
+
+ SeqPiecePicker (FileTransfer* file_) : file(file_) {
+ diho(file->ack_out);
+ }
+
+ virtual bin64_t Pick (bins& from, uint8_t layer) {
+ bins may_pick = ~ file->ack_out;
+ may_pick &= from;
+ may_pick -= hint_out;
+ bin64_t pick = may_pick.find(file->top,bins::FILLED);
+ if ( pick==bin64_t::NONE || pick.right_foot() > file->size() )
+ if (layer)
+ return Pick(from,layer-1);
+ else
+ return bin64_t::NONE;
+ return pick;
+ }
+
+ virtual void Received (bin64_t b) {
+ diho.set(b,bins::FILLED);
+ }
+
+ virtual void Snubbed (bin64_t b) {
+ diho.set(b,bins::EMPTY);
+ }
+
+};
\ No newline at end of file
--- /dev/null
+/*
+ * simple_selector.cpp
+ * p2tp
+ *
+ * Created by Victor Grishchenko on 10/6/09.
+ * Copyright 2009 Delft Technical University. All rights reserved.
+ *
+ */
+
+#include "p2tp.h"
+
+using namespace p2tp;
+
+class SimpleSelector : public PeerSelector {
+ typedef std::pair<sockaddr_in,uint32_t> memo_t;
+ std::queue<memo_t> peers;
+public:
+ virtual void PeerKnown (const Sha1Hash& root, struct sockaddr_in& addr) {
+ peers.push_front(memo_t(addr,root.fingerprint()));
+ }
+ virtual sockaddr_in GetPeer (const Sha1Hash& for_root) {
+ uint32_t fp = for_root.fingerprint();
+ for(std::queue<memo_t>::iterator i=peers.begin(); i!=peers.end(); i++)
+ if (i->second==fp) {
+ i->second = 0;
+ sockaddr_in ret = i->first;
+ while (peers.begin()->second==0)
+ peers.pop_front();
+ return ret;
+ }
+ }
+};
+
+static Channel::peer_selector = new SimpleSelector();
\ No newline at end of file
LIBS=['p2tp','stdc++','gtest','glog','crypto'],
LIBPATH='..' )
+env.Program(
+ target='freemap',
+ source=['freemap.cpp'],
+ CPPPATH=['..'],
+ LIBS=['p2tp','stdc++','gtest','glog'],
+ LIBPATH='..' )
+
env.Program(
target='bin64test',
source=['bin64test.cpp','../bin64.h'],
--- /dev/null
+/*
+ * binstest.cpp
+ * serp++
+ *
+ * Created by Victor Grishchenko on 3/22/09.
+ * Copyright 2009 Delft Technical University. All rights reserved.
+ *
+ */
+#include <time.h>
+#include <gtest/gtest.h>
+#include <set>
+#include "bins.h"
+
+int bins_stripe_count (bins& b) {
+ int stripe_count;
+ uint64_t * stripes = b.get_stripes(stripe_count);
+ free(stripes);
+ return stripe_count;
+}
+
+uint8_t rand_norm (uint8_t lim) {
+ long rnd = random() & ((1<<lim)-1);
+ uint8_t bits = 0;
+ while (rnd) {
+ bits += rnd&1;
+ rnd >>= 1;
+ }
+ return bits;
+}
+
+TEST(FreemapTest,Freemap) {
+ bins space;
+ const bin64_t top(30,0);
+ space.set(top,bins::EMPTY);
+ typedef std::pair<int,bin64_t> timebin_t;
+ typedef std::set<timebin_t> ts_t;
+ ts_t to_free;
+ for (int t=0; t<1000000; t++) {
+ if (t<500000 || t>504000) {
+ uint8_t lr = rand_norm(28);
+ bin64_t alloc = space.find(top,lr);
+ ASSERT_NE(0ULL,~alloc);
+ EXPECT_EQ(bins::EMPTY, space.get(alloc));
+ space.set(alloc,bins::FILLED);
+ long dealloc_time = 1<<rand_norm(22);
+ printf("alloc 2**%i starting at %lli for %li ticks\n",
+ (int)lr,(uint64_t)alloc,dealloc_time);
+ dealloc_time += t;
+ to_free.insert(timebin_t(dealloc_time,alloc));
+ }
+ // now, the red-black tree
+ while (to_free.begin()->first<=t) {
+ bin64_t freebin = to_free.begin()->second;
+ to_free.erase(to_free.begin());
+ space.set(freebin,bins::EMPTY);
+ printf("freed at %lli\n",
+ (uint64_t)freebin);
+ }
+ // log: space taken, gaps, binmap cells, tree cells
+ int cells = space.size();
+ int intervals = bins_stripe_count(space);
+ printf("time %i cells used %i intervals %i blocks %i\n",
+ t,cells,intervals,(int)to_free.size());
+ //space.dump("space");
+ }
+ for(ts_t::iterator i=to_free.begin(); i!=to_free.end(); i++)
+ space.set(i->second,bins::EMPTY);
+ EXPECT_EQ(bins::EMPTY,space.get(top));
+}
+
+int main (int argc, char** argv) {
+ testing::InitGoogleTest(&argc, argv);
+ return RUN_ALL_TESTS();
+}
--- /dev/null
+/*
+ * transfer.cpp
+ * p2tp
+ *
+ * Created by Victor Grishchenko on 10/6/09.
+ * Copyright 2009 Delft Technical University. All rights reserved.
+ *
+ */
+
+#include "p2tp.h"
+
+
+File::File (int _fd) : fd(_fd), status_(DONE), hashes(_fd)
+{
+ bin::vec peaks = bin::peaks(hashes.data_size());
+ history.insert(history.end(),peaks.begin(),peaks.end());
+ for(bin::vec::iterator i=peaks.begin(); i!=peaks.end(); i++)
+ ack_out.set(*i);
+}
+
+File::File (Sha1Hash hash, int _fd) : hashes(hash), fd(_fd), status_(EMPTY) {
+ // TODO resubmit data
+}
+
+File::~File() {
+ if (fd>0) ::close(fd);
+}
+
+
+bool File::OfferHash (bin pos, const Sha1Hash& hash) {
+ HashTree::hashres_t res = hashes.offer(pos,hash);
+ if (res==HashTree::PEAK_ACCEPT) { // file size is finally known
+ ftruncate(fd, size());
+ LOG(INFO)<<fd<<" file size is set to "<<size();
+ history.push_back(0);
+ status_ = IN_PROGRESS;
+ }
+ return res==HashTree::PEAK_ACCEPT || res==HashTree::ACCEPT;
+}
+
+
+File* File::find (const Sha1Hash& hash) {
+ for(vector<File*>::iterator i=files.begin(); i!=files.end(); i++)
+ if (*i && (*i)->hashes.root==hash)
+ return *i;
+ return NULL;
+}
+
+
+int p2tp::Open (const char* filename) {
+ int fd = ::open(filename,O_RDONLY);
+ if (fd<0)
+ return -1;
+ if (File::files.size()<fd+1)
+ File::files.resize(fd+1);
+ File::files[fd] = new File(fd);
+ return fd;
+}
+
+int p2tp::Open (const Sha1Hash& root_hash, const char* filename) {
+ int fd = ::open(filename,O_RDWR|O_CREAT,S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
+ if (fd<0)
+ return -1;
+ if (File::files.size()<fd+1)
+ File::files.resize(fd+1);
+ File::files[fd] = new File(root_hash,fd);
+ return fd;
+}
+
+size_t p2tp::file_size (int fd) { return File::file(fd)->size(); }
+
+void p2tp::Close (int fid) {
+ if (!File::files[fid])
+ return;
+ delete File::files[fid];
+ File::files[fid] = NULL;
+}
+
+
+Sha1Hash HashTree::deriveRoot () {
+ int i = peak_count-1;
+ bin64_t p = peaks[i].first;
+ Sha1Hash hash = peak_hashes[i].second;
+ i--;
+ while (p<bin64_t::ALL) {
+ if (p.is_left()) {
+ p = p.parent();
+ hash = Sha1Hash(hash,Sha1Hash::ZERO);
+ } else {
+ if (i<0 || peaks[i].first!=p.sibling())
+ return Sha1Hash::ZERO;
+ hash = Sha1Hash(peak_hashes[i].second,hash);
+ p = p.parent();
+ i--;
+ }
+ }
+ return hash;
+}
+
+/** Three stages: have file, have root hash, have peaks. */
+HashTree::HashTree (int _datafd, int _hashfd, Sha1Hash _root)
+: datafd(_datafd), hashfd(_hashfd), root(_root)
+{
+ if (root==Sha1Hash::ZERO) { // fresh file; derive the root hash
+ struct stat st;
+ if (fstat(fd, &st)!=0)
+ return;
+ resize(st.st_size);
+ lseek(datafd,0,SEEK_SET);
+ for(bin64_t k=0; k!=toppeak.right(); k=k.dfs_next()) {
+ if (k.is_base()) {
+ uint8_t data[1024];
+ int rd = read(datafd,data,1024);
+ if (rd<=0)
+ return; // FIXME
+ hashes[k] = Sha1Hash(data,rd);
+ } else
+ hashes[k] = Sha1Hash(hashes[k.left()],hashes[k.right()]);
+ }
+ // zeros
+ root = hashes[toppeak];
+ for(bin64_t p=toppeak; p!=bin64_t::ALL; p=p.parent())
+ root = Sha1Hash(root,Sha1Hash::ZERO);
+ }
+ // TODO: THIS MUST BE THE "Transfer"/"File" CLASS
+ if (file_size==0) { // hash only, no file, no peak hashes
+ if (root==Sha1Hash::ZERO)
+ return; // FIXME
+ resize(0); // anyway, 1cell for the root, 63 for peaks
+ }
+
+}
+
+bool FileTransfer::acceptData (uint64_t bin, uint8_t* data, size_t len) {
+}
+
+
+HashTree::~HashTree () {
+ close(fd);
+}
+
+HashTree::hashres_t HashTree::offerPeak (bin pos, Sha1Hash hash) {
+ if (bin(pos+1).layer())
+ return REJECT;
+ if (bin::all1(pos))
+ peaks.clear();
+ peaks.push_back(binhash(pos,hash));
+ if (deriveRoot()==root) { // bingo
+ mass = peaks.back().first;
+ length = mass.length();
+ status.resize(mass+1);
+ bits.resize(mass+1);
+ for(int i=0; i<peaks.size(); i++) {
+ bits[peaks[i].first] = peaks[i].second;
+ status[peaks[i].first] = true;
+ }
+ return PEAK_ACCEPT;
+ } else
+ return pos.layer() ? DUNNO : REJECT;
+}
+
+HashTree::hashres_t HashTree::offer (bin pos, const Sha1Hash& hash) {
+ if (!length) // only peak hashes are accepted at this point
+ return offerPeak(pos,hash);
+ if (pos>mass)
+ return REJECT;
+ if (status[pos])
+ return bits[pos]==hash ? ACCEPT : REJECT;
+ bits[pos] = hash;
+ // walk to the nearest proven hash
+ if (bits[pos.sibling()]==Sha1Hash::ZERO)
+ return DUNNO;
+ bin p = pos.parent();
+ while (!status[p]) {
+ bits[p] = Sha1Hash(bits[p.left()],bits[p.right()]);
+ p = p.parent();
+ }
+ if ( bits[p] == Sha1Hash(bits[p.left()],bits[p.right()]) ) {
+ for(bin i=pos; i<p; i=i.parent())
+ status[i] = status[i.sibling()] = true;
+ return ACCEPT;
+ } else
+ return REJECT;
+
+}
+
+
+
projects / swift-upb.git / commitdiff