0000: 23 20 46 6c 6f 77 20 43 6f 6e 74 72 6f 6c 20 23 # Flow Control #
0010: 0a 0a 54 68 65 20 61 73 73 75 6d 70 74 69 6f 6e ..The assumption
0020: 73 20 6f 66 20 54 43 50 20 61 72 65 20 77 72 6f s of TCP are wro
0030: 6e 67 2c 20 73 6f 20 54 43 50 27 73 20 66 6c 6f ng, so TCP's flo
0040: 77 20 63 6f 6e 74 72 6f 6c 20 69 73 20 62 72 6f w control is bro
0050: 6b 65 6e 20 2d 20 74 68 61 74 27 73 0a 6f 6e 65 ken - that's.one
0060: 20 6f 66 20 6d 79 20 72 61 74 69 6f 6e 61 6c 65 of my rationale
0070: 20 74 6f 20 63 72 65 61 74 65 20 61 20 6e 65 77 to create a new
0080: 20 70 72 6f 74 6f 63 6f 6c 2e 20 53 6f 20 77 68 protocol. So wh
0090: 61 74 20 61 72 65 20 6d 79 20 61 73 73 75 6d 70 at are my assump
00a0: 74 69 6f 6e 73 2c 20 61 6e 64 0a 77 68 61 74 20 tions, and.what
00b0: 64 6f 20 49 20 70 72 6f 70 6f 73 65 20 69 6e 73 do I propose ins
00c0: 74 65 61 64 3f 0a 0a 4c 65 74 27 73 20 66 69 72 tead?..Let's fir
00d0: 73 74 20 6c 6f 6f 6b 20 61 74 20 54 43 50 3a 20 st look at TCP:
00e0: 57 68 61 74 20 64 6f 65 73 20 54 43 50 20 61 73 What does TCP as
00f0: 73 75 6d 65 3f 0a 0a 54 43 50 20 64 65 66 69 6e sume?..TCP defin
0100: 65 73 20 61 20 77 69 6e 64 6f 77 20 73 69 7a 65 es a window size
0110: 2e 20 54 68 69 73 20 65 71 75 61 6c 73 20 74 68 . This equals th
0120: 65 20 61 6d 6f 75 6e 74 20 6f 66 20 64 61 74 61 e amount of data
0130: 20 77 68 69 63 68 20 69 73 20 69 6e 0a 66 6c 69 which is in.fli
0140: 67 68 74 2c 20 73 6f 20 73 75 70 70 6f 73 65 64 ght, so supposed
0150: 20 74 68 65 72 65 20 69 73 20 6e 6f 20 70 61 63 there is no pac
0160: 6b 65 74 20 64 72 6f 70 2e 20 26 6e 62 73 70 3b ket drop.
0170: 54 68 65 20 74 61 69 6c 20 69 73 20 74 68 65 20 The tail is the
0180: 6c 61 73 74 0a 61 63 6b 6e 6f 77 6c 65 64 67 65 last.acknowledge
0190: 64 20 70 61 63 6b 65 74 2c 20 61 6e 64 20 74 68 d packet, and th
01a0: 65 20 68 65 61 64 20 69 73 20 74 68 65 20 6c 61 e head is the la
01b0: 73 74 20 73 65 6e 74 20 70 61 63 6b 65 74 2e 20 st sent packet.
01c0: 26 6e 62 73 70 3b 54 43 50 20 68 61 73 20 61 0a TCP has a.
01d0: 22 73 6c 6f 77 20 73 74 61 72 74 22 2c 20 73 6f "slow start", so
01e0: 20 69 74 20 73 74 61 72 74 73 20 77 69 74 68 20 it starts with
01f0: 6f 6e 65 20 70 61 63 6b 65 74 2c 20 61 6e 64 20 one packet, and
0200: 69 6e 63 72 65 61 73 65 73 20 74 68 65 20 6e 75 increases the nu
0210: 6d 62 65 72 20 6f 66 0a 73 65 67 6d 65 6e 74 73 mber of.segments
0220: 20 69 6e 20 74 68 65 20 77 69 6e 64 6f 77 20 64 in the window d
0230: 65 70 65 6e 64 69 6e 67 20 6f 6e 20 68 6f 77 20 epending on how
0240: 6d 61 6e 79 20 73 65 67 6d 65 6e 74 73 20 61 72 many segments ar
0250: 65 20 61 63 6b 6e 6f 77 6c 65 64 67 65 64 2e 20 e acknowledged.
0260: 26 6e 62 73 70 3b 54 68 69 73 0a 67 69 76 65 73 This.gives
0270: 20 61 6e 20 65 78 70 6f 6e 65 6e 74 69 61 6c 20 an exponential
0280: 67 72 6f 77 74 68 20 70 68 61 73 65 2c 20 75 6e growth phase, un
0290: 74 69 6c 20 74 68 65 72 65 27 73 20 6f 6e 65 20 til there's one
02a0: 70 61 63 6b 65 74 20 64 72 6f 70 2e 20 26 6e 62 packet drop. &nb
02b0: 73 70 3b 54 68 65 0a 61 73 73 75 6d 70 74 69 6f sp;The.assumptio
02c0: 6e 20 68 65 72 65 20 69 73 20 74 68 61 74 20 77 n here is that w
02d0: 68 65 6e 20 74 68 65 20 73 65 6e 64 65 72 20 73 hen the sender s
02e0: 65 6e 64 73 20 74 6f 6f 20 66 61 73 74 2c 20 70 ends too fast, p
02f0: 61 63 6b 65 74 73 20 61 72 65 20 64 72 6f 70 70 ackets are dropp
0300: 65 64 2e 0a 26 6e 62 73 70 3b 49 66 20 61 20 70 ed.. If a p
0310: 61 63 6b 65 74 20 69 73 20 64 72 6f 70 70 65 64 acket is dropped
0320: 2c 20 54 43 50 20 77 69 6c 6c 20 68 61 6c 66 20 , TCP will half
0330: 74 68 65 20 77 69 6e 64 6f 77 20 73 69 7a 65 2c the window size,
0340: 20 61 6e 64 20 73 6c 6f 77 6c 79 20 67 72 6f 77 and slowly grow
0350: 20 69 74 0a 62 79 20 6f 6e 65 20 73 65 67 6d 65 it.by one segme
0360: 6e 74 20 70 65 72 20 72 6f 75 6e 64 2d 74 72 69 nt per round-tri
0370: 70 20 2d 20 75 6e 74 69 6c 20 61 67 61 69 6e 20 p - until again
0380: 61 20 70 61 63 6b 65 74 20 69 73 20 6c 6f 73 74 a packet is lost
0390: 2e 0a 0a 54 68 69 73 20 6d 65 61 6e 73 20 74 68 ...This means th
03a0: 65 20 6e 75 6d 62 65 72 20 6f 66 20 70 61 63 6b e number of pack
03b0: 65 74 73 20 69 6e 20 66 6c 69 67 68 74 20 6f 73 ets in flight os
03c0: 63 69 6c 6c 61 74 65 73 20 62 79 20 61 20 66 61 cillates by a fa
03d0: 63 74 6f 72 20 6f 66 20 74 77 6f 2e 0a 26 6e 62 ctor of two..&nb
03e0: 73 70 3b 53 6f 20 77 68 61 74 20 61 62 6f 75 74 sp;So what about
03f0: 20 6f 70 74 69 6d 61 6c 20 62 75 66 66 65 72 20 optimal buffer
0400: 73 69 7a 65 3f 20 26 6e 62 73 70 3b 54 68 65 20 size? The
0410: 6f 70 74 69 6d 61 6c 20 62 75 66 66 65 72 20 73 optimal buffer s
0420: 69 7a 65 20 66 6f 72 20 61 0a 54 43 50 20 63 6f ize for a.TCP co
0430: 6e 6e 65 63 74 69 6f 6e 20 69 73 20 63 61 70 61 nnection is capa
0440: 62 6c 65 20 74 6f 20 6b 65 65 70 20 70 61 63 6b ble to keep pack
0450: 65 74 73 20 66 6f 72 20 30 2e 35 20 52 54 54 20 ets for 0.5 RTT
0460: 2d 20 62 65 63 61 75 73 65 20 74 68 61 74 27 73 - because that's
0470: 20 74 68 65 20 73 61 6d 65 0a 61 6d 6f 75 6e 74 the same.amount
0480: 20 6f 66 20 70 61 63 6b 65 74 73 20 74 68 61 74 of packets that
0490: 20 66 69 74 20 6f 6e 74 6f 20 74 68 65 20 77 69 fit onto the wi
04a0: 72 65 20 28 74 68 65 79 20 74 61 6b 65 20 30 2e re (they take 0.
04b0: 35 20 52 54 54 20 66 72 6f 6d 20 73 6f 75 72 63 5 RTT from sourc
04c0: 65 20 74 6f 0a 64 65 73 74 69 6e 61 74 69 6f 6e e to.destination
04d0: 29 2e 20 26 6e 62 73 70 3b 54 68 69 73 20 69 73 ). This is
04e0: 20 73 6f 6d 65 74 68 69 6e 67 20 73 6f 20 66 61 something so fa
04f0: 72 20 6e 6f 62 6f 64 79 20 68 61 73 20 69 6d 70 r nobody has imp
0500: 6c 65 6d 65 6e 74 65 64 2c 20 62 65 63 61 75 73 lemented, becaus
0510: 65 20 69 74 0a 72 65 71 75 69 72 65 73 20 74 68 e it.requires th
0520: 61 74 20 74 68 65 20 62 75 66 66 65 72 69 6e 67 at the buffering
0530: 20 72 6f 75 74 65 72 20 6d 65 61 73 75 72 65 73 router measures
0540: 20 74 68 65 20 52 54 54 20 66 6f 72 20 65 61 63 the RTT for eac
0550: 68 20 54 43 50 20 63 6f 6e 6e 65 63 74 69 6f 6e h TCP connection
0560: 2e 0a 26 6e 62 73 70 3b 54 68 69 73 20 6d 65 61 .. This mea
0570: 73 75 72 65 6d 65 6e 74 20 69 73 20 70 6f 73 73 surement is poss
0580: 69 62 6c 65 20 69 6e 20 74 68 65 6f 72 79 2c 20 ible in theory,
0590: 69 74 20 63 61 6e 20 62 65 20 6d 65 61 73 75 72 it can be measur
05a0: 65 64 20 66 72 6f 6d 20 74 68 65 20 66 69 72 73 ed from the firs
05b0: 74 0a 73 79 6e 20 74 6f 20 74 68 65 20 66 69 72 t.syn to the fir
05c0: 73 74 20 61 63 6b 2e 20 26 6e 62 73 70 3b 49 6e st ack. In
05d0: 20 70 72 61 63 74 69 63 65 2c 20 74 68 65 72 65 practice, there
05e0: 27 73 20 75 73 75 61 6c 6c 79 20 6e 6f 20 73 63 's usually no sc
05f0: 69 65 6e 74 69 66 69 63 20 6d 65 74 68 6f 64 0a ientific method.
0600: 61 70 70 6c 69 65 64 20 74 6f 20 63 68 6f 6f 73 applied to choos
0610: 65 20 74 68 65 20 72 69 67 68 74 20 62 75 66 66 e the right buff
0620: 65 72 20 73 69 7a 65 3b 20 69 66 20 79 6f 75 20 er size; if you
0630: 61 72 65 20 6c 75 63 6b 79 2c 20 74 68 65 72 65 are lucky, there
0640: 20 68 61 64 20 62 65 65 6e 20 61 20 66 65 77 0a had been a few.
0650: 65 78 70 65 72 69 6d 65 6e 74 73 2c 20 73 65 6c experiments, sel
0660: 65 63 74 69 6e 67 20 73 6f 6d 65 20 62 75 66 66 ecting some buff
0670: 65 72 20 73 69 7a 65 20 6f 6e 20 61 6e 20 65 64 er size on an ed
0680: 75 63 61 74 65 64 20 67 75 65 73 73 2c 20 61 6e ucated guess, an
0690: 64 20 74 68 65 20 72 6f 75 74 65 72 0a 68 61 73 d the router.has
06a0: 20 61 20 67 6c 6f 62 61 6c 20 46 49 46 4f 2e 0a a global FIFO..
06b0: 0a 54 68 65 20 77 6f 72 73 74 20 70 72 6f 62 6c .The worst probl
06c0: 65 6d 20 77 69 74 68 20 74 68 69 73 20 61 6c 67 em with this alg
06d0: 6f 72 69 74 68 6d 20 69 73 20 6f 6e 20 6e 65 74 orithm is on net
06e0: 77 6f 72 6b 73 20 77 69 74 68 20 70 6f 6f 72 20 works with poor
06f0: 71 75 61 6c 69 74 79 2c 20 73 75 63 68 0a 61 73 quality, such.as
0700: 20 77 69 72 65 6c 65 73 73 20 6e 65 74 77 6f 72 wireless networ
0710: 6b 73 2c 20 77 68 65 72 65 20 70 61 63 6b 65 74 ks, where packet
0720: 20 64 72 6f 70 73 20 61 72 65 20 72 65 6c 61 74 drops are relat
0730: 69 76 65 6c 79 20 66 72 65 71 75 65 6e 74 2c 20 ively frequent,
0740: 61 6e 64 20 68 61 76 65 0a 6e 6f 74 68 69 6e 67 and have.nothing
0750: 20 74 6f 20 64 6f 20 77 69 74 68 20 74 68 65 20 to do with the
0760: 73 65 6e 64 65 72 20 72 61 74 65 2e 20 26 6e 62 sender rate. &nb
0770: 73 70 3b 54 68 65 20 6e 65 78 74 20 70 72 6f 62 sp;The next prob
0780: 6c 65 6d 20 69 73 20 74 68 61 74 20 61 20 66 69 lem is that a fi
0790: 6c 6c 65 64 20 75 70 0a 62 75 66 66 65 72 20 69 lled up.buffer i
07a0: 6e 20 74 68 65 20 72 6f 75 74 65 72 20 64 65 6c n the router del
07b0: 61 79 73 20 61 6c 6c 20 6f 74 68 65 72 20 63 6f ays all other co
07c0: 6e 6e 65 63 74 69 6f 6e 73 2c 20 69 6e 63 6c 75 nnections, inclu
07d0: 64 69 6e 67 20 6c 6f 77 2d 6c 61 74 65 6e 63 79 ding low-latency
07e0: 0a 6c 6f 77 2d 62 61 6e 64 77 69 64 74 68 20 72 .low-bandwidth r
07f0: 65 61 6c 2d 74 69 6d 65 20 70 72 6f 74 6f 63 6f eal-time protoco
0800: 6c 73 2e 0a 0a 53 6f 20 74 68 61 74 27 73 20 6e ls...So that's n
0810: 6f 74 20 74 68 65 20 77 61 79 20 74 6f 20 67 6f ot the way to go
0820: 2e 20 26 6e 62 73 70 3b 57 65 20 6d 75 73 74 20 . We must
0830: 72 65 65 76 61 6c 75 61 74 65 20 74 68 65 20 61 reevaluate the a
0840: 73 73 75 6d 70 74 69 6f 6e 73 20 61 6e 64 0a 66 ssumptions and.f
0850: 69 6e 64 20 61 20 73 6f 6c 75 74 69 6f 6e 2e 0a ind a solution..
0860: 0a 23 23 20 54 68 65 20 61 73 73 75 6d 70 74 69 .## The assumpti
0870: 6f 6e 73 20 23 23 0a 0a 2b 20 4e 65 74 77 6f 72 ons ##..+ Networ
0880: 6b 20 64 65 76 69 63 65 73 20 64 6f 20 68 61 76 k devices do hav
0890: 65 20 62 75 66 66 65 72 73 2c 20 6d 6f 73 74 20 e buffers, most
08a0: 6f 66 20 74 68 65 6d 20 22 74 6f 6f 20 6c 61 72 of them "too lar
08b0: 67 65 20 62 75 66 66 65 72 73 22 20 66 6f 72 20 ge buffers" for
08c0: 54 43 50 0a 20 20 74 6f 20 77 6f 72 6b 20 72 65 TCP. to work re
08d0: 61 73 6f 6e 61 62 6c 65 0a 2b 20 54 68 65 20 62 asonable.+ The b
08e0: 75 66 66 65 72 73 20 61 72 65 20 79 6f 75 72 20 uffers are your
08f0: 66 72 69 65 6e 64 2c 20 6e 6f 74 20 79 6f 75 72 friend, not your
0900: 20 65 6e 65 6d 79 2c 20 74 68 65 79 20 61 76 6f enemy, they avo
0910: 69 64 20 72 65 74 72 61 6e 73 6d 69 73 73 69 6f id retransmissio
0920: 6e 73 0a 2b 20 42 75 66 66 65 72 73 20 73 68 6f ns.+ Buffers sho
0930: 75 6c 64 20 75 73 75 61 6c 6c 79 20 73 74 61 79 uld usually stay
0940: 20 61 6c 6d 6f 73 74 20 65 6d 70 74 79 0a 2b 20 almost empty.+
0950: 50 61 63 6b 65 74 20 64 72 6f 70 73 20 61 72 65 Packet drops are
0960: 20 6e 6f 74 20 72 65 6c 61 74 65 64 20 74 6f 20 not related to
0970: 66 6c 6f 77 20 63 6f 6e 74 72 6f 6c 20 70 72 6f flow control pro
0980: 62 6c 65 6d 73 0a 2b 20 49 6e 74 65 72 6d 65 64 blems.+ Intermed
0990: 69 61 74 65 20 6e 65 74 77 6f 72 6b 20 68 6f 70 iate network hop
09a0: 73 20 63 61 6e 20 68 65 6c 70 20 66 61 69 72 6e s can help fairn
09b0: 65 73 73 2c 20 62 79 20 70 72 6f 76 69 64 69 6e ess, by providin
09c0: 67 20 22 66 61 69 72 20 71 75 65 75 69 6e 67 22 g "fair queuing"
09d0: 0a 0a 23 23 20 54 68 65 20 73 6f 6c 75 74 69 6f ..## The solutio
09e0: 6e 20 23 23 0a 0a 53 69 6e 63 65 20 6e 65 74 77 n ##..Since netw
09f0: 6f 72 6b 20 68 6f 70 73 20 77 68 69 63 68 20 6d ork hops which m
0a00: 61 79 20 68 65 6c 70 20 77 69 74 68 20 66 6c 6f ay help with flo
0a10: 77 20 63 6f 6e 74 72 6f 6c 20 61 72 65 20 6e 6f w control are no
0a20: 74 20 6c 69 6b 65 6c 79 20 74 6f 20 62 65 0a 61 t likely to be.a
0a30: 76 61 69 6c 61 62 6c 65 20 73 6f 6f 6e 20 28 61 vailable soon (a
0a40: 6e 64 20 70 72 6f 62 61 62 6c 79 20 61 6c 73 6f nd probably also
0a50: 20 6e 6f 74 20 74 68 65 20 72 69 67 68 74 20 73 not the right s
0a60: 6f 6c 75 74 69 6f 6e 29 2c 20 74 68 65 20 73 6f olution), the so
0a70: 6c 75 74 69 6f 6e 20 68 61 73 20 74 6f 0a 64 6f lution has to.do
0a80: 20 65 6e 64 2d 74 6f 2d 65 6e 64 20 66 6c 6f 77 end-to-end flow
0a90: 20 63 6f 6e 74 72 6f 6c 20 28 6c 69 6b 65 20 54 control (like T
0aa0: 43 50 2f 49 50 29 2c 20 77 6f 72 6b 69 6e 67 20 CP/IP), working
0ab0: 77 69 74 68 20 73 69 6e 67 6c 65 20 28 75 6e 66 with single (unf
0ac0: 61 69 72 29 20 46 49 46 4f 0a 71 75 65 75 69 6e air) FIFO.queuin
0ad0: 67 2e 20 54 68 65 20 66 6c 6f 77 20 63 6f 6e 74 g. The flow cont
0ae0: 72 6f 6c 20 73 68 6f 75 6c 64 20 62 65 20 66 61 rol should be fa
0af0: 69 72 20 28 5f 6e 5f 20 63 6f 6d 70 65 74 69 6e ir (_n_ competin
0b00: 67 20 63 6f 6e 6e 65 63 74 69 6f 6e 73 20 73 68 g connections sh
0b10: 6f 75 6c 64 0a 67 65 74 20 5f 6e 5f 20 6f 66 20 ould.get _n_ of
0b20: 74 68 65 20 64 61 74 61 20 72 61 74 65 20 65 61 the data rate ea
0b30: 63 68 29 2c 20 61 6e 64 20 69 74 20 73 68 6f 75 ch), and it shou
0b40: 6c 64 20 6e 6f 74 20 63 6f 6d 70 6c 65 74 65 6c ld not completel
0b50: 79 20 79 69 65 6c 64 20 74 6f 0a 54 43 50 2c 20 y yield to.TCP,
0b60: 65 76 65 6e 20 69 6e 20 61 20 62 75 66 66 65 72 even in a buffer
0b70: 2d 62 6c 6f 61 74 20 63 6f 6e 66 69 67 75 72 61 -bloat configura
0b80: 74 69 6f 6e 2e 0a 0a 54 68 65 20 61 70 70 72 6f tion...The appro
0b90: 61 63 68 20 69 73 20 74 68 65 20 66 6f 6c 6c 6f ach is the follo
0ba0: 77 69 6e 67 3a 20 54 68 65 20 73 65 6e 64 65 72 wing: The sender
0bb0: 20 73 65 6e 64 73 20 73 68 6f 72 74 20 62 75 72 sends short bur
0bc0: 73 74 73 20 6f 66 0a 70 61 63 6b 65 74 73 20 28 sts of.packets (
0bd0: 64 65 66 61 75 6c 74 3a 20 38 20 70 61 63 6b 65 default: 8 packe
0be0: 74 73 20 70 65 72 20 62 75 72 73 74 29 2c 20 61 ts per burst), a
0bf0: 6e 64 20 74 68 65 20 72 65 63 65 69 76 65 72 20 nd the receiver
0c00: 6d 65 61 73 75 72 65 73 20 74 68 65 0a 74 69 6d measures the.tim
0c10: 69 6e 67 20 77 68 65 6e 20 74 68 65 73 65 20 70 ing when these p
0c20: 61 63 6b 65 74 73 20 61 72 72 69 76 65 20 2d 20 ackets arrive -
0c30: 66 72 6f 6d 20 65 61 72 6c 69 65 73 74 20 74 6f from earliest to
0c40: 20 6c 61 74 65 73 74 2c 20 61 6e 64 0a 63 61 6c latest, and.cal
0c50: 63 75 6c 61 74 65 73 20 74 68 65 20 61 63 68 69 culates the achi
0c60: 65 76 61 62 6c 65 20 64 61 74 61 20 72 61 74 65 evable data rate
0c70: 2e 20 54 68 65 20 72 65 63 65 69 76 65 72 20 73 . The receiver s
0c80: 65 6e 64 73 20 74 68 69 73 20 64 61 74 61 20 72 ends this data r
0c90: 61 74 65 0a 62 61 63 6b 20 74 6f 20 74 68 65 20 ate.back to the
0ca0: 73 65 6e 64 65 72 2c 20 77 68 69 63 68 20 61 64 sender, which ad
0cb0: 6a 75 73 74 73 20 69 74 73 20 73 65 6e 64 69 6e justs its sendin
0cc0: 67 20 72 61 74 65 20 28 74 6f 20 6d 61 6b 65 20 g rate (to make
0cd0: 73 75 72 65 20 74 68 65 0a 72 61 74 65 20 69 73 sure the.rate is
0ce0: 20 6e 6f 74 20 66 61 6b 65 64 2c 20 74 68 65 20 not faked, the
0cf0: 72 65 63 65 69 76 65 72 20 6d 75 73 74 20 70 72 receiver must pr
0d00: 6f 76 65 20 69 74 20 68 61 73 20 72 65 63 65 69 ove it has recei
0d10: 76 65 64 20 61 74 20 6c 65 61 73 74 0a 6d 6f 73 ved at least.mos
0d20: 74 20 70 61 63 6b 65 74 73 29 2e 20 44 61 74 61 t packets). Data
0d30: 20 72 61 74 65 20 63 61 6c 63 75 6c 61 74 69 6f rate calculatio
0d40: 6e 20 61 63 63 75 6d 75 6c 61 74 65 73 20 72 61 n accumulates ra
0d50: 74 65 73 20 6f 76 65 72 20 73 65 76 65 72 61 6c tes over several
0d60: 0a 62 75 72 73 74 73 20 28 64 65 66 61 75 6c 74 .bursts (default
0d70: 3a 20 34 20 62 75 72 73 74 73 20 70 65 72 20 62 : 4 bursts per b
0d80: 6c 6f 63 6b 29 2c 20 61 6e 64 20 73 65 6e 64 73 lock), and sends
0d90: 20 6f 6e 6c 79 20 61 20 66 69 6e 61 6c 20 72 65 only a final re
0da0: 73 75 6c 74 2c 0a 69 2e 65 2e 20 6f 6e 65 20 61 sult,.i.e. one a
0db0: 63 6b 6e 6f 77 6c 65 64 67 65 20 70 65 72 20 33 cknowledge per 3
0dc0: 32 20 70 61 63 6b 65 74 73 2e 20 54 68 69 73 20 2 packets. This
0dd0: 69 73 20 74 68 65 20 50 20 70 61 72 74 20 6f 66 is the P part of
0de0: 20 61 20 50 49 44 0a 63 6f 6e 74 72 6f 6c 6c 65 a PID.controlle
0df0: 72 3b 20 74 68 65 20 72 65 63 65 69 76 65 72 20 r; the receiver
0e00: 63 6f 6e 73 74 61 6e 74 6c 79 20 70 72 6f 76 69 constantly provi
0e10: 64 65 73 20 6d 65 61 73 75 72 65 6d 65 6e 74 73 des measurements
0e20: 20 6f 66 0a 61 63 68 65 69 76 61 62 6c 65 20 72 of.acheivable r
0e30: 61 74 65 73 2c 20 61 6e 64 20 74 68 65 20 73 65 ates, and the se
0e40: 6e 64 65 72 20 61 64 6a 75 73 74 73 20 74 68 69 nder adjusts thi
0e50: 73 20 72 61 74 65 20 6f 6e 20 65 76 65 72 79 20 s rate on every
0e60: 61 63 6b 0a 72 65 63 65 69 76 65 64 2e 0a 0a 54 ack.received...T
0e70: 68 65 20 73 65 6e 64 65 72 20 74 72 61 63 6b 73 he sender tracks
0e80: 20 74 77 6f 20 74 68 69 6e 67 73 3a 20 53 6c 61 two things: Sla
0e90: 63 6b 20 61 6e 64 20 73 6c 61 63 6b 2d 67 72 6f ck and slack-gro
0ea0: 77 74 68 20 28 49 20 61 6e 64 20 44 20 6f 66 20 wth (I and D of
0eb0: 74 68 65 20 50 49 44 0a 63 6f 6e 74 72 6f 6c 6c the PID.controll
0ec0: 65 72 29 2e 20 53 6c 61 63 6b 2c 20 69 2e 65 2e er). Slack, i.e.
0ed0: 20 61 63 63 75 6d 75 6c 61 74 65 64 20 62 75 66 accumulated buf
0ee0: 66 65 72 20 73 70 61 63 65 2c 20 70 72 6f 76 69 fer space, provi
0ef0: 64 65 73 20 61 6e 20 65 78 70 6f 6e 65 6e 74 69 des an exponenti
0f00: 61 6c 0a 73 6c 6f 77 64 6f 77 6e 2c 20 77 68 65 al.slowdown, whe
0f10: 72 65 20 61 20 66 61 63 74 6f 72 20 6f 66 20 74 re a factor of t
0f20: 77 6f 20 65 71 75 61 74 65 73 20 74 6f 20 65 69 wo equates to ei
0f30: 74 68 65 72 20 68 61 6c 66 20 74 68 65 20 64 69 ther half the di
0f40: 66 66 65 72 65 6e 63 65 20 6f 66 0a 6d 61 78 69 fference of.maxi
0f50: 6d 75 6d 20 61 6e 64 20 6d 69 6e 69 6d 75 6d 20 mum and minimum
0f60: 6f 62 73 65 72 76 65 64 20 73 6c 61 63 6b 20 6f observed slack o
0f70: 72 20 32 30 6d 73 20 28 77 68 61 74 65 76 65 72 r 20ms (whatever
0f80: 20 69 73 20 6c 61 72 67 65 72 29 2e 0a 0a 53 6c is larger)...Sl
0f90: 61 63 6b 2d 67 72 6f 77 74 68 20 69 73 20 6f 62 ack-growth is ob
0fa0: 73 65 72 76 65 64 20 62 79 20 74 68 65 20 74 69 served by the ti
0fb0: 6d 69 6e 67 20 6f 66 20 74 68 65 20 6c 61 73 74 ming of the last
0fc0: 20 62 75 72 73 74 20 63 6f 6d 70 61 72 65 64 20 burst compared
0fd0: 77 69 74 68 20 74 68 65 0a 66 69 72 73 74 20 62 with the.first b
0fe0: 75 72 73 74 20 69 6e 20 74 68 65 20 66 6f 75 72 urst in the four
0ff0: 20 62 75 72 73 74 20 73 65 71 75 65 6e 63 65 2e burst sequence.
1000: 20 54 68 69 73 20 74 65 6c 6c 73 20 75 73 20 68 This tells us h
1010: 6f 77 20 65 78 63 65 73 73 69 76 65 20 6f 75 72 ow excessive our
1020: 20 64 61 74 61 0a 72 61 74 65 20 69 73 2e 20 54 data.rate is. T
1030: 6f 20 63 6f 6d 70 65 6e 73 61 74 65 2c 20 77 65 o compensate, we
1040: 20 6e 65 65 64 20 74 6f 20 6d 75 6c 74 69 70 6c need to multipl
1050: 79 20 74 68 61 74 20 74 69 6d 65 20 77 69 74 68 y that time with
1060: 20 74 68 65 20 62 75 72 73 74 73 20 69 6e 0a 66 the bursts in.f
1070: 6c 69 67 68 74 2c 20 61 6e 64 20 61 64 64 20 74 light, and add t
1080: 68 61 74 20 61 73 20 65 78 74 72 61 20 64 65 6c hat as extra del
1090: 61 79 20 61 66 74 65 72 20 74 68 65 20 6e 65 78 ay after the nex
10a0: 74 20 62 75 72 73 74 20 77 65 20 73 65 6e 64 2e t burst we send.
10b0: 20 54 68 69 73 20 61 6c 6c 6f 77 73 0a 74 68 65 This allows.the
10c0: 20 62 75 66 66 65 72 20 74 6f 20 72 65 63 6f 76 buffer to recov
10d0: 65 72 2e 0a 0a 54 68 65 20 77 68 6f 6c 65 20 61 er...The whole a
10e0: 6c 67 6f 72 69 74 68 6d 2c 20 73 65 6e 64 65 72 lgorithm, sender
10f0: 20 61 6e 64 20 72 65 63 65 69 76 65 72 20 73 69 and receiver si
1100: 64 65 2c 20 66 69 74 73 20 69 6e 74 6f 20 61 62 de, fits into ab
1110: 6f 75 74 20 31 30 30 20 6c 69 6e 65 73 20 61 74 out 100 lines at
1120: 0a 74 68 65 20 6d 6f 6d 65 6e 74 2c 20 77 68 69 .the moment, whi
1130: 63 68 20 69 6e 63 6c 75 64 65 73 20 74 68 65 20 ch includes the
1140: 72 61 74 65 20 63 6f 6e 74 72 6f 6c 20 61 6e 64 rate control and
1150: 20 62 75 72 73 74 20 67 65 6e 65 72 61 74 69 6f burst generatio
1160: 6e 20 6f 6e 20 74 68 65 20 73 65 6e 64 65 72 0a n on the sender.
1170: 73 69 64 65 2c 20 62 75 74 20 64 6f 65 73 20 6e side, but does n
1180: 6f 74 20 69 6e 63 6c 75 64 65 20 61 6c 6c 20 74 ot include all t
1190: 68 65 20 64 65 62 75 67 67 69 6e 67 20 61 6e 64 he debugging and
11a0: 20 73 74 61 74 69 73 74 69 63 73 20 63 6f 64 65 statistics code
11b0: 20 74 6f 20 6f 62 73 65 72 76 65 0a 77 68 61 74 to observe.what
11c0: 20 68 61 70 70 65 6e 73 2e 0a 0a 54 6f 20 67 65 happens...To ge
11d0: 74 20 66 61 73 74 20 6c 6f 6e 67 2d 64 69 73 74 t fast long-dist
11e0: 61 6e 63 65 20 63 6f 6e 6e 65 63 74 69 6f 6e 73 ance connections
11f0: 20 75 70 20 74 6f 20 73 70 65 65 64 20 71 75 69 up to speed qui
1200: 63 6b 6c 79 2c 20 74 68 65 20 66 69 72 73 74 20 ckly, the first
1210: 72 61 74 65 0a 61 64 6a 75 73 74 6d 65 6e 74 20 rate.adjustment
1220: 77 69 6c 6c 20 61 6c 73 6f 20 75 70 20 74 68 65 will also up the
1230: 20 70 61 63 6b 65 74 73 20 69 6e 20 66 6c 69 67 packets in flig
1240: 68 74 2e 20 4c 61 74 65 72 2c 20 65 61 63 68 20 ht. Later, each
1250: 61 63 6b 20 61 6c 6c 6f 77 73 20 66 6f 72 0a 66 ack allows for.f
1260: 75 72 74 68 65 72 20 70 61 63 6b 65 74 73 20 69 urther packets i
1270: 6e 20 66 6c 69 67 68 74 20 28 64 65 66 61 75 6c n flight (defaul
1280: 74 3a 20 61 20 6d 61 78 69 6d 75 6d 20 6f 66 20 t: a maximum of
1290: 74 77 6f 20 62 75 72 73 74 73 2c 20 69 2e 65 2e two bursts, i.e.
12a0: 20 36 34 20 70 61 63 6b 65 74 73 29 0a 62 65 66 64 packets).bef
12b0: 6f 72 65 20 74 68 65 20 6e 65 78 74 20 61 63 6b ore the next ack
12c0: 20 69 73 20 65 78 70 65 63 74 65 64 2e 20 54 6f is expected. To
12d0: 20 61 63 68 69 65 76 65 20 74 68 69 73 2c 20 74 achieve this, t
12e0: 68 65 20 73 65 6e 64 65 72 20 6d 65 61 73 75 72 he sender measur
12f0: 65 73 20 72 6f 75 6e 64 0a 74 72 69 70 20 64 65 es round.trip de
1300: 6c 61 79 2e 0a 0a 54 68 69 73 20 68 65 6c 70 73 lay...This helps
1310: 20 74 6f 20 64 65 74 65 63 74 20 62 72 6f 6b 65 to detect broke
1320: 6e 20 63 6f 6e 6e 65 63 74 69 6f 6e 73 20 2d 20 n connections -
1330: 69 66 20 74 68 65 20 72 65 63 65 69 76 65 72 20 if the receiver
1340: 67 6f 65 73 20 6f 66 66 6c 69 6e 65 20 6f 72 0a goes offline or.
1350: 68 61 73 20 62 65 65 6e 20 73 75 73 70 65 6e 64 has been suspend
1360: 65 64 20 74 65 6d 70 6f 72 61 72 69 6c 79 2c 20 ed temporarily,
1370: 74 68 65 20 73 65 6e 64 65 72 20 73 74 6f 70 73 the sender stops
1380: 2e 20 49 74 20 63 61 6e 27 74 20 63 61 6c 6c 20 . It can't call
1390: 62 61 63 6b 20 74 68 65 0a 70 61 63 6b 65 74 73 back the.packets
13a0: 20 69 6e 20 66 6c 69 67 68 74 2c 20 66 6f 72 20 in flight, for
13b0: 73 75 72 65 2c 20 74 68 65 73 65 20 77 69 6c 6c sure, these will
13c0: 20 67 65 74 20 6c 6f 73 74 2c 20 61 6e 64 20 6d get lost, and m
13d0: 69 67 68 74 20 74 65 6d 70 6f 72 61 72 69 6c 79 ight temporarily
13e0: 20 66 69 6c 6c 20 75 70 0a 62 75 66 66 65 72 73 fill up.buffers
13f0: 2e 0a 0a 54 68 65 20 61 6c 67 6f 72 69 74 68 6d ...The algorithm
1400: 20 69 73 20 6d 65 61 73 75 72 65 64 20 61 73 20 is measured as
1410: 66 61 69 72 20 61 6e 64 20 73 75 66 66 69 63 69 fair and suffici
1420: 65 6e 74 6c 79 20 73 74 61 62 6c 65 20 66 6f 72 ently stable for
1430: 20 73 65 76 65 72 61 6c 0a 70 61 72 61 6c 6c 65 several.paralle
1440: 6c 20 63 6f 6e 6e 65 63 74 69 6f 6e 73 20 74 6f l connections to
1450: 20 74 68 65 20 73 61 6d 65 20 73 65 6e 64 65 72 the same sender
1460: 2c 20 61 6e 64 20 69 74 20 77 6f 72 6b 73 20 74 , and it works t
1470: 6f 67 65 74 68 65 72 20 77 69 74 68 20 70 61 72 ogether with par
1480: 61 6c 6c 65 6c 0a 54 43 50 20 61 6e 64 20 4c 45 allel.TCP and LE
1490: 44 42 41 54 20 74 72 61 66 66 69 63 2e 0a 0a 23 DBAT traffic...#
14a0: 23 20 46 61 69 72 20 51 75 65 75 69 6e 67 20 23 # Fair Queuing #
14b0: 23 0a 0a 49 6e 73 74 65 61 64 20 6f 66 20 75 73 #..Instead of us
14c0: 69 6e 67 20 61 20 73 69 6e 67 6c 65 20 46 49 46 ing a single FIF
14d0: 4f 20 62 75 66 66 65 72 20 70 6f 6c 69 63 79 2c O buffer policy,
14e0: 20 72 6f 75 74 65 72 73 20 28 6f 72 2c 20 69 6e routers (or, in
14f0: 20 6e 65 74 32 6f 0a 74 65 72 6d 69 6e 6f 6c 6f net2o.terminolo
1500: 67 79 2c 20 73 77 69 74 63 68 65 73 29 20 63 61 gy, switches) ca
1510: 6e 20 68 65 6c 70 20 66 61 69 72 6e 65 73 73 3a n help fairness:
1520: 20 55 6e 64 65 72 20 63 6f 6e 67 65 73 74 69 6f Under congestio
1530: 6e 2c 20 65 61 63 68 20 63 6f 6e 6e 65 63 74 69 n, each connecti
1540: 6f 6e 20 69 73 0a 67 69 76 65 6e 20 69 74 73 20 on is.given its
1550: 6f 77 6e 20 46 49 46 4f 2c 20 61 6e 64 20 61 6c own FIFO, and al
1560: 6c 20 66 69 6c 6c 65 64 20 46 49 46 4f 73 20 6f l filled FIFOs o
1570: 66 20 74 68 65 20 73 61 6d 65 20 51 6f 53 20 6c f the same QoS l
1580: 65 76 65 6c 20 61 72 65 20 73 65 72 76 65 64 20 evel are served
1590: 69 6e 20 61 0a 72 6f 75 6e 64 2d 72 6f 62 69 6e in a.round-robin
15a0: 20 66 61 73 68 69 6f 6e 2e 20 26 6e 62 73 70 3b fashion.
15b0: 54 68 69 73 20 61 6c 6c 6f 77 73 20 74 68 65 20 This allows the
15c0: 72 65 63 65 69 76 65 72 20 74 6f 20 61 63 63 75 receiver to accu
15d0: 72 61 74 65 6c 79 20 64 65 74 65 72 6d 69 6e 65 rately determine
15e0: 20 74 68 65 0a 61 63 74 75 61 6c 20 61 63 68 69 the.actual achi
15f0: 65 76 61 62 6c 65 20 62 61 6e 64 77 69 64 74 68 evable bandwidth
1600: 2c 20 61 6e 64 20 64 6f 65 73 20 6e 6f 74 20 74 , and does not t
1610: 72 69 67 67 65 72 20 74 68 65 20 6d 6f 72 65 20 rigger the more
1620: 68 65 75 72 69 73 74 69 63 61 6c 0a 64 65 6c 61 heuristical.dela
1630: 79 2d 6f 70 74 69 6d 69 7a 69 6e 67 20 73 74 72 y-optimizing str
1640: 61 74 65 67 69 65 73 2e 20 26 6e 62 73 70 3b 41 ategies. A
1650: 6c 73 6f 2c 20 74 68 69 73 20 62 75 66 66 65 72 lso, this buffer
1660: 20 70 6f 6c 69 63 79 20 61 6c 6c 6f 77 73 20 74 policy allows t
1670: 6f 20 68 61 76 65 0a 64 69 66 66 65 72 65 6e 74 o have.different
1680: 20 66 6c 6f 77 20 63 6f 6e 74 72 6f 6c 20 61 6c flow control al
1690: 67 6f 72 69 74 68 6d 73 20 6f 6e 20 64 69 66 66 gorithms on diff
16a0: 65 72 65 6e 74 20 70 72 6f 74 6f 63 6f 6c 73 2c erent protocols,
16b0: 20 61 6e 64 20 73 74 69 6c 6c 20 68 61 76 65 0a and still have.
16c0: 66 61 69 72 6e 65 73 73 2e fairness.