rwebtransport 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/Cargo.lock +666 -666
  2. package/Cargo.toml +31 -31
  3. package/LICENSE +202 -202
  4. package/NOTICE +41 -41
  5. package/README.md +550 -550
  6. package/crates/echo-server/Cargo.toml +20 -20
  7. package/crates/echo-server/src/main.rs +666 -666
  8. package/crates/native/Cargo.toml +23 -23
  9. package/crates/native/src/config.rs +129 -129
  10. package/crates/native/src/driver.rs +463 -463
  11. package/crates/native/src/lib.rs +821 -821
  12. package/crates/native/src/server.rs +718 -718
  13. package/crates/native/src/session.rs +1191 -1191
  14. package/crates/native/src/tls.rs +96 -96
  15. package/crates/wtcore/Cargo.toml +15 -15
  16. package/crates/wtcore/src/h3.rs +226 -226
  17. package/crates/wtcore/src/lib.rs +9 -9
  18. package/dist/index.js.map +1 -1
  19. package/dist/index.mjs.map +1 -1
  20. package/package.json +80 -79
  21. package/prebuilds/darwin-arm64/rwebtransport.node +0 -0
  22. package/prebuilds/linux-arm64/rwebtransport.node +0 -0
  23. package/prebuilds/linux-x64/rwebtransport.node +0 -0
  24. package/prebuilds/win32-x64/rwebtransport.node +0 -0
  25. package/scripts/bench.mjs +180 -180
  26. package/scripts/build.js +67 -67
  27. package/scripts/postinstall.js +67 -67
  28. package/scripts/setup-unix.sh +129 -129
  29. package/scripts/setup-windows.ps1 +229 -229
  30. package/vendor/quiche/AGENTS.md +75 -75
  31. package/vendor/quiche/COPYING +23 -23
  32. package/vendor/quiche/Cargo.lock +1212 -1212
  33. package/vendor/quiche/Cargo.toml +215 -215
  34. package/vendor/quiche/README.md +466 -466
  35. package/vendor/quiche/examples/Makefile +46 -46
  36. package/vendor/quiche/examples/README.md +20 -20
  37. package/vendor/quiche/examples/cert-big.crt +90 -90
  38. package/vendor/quiche/examples/cert.crt +18 -18
  39. package/vendor/quiche/examples/cert.key +28 -28
  40. package/vendor/quiche/examples/client.c +343 -343
  41. package/vendor/quiche/examples/client.rs +281 -281
  42. package/vendor/quiche/examples/gen-certs.sh +15 -15
  43. package/vendor/quiche/examples/http3-client.c +473 -473
  44. package/vendor/quiche/examples/http3-client.rs +349 -349
  45. package/vendor/quiche/examples/http3-server.c +614 -614
  46. package/vendor/quiche/examples/http3-server.rs +680 -680
  47. package/vendor/quiche/examples/qpack-decode.rs +87 -87
  48. package/vendor/quiche/examples/qpack-encode.rs +88 -88
  49. package/vendor/quiche/examples/rootca.crt +21 -21
  50. package/vendor/quiche/examples/server.c +529 -529
  51. package/vendor/quiche/examples/server.rs +529 -529
  52. package/vendor/quiche/include/quiche.h +1282 -1282
  53. package/vendor/quiche/src/buffers.rs +105 -105
  54. package/vendor/quiche/src/build.rs +66 -66
  55. package/vendor/quiche/src/cid.rs +1079 -1079
  56. package/vendor/quiche/src/crypto/boringssl.rs +373 -373
  57. package/vendor/quiche/src/crypto/mod.rs +671 -671
  58. package/vendor/quiche/src/dgram.rs +118 -118
  59. package/vendor/quiche/src/error.rs +263 -263
  60. package/vendor/quiche/src/ffi.rs +2290 -2290
  61. package/vendor/quiche/src/flowcontrol.rs +246 -246
  62. package/vendor/quiche/src/frame.rs +2172 -2172
  63. package/vendor/quiche/src/h3/AGENTS.md +54 -54
  64. package/vendor/quiche/src/h3/ffi.rs +442 -442
  65. package/vendor/quiche/src/h3/frame.rs +1404 -1404
  66. package/vendor/quiche/src/h3/mod.rs +8223 -8223
  67. package/vendor/quiche/src/h3/qpack/decoder.rs +288 -288
  68. package/vendor/quiche/src/h3/qpack/encoder.rs +235 -235
  69. package/vendor/quiche/src/h3/qpack/mod.rs +202 -202
  70. package/vendor/quiche/src/h3/qpack/static_table.rs +323 -323
  71. package/vendor/quiche/src/h3/stream.rs +1595 -1595
  72. package/vendor/quiche/src/lib.rs +9556 -9556
  73. package/vendor/quiche/src/minmax.rs +446 -446
  74. package/vendor/quiche/src/packet.rs +2334 -2334
  75. package/vendor/quiche/src/path.rs +1374 -1374
  76. package/vendor/quiche/src/pmtud.rs +652 -652
  77. package/vendor/quiche/src/rand.rs +64 -64
  78. package/vendor/quiche/src/range_buf.rs +186 -186
  79. package/vendor/quiche/src/ranges.rs +755 -755
  80. package/vendor/quiche/src/recovery/AGENTS.md +80 -80
  81. package/vendor/quiche/src/recovery/bandwidth.rs +565 -565
  82. package/vendor/quiche/src/recovery/bytes_in_flight.rs +179 -179
  83. package/vendor/quiche/src/recovery/congestion/cubic.rs +1038 -1038
  84. package/vendor/quiche/src/recovery/congestion/delivery_rate.rs +514 -514
  85. package/vendor/quiche/src/recovery/congestion/hystart.rs +241 -241
  86. package/vendor/quiche/src/recovery/congestion/mod.rs +361 -361
  87. package/vendor/quiche/src/recovery/congestion/prr.rs +238 -238
  88. package/vendor/quiche/src/recovery/congestion/recovery.rs +1170 -1170
  89. package/vendor/quiche/src/recovery/congestion/reno.rs +291 -291
  90. package/vendor/quiche/src/recovery/congestion/test_sender.rs +201 -201
  91. package/vendor/quiche/src/recovery/gcongestion/bbr/bandwidth_sampler.rs +1947 -1947
  92. package/vendor/quiche/src/recovery/gcongestion/bbr/windowed_filter.rs +158 -158
  93. package/vendor/quiche/src/recovery/gcongestion/bbr.rs +37 -37
  94. package/vendor/quiche/src/recovery/gcongestion/bbr2/drain.rs +120 -120
  95. package/vendor/quiche/src/recovery/gcongestion/bbr2/mode.rs +318 -318
  96. package/vendor/quiche/src/recovery/gcongestion/bbr2/network_model.rs +784 -784
  97. package/vendor/quiche/src/recovery/gcongestion/bbr2/probe_bw.rs +729 -729
  98. package/vendor/quiche/src/recovery/gcongestion/bbr2/probe_rtt.rs +175 -175
  99. package/vendor/quiche/src/recovery/gcongestion/bbr2/startup.rs +190 -190
  100. package/vendor/quiche/src/recovery/gcongestion/bbr2.rs +855 -855
  101. package/vendor/quiche/src/recovery/gcongestion/mod.rs +275 -275
  102. package/vendor/quiche/src/recovery/gcongestion/pacer.rs +294 -294
  103. package/vendor/quiche/src/recovery/gcongestion/recovery.rs +1384 -1384
  104. package/vendor/quiche/src/recovery/mod.rs +3086 -3086
  105. package/vendor/quiche/src/recovery/rtt.rs +152 -152
  106. package/vendor/quiche/src/stream/mod.rs +2210 -2210
  107. package/vendor/quiche/src/stream/recv_buf.rs +1236 -1236
  108. package/vendor/quiche/src/stream/send_buf.rs +849 -849
  109. package/vendor/quiche/src/test_utils.rs +681 -681
  110. package/vendor/quiche/src/tests.rs +12922 -12922
  111. package/vendor/quiche/src/tls/boringssl.rs +375 -375
  112. package/vendor/quiche/src/tls/mod.rs +1245 -1245
  113. package/vendor/quiche/src/transport_params.rs +628 -628
@@ -1,1384 +1,1384 @@
1
- use crate::packet;
2
- use crate::recovery::OnLossDetectionTimeoutOutcome;
3
- use crate::recovery::INITIAL_TIME_THRESHOLD_OVERHEAD;
4
- use crate::recovery::TIME_THRESHOLD_OVERHEAD_MULTIPLIER;
5
- use crate::Error;
6
- use crate::Result;
7
-
8
- use std::collections::VecDeque;
9
- use std::time::Duration;
10
- use std::time::Instant;
11
-
12
- use smallvec::SmallVec;
13
-
14
- #[cfg(feature = "qlog")]
15
- use qlog::events::EventData;
16
-
17
- #[cfg(feature = "qlog")]
18
- use crate::recovery::QlogMetrics;
19
-
20
- use crate::frame;
21
-
22
- use crate::recovery::bytes_in_flight::BytesInFlight;
23
- use crate::recovery::gcongestion::Bandwidth;
24
- use crate::recovery::rtt::RttStats;
25
- use crate::recovery::CongestionControlAlgorithm;
26
- use crate::recovery::HandshakeStatus;
27
- use crate::recovery::LossDetectionTimer;
28
- use crate::recovery::OnAckReceivedOutcome;
29
- use crate::recovery::RangeSet;
30
- use crate::recovery::RecoveryConfig;
31
- use crate::recovery::RecoveryOps;
32
- use crate::recovery::RecoveryStats;
33
- use crate::recovery::ReleaseDecision;
34
- use crate::recovery::Sent;
35
- use crate::recovery::StartupExit;
36
- use crate::recovery::GRANULARITY;
37
- use crate::recovery::INITIAL_PACKET_THRESHOLD;
38
- use crate::recovery::INITIAL_TIME_THRESHOLD;
39
- use crate::recovery::MAX_OUTSTANDING_NON_ACK_ELICITING;
40
- use crate::recovery::MAX_PACKET_THRESHOLD;
41
- use crate::recovery::MAX_PTO_EXPONENT;
42
- use crate::recovery::MAX_PTO_PROBES_COUNT;
43
- use crate::recovery::PACKET_REORDER_TIME_THRESHOLD;
44
-
45
- use super::bbr2::BBRv2;
46
- use super::pacer::Pacer;
47
- use super::Acked;
48
- use super::Lost;
49
-
50
- // Congestion Control
51
- const MAX_WINDOW_PACKETS: usize = 20_000;
52
-
53
- #[derive(Debug)]
54
- struct SentPacket {
55
- pkt_num: u64,
56
- status: SentStatus,
57
- }
58
-
59
- #[derive(Debug)]
60
- enum SentStatus {
61
- Sent {
62
- time_sent: Instant,
63
- ack_eliciting: bool,
64
- in_flight: bool,
65
- has_data: bool,
66
- is_pmtud_probe: bool,
67
- sent_bytes: usize,
68
- frames: SmallVec<[frame::Frame; 1]>,
69
- },
70
- Acked,
71
- Lost,
72
- }
73
-
74
- impl SentStatus {
75
- fn ack(&mut self) -> Self {
76
- std::mem::replace(self, SentStatus::Acked)
77
- }
78
-
79
- fn lose(&mut self) -> Self {
80
- if !matches!(self, SentStatus::Acked) {
81
- std::mem::replace(self, SentStatus::Lost)
82
- } else {
83
- SentStatus::Acked
84
- }
85
- }
86
- }
87
-
88
- #[derive(Default)]
89
- struct RecoveryEpoch {
90
- /// The time the most recent ack-eliciting packet was sent.
91
- time_of_last_ack_eliciting_packet: Option<Instant>,
92
-
93
- /// The largest packet number acknowledged in the packet number space so
94
- /// far.
95
- largest_acked_packet: Option<u64>,
96
-
97
- /// The time at which the next packet in that packet number space can be
98
- /// considered lost based on exceeding the reordering window in time.
99
- loss_time: Option<Instant>,
100
-
101
- /// An association of packet numbers in a packet number space to information
102
- /// about them.
103
- sent_packets: VecDeque<SentPacket>,
104
-
105
- loss_probes: usize,
106
- pkts_in_flight: usize,
107
-
108
- acked_frames: VecDeque<frame::Frame>,
109
-
110
- // Frames scheduled for retransmission due to PTO are tracked
111
- // separately so we can check that frames were drained before
112
- // generating more PTO probes.
113
- lost_frames_ack: VecDeque<frame::Frame>,
114
- lost_frames_pto: VecDeque<frame::Frame>,
115
-
116
- /// The largest packet number sent in the packet number space so far.
117
- #[allow(dead_code)]
118
- test_largest_sent_pkt_num_on_path: Option<u64>,
119
- }
120
-
121
- struct AckedDetectionResult {
122
- acked_bytes: usize,
123
- spurious_losses: usize,
124
- spurious_pkt_thresh: Option<u64>,
125
- has_ack_eliciting: bool,
126
- }
127
-
128
- struct LossDetectionResult {
129
- lost_bytes: usize,
130
- lost_packets: usize,
131
-
132
- pmtud_lost_bytes: usize,
133
- pmtud_lost_packets: SmallVec<[u64; 1]>,
134
- }
135
-
136
- impl RecoveryEpoch {
137
- /// Discard the Epoch state and return the total size of unacked packets
138
- /// that were discarded
139
- fn discard(&mut self, cc: &mut Pacer) -> usize {
140
- let unacked_bytes = self
141
- .sent_packets
142
- .drain(..)
143
- .map(|p| {
144
- if let SentPacket {
145
- status:
146
- SentStatus::Sent {
147
- in_flight,
148
- sent_bytes,
149
- ..
150
- },
151
- pkt_num,
152
- } = p
153
- {
154
- cc.on_packet_neutered(pkt_num);
155
- if in_flight {
156
- return sent_bytes;
157
- }
158
- }
159
- 0
160
- })
161
- .sum();
162
-
163
- std::mem::take(&mut self.sent_packets);
164
- self.clear_lost_frames();
165
- std::mem::take(&mut self.acked_frames);
166
- self.time_of_last_ack_eliciting_packet = None;
167
- self.loss_time = None;
168
- self.loss_probes = 0;
169
- self.pkts_in_flight = 0;
170
-
171
- unacked_bytes
172
- }
173
-
174
- // `peer_sent_ack_ranges` should not be used without validation.
175
- fn detect_and_remove_acked_packets(
176
- &mut self, peer_sent_ack_ranges: &RangeSet, newly_acked: &mut Vec<Acked>,
177
- skip_pn: Option<u64>, trace_id: &str,
178
- ) -> Result<AckedDetectionResult> {
179
- newly_acked.clear();
180
-
181
- let mut acked_bytes = 0;
182
- let mut spurious_losses = 0;
183
- let mut spurious_pkt_thresh = None;
184
- let mut has_ack_eliciting = false;
185
-
186
- let largest_ack_received = peer_sent_ack_ranges.last().unwrap();
187
- let largest_acked = self
188
- .largest_acked_packet
189
- .unwrap_or(0)
190
- .max(largest_ack_received);
191
-
192
- for peer_sent_range in peer_sent_ack_ranges.iter() {
193
- if skip_pn.is_some_and(|skip_pn| peer_sent_range.contains(&skip_pn)) {
194
- // https://www.rfc-editor.org/rfc/rfc9000#section-13.1
195
- // An endpoint SHOULD treat receipt of an acknowledgment
196
- // for a packet it did not send as
197
- // a connection error of type PROTOCOL_VIOLATION
198
- return Err(Error::OptimisticAckDetected);
199
- }
200
-
201
- // Because packets always have incrementing numbers, they are always
202
- // in sorted order.
203
- let start = if self
204
- .sent_packets
205
- .front()
206
- .filter(|e| e.pkt_num >= peer_sent_range.start)
207
- .is_some()
208
- {
209
- // Usually it will be the first packet.
210
- 0
211
- } else {
212
- self.sent_packets
213
- .binary_search_by_key(&peer_sent_range.start, |p| p.pkt_num)
214
- .unwrap_or_else(|e| e)
215
- };
216
-
217
- for SentPacket { pkt_num, status } in
218
- self.sent_packets.range_mut(start..)
219
- {
220
- if *pkt_num < peer_sent_range.end {
221
- match status.ack() {
222
- SentStatus::Sent {
223
- time_sent,
224
- in_flight,
225
- sent_bytes,
226
- frames,
227
- ack_eliciting,
228
- ..
229
- } => {
230
- if in_flight {
231
- self.pkts_in_flight -= 1;
232
- acked_bytes += sent_bytes;
233
- }
234
- newly_acked.push(Acked {
235
- pkt_num: *pkt_num,
236
- time_sent,
237
- });
238
-
239
- self.acked_frames.extend(frames);
240
-
241
- has_ack_eliciting |= ack_eliciting;
242
-
243
- trace!("{trace_id} packet newly acked {pkt_num}");
244
- },
245
-
246
- SentStatus::Acked => {},
247
- SentStatus::Lost => {
248
- // An acked packet was already declared lost
249
- spurious_losses += 1;
250
- spurious_pkt_thresh
251
- .get_or_insert(largest_acked - *pkt_num + 1);
252
- },
253
- }
254
- } else {
255
- break;
256
- }
257
- }
258
- }
259
-
260
- self.drain_acked_and_lost_packets();
261
-
262
- Ok(AckedDetectionResult {
263
- acked_bytes,
264
- spurious_losses,
265
- spurious_pkt_thresh,
266
- has_ack_eliciting,
267
- })
268
- }
269
-
270
- fn detect_and_remove_lost_packets(
271
- &mut self, loss_delay: Duration, pkt_thresh: Option<u64>, now: Instant,
272
- newly_lost: &mut Vec<Lost>,
273
- ) -> LossDetectionResult {
274
- newly_lost.clear();
275
- let mut lost_bytes = 0;
276
- self.loss_time = None;
277
-
278
- let lost_send_time = now.checked_sub(loss_delay).unwrap();
279
- let largest_acked = self.largest_acked_packet.unwrap_or(0);
280
- let mut pmtud_lost_bytes = 0;
281
- let mut pmtud_lost_packets = SmallVec::new();
282
-
283
- for SentPacket { pkt_num, status } in &mut self.sent_packets {
284
- if *pkt_num > largest_acked {
285
- break;
286
- }
287
-
288
- if let SentStatus::Sent { time_sent, .. } = status {
289
- let loss_by_time = *time_sent <= lost_send_time;
290
- let loss_by_pkt = match pkt_thresh {
291
- Some(pkt_thresh) => largest_acked >= *pkt_num + pkt_thresh,
292
- None => false,
293
- };
294
-
295
- if loss_by_time || loss_by_pkt {
296
- if let SentStatus::Sent {
297
- in_flight,
298
- sent_bytes,
299
- frames,
300
- is_pmtud_probe,
301
- ..
302
- } = status.lose()
303
- {
304
- self.lost_frames_ack.extend(frames);
305
-
306
- if in_flight {
307
- self.pkts_in_flight -= 1;
308
-
309
- if is_pmtud_probe {
310
- pmtud_lost_bytes += sent_bytes;
311
- pmtud_lost_packets.push(*pkt_num);
312
- // Do not track PMTUD probes losses
313
- continue;
314
- }
315
-
316
- lost_bytes += sent_bytes;
317
- }
318
-
319
- newly_lost.push(Lost {
320
- packet_number: *pkt_num,
321
- bytes_lost: sent_bytes,
322
- });
323
- }
324
- } else {
325
- self.loss_time = Some(*time_sent + loss_delay);
326
- break;
327
- }
328
- }
329
- }
330
-
331
- LossDetectionResult {
332
- lost_bytes,
333
- lost_packets: newly_lost.len(),
334
-
335
- pmtud_lost_bytes,
336
- pmtud_lost_packets,
337
- }
338
- }
339
-
340
- /// Remove packets that were already handled from the front of the queue,
341
- /// but avoid removing packets from the middle of the queue to avoid
342
- /// compaction
343
- fn drain_acked_and_lost_packets(&mut self) {
344
- while let Some(SentPacket {
345
- status: SentStatus::Acked | SentStatus::Lost,
346
- ..
347
- }) = self.sent_packets.front()
348
- {
349
- self.sent_packets.pop_front();
350
- }
351
- }
352
-
353
- fn least_unacked(&self) -> u64 {
354
- for pkt in self.sent_packets.iter() {
355
- if let SentPacket {
356
- pkt_num,
357
- status: SentStatus::Sent { .. },
358
- } = pkt
359
- {
360
- return *pkt_num;
361
- }
362
- }
363
-
364
- self.largest_acked_packet.unwrap_or(0) + 1
365
- }
366
-
367
- /// Returns the next lost frame, trying ACK-based lost frames first,
368
- /// then PTO-based lost frames.
369
- fn next_lost_frame(&mut self) -> Option<frame::Frame> {
370
- self.lost_frames_ack
371
- .pop_front()
372
- .or_else(|| self.lost_frames_pto.pop_front())
373
- }
374
-
375
- /// Returns true if there are any lost frames (ACK or PTO).
376
- fn has_lost_frames(&self) -> bool {
377
- !self.lost_frames_ack.is_empty() || !self.lost_frames_pto.is_empty()
378
- }
379
-
380
- /// Returns the total count of lost frames (ACK + PTO).
381
- #[cfg(test)]
382
- fn lost_frames_count(&self) -> usize {
383
- self.lost_frames_ack.len() + self.lost_frames_pto.len()
384
- }
385
-
386
- /// Clears all lost frames (both ACK and PTO).
387
- fn clear_lost_frames(&mut self) {
388
- self.lost_frames_ack.clear();
389
- self.lost_frames_pto.clear();
390
- }
391
- }
392
-
393
- struct LossThreshold {
394
- pkt_thresh: Option<u64>,
395
- time_thresh: f64,
396
-
397
- // # Experiment: enable_relaxed_loss_threshold
398
- //
399
- // If `Some` this will disable pkt_thresh on the first loss and then double
400
- // time_thresh on subsequent loss.
401
- //
402
- // The actual threshold is calcualted as `1.0 +
403
- // INITIAL_TIME_THRESHOLD_OVERHEAD` and equivalent to the initial value
404
- // of INITIAL_TIME_THRESHOLD.
405
- time_thresh_overhead: Option<f64>,
406
- }
407
-
408
- impl LossThreshold {
409
- fn new(recovery_config: &RecoveryConfig) -> Self {
410
- let time_thresh_overhead =
411
- if recovery_config.enable_relaxed_loss_threshold {
412
- Some(INITIAL_TIME_THRESHOLD_OVERHEAD)
413
- } else {
414
- None
415
- };
416
- LossThreshold {
417
- pkt_thresh: Some(INITIAL_PACKET_THRESHOLD),
418
- time_thresh: INITIAL_TIME_THRESHOLD,
419
- time_thresh_overhead,
420
- }
421
- }
422
-
423
- fn pkt_thresh(&self) -> Option<u64> {
424
- self.pkt_thresh
425
- }
426
-
427
- fn time_thresh(&self) -> f64 {
428
- self.time_thresh
429
- }
430
-
431
- fn on_spurious_loss(&mut self, new_pkt_thresh: u64) {
432
- match &mut self.time_thresh_overhead {
433
- Some(time_thresh_overhead) => {
434
- if self.pkt_thresh.is_some() {
435
- // Disable packet threshold on first spurious loss.
436
- self.pkt_thresh = None;
437
- } else {
438
- // Double time threshold but cap it at `1.0`, which ends up
439
- // being 2x the RTT.
440
- *time_thresh_overhead *= TIME_THRESHOLD_OVERHEAD_MULTIPLIER;
441
- *time_thresh_overhead = time_thresh_overhead.min(1.0);
442
-
443
- self.time_thresh = 1.0 + *time_thresh_overhead;
444
- }
445
- },
446
- None => {
447
- let new_packet_threshold = self
448
- .pkt_thresh
449
- .expect("packet threshold should always be Some when `enable_relaxed_loss_threshold` is false")
450
- .max(new_pkt_thresh.min(MAX_PACKET_THRESHOLD));
451
- self.pkt_thresh = Some(new_packet_threshold);
452
-
453
- self.time_thresh = PACKET_REORDER_TIME_THRESHOLD;
454
- },
455
- }
456
- }
457
- }
458
-
459
- pub struct GRecovery {
460
- epochs: [RecoveryEpoch; packet::Epoch::count()],
461
-
462
- loss_timer: LossDetectionTimer,
463
-
464
- pto_count: u32,
465
-
466
- rtt_stats: RttStats,
467
-
468
- recovery_stats: RecoveryStats,
469
-
470
- pub lost_count: usize,
471
-
472
- pub lost_spurious_count: usize,
473
-
474
- loss_thresh: LossThreshold,
475
-
476
- bytes_in_flight: BytesInFlight,
477
-
478
- bytes_sent: usize,
479
-
480
- pub bytes_lost: u64,
481
-
482
- max_datagram_size: usize,
483
- time_sent_set_to_now: bool,
484
-
485
- #[cfg(feature = "qlog")]
486
- qlog_metrics: QlogMetrics,
487
-
488
- #[cfg(feature = "qlog")]
489
- qlog_prev_cc_state: &'static str,
490
-
491
- /// How many non-ack-eliciting packets have been sent.
492
- outstanding_non_ack_eliciting: usize,
493
-
494
- /// A resusable list of acks.
495
- newly_acked: Vec<Acked>,
496
-
497
- /// A [`Vec`] that can be reused for calls of
498
- /// [`Self::detect_and_remove_lost_packets`] to avoid allocations
499
- lost_reuse: Vec<Lost>,
500
-
501
- pacer: Pacer,
502
- }
503
-
504
- impl GRecovery {
505
- #[cfg(feature = "qlog")]
506
- fn send_rate(&self) -> Bandwidth {
507
- self.pacer.send_rate().unwrap_or(Bandwidth::zero())
508
- }
509
-
510
- #[cfg(feature = "qlog")]
511
- fn ack_rate(&self) -> Bandwidth {
512
- self.pacer.ack_rate().unwrap_or(Bandwidth::zero())
513
- }
514
-
515
- pub fn new(recovery_config: &RecoveryConfig) -> Option<Self> {
516
- let cc = match recovery_config.cc_algorithm {
517
- CongestionControlAlgorithm::Bbr2Gcongestion => BBRv2::new(
518
- recovery_config.initial_congestion_window_packets,
519
- MAX_WINDOW_PACKETS,
520
- recovery_config.max_send_udp_payload_size,
521
- recovery_config.initial_rtt,
522
- recovery_config.custom_bbr_params.as_ref(),
523
- ),
524
- _ => return None,
525
- };
526
-
527
- Some(Self {
528
- epochs: Default::default(),
529
- rtt_stats: RttStats::new(
530
- recovery_config.initial_rtt,
531
- recovery_config.max_ack_delay,
532
- ),
533
- recovery_stats: RecoveryStats::default(),
534
- loss_timer: Default::default(),
535
- pto_count: 0,
536
-
537
- lost_count: 0,
538
- lost_spurious_count: 0,
539
-
540
- loss_thresh: LossThreshold::new(recovery_config),
541
- bytes_in_flight: Default::default(),
542
- bytes_sent: 0,
543
- bytes_lost: 0,
544
-
545
- max_datagram_size: recovery_config.max_send_udp_payload_size,
546
- time_sent_set_to_now: cc.time_sent_set_to_now(),
547
-
548
- #[cfg(feature = "qlog")]
549
- qlog_metrics: QlogMetrics::default(),
550
-
551
- #[cfg(feature = "qlog")]
552
- qlog_prev_cc_state: "",
553
-
554
- outstanding_non_ack_eliciting: 0,
555
-
556
- pacer: Pacer::new(
557
- recovery_config.pacing,
558
- cc,
559
- recovery_config
560
- .max_pacing_rate
561
- .map(Bandwidth::from_mbits_per_second),
562
- ),
563
-
564
- newly_acked: Vec::new(),
565
- lost_reuse: Vec::new(),
566
- })
567
- }
568
-
569
- fn detect_and_remove_lost_packets(
570
- &mut self, epoch: packet::Epoch, now: Instant,
571
- ) -> (usize, usize) {
572
- let loss_delay =
573
- self.rtt_stats.loss_delay(self.loss_thresh.time_thresh());
574
- let lost = &mut self.lost_reuse;
575
-
576
- let LossDetectionResult {
577
- lost_bytes,
578
- lost_packets,
579
- pmtud_lost_bytes,
580
- pmtud_lost_packets,
581
- } = self.epochs[epoch].detect_and_remove_lost_packets(
582
- loss_delay,
583
- self.loss_thresh.pkt_thresh(),
584
- now,
585
- lost,
586
- );
587
-
588
- self.bytes_in_flight
589
- .saturating_subtract(lost_bytes + pmtud_lost_bytes, now);
590
-
591
- for pkt in pmtud_lost_packets {
592
- self.pacer.on_packet_neutered(pkt);
593
- }
594
-
595
- (lost_bytes, lost_packets)
596
- }
597
-
598
- fn loss_time_and_space(&self) -> (Option<Instant>, packet::Epoch) {
599
- let mut epoch = packet::Epoch::Initial;
600
- let mut time = self.epochs[epoch].loss_time;
601
-
602
- // Iterate over all packet number spaces starting from Handshake.
603
- for e in [packet::Epoch::Handshake, packet::Epoch::Application] {
604
- let new_time = self.epochs[e].loss_time;
605
- if time.is_none() || new_time < time {
606
- time = new_time;
607
- epoch = e;
608
- }
609
- }
610
-
611
- (time, epoch)
612
- }
613
-
614
- fn pto_time_and_space(
615
- &self, handshake_status: HandshakeStatus, now: Instant,
616
- ) -> (Option<Instant>, packet::Epoch) {
617
- let mut duration =
618
- self.pto() * 2_u32.pow(self.pto_count.min(MAX_PTO_EXPONENT));
619
-
620
- // Arm PTO from now when there are no inflight packets.
621
- if self.bytes_in_flight.is_zero() {
622
- if handshake_status.has_handshake_keys {
623
- return (Some(now + duration), packet::Epoch::Handshake);
624
- } else {
625
- return (Some(now + duration), packet::Epoch::Initial);
626
- }
627
- }
628
-
629
- let mut pto_timeout = None;
630
- let mut pto_space = packet::Epoch::Initial;
631
-
632
- // Iterate over all packet number spaces.
633
- for &e in packet::Epoch::epochs(
634
- packet::Epoch::Initial..=packet::Epoch::Application,
635
- ) {
636
- if self.epochs[e].pkts_in_flight == 0 {
637
- continue;
638
- }
639
-
640
- if e == packet::Epoch::Application {
641
- // Skip Application Data until handshake completes.
642
- if !handshake_status.completed {
643
- return (pto_timeout, pto_space);
644
- }
645
-
646
- // Include max_ack_delay and backoff for Application Data.
647
- duration += self.rtt_stats.max_ack_delay *
648
- 2_u32.pow(self.pto_count.min(MAX_PTO_EXPONENT));
649
- }
650
-
651
- let new_time = self.epochs[e]
652
- .time_of_last_ack_eliciting_packet
653
- .map(|t| t + duration);
654
-
655
- if pto_timeout.is_none() || new_time < pto_timeout {
656
- pto_timeout = new_time;
657
- pto_space = e;
658
- }
659
- }
660
-
661
- (pto_timeout, pto_space)
662
- }
663
-
664
- fn set_loss_detection_timer(
665
- &mut self, handshake_status: HandshakeStatus, now: Instant,
666
- ) {
667
- if let (Some(earliest_loss_time), _) = self.loss_time_and_space() {
668
- // Time threshold loss detection.
669
- self.loss_timer.update(earliest_loss_time);
670
- return;
671
- }
672
-
673
- if self.bytes_in_flight.is_zero() &&
674
- handshake_status.peer_verified_address
675
- {
676
- self.loss_timer.clear();
677
- return;
678
- }
679
-
680
- // PTO timer.
681
- if let (Some(timeout), _) = self.pto_time_and_space(handshake_status, now)
682
- {
683
- self.loss_timer.update(timeout);
684
- } else {
685
- self.loss_timer.clear();
686
- }
687
- }
688
- }
689
-
690
- impl RecoveryOps for GRecovery {
691
- fn lost_count(&self) -> usize {
692
- self.lost_count
693
- }
694
-
695
- fn bytes_lost(&self) -> u64 {
696
- self.bytes_lost
697
- }
698
-
699
- fn should_elicit_ack(&self, epoch: packet::Epoch) -> bool {
700
- self.epochs[epoch].loss_probes > 0 ||
701
- self.outstanding_non_ack_eliciting >=
702
- MAX_OUTSTANDING_NON_ACK_ELICITING
703
- }
704
-
705
- fn next_acked_frame(&mut self, epoch: packet::Epoch) -> Option<frame::Frame> {
706
- self.epochs[epoch].acked_frames.pop_front()
707
- }
708
-
709
- fn next_lost_frame(&mut self, epoch: packet::Epoch) -> Option<frame::Frame> {
710
- self.epochs[epoch].next_lost_frame()
711
- }
712
-
713
- fn get_largest_acked_on_epoch(&self, epoch: packet::Epoch) -> Option<u64> {
714
- self.epochs[epoch].largest_acked_packet
715
- }
716
-
717
- fn has_lost_frames(&self, epoch: packet::Epoch) -> bool {
718
- self.epochs[epoch].has_lost_frames()
719
- }
720
-
721
- fn loss_probes(&self, epoch: packet::Epoch) -> usize {
722
- self.epochs[epoch].loss_probes
723
- }
724
-
725
- #[cfg(test)]
726
- fn inc_loss_probes(&mut self, epoch: packet::Epoch) {
727
- self.epochs[epoch].loss_probes += 1;
728
- }
729
-
730
- #[cfg(test)]
731
- fn lost_frames_count(&self, epoch: packet::Epoch) -> usize {
732
- self.epochs[epoch].lost_frames_count()
733
- }
734
-
735
- fn ping_sent(&mut self, epoch: packet::Epoch) {
736
- self.epochs[epoch].loss_probes =
737
- self.epochs[epoch].loss_probes.saturating_sub(1);
738
- }
739
-
740
- fn on_packet_sent(
741
- &mut self, pkt: Sent, epoch: packet::Epoch,
742
- handshake_status: HandshakeStatus, now: Instant, trace_id: &str,
743
- ) {
744
- let time_sent = if self.time_sent_set_to_now {
745
- now
746
- } else {
747
- self.get_next_release_time().time(now).unwrap_or(now)
748
- };
749
-
750
- let epoch = &mut self.epochs[epoch];
751
-
752
- let ack_eliciting = pkt.ack_eliciting;
753
- let in_flight = pkt.in_flight;
754
- let is_pmtud_probe = pkt.is_pmtud_probe;
755
- let pkt_num = pkt.pkt_num;
756
- let sent_bytes = pkt.size;
757
-
758
- if let Some(SentPacket { pkt_num, .. }) = epoch.sent_packets.back() {
759
- assert!(*pkt_num < pkt.pkt_num, "Packet numbers must increase");
760
- }
761
-
762
- let status = SentStatus::Sent {
763
- time_sent,
764
- ack_eliciting,
765
- in_flight,
766
- is_pmtud_probe,
767
- has_data: pkt.has_data,
768
- sent_bytes,
769
- frames: pkt.frames,
770
- };
771
-
772
- #[cfg(test)]
773
- {
774
- epoch.test_largest_sent_pkt_num_on_path = epoch
775
- .test_largest_sent_pkt_num_on_path
776
- .max(Some(pkt.pkt_num));
777
- }
778
-
779
- epoch.sent_packets.push_back(SentPacket { pkt_num, status });
780
-
781
- if ack_eliciting {
782
- epoch.time_of_last_ack_eliciting_packet = Some(time_sent);
783
- self.outstanding_non_ack_eliciting = 0;
784
- } else {
785
- self.outstanding_non_ack_eliciting += 1;
786
- }
787
-
788
- if in_flight {
789
- self.pacer.on_packet_sent(
790
- time_sent,
791
- self.bytes_in_flight.get(),
792
- pkt_num,
793
- sent_bytes,
794
- pkt.has_data,
795
- &self.rtt_stats,
796
- );
797
-
798
- self.bytes_in_flight.add(sent_bytes, now);
799
- epoch.pkts_in_flight += 1;
800
- self.set_loss_detection_timer(handshake_status, time_sent);
801
- }
802
-
803
- self.bytes_sent += sent_bytes;
804
-
805
- trace!("{trace_id} {self:?}");
806
- }
807
-
808
- fn get_packet_send_time(&self, now: Instant) -> Instant {
809
- self.pacer.get_next_release_time().time(now).unwrap_or(now)
810
- }
811
-
812
- // `peer_sent_ack_ranges` should not be used without validation.
813
- fn on_ack_received(
814
- &mut self, peer_sent_ack_ranges: &RangeSet, ack_delay: u64,
815
- epoch: packet::Epoch, handshake_status: HandshakeStatus, now: Instant,
816
- skip_pn: Option<u64>, trace_id: &str,
817
- ) -> Result<OnAckReceivedOutcome> {
818
- let prior_in_flight = self.bytes_in_flight.get();
819
-
820
- let AckedDetectionResult {
821
- acked_bytes,
822
- spurious_losses,
823
- spurious_pkt_thresh,
824
- has_ack_eliciting,
825
- } = self.epochs[epoch].detect_and_remove_acked_packets(
826
- peer_sent_ack_ranges,
827
- &mut self.newly_acked,
828
- skip_pn,
829
- trace_id,
830
- )?;
831
-
832
- self.lost_spurious_count += spurious_losses;
833
- if let Some(thresh) = spurious_pkt_thresh {
834
- self.loss_thresh.on_spurious_loss(thresh);
835
- }
836
-
837
- if self.newly_acked.is_empty() {
838
- return Ok(OnAckReceivedOutcome {
839
- acked_bytes,
840
- spurious_losses,
841
- ..Default::default()
842
- });
843
- }
844
-
845
- self.bytes_in_flight.saturating_subtract(acked_bytes, now);
846
-
847
- let largest_newly_acked = self.newly_acked.last().unwrap();
848
-
849
- // Update `largest_acked_packet` based on the validated `newly_acked`
850
- // value.
851
- let largest_acked_pkt_num = self.epochs[epoch]
852
- .largest_acked_packet
853
- .unwrap_or(0)
854
- .max(largest_newly_acked.pkt_num);
855
- self.epochs[epoch].largest_acked_packet = Some(largest_acked_pkt_num);
856
-
857
- // Check if largest packet is newly acked.
858
- let update_rtt = largest_newly_acked.pkt_num == largest_acked_pkt_num &&
859
- has_ack_eliciting;
860
- if update_rtt {
861
- let latest_rtt = now - largest_newly_acked.time_sent;
862
- self.rtt_stats.update_rtt(
863
- latest_rtt,
864
- Duration::from_micros(ack_delay),
865
- now,
866
- handshake_status.completed,
867
- );
868
- }
869
-
870
- let (lost_bytes, lost_packets) =
871
- self.detect_and_remove_lost_packets(epoch, now);
872
-
873
- self.pacer.on_congestion_event(
874
- update_rtt,
875
- prior_in_flight,
876
- self.bytes_in_flight.get(),
877
- now,
878
- &self.newly_acked,
879
- &self.lost_reuse,
880
- self.epochs[epoch].least_unacked(),
881
- &self.rtt_stats,
882
- &mut self.recovery_stats,
883
- );
884
-
885
- self.pto_count = 0;
886
- self.lost_count += lost_packets;
887
-
888
- self.set_loss_detection_timer(handshake_status, now);
889
-
890
- trace!("{trace_id} {self:?}");
891
-
892
- Ok(OnAckReceivedOutcome {
893
- lost_packets,
894
- lost_bytes,
895
- acked_bytes,
896
- spurious_losses,
897
- })
898
- }
899
-
900
- fn on_loss_detection_timeout(
901
- &mut self, handshake_status: HandshakeStatus, now: Instant,
902
- trace_id: &str,
903
- ) -> OnLossDetectionTimeoutOutcome {
904
- let (earliest_loss_time, epoch) = self.loss_time_and_space();
905
-
906
- if earliest_loss_time.is_some() {
907
- let prior_in_flight = self.bytes_in_flight.get();
908
-
909
- let (lost_bytes, lost_packets) =
910
- self.detect_and_remove_lost_packets(epoch, now);
911
-
912
- self.pacer.on_congestion_event(
913
- false,
914
- prior_in_flight,
915
- self.bytes_in_flight.get(),
916
- now,
917
- &[],
918
- &self.lost_reuse,
919
- self.epochs[epoch].least_unacked(),
920
- &self.rtt_stats,
921
- &mut self.recovery_stats,
922
- );
923
-
924
- self.lost_count += lost_packets;
925
-
926
- self.set_loss_detection_timer(handshake_status, now);
927
-
928
- trace!("{trace_id} {self:?}");
929
- return OnLossDetectionTimeoutOutcome {
930
- lost_packets,
931
- lost_bytes,
932
- };
933
- }
934
-
935
- let epoch = if self.bytes_in_flight.get() > 0 {
936
- // Send new data if available, else retransmit old data. If neither
937
- // is available, send a single PING frame.
938
- let (_, e) = self.pto_time_and_space(handshake_status, now);
939
-
940
- e
941
- } else {
942
- // Client sends an anti-deadlock packet: Initial is padded to earn
943
- // more anti-amplification credit, a Handshake packet proves address
944
- // ownership.
945
- if handshake_status.has_handshake_keys {
946
- packet::Epoch::Handshake
947
- } else {
948
- packet::Epoch::Initial
949
- }
950
- };
951
-
952
- self.pto_count += 1;
953
-
954
- let epoch = &mut self.epochs[epoch];
955
-
956
- epoch.loss_probes = MAX_PTO_PROBES_COUNT.min(self.pto_count as usize);
957
-
958
- let sent_packets_iter_limit = if !epoch.lost_frames_pto.is_empty() {
959
- // Skip the search for frames to add to PTO probes if frames
960
- // added in a prior PTO haven't been processed yet.
961
- 0
962
- } else {
963
- usize::MAX
964
- };
965
-
966
- // Skip packets that have already been acked or lost, and packets
967
- // that don't contain either CRYPTO or STREAM frames and only return as
968
- // many packets as the number of probe packets that will be sent.
969
- let unacked_frames = epoch
970
- .sent_packets
971
- .iter()
972
- .take(sent_packets_iter_limit)
973
- .filter_map(|p| {
974
- if let SentStatus::Sent {
975
- has_data: true,
976
- frames,
977
- ..
978
- } = &p.status
979
- {
980
- Some(frames)
981
- } else {
982
- None
983
- }
984
- })
985
- .take(epoch.loss_probes)
986
- .flatten()
987
- .filter(|f| !matches!(f, frame::Frame::DatagramHeader { .. }));
988
-
989
- // Retransmit the frames from the oldest sent packets on PTO. However
990
- // the packets are not actually declared lost (so there is no effect to
991
- // congestion control), we just reschedule the data they carried.
992
- //
993
- // This will also trigger sending an ACK and retransmitting frames like
994
- // HANDSHAKE_DONE and MAX_DATA / MAX_STREAM_DATA as well, in addition
995
- // to CRYPTO and STREAM, if the original packet carried them.
996
- epoch.lost_frames_pto.extend(unacked_frames.cloned());
997
-
998
- self.pacer
999
- .on_retransmission_timeout(epoch.has_lost_frames());
1000
-
1001
- self.set_loss_detection_timer(handshake_status, now);
1002
-
1003
- trace!("{trace_id} {self:?}");
1004
- OnLossDetectionTimeoutOutcome {
1005
- lost_packets: 0,
1006
- lost_bytes: 0,
1007
- }
1008
- }
1009
-
1010
- fn on_pkt_num_space_discarded(
1011
- &mut self, epoch: packet::Epoch, handshake_status: HandshakeStatus,
1012
- now: Instant,
1013
- ) {
1014
- let epoch = &mut self.epochs[epoch];
1015
- self.bytes_in_flight
1016
- .saturating_subtract(epoch.discard(&mut self.pacer), now);
1017
- self.set_loss_detection_timer(handshake_status, now);
1018
- }
1019
-
1020
- fn on_path_change(
1021
- &mut self, epoch: packet::Epoch, now: Instant, _trace_id: &str,
1022
- ) -> (usize, usize) {
1023
- let (lost_bytes, lost_packets) =
1024
- self.detect_and_remove_lost_packets(epoch, now);
1025
-
1026
- (lost_packets, lost_bytes)
1027
- }
1028
-
1029
- fn loss_detection_timer(&self) -> Option<Instant> {
1030
- self.loss_timer.time
1031
- }
1032
-
1033
- fn cwnd(&self) -> usize {
1034
- self.pacer.get_congestion_window()
1035
- }
1036
-
1037
- fn cwnd_available(&self) -> usize {
1038
- // Ignore cwnd when sending probe packets.
1039
- if self.epochs.iter().any(|e| e.loss_probes > 0) {
1040
- return usize::MAX;
1041
- }
1042
-
1043
- self.cwnd().saturating_sub(self.bytes_in_flight.get())
1044
- }
1045
-
1046
- fn rtt(&self) -> Duration {
1047
- self.rtt_stats.rtt()
1048
- }
1049
-
1050
- fn min_rtt(&self) -> Option<Duration> {
1051
- self.rtt_stats.min_rtt()
1052
- }
1053
-
1054
- fn max_rtt(&self) -> Option<Duration> {
1055
- self.rtt_stats.max_rtt()
1056
- }
1057
-
1058
- fn rttvar(&self) -> Duration {
1059
- self.rtt_stats.rttvar()
1060
- }
1061
-
1062
- fn pto(&self) -> Duration {
1063
- let r = &self.rtt_stats;
1064
- r.rtt() + (r.rttvar() * 4).max(GRANULARITY)
1065
- }
1066
-
1067
- /// The most recent data delivery rate estimate.
1068
- fn delivery_rate(&self) -> Bandwidth {
1069
- self.pacer.bandwidth_estimate(&self.rtt_stats)
1070
- }
1071
-
1072
- fn max_bandwidth(&self) -> Option<Bandwidth> {
1073
- Some(self.pacer.max_bandwidth())
1074
- }
1075
-
1076
- /// Statistics from when a CCA first exited the startup phase.
1077
- fn startup_exit(&self) -> Option<StartupExit> {
1078
- self.recovery_stats.startup_exit
1079
- }
1080
-
1081
- fn max_datagram_size(&self) -> usize {
1082
- self.max_datagram_size
1083
- }
1084
-
1085
- fn pmtud_update_max_datagram_size(&mut self, new_max_datagram_size: usize) {
1086
- self.max_datagram_size = new_max_datagram_size;
1087
- self.pacer.update_mss(self.max_datagram_size);
1088
- }
1089
-
1090
- fn update_max_datagram_size(&mut self, new_max_datagram_size: usize) {
1091
- self.pmtud_update_max_datagram_size(
1092
- self.max_datagram_size.min(new_max_datagram_size),
1093
- )
1094
- }
1095
-
1096
- // FIXME only used by gcongestion
1097
- fn on_app_limited(&mut self) {
1098
- self.pacer.on_app_limited(self.bytes_in_flight.get())
1099
- }
1100
-
1101
- #[cfg(test)]
1102
- fn sent_packets_len(&self, epoch: packet::Epoch) -> usize {
1103
- self.epochs[epoch].sent_packets.len()
1104
- }
1105
-
1106
- #[cfg(test)]
1107
- fn in_flight_count(&self, epoch: packet::Epoch) -> usize {
1108
- self.epochs[epoch].pkts_in_flight
1109
- }
1110
-
1111
- fn bytes_in_flight(&self) -> usize {
1112
- self.bytes_in_flight.get()
1113
- }
1114
-
1115
- fn bytes_in_flight_duration(&self) -> Duration {
1116
- self.bytes_in_flight.get_duration()
1117
- }
1118
-
1119
- #[cfg(test)]
1120
- fn pacing_rate(&self) -> u64 {
1121
- self.pacer
1122
- .pacing_rate(self.bytes_in_flight.get(), &self.rtt_stats)
1123
- .to_bytes_per_period(Duration::from_secs(1))
1124
- }
1125
-
1126
- #[cfg(test)]
1127
- fn pto_count(&self) -> u32 {
1128
- self.pto_count
1129
- }
1130
-
1131
- #[cfg(test)]
1132
- fn pkt_thresh(&self) -> Option<u64> {
1133
- self.loss_thresh.pkt_thresh()
1134
- }
1135
-
1136
- #[cfg(test)]
1137
- fn time_thresh(&self) -> f64 {
1138
- self.loss_thresh.time_thresh()
1139
- }
1140
-
1141
- #[cfg(test)]
1142
- fn lost_spurious_count(&self) -> usize {
1143
- self.lost_spurious_count
1144
- }
1145
-
1146
- #[cfg(test)]
1147
- fn detect_lost_packets_for_test(
1148
- &mut self, epoch: packet::Epoch, now: Instant,
1149
- ) -> (usize, usize) {
1150
- let ret = self.detect_and_remove_lost_packets(epoch, now);
1151
- self.epochs[epoch].drain_acked_and_lost_packets();
1152
- ret
1153
- }
1154
-
1155
- #[cfg(test)]
1156
- fn largest_sent_pkt_num_on_path(&self, epoch: packet::Epoch) -> Option<u64> {
1157
- self.epochs[epoch].test_largest_sent_pkt_num_on_path
1158
- }
1159
-
1160
- #[cfg(test)]
1161
- fn app_limited(&self) -> bool {
1162
- self.pacer.is_app_limited(self.bytes_in_flight.get())
1163
- }
1164
-
1165
- // FIXME only used by congestion
1166
- fn update_app_limited(&mut self, _v: bool) {
1167
- // TODO
1168
- }
1169
-
1170
- // FIXME only used by congestion
1171
- fn delivery_rate_update_app_limited(&mut self, _v: bool) {
1172
- // TODO
1173
- }
1174
-
1175
- fn update_max_ack_delay(&mut self, max_ack_delay: Duration) {
1176
- self.rtt_stats.max_ack_delay = max_ack_delay;
1177
- }
1178
-
1179
- fn get_next_release_time(&self) -> ReleaseDecision {
1180
- self.pacer.get_next_release_time()
1181
- }
1182
-
1183
- fn gcongestion_enabled(&self) -> bool {
1184
- true
1185
- }
1186
-
1187
- #[cfg(feature = "qlog")]
1188
- fn state_str(&self, _now: Instant) -> &'static str {
1189
- self.pacer.state_str()
1190
- }
1191
-
1192
- #[cfg(feature = "qlog")]
1193
- fn get_updated_qlog_event_data(&mut self) -> Option<EventData> {
1194
- let qlog_metrics = QlogMetrics {
1195
- min_rtt: *self.rtt_stats.min_rtt,
1196
- smoothed_rtt: self.rtt(),
1197
- latest_rtt: self.rtt_stats.latest_rtt(),
1198
- rttvar: self.rtt_stats.rttvar(),
1199
- cwnd: self.cwnd() as u64,
1200
- bytes_in_flight: self.bytes_in_flight.get() as u64,
1201
- ssthresh: self.pacer.ssthresh(),
1202
-
1203
- pacing_rate: Some(
1204
- self.pacer
1205
- .pacing_rate(self.bytes_in_flight.get(), &self.rtt_stats)
1206
- .to_bytes_per_second(),
1207
- ),
1208
- delivery_rate: Some(self.delivery_rate().to_bytes_per_second()),
1209
- send_rate: Some(self.send_rate().to_bytes_per_second()),
1210
- ack_rate: Some(self.ack_rate().to_bytes_per_second()),
1211
- lost_packets: Some(self.lost_count as u64),
1212
- lost_bytes: Some(self.bytes_lost),
1213
- pto_count: Some(self.pto_count),
1214
- };
1215
-
1216
- self.qlog_metrics.maybe_update(qlog_metrics)
1217
- }
1218
-
1219
- #[cfg(feature = "qlog")]
1220
- fn get_updated_qlog_cc_state(
1221
- &mut self, now: Instant,
1222
- ) -> Option<&'static str> {
1223
- let cc_state = self.state_str(now);
1224
- if cc_state != self.qlog_prev_cc_state {
1225
- self.qlog_prev_cc_state = cc_state;
1226
- Some(cc_state)
1227
- } else {
1228
- None
1229
- }
1230
- }
1231
-
1232
- fn send_quantum(&self) -> usize {
1233
- let pacing_rate = self
1234
- .pacer
1235
- .pacing_rate(self.bytes_in_flight.get(), &self.rtt_stats);
1236
-
1237
- let floor = if pacing_rate < Bandwidth::from_kbits_per_second(1200) {
1238
- self.max_datagram_size
1239
- } else {
1240
- 2 * self.max_datagram_size
1241
- };
1242
-
1243
- pacing_rate
1244
- .to_bytes_per_period(ReleaseDecision::EQUAL_THRESHOLD)
1245
- .min(64 * 1024)
1246
- .max(floor as u64) as usize
1247
- }
1248
- }
1249
-
1250
- impl std::fmt::Debug for GRecovery {
1251
- fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
1252
- write!(f, "timer={:?} ", self.loss_detection_timer())?;
1253
- write!(f, "rtt_stats={:?} ", self.rtt_stats)?;
1254
- write!(f, "bytes_in_flight={} ", self.bytes_in_flight.get())?;
1255
- write!(f, "{:?} ", self.pacer)?;
1256
- Ok(())
1257
- }
1258
- }
1259
-
1260
- #[cfg(test)]
1261
- mod tests {
1262
- use super::*;
1263
- use crate::Config;
1264
-
1265
- #[test]
1266
- fn loss_threshold() {
1267
- let config = Config::new(crate::PROTOCOL_VERSION).unwrap();
1268
- let recovery_config = RecoveryConfig::from_config(&config);
1269
- assert!(!recovery_config.enable_relaxed_loss_threshold);
1270
-
1271
- let mut loss_thresh = LossThreshold::new(&recovery_config);
1272
- assert_eq!(loss_thresh.time_thresh_overhead, None);
1273
- assert_eq!(loss_thresh.pkt_thresh().unwrap(), INITIAL_PACKET_THRESHOLD);
1274
- assert_eq!(loss_thresh.time_thresh(), INITIAL_TIME_THRESHOLD);
1275
-
1276
- // First spurious loss.
1277
- loss_thresh.on_spurious_loss(INITIAL_PACKET_THRESHOLD);
1278
- assert_eq!(loss_thresh.pkt_thresh().unwrap(), INITIAL_PACKET_THRESHOLD);
1279
- assert_eq!(loss_thresh.time_thresh(), PACKET_REORDER_TIME_THRESHOLD);
1280
-
1281
- // Packet gaps < INITIAL_PACKET_THRESHOLD will NOT change packet
1282
- // threshold.
1283
- for packet_gap in 0..INITIAL_PACKET_THRESHOLD {
1284
- loss_thresh.on_spurious_loss(packet_gap);
1285
-
1286
- // Packet threshold only increases once the packet gap increases.
1287
- assert_eq!(
1288
- loss_thresh.pkt_thresh().unwrap(),
1289
- INITIAL_PACKET_THRESHOLD
1290
- );
1291
- assert_eq!(loss_thresh.time_thresh(), PACKET_REORDER_TIME_THRESHOLD);
1292
- }
1293
-
1294
- // Subsequent spurious loss with packet_gaps > INITIAL_PACKET_THRESHOLD.
1295
- // Test values much larger than MAX_PACKET_THRESHOLD, i.e.
1296
- // `MAX_PACKET_THRESHOLD * 2`
1297
- for packet_gap in INITIAL_PACKET_THRESHOLD + 1..MAX_PACKET_THRESHOLD * 2 {
1298
- loss_thresh.on_spurious_loss(packet_gap);
1299
-
1300
- // Packet threshold is equal to packet gap beyond
1301
- // INITIAL_PACKET_THRESHOLD, but capped
1302
- // at MAX_PACKET_THRESHOLD.
1303
- let new_packet_threshold = if packet_gap < MAX_PACKET_THRESHOLD {
1304
- packet_gap
1305
- } else {
1306
- MAX_PACKET_THRESHOLD
1307
- };
1308
- assert_eq!(loss_thresh.pkt_thresh().unwrap(), new_packet_threshold);
1309
- assert_eq!(loss_thresh.time_thresh(), PACKET_REORDER_TIME_THRESHOLD);
1310
- }
1311
- // Packet threshold is capped at MAX_PACKET_THRESHOLD
1312
- assert_eq!(loss_thresh.pkt_thresh().unwrap(), MAX_PACKET_THRESHOLD);
1313
- assert_eq!(loss_thresh.time_thresh(), PACKET_REORDER_TIME_THRESHOLD);
1314
-
1315
- // Packet threshold is monotonically increasing
1316
- loss_thresh.on_spurious_loss(INITIAL_PACKET_THRESHOLD);
1317
- assert_eq!(loss_thresh.pkt_thresh().unwrap(), MAX_PACKET_THRESHOLD);
1318
- assert_eq!(loss_thresh.time_thresh(), PACKET_REORDER_TIME_THRESHOLD);
1319
- }
1320
-
1321
- #[test]
1322
- fn relaxed_loss_threshold() {
1323
- // The max time threshold when operating in relaxed loss mode.
1324
- const MAX_TIME_THRESHOLD: f64 = 2.0;
1325
-
1326
- let mut config = Config::new(crate::PROTOCOL_VERSION).unwrap();
1327
- config.set_enable_relaxed_loss_threshold(true);
1328
- let recovery_config = RecoveryConfig::from_config(&config);
1329
- assert!(recovery_config.enable_relaxed_loss_threshold);
1330
-
1331
- let mut loss_thresh = LossThreshold::new(&recovery_config);
1332
- assert_eq!(
1333
- loss_thresh.time_thresh_overhead,
1334
- Some(INITIAL_TIME_THRESHOLD_OVERHEAD)
1335
- );
1336
- assert_eq!(loss_thresh.pkt_thresh().unwrap(), INITIAL_PACKET_THRESHOLD);
1337
- assert_eq!(loss_thresh.time_thresh(), INITIAL_TIME_THRESHOLD);
1338
-
1339
- // First spurious loss.
1340
- loss_thresh.on_spurious_loss(INITIAL_PACKET_THRESHOLD);
1341
- assert_eq!(loss_thresh.pkt_thresh(), None);
1342
- assert_eq!(loss_thresh.time_thresh(), INITIAL_TIME_THRESHOLD);
1343
-
1344
- // Subsequent spurious loss.
1345
- for subsequent_loss_count in 1..100 {
1346
- // Double the overhead until it caps at `2.0`.
1347
- //
1348
- // It takes `3` rounds of doubling for INITIAL_TIME_THRESHOLD_OVERHEAD
1349
- // to equal `1.0`.
1350
- let new_time_threshold = if subsequent_loss_count <= 3 {
1351
- 1.0 + INITIAL_TIME_THRESHOLD_OVERHEAD *
1352
- 2_f64.powi(subsequent_loss_count as i32)
1353
- } else {
1354
- 2.0
1355
- };
1356
-
1357
- loss_thresh.on_spurious_loss(subsequent_loss_count);
1358
- assert_eq!(loss_thresh.pkt_thresh(), None);
1359
- assert_eq!(loss_thresh.time_thresh(), new_time_threshold);
1360
- }
1361
- // Time threshold is capped at 2.0.
1362
- assert_eq!(loss_thresh.pkt_thresh(), None);
1363
- assert_eq!(loss_thresh.time_thresh(), MAX_TIME_THRESHOLD);
1364
- }
1365
-
1366
- #[test]
1367
- fn test_high_pto_count_no_panic() {
1368
- let mut config = Config::new(crate::PROTOCOL_VERSION).unwrap();
1369
- config.set_cc_algorithm(CongestionControlAlgorithm::Bbr2Gcongestion);
1370
- let recovery_config = RecoveryConfig::from_config(&config);
1371
- let mut r = GRecovery::new(&recovery_config).unwrap();
1372
-
1373
- r.pto_count = 99999;
1374
-
1375
- let handshake_status = HandshakeStatus {
1376
- completed: true,
1377
- has_handshake_keys: true,
1378
- peer_verified_address: true,
1379
- };
1380
- let now = Instant::now();
1381
-
1382
- let _ = r.pto_time_and_space(handshake_status, now);
1383
- }
1384
- }
1
+ use crate::packet;
2
+ use crate::recovery::OnLossDetectionTimeoutOutcome;
3
+ use crate::recovery::INITIAL_TIME_THRESHOLD_OVERHEAD;
4
+ use crate::recovery::TIME_THRESHOLD_OVERHEAD_MULTIPLIER;
5
+ use crate::Error;
6
+ use crate::Result;
7
+
8
+ use std::collections::VecDeque;
9
+ use std::time::Duration;
10
+ use std::time::Instant;
11
+
12
+ use smallvec::SmallVec;
13
+
14
+ #[cfg(feature = "qlog")]
15
+ use qlog::events::EventData;
16
+
17
+ #[cfg(feature = "qlog")]
18
+ use crate::recovery::QlogMetrics;
19
+
20
+ use crate::frame;
21
+
22
+ use crate::recovery::bytes_in_flight::BytesInFlight;
23
+ use crate::recovery::gcongestion::Bandwidth;
24
+ use crate::recovery::rtt::RttStats;
25
+ use crate::recovery::CongestionControlAlgorithm;
26
+ use crate::recovery::HandshakeStatus;
27
+ use crate::recovery::LossDetectionTimer;
28
+ use crate::recovery::OnAckReceivedOutcome;
29
+ use crate::recovery::RangeSet;
30
+ use crate::recovery::RecoveryConfig;
31
+ use crate::recovery::RecoveryOps;
32
+ use crate::recovery::RecoveryStats;
33
+ use crate::recovery::ReleaseDecision;
34
+ use crate::recovery::Sent;
35
+ use crate::recovery::StartupExit;
36
+ use crate::recovery::GRANULARITY;
37
+ use crate::recovery::INITIAL_PACKET_THRESHOLD;
38
+ use crate::recovery::INITIAL_TIME_THRESHOLD;
39
+ use crate::recovery::MAX_OUTSTANDING_NON_ACK_ELICITING;
40
+ use crate::recovery::MAX_PACKET_THRESHOLD;
41
+ use crate::recovery::MAX_PTO_EXPONENT;
42
+ use crate::recovery::MAX_PTO_PROBES_COUNT;
43
+ use crate::recovery::PACKET_REORDER_TIME_THRESHOLD;
44
+
45
+ use super::bbr2::BBRv2;
46
+ use super::pacer::Pacer;
47
+ use super::Acked;
48
+ use super::Lost;
49
+
50
+ // Congestion Control
51
+ const MAX_WINDOW_PACKETS: usize = 20_000;
52
+
53
+ #[derive(Debug)]
54
+ struct SentPacket {
55
+ pkt_num: u64,
56
+ status: SentStatus,
57
+ }
58
+
59
+ #[derive(Debug)]
60
+ enum SentStatus {
61
+ Sent {
62
+ time_sent: Instant,
63
+ ack_eliciting: bool,
64
+ in_flight: bool,
65
+ has_data: bool,
66
+ is_pmtud_probe: bool,
67
+ sent_bytes: usize,
68
+ frames: SmallVec<[frame::Frame; 1]>,
69
+ },
70
+ Acked,
71
+ Lost,
72
+ }
73
+
74
+ impl SentStatus {
75
+ fn ack(&mut self) -> Self {
76
+ std::mem::replace(self, SentStatus::Acked)
77
+ }
78
+
79
+ fn lose(&mut self) -> Self {
80
+ if !matches!(self, SentStatus::Acked) {
81
+ std::mem::replace(self, SentStatus::Lost)
82
+ } else {
83
+ SentStatus::Acked
84
+ }
85
+ }
86
+ }
87
+
88
+ #[derive(Default)]
89
+ struct RecoveryEpoch {
90
+ /// The time the most recent ack-eliciting packet was sent.
91
+ time_of_last_ack_eliciting_packet: Option<Instant>,
92
+
93
+ /// The largest packet number acknowledged in the packet number space so
94
+ /// far.
95
+ largest_acked_packet: Option<u64>,
96
+
97
+ /// The time at which the next packet in that packet number space can be
98
+ /// considered lost based on exceeding the reordering window in time.
99
+ loss_time: Option<Instant>,
100
+
101
+ /// An association of packet numbers in a packet number space to information
102
+ /// about them.
103
+ sent_packets: VecDeque<SentPacket>,
104
+
105
+ loss_probes: usize,
106
+ pkts_in_flight: usize,
107
+
108
+ acked_frames: VecDeque<frame::Frame>,
109
+
110
+ // Frames scheduled for retransmission due to PTO are tracked
111
+ // separately so we can check that frames were drained before
112
+ // generating more PTO probes.
113
+ lost_frames_ack: VecDeque<frame::Frame>,
114
+ lost_frames_pto: VecDeque<frame::Frame>,
115
+
116
+ /// The largest packet number sent in the packet number space so far.
117
+ #[allow(dead_code)]
118
+ test_largest_sent_pkt_num_on_path: Option<u64>,
119
+ }
120
+
121
+ struct AckedDetectionResult {
122
+ acked_bytes: usize,
123
+ spurious_losses: usize,
124
+ spurious_pkt_thresh: Option<u64>,
125
+ has_ack_eliciting: bool,
126
+ }
127
+
128
+ struct LossDetectionResult {
129
+ lost_bytes: usize,
130
+ lost_packets: usize,
131
+
132
+ pmtud_lost_bytes: usize,
133
+ pmtud_lost_packets: SmallVec<[u64; 1]>,
134
+ }
135
+
136
+ impl RecoveryEpoch {
137
+ /// Discard the Epoch state and return the total size of unacked packets
138
+ /// that were discarded
139
+ fn discard(&mut self, cc: &mut Pacer) -> usize {
140
+ let unacked_bytes = self
141
+ .sent_packets
142
+ .drain(..)
143
+ .map(|p| {
144
+ if let SentPacket {
145
+ status:
146
+ SentStatus::Sent {
147
+ in_flight,
148
+ sent_bytes,
149
+ ..
150
+ },
151
+ pkt_num,
152
+ } = p
153
+ {
154
+ cc.on_packet_neutered(pkt_num);
155
+ if in_flight {
156
+ return sent_bytes;
157
+ }
158
+ }
159
+ 0
160
+ })
161
+ .sum();
162
+
163
+ std::mem::take(&mut self.sent_packets);
164
+ self.clear_lost_frames();
165
+ std::mem::take(&mut self.acked_frames);
166
+ self.time_of_last_ack_eliciting_packet = None;
167
+ self.loss_time = None;
168
+ self.loss_probes = 0;
169
+ self.pkts_in_flight = 0;
170
+
171
+ unacked_bytes
172
+ }
173
+
174
+ // `peer_sent_ack_ranges` should not be used without validation.
175
+ fn detect_and_remove_acked_packets(
176
+ &mut self, peer_sent_ack_ranges: &RangeSet, newly_acked: &mut Vec<Acked>,
177
+ skip_pn: Option<u64>, trace_id: &str,
178
+ ) -> Result<AckedDetectionResult> {
179
+ newly_acked.clear();
180
+
181
+ let mut acked_bytes = 0;
182
+ let mut spurious_losses = 0;
183
+ let mut spurious_pkt_thresh = None;
184
+ let mut has_ack_eliciting = false;
185
+
186
+ let largest_ack_received = peer_sent_ack_ranges.last().unwrap();
187
+ let largest_acked = self
188
+ .largest_acked_packet
189
+ .unwrap_or(0)
190
+ .max(largest_ack_received);
191
+
192
+ for peer_sent_range in peer_sent_ack_ranges.iter() {
193
+ if skip_pn.is_some_and(|skip_pn| peer_sent_range.contains(&skip_pn)) {
194
+ // https://www.rfc-editor.org/rfc/rfc9000#section-13.1
195
+ // An endpoint SHOULD treat receipt of an acknowledgment
196
+ // for a packet it did not send as
197
+ // a connection error of type PROTOCOL_VIOLATION
198
+ return Err(Error::OptimisticAckDetected);
199
+ }
200
+
201
+ // Because packets always have incrementing numbers, they are always
202
+ // in sorted order.
203
+ let start = if self
204
+ .sent_packets
205
+ .front()
206
+ .filter(|e| e.pkt_num >= peer_sent_range.start)
207
+ .is_some()
208
+ {
209
+ // Usually it will be the first packet.
210
+ 0
211
+ } else {
212
+ self.sent_packets
213
+ .binary_search_by_key(&peer_sent_range.start, |p| p.pkt_num)
214
+ .unwrap_or_else(|e| e)
215
+ };
216
+
217
+ for SentPacket { pkt_num, status } in
218
+ self.sent_packets.range_mut(start..)
219
+ {
220
+ if *pkt_num < peer_sent_range.end {
221
+ match status.ack() {
222
+ SentStatus::Sent {
223
+ time_sent,
224
+ in_flight,
225
+ sent_bytes,
226
+ frames,
227
+ ack_eliciting,
228
+ ..
229
+ } => {
230
+ if in_flight {
231
+ self.pkts_in_flight -= 1;
232
+ acked_bytes += sent_bytes;
233
+ }
234
+ newly_acked.push(Acked {
235
+ pkt_num: *pkt_num,
236
+ time_sent,
237
+ });
238
+
239
+ self.acked_frames.extend(frames);
240
+
241
+ has_ack_eliciting |= ack_eliciting;
242
+
243
+ trace!("{trace_id} packet newly acked {pkt_num}");
244
+ },
245
+
246
+ SentStatus::Acked => {},
247
+ SentStatus::Lost => {
248
+ // An acked packet was already declared lost
249
+ spurious_losses += 1;
250
+ spurious_pkt_thresh
251
+ .get_or_insert(largest_acked - *pkt_num + 1);
252
+ },
253
+ }
254
+ } else {
255
+ break;
256
+ }
257
+ }
258
+ }
259
+
260
+ self.drain_acked_and_lost_packets();
261
+
262
+ Ok(AckedDetectionResult {
263
+ acked_bytes,
264
+ spurious_losses,
265
+ spurious_pkt_thresh,
266
+ has_ack_eliciting,
267
+ })
268
+ }
269
+
270
+ fn detect_and_remove_lost_packets(
271
+ &mut self, loss_delay: Duration, pkt_thresh: Option<u64>, now: Instant,
272
+ newly_lost: &mut Vec<Lost>,
273
+ ) -> LossDetectionResult {
274
+ newly_lost.clear();
275
+ let mut lost_bytes = 0;
276
+ self.loss_time = None;
277
+
278
+ let lost_send_time = now.checked_sub(loss_delay).unwrap();
279
+ let largest_acked = self.largest_acked_packet.unwrap_or(0);
280
+ let mut pmtud_lost_bytes = 0;
281
+ let mut pmtud_lost_packets = SmallVec::new();
282
+
283
+ for SentPacket { pkt_num, status } in &mut self.sent_packets {
284
+ if *pkt_num > largest_acked {
285
+ break;
286
+ }
287
+
288
+ if let SentStatus::Sent { time_sent, .. } = status {
289
+ let loss_by_time = *time_sent <= lost_send_time;
290
+ let loss_by_pkt = match pkt_thresh {
291
+ Some(pkt_thresh) => largest_acked >= *pkt_num + pkt_thresh,
292
+ None => false,
293
+ };
294
+
295
+ if loss_by_time || loss_by_pkt {
296
+ if let SentStatus::Sent {
297
+ in_flight,
298
+ sent_bytes,
299
+ frames,
300
+ is_pmtud_probe,
301
+ ..
302
+ } = status.lose()
303
+ {
304
+ self.lost_frames_ack.extend(frames);
305
+
306
+ if in_flight {
307
+ self.pkts_in_flight -= 1;
308
+
309
+ if is_pmtud_probe {
310
+ pmtud_lost_bytes += sent_bytes;
311
+ pmtud_lost_packets.push(*pkt_num);
312
+ // Do not track PMTUD probes losses
313
+ continue;
314
+ }
315
+
316
+ lost_bytes += sent_bytes;
317
+ }
318
+
319
+ newly_lost.push(Lost {
320
+ packet_number: *pkt_num,
321
+ bytes_lost: sent_bytes,
322
+ });
323
+ }
324
+ } else {
325
+ self.loss_time = Some(*time_sent + loss_delay);
326
+ break;
327
+ }
328
+ }
329
+ }
330
+
331
+ LossDetectionResult {
332
+ lost_bytes,
333
+ lost_packets: newly_lost.len(),
334
+
335
+ pmtud_lost_bytes,
336
+ pmtud_lost_packets,
337
+ }
338
+ }
339
+
340
+ /// Remove packets that were already handled from the front of the queue,
341
+ /// but avoid removing packets from the middle of the queue to avoid
342
+ /// compaction
343
+ fn drain_acked_and_lost_packets(&mut self) {
344
+ while let Some(SentPacket {
345
+ status: SentStatus::Acked | SentStatus::Lost,
346
+ ..
347
+ }) = self.sent_packets.front()
348
+ {
349
+ self.sent_packets.pop_front();
350
+ }
351
+ }
352
+
353
+ fn least_unacked(&self) -> u64 {
354
+ for pkt in self.sent_packets.iter() {
355
+ if let SentPacket {
356
+ pkt_num,
357
+ status: SentStatus::Sent { .. },
358
+ } = pkt
359
+ {
360
+ return *pkt_num;
361
+ }
362
+ }
363
+
364
+ self.largest_acked_packet.unwrap_or(0) + 1
365
+ }
366
+
367
+ /// Returns the next lost frame, trying ACK-based lost frames first,
368
+ /// then PTO-based lost frames.
369
+ fn next_lost_frame(&mut self) -> Option<frame::Frame> {
370
+ self.lost_frames_ack
371
+ .pop_front()
372
+ .or_else(|| self.lost_frames_pto.pop_front())
373
+ }
374
+
375
+ /// Returns true if there are any lost frames (ACK or PTO).
376
+ fn has_lost_frames(&self) -> bool {
377
+ !self.lost_frames_ack.is_empty() || !self.lost_frames_pto.is_empty()
378
+ }
379
+
380
+ /// Returns the total count of lost frames (ACK + PTO).
381
+ #[cfg(test)]
382
+ fn lost_frames_count(&self) -> usize {
383
+ self.lost_frames_ack.len() + self.lost_frames_pto.len()
384
+ }
385
+
386
+ /// Clears all lost frames (both ACK and PTO).
387
+ fn clear_lost_frames(&mut self) {
388
+ self.lost_frames_ack.clear();
389
+ self.lost_frames_pto.clear();
390
+ }
391
+ }
392
+
393
+ struct LossThreshold {
394
+ pkt_thresh: Option<u64>,
395
+ time_thresh: f64,
396
+
397
+ // # Experiment: enable_relaxed_loss_threshold
398
+ //
399
+ // If `Some` this will disable pkt_thresh on the first loss and then double
400
+ // time_thresh on subsequent loss.
401
+ //
402
+ // The actual threshold is calcualted as `1.0 +
403
+ // INITIAL_TIME_THRESHOLD_OVERHEAD` and equivalent to the initial value
404
+ // of INITIAL_TIME_THRESHOLD.
405
+ time_thresh_overhead: Option<f64>,
406
+ }
407
+
408
+ impl LossThreshold {
409
+ fn new(recovery_config: &RecoveryConfig) -> Self {
410
+ let time_thresh_overhead =
411
+ if recovery_config.enable_relaxed_loss_threshold {
412
+ Some(INITIAL_TIME_THRESHOLD_OVERHEAD)
413
+ } else {
414
+ None
415
+ };
416
+ LossThreshold {
417
+ pkt_thresh: Some(INITIAL_PACKET_THRESHOLD),
418
+ time_thresh: INITIAL_TIME_THRESHOLD,
419
+ time_thresh_overhead,
420
+ }
421
+ }
422
+
423
+ fn pkt_thresh(&self) -> Option<u64> {
424
+ self.pkt_thresh
425
+ }
426
+
427
+ fn time_thresh(&self) -> f64 {
428
+ self.time_thresh
429
+ }
430
+
431
+ fn on_spurious_loss(&mut self, new_pkt_thresh: u64) {
432
+ match &mut self.time_thresh_overhead {
433
+ Some(time_thresh_overhead) => {
434
+ if self.pkt_thresh.is_some() {
435
+ // Disable packet threshold on first spurious loss.
436
+ self.pkt_thresh = None;
437
+ } else {
438
+ // Double time threshold but cap it at `1.0`, which ends up
439
+ // being 2x the RTT.
440
+ *time_thresh_overhead *= TIME_THRESHOLD_OVERHEAD_MULTIPLIER;
441
+ *time_thresh_overhead = time_thresh_overhead.min(1.0);
442
+
443
+ self.time_thresh = 1.0 + *time_thresh_overhead;
444
+ }
445
+ },
446
+ None => {
447
+ let new_packet_threshold = self
448
+ .pkt_thresh
449
+ .expect("packet threshold should always be Some when `enable_relaxed_loss_threshold` is false")
450
+ .max(new_pkt_thresh.min(MAX_PACKET_THRESHOLD));
451
+ self.pkt_thresh = Some(new_packet_threshold);
452
+
453
+ self.time_thresh = PACKET_REORDER_TIME_THRESHOLD;
454
+ },
455
+ }
456
+ }
457
+ }
458
+
459
+ pub struct GRecovery {
460
+ epochs: [RecoveryEpoch; packet::Epoch::count()],
461
+
462
+ loss_timer: LossDetectionTimer,
463
+
464
+ pto_count: u32,
465
+
466
+ rtt_stats: RttStats,
467
+
468
+ recovery_stats: RecoveryStats,
469
+
470
+ pub lost_count: usize,
471
+
472
+ pub lost_spurious_count: usize,
473
+
474
+ loss_thresh: LossThreshold,
475
+
476
+ bytes_in_flight: BytesInFlight,
477
+
478
+ bytes_sent: usize,
479
+
480
+ pub bytes_lost: u64,
481
+
482
+ max_datagram_size: usize,
483
+ time_sent_set_to_now: bool,
484
+
485
+ #[cfg(feature = "qlog")]
486
+ qlog_metrics: QlogMetrics,
487
+
488
+ #[cfg(feature = "qlog")]
489
+ qlog_prev_cc_state: &'static str,
490
+
491
+ /// How many non-ack-eliciting packets have been sent.
492
+ outstanding_non_ack_eliciting: usize,
493
+
494
+ /// A resusable list of acks.
495
+ newly_acked: Vec<Acked>,
496
+
497
+ /// A [`Vec`] that can be reused for calls of
498
+ /// [`Self::detect_and_remove_lost_packets`] to avoid allocations
499
+ lost_reuse: Vec<Lost>,
500
+
501
+ pacer: Pacer,
502
+ }
503
+
504
+ impl GRecovery {
505
+ #[cfg(feature = "qlog")]
506
+ fn send_rate(&self) -> Bandwidth {
507
+ self.pacer.send_rate().unwrap_or(Bandwidth::zero())
508
+ }
509
+
510
+ #[cfg(feature = "qlog")]
511
+ fn ack_rate(&self) -> Bandwidth {
512
+ self.pacer.ack_rate().unwrap_or(Bandwidth::zero())
513
+ }
514
+
515
+ pub fn new(recovery_config: &RecoveryConfig) -> Option<Self> {
516
+ let cc = match recovery_config.cc_algorithm {
517
+ CongestionControlAlgorithm::Bbr2Gcongestion => BBRv2::new(
518
+ recovery_config.initial_congestion_window_packets,
519
+ MAX_WINDOW_PACKETS,
520
+ recovery_config.max_send_udp_payload_size,
521
+ recovery_config.initial_rtt,
522
+ recovery_config.custom_bbr_params.as_ref(),
523
+ ),
524
+ _ => return None,
525
+ };
526
+
527
+ Some(Self {
528
+ epochs: Default::default(),
529
+ rtt_stats: RttStats::new(
530
+ recovery_config.initial_rtt,
531
+ recovery_config.max_ack_delay,
532
+ ),
533
+ recovery_stats: RecoveryStats::default(),
534
+ loss_timer: Default::default(),
535
+ pto_count: 0,
536
+
537
+ lost_count: 0,
538
+ lost_spurious_count: 0,
539
+
540
+ loss_thresh: LossThreshold::new(recovery_config),
541
+ bytes_in_flight: Default::default(),
542
+ bytes_sent: 0,
543
+ bytes_lost: 0,
544
+
545
+ max_datagram_size: recovery_config.max_send_udp_payload_size,
546
+ time_sent_set_to_now: cc.time_sent_set_to_now(),
547
+
548
+ #[cfg(feature = "qlog")]
549
+ qlog_metrics: QlogMetrics::default(),
550
+
551
+ #[cfg(feature = "qlog")]
552
+ qlog_prev_cc_state: "",
553
+
554
+ outstanding_non_ack_eliciting: 0,
555
+
556
+ pacer: Pacer::new(
557
+ recovery_config.pacing,
558
+ cc,
559
+ recovery_config
560
+ .max_pacing_rate
561
+ .map(Bandwidth::from_mbits_per_second),
562
+ ),
563
+
564
+ newly_acked: Vec::new(),
565
+ lost_reuse: Vec::new(),
566
+ })
567
+ }
568
+
569
+ fn detect_and_remove_lost_packets(
570
+ &mut self, epoch: packet::Epoch, now: Instant,
571
+ ) -> (usize, usize) {
572
+ let loss_delay =
573
+ self.rtt_stats.loss_delay(self.loss_thresh.time_thresh());
574
+ let lost = &mut self.lost_reuse;
575
+
576
+ let LossDetectionResult {
577
+ lost_bytes,
578
+ lost_packets,
579
+ pmtud_lost_bytes,
580
+ pmtud_lost_packets,
581
+ } = self.epochs[epoch].detect_and_remove_lost_packets(
582
+ loss_delay,
583
+ self.loss_thresh.pkt_thresh(),
584
+ now,
585
+ lost,
586
+ );
587
+
588
+ self.bytes_in_flight
589
+ .saturating_subtract(lost_bytes + pmtud_lost_bytes, now);
590
+
591
+ for pkt in pmtud_lost_packets {
592
+ self.pacer.on_packet_neutered(pkt);
593
+ }
594
+
595
+ (lost_bytes, lost_packets)
596
+ }
597
+
598
+ fn loss_time_and_space(&self) -> (Option<Instant>, packet::Epoch) {
599
+ let mut epoch = packet::Epoch::Initial;
600
+ let mut time = self.epochs[epoch].loss_time;
601
+
602
+ // Iterate over all packet number spaces starting from Handshake.
603
+ for e in [packet::Epoch::Handshake, packet::Epoch::Application] {
604
+ let new_time = self.epochs[e].loss_time;
605
+ if time.is_none() || new_time < time {
606
+ time = new_time;
607
+ epoch = e;
608
+ }
609
+ }
610
+
611
+ (time, epoch)
612
+ }
613
+
614
+ fn pto_time_and_space(
615
+ &self, handshake_status: HandshakeStatus, now: Instant,
616
+ ) -> (Option<Instant>, packet::Epoch) {
617
+ let mut duration =
618
+ self.pto() * 2_u32.pow(self.pto_count.min(MAX_PTO_EXPONENT));
619
+
620
+ // Arm PTO from now when there are no inflight packets.
621
+ if self.bytes_in_flight.is_zero() {
622
+ if handshake_status.has_handshake_keys {
623
+ return (Some(now + duration), packet::Epoch::Handshake);
624
+ } else {
625
+ return (Some(now + duration), packet::Epoch::Initial);
626
+ }
627
+ }
628
+
629
+ let mut pto_timeout = None;
630
+ let mut pto_space = packet::Epoch::Initial;
631
+
632
+ // Iterate over all packet number spaces.
633
+ for &e in packet::Epoch::epochs(
634
+ packet::Epoch::Initial..=packet::Epoch::Application,
635
+ ) {
636
+ if self.epochs[e].pkts_in_flight == 0 {
637
+ continue;
638
+ }
639
+
640
+ if e == packet::Epoch::Application {
641
+ // Skip Application Data until handshake completes.
642
+ if !handshake_status.completed {
643
+ return (pto_timeout, pto_space);
644
+ }
645
+
646
+ // Include max_ack_delay and backoff for Application Data.
647
+ duration += self.rtt_stats.max_ack_delay *
648
+ 2_u32.pow(self.pto_count.min(MAX_PTO_EXPONENT));
649
+ }
650
+
651
+ let new_time = self.epochs[e]
652
+ .time_of_last_ack_eliciting_packet
653
+ .map(|t| t + duration);
654
+
655
+ if pto_timeout.is_none() || new_time < pto_timeout {
656
+ pto_timeout = new_time;
657
+ pto_space = e;
658
+ }
659
+ }
660
+
661
+ (pto_timeout, pto_space)
662
+ }
663
+
664
+ fn set_loss_detection_timer(
665
+ &mut self, handshake_status: HandshakeStatus, now: Instant,
666
+ ) {
667
+ if let (Some(earliest_loss_time), _) = self.loss_time_and_space() {
668
+ // Time threshold loss detection.
669
+ self.loss_timer.update(earliest_loss_time);
670
+ return;
671
+ }
672
+
673
+ if self.bytes_in_flight.is_zero() &&
674
+ handshake_status.peer_verified_address
675
+ {
676
+ self.loss_timer.clear();
677
+ return;
678
+ }
679
+
680
+ // PTO timer.
681
+ if let (Some(timeout), _) = self.pto_time_and_space(handshake_status, now)
682
+ {
683
+ self.loss_timer.update(timeout);
684
+ } else {
685
+ self.loss_timer.clear();
686
+ }
687
+ }
688
+ }
689
+
690
+ impl RecoveryOps for GRecovery {
691
+ fn lost_count(&self) -> usize {
692
+ self.lost_count
693
+ }
694
+
695
+ fn bytes_lost(&self) -> u64 {
696
+ self.bytes_lost
697
+ }
698
+
699
+ fn should_elicit_ack(&self, epoch: packet::Epoch) -> bool {
700
+ self.epochs[epoch].loss_probes > 0 ||
701
+ self.outstanding_non_ack_eliciting >=
702
+ MAX_OUTSTANDING_NON_ACK_ELICITING
703
+ }
704
+
705
+ fn next_acked_frame(&mut self, epoch: packet::Epoch) -> Option<frame::Frame> {
706
+ self.epochs[epoch].acked_frames.pop_front()
707
+ }
708
+
709
+ fn next_lost_frame(&mut self, epoch: packet::Epoch) -> Option<frame::Frame> {
710
+ self.epochs[epoch].next_lost_frame()
711
+ }
712
+
713
+ fn get_largest_acked_on_epoch(&self, epoch: packet::Epoch) -> Option<u64> {
714
+ self.epochs[epoch].largest_acked_packet
715
+ }
716
+
717
+ fn has_lost_frames(&self, epoch: packet::Epoch) -> bool {
718
+ self.epochs[epoch].has_lost_frames()
719
+ }
720
+
721
+ fn loss_probes(&self, epoch: packet::Epoch) -> usize {
722
+ self.epochs[epoch].loss_probes
723
+ }
724
+
725
+ #[cfg(test)]
726
+ fn inc_loss_probes(&mut self, epoch: packet::Epoch) {
727
+ self.epochs[epoch].loss_probes += 1;
728
+ }
729
+
730
+ #[cfg(test)]
731
+ fn lost_frames_count(&self, epoch: packet::Epoch) -> usize {
732
+ self.epochs[epoch].lost_frames_count()
733
+ }
734
+
735
+ fn ping_sent(&mut self, epoch: packet::Epoch) {
736
+ self.epochs[epoch].loss_probes =
737
+ self.epochs[epoch].loss_probes.saturating_sub(1);
738
+ }
739
+
740
+ fn on_packet_sent(
741
+ &mut self, pkt: Sent, epoch: packet::Epoch,
742
+ handshake_status: HandshakeStatus, now: Instant, trace_id: &str,
743
+ ) {
744
+ let time_sent = if self.time_sent_set_to_now {
745
+ now
746
+ } else {
747
+ self.get_next_release_time().time(now).unwrap_or(now)
748
+ };
749
+
750
+ let epoch = &mut self.epochs[epoch];
751
+
752
+ let ack_eliciting = pkt.ack_eliciting;
753
+ let in_flight = pkt.in_flight;
754
+ let is_pmtud_probe = pkt.is_pmtud_probe;
755
+ let pkt_num = pkt.pkt_num;
756
+ let sent_bytes = pkt.size;
757
+
758
+ if let Some(SentPacket { pkt_num, .. }) = epoch.sent_packets.back() {
759
+ assert!(*pkt_num < pkt.pkt_num, "Packet numbers must increase");
760
+ }
761
+
762
+ let status = SentStatus::Sent {
763
+ time_sent,
764
+ ack_eliciting,
765
+ in_flight,
766
+ is_pmtud_probe,
767
+ has_data: pkt.has_data,
768
+ sent_bytes,
769
+ frames: pkt.frames,
770
+ };
771
+
772
+ #[cfg(test)]
773
+ {
774
+ epoch.test_largest_sent_pkt_num_on_path = epoch
775
+ .test_largest_sent_pkt_num_on_path
776
+ .max(Some(pkt.pkt_num));
777
+ }
778
+
779
+ epoch.sent_packets.push_back(SentPacket { pkt_num, status });
780
+
781
+ if ack_eliciting {
782
+ epoch.time_of_last_ack_eliciting_packet = Some(time_sent);
783
+ self.outstanding_non_ack_eliciting = 0;
784
+ } else {
785
+ self.outstanding_non_ack_eliciting += 1;
786
+ }
787
+
788
+ if in_flight {
789
+ self.pacer.on_packet_sent(
790
+ time_sent,
791
+ self.bytes_in_flight.get(),
792
+ pkt_num,
793
+ sent_bytes,
794
+ pkt.has_data,
795
+ &self.rtt_stats,
796
+ );
797
+
798
+ self.bytes_in_flight.add(sent_bytes, now);
799
+ epoch.pkts_in_flight += 1;
800
+ self.set_loss_detection_timer(handshake_status, time_sent);
801
+ }
802
+
803
+ self.bytes_sent += sent_bytes;
804
+
805
+ trace!("{trace_id} {self:?}");
806
+ }
807
+
808
+ fn get_packet_send_time(&self, now: Instant) -> Instant {
809
+ self.pacer.get_next_release_time().time(now).unwrap_or(now)
810
+ }
811
+
812
+ // `peer_sent_ack_ranges` should not be used without validation.
813
+ fn on_ack_received(
814
+ &mut self, peer_sent_ack_ranges: &RangeSet, ack_delay: u64,
815
+ epoch: packet::Epoch, handshake_status: HandshakeStatus, now: Instant,
816
+ skip_pn: Option<u64>, trace_id: &str,
817
+ ) -> Result<OnAckReceivedOutcome> {
818
+ let prior_in_flight = self.bytes_in_flight.get();
819
+
820
+ let AckedDetectionResult {
821
+ acked_bytes,
822
+ spurious_losses,
823
+ spurious_pkt_thresh,
824
+ has_ack_eliciting,
825
+ } = self.epochs[epoch].detect_and_remove_acked_packets(
826
+ peer_sent_ack_ranges,
827
+ &mut self.newly_acked,
828
+ skip_pn,
829
+ trace_id,
830
+ )?;
831
+
832
+ self.lost_spurious_count += spurious_losses;
833
+ if let Some(thresh) = spurious_pkt_thresh {
834
+ self.loss_thresh.on_spurious_loss(thresh);
835
+ }
836
+
837
+ if self.newly_acked.is_empty() {
838
+ return Ok(OnAckReceivedOutcome {
839
+ acked_bytes,
840
+ spurious_losses,
841
+ ..Default::default()
842
+ });
843
+ }
844
+
845
+ self.bytes_in_flight.saturating_subtract(acked_bytes, now);
846
+
847
+ let largest_newly_acked = self.newly_acked.last().unwrap();
848
+
849
+ // Update `largest_acked_packet` based on the validated `newly_acked`
850
+ // value.
851
+ let largest_acked_pkt_num = self.epochs[epoch]
852
+ .largest_acked_packet
853
+ .unwrap_or(0)
854
+ .max(largest_newly_acked.pkt_num);
855
+ self.epochs[epoch].largest_acked_packet = Some(largest_acked_pkt_num);
856
+
857
+ // Check if largest packet is newly acked.
858
+ let update_rtt = largest_newly_acked.pkt_num == largest_acked_pkt_num &&
859
+ has_ack_eliciting;
860
+ if update_rtt {
861
+ let latest_rtt = now - largest_newly_acked.time_sent;
862
+ self.rtt_stats.update_rtt(
863
+ latest_rtt,
864
+ Duration::from_micros(ack_delay),
865
+ now,
866
+ handshake_status.completed,
867
+ );
868
+ }
869
+
870
+ let (lost_bytes, lost_packets) =
871
+ self.detect_and_remove_lost_packets(epoch, now);
872
+
873
+ self.pacer.on_congestion_event(
874
+ update_rtt,
875
+ prior_in_flight,
876
+ self.bytes_in_flight.get(),
877
+ now,
878
+ &self.newly_acked,
879
+ &self.lost_reuse,
880
+ self.epochs[epoch].least_unacked(),
881
+ &self.rtt_stats,
882
+ &mut self.recovery_stats,
883
+ );
884
+
885
+ self.pto_count = 0;
886
+ self.lost_count += lost_packets;
887
+
888
+ self.set_loss_detection_timer(handshake_status, now);
889
+
890
+ trace!("{trace_id} {self:?}");
891
+
892
+ Ok(OnAckReceivedOutcome {
893
+ lost_packets,
894
+ lost_bytes,
895
+ acked_bytes,
896
+ spurious_losses,
897
+ })
898
+ }
899
+
900
+ fn on_loss_detection_timeout(
901
+ &mut self, handshake_status: HandshakeStatus, now: Instant,
902
+ trace_id: &str,
903
+ ) -> OnLossDetectionTimeoutOutcome {
904
+ let (earliest_loss_time, epoch) = self.loss_time_and_space();
905
+
906
+ if earliest_loss_time.is_some() {
907
+ let prior_in_flight = self.bytes_in_flight.get();
908
+
909
+ let (lost_bytes, lost_packets) =
910
+ self.detect_and_remove_lost_packets(epoch, now);
911
+
912
+ self.pacer.on_congestion_event(
913
+ false,
914
+ prior_in_flight,
915
+ self.bytes_in_flight.get(),
916
+ now,
917
+ &[],
918
+ &self.lost_reuse,
919
+ self.epochs[epoch].least_unacked(),
920
+ &self.rtt_stats,
921
+ &mut self.recovery_stats,
922
+ );
923
+
924
+ self.lost_count += lost_packets;
925
+
926
+ self.set_loss_detection_timer(handshake_status, now);
927
+
928
+ trace!("{trace_id} {self:?}");
929
+ return OnLossDetectionTimeoutOutcome {
930
+ lost_packets,
931
+ lost_bytes,
932
+ };
933
+ }
934
+
935
+ let epoch = if self.bytes_in_flight.get() > 0 {
936
+ // Send new data if available, else retransmit old data. If neither
937
+ // is available, send a single PING frame.
938
+ let (_, e) = self.pto_time_and_space(handshake_status, now);
939
+
940
+ e
941
+ } else {
942
+ // Client sends an anti-deadlock packet: Initial is padded to earn
943
+ // more anti-amplification credit, a Handshake packet proves address
944
+ // ownership.
945
+ if handshake_status.has_handshake_keys {
946
+ packet::Epoch::Handshake
947
+ } else {
948
+ packet::Epoch::Initial
949
+ }
950
+ };
951
+
952
+ self.pto_count += 1;
953
+
954
+ let epoch = &mut self.epochs[epoch];
955
+
956
+ epoch.loss_probes = MAX_PTO_PROBES_COUNT.min(self.pto_count as usize);
957
+
958
+ let sent_packets_iter_limit = if !epoch.lost_frames_pto.is_empty() {
959
+ // Skip the search for frames to add to PTO probes if frames
960
+ // added in a prior PTO haven't been processed yet.
961
+ 0
962
+ } else {
963
+ usize::MAX
964
+ };
965
+
966
+ // Skip packets that have already been acked or lost, and packets
967
+ // that don't contain either CRYPTO or STREAM frames and only return as
968
+ // many packets as the number of probe packets that will be sent.
969
+ let unacked_frames = epoch
970
+ .sent_packets
971
+ .iter()
972
+ .take(sent_packets_iter_limit)
973
+ .filter_map(|p| {
974
+ if let SentStatus::Sent {
975
+ has_data: true,
976
+ frames,
977
+ ..
978
+ } = &p.status
979
+ {
980
+ Some(frames)
981
+ } else {
982
+ None
983
+ }
984
+ })
985
+ .take(epoch.loss_probes)
986
+ .flatten()
987
+ .filter(|f| !matches!(f, frame::Frame::DatagramHeader { .. }));
988
+
989
+ // Retransmit the frames from the oldest sent packets on PTO. However
990
+ // the packets are not actually declared lost (so there is no effect to
991
+ // congestion control), we just reschedule the data they carried.
992
+ //
993
+ // This will also trigger sending an ACK and retransmitting frames like
994
+ // HANDSHAKE_DONE and MAX_DATA / MAX_STREAM_DATA as well, in addition
995
+ // to CRYPTO and STREAM, if the original packet carried them.
996
+ epoch.lost_frames_pto.extend(unacked_frames.cloned());
997
+
998
+ self.pacer
999
+ .on_retransmission_timeout(epoch.has_lost_frames());
1000
+
1001
+ self.set_loss_detection_timer(handshake_status, now);
1002
+
1003
+ trace!("{trace_id} {self:?}");
1004
+ OnLossDetectionTimeoutOutcome {
1005
+ lost_packets: 0,
1006
+ lost_bytes: 0,
1007
+ }
1008
+ }
1009
+
1010
+ fn on_pkt_num_space_discarded(
1011
+ &mut self, epoch: packet::Epoch, handshake_status: HandshakeStatus,
1012
+ now: Instant,
1013
+ ) {
1014
+ let epoch = &mut self.epochs[epoch];
1015
+ self.bytes_in_flight
1016
+ .saturating_subtract(epoch.discard(&mut self.pacer), now);
1017
+ self.set_loss_detection_timer(handshake_status, now);
1018
+ }
1019
+
1020
+ fn on_path_change(
1021
+ &mut self, epoch: packet::Epoch, now: Instant, _trace_id: &str,
1022
+ ) -> (usize, usize) {
1023
+ let (lost_bytes, lost_packets) =
1024
+ self.detect_and_remove_lost_packets(epoch, now);
1025
+
1026
+ (lost_packets, lost_bytes)
1027
+ }
1028
+
1029
+ fn loss_detection_timer(&self) -> Option<Instant> {
1030
+ self.loss_timer.time
1031
+ }
1032
+
1033
+ fn cwnd(&self) -> usize {
1034
+ self.pacer.get_congestion_window()
1035
+ }
1036
+
1037
+ fn cwnd_available(&self) -> usize {
1038
+ // Ignore cwnd when sending probe packets.
1039
+ if self.epochs.iter().any(|e| e.loss_probes > 0) {
1040
+ return usize::MAX;
1041
+ }
1042
+
1043
+ self.cwnd().saturating_sub(self.bytes_in_flight.get())
1044
+ }
1045
+
1046
+ fn rtt(&self) -> Duration {
1047
+ self.rtt_stats.rtt()
1048
+ }
1049
+
1050
+ fn min_rtt(&self) -> Option<Duration> {
1051
+ self.rtt_stats.min_rtt()
1052
+ }
1053
+
1054
+ fn max_rtt(&self) -> Option<Duration> {
1055
+ self.rtt_stats.max_rtt()
1056
+ }
1057
+
1058
+ fn rttvar(&self) -> Duration {
1059
+ self.rtt_stats.rttvar()
1060
+ }
1061
+
1062
+ fn pto(&self) -> Duration {
1063
+ let r = &self.rtt_stats;
1064
+ r.rtt() + (r.rttvar() * 4).max(GRANULARITY)
1065
+ }
1066
+
1067
+ /// The most recent data delivery rate estimate.
1068
+ fn delivery_rate(&self) -> Bandwidth {
1069
+ self.pacer.bandwidth_estimate(&self.rtt_stats)
1070
+ }
1071
+
1072
+ fn max_bandwidth(&self) -> Option<Bandwidth> {
1073
+ Some(self.pacer.max_bandwidth())
1074
+ }
1075
+
1076
+ /// Statistics from when a CCA first exited the startup phase.
1077
+ fn startup_exit(&self) -> Option<StartupExit> {
1078
+ self.recovery_stats.startup_exit
1079
+ }
1080
+
1081
+ fn max_datagram_size(&self) -> usize {
1082
+ self.max_datagram_size
1083
+ }
1084
+
1085
+ fn pmtud_update_max_datagram_size(&mut self, new_max_datagram_size: usize) {
1086
+ self.max_datagram_size = new_max_datagram_size;
1087
+ self.pacer.update_mss(self.max_datagram_size);
1088
+ }
1089
+
1090
+ fn update_max_datagram_size(&mut self, new_max_datagram_size: usize) {
1091
+ self.pmtud_update_max_datagram_size(
1092
+ self.max_datagram_size.min(new_max_datagram_size),
1093
+ )
1094
+ }
1095
+
1096
+ // FIXME only used by gcongestion
1097
+ fn on_app_limited(&mut self) {
1098
+ self.pacer.on_app_limited(self.bytes_in_flight.get())
1099
+ }
1100
+
1101
+ #[cfg(test)]
1102
+ fn sent_packets_len(&self, epoch: packet::Epoch) -> usize {
1103
+ self.epochs[epoch].sent_packets.len()
1104
+ }
1105
+
1106
+ #[cfg(test)]
1107
+ fn in_flight_count(&self, epoch: packet::Epoch) -> usize {
1108
+ self.epochs[epoch].pkts_in_flight
1109
+ }
1110
+
1111
+ fn bytes_in_flight(&self) -> usize {
1112
+ self.bytes_in_flight.get()
1113
+ }
1114
+
1115
+ fn bytes_in_flight_duration(&self) -> Duration {
1116
+ self.bytes_in_flight.get_duration()
1117
+ }
1118
+
1119
+ #[cfg(test)]
1120
+ fn pacing_rate(&self) -> u64 {
1121
+ self.pacer
1122
+ .pacing_rate(self.bytes_in_flight.get(), &self.rtt_stats)
1123
+ .to_bytes_per_period(Duration::from_secs(1))
1124
+ }
1125
+
1126
+ #[cfg(test)]
1127
+ fn pto_count(&self) -> u32 {
1128
+ self.pto_count
1129
+ }
1130
+
1131
+ #[cfg(test)]
1132
+ fn pkt_thresh(&self) -> Option<u64> {
1133
+ self.loss_thresh.pkt_thresh()
1134
+ }
1135
+
1136
+ #[cfg(test)]
1137
+ fn time_thresh(&self) -> f64 {
1138
+ self.loss_thresh.time_thresh()
1139
+ }
1140
+
1141
+ #[cfg(test)]
1142
+ fn lost_spurious_count(&self) -> usize {
1143
+ self.lost_spurious_count
1144
+ }
1145
+
1146
+ #[cfg(test)]
1147
+ fn detect_lost_packets_for_test(
1148
+ &mut self, epoch: packet::Epoch, now: Instant,
1149
+ ) -> (usize, usize) {
1150
+ let ret = self.detect_and_remove_lost_packets(epoch, now);
1151
+ self.epochs[epoch].drain_acked_and_lost_packets();
1152
+ ret
1153
+ }
1154
+
1155
+ #[cfg(test)]
1156
+ fn largest_sent_pkt_num_on_path(&self, epoch: packet::Epoch) -> Option<u64> {
1157
+ self.epochs[epoch].test_largest_sent_pkt_num_on_path
1158
+ }
1159
+
1160
+ #[cfg(test)]
1161
+ fn app_limited(&self) -> bool {
1162
+ self.pacer.is_app_limited(self.bytes_in_flight.get())
1163
+ }
1164
+
1165
+ // FIXME only used by congestion
1166
+ fn update_app_limited(&mut self, _v: bool) {
1167
+ // TODO
1168
+ }
1169
+
1170
+ // FIXME only used by congestion
1171
+ fn delivery_rate_update_app_limited(&mut self, _v: bool) {
1172
+ // TODO
1173
+ }
1174
+
1175
+ fn update_max_ack_delay(&mut self, max_ack_delay: Duration) {
1176
+ self.rtt_stats.max_ack_delay = max_ack_delay;
1177
+ }
1178
+
1179
+ fn get_next_release_time(&self) -> ReleaseDecision {
1180
+ self.pacer.get_next_release_time()
1181
+ }
1182
+
1183
+ fn gcongestion_enabled(&self) -> bool {
1184
+ true
1185
+ }
1186
+
1187
+ #[cfg(feature = "qlog")]
1188
+ fn state_str(&self, _now: Instant) -> &'static str {
1189
+ self.pacer.state_str()
1190
+ }
1191
+
1192
+ #[cfg(feature = "qlog")]
1193
+ fn get_updated_qlog_event_data(&mut self) -> Option<EventData> {
1194
+ let qlog_metrics = QlogMetrics {
1195
+ min_rtt: *self.rtt_stats.min_rtt,
1196
+ smoothed_rtt: self.rtt(),
1197
+ latest_rtt: self.rtt_stats.latest_rtt(),
1198
+ rttvar: self.rtt_stats.rttvar(),
1199
+ cwnd: self.cwnd() as u64,
1200
+ bytes_in_flight: self.bytes_in_flight.get() as u64,
1201
+ ssthresh: self.pacer.ssthresh(),
1202
+
1203
+ pacing_rate: Some(
1204
+ self.pacer
1205
+ .pacing_rate(self.bytes_in_flight.get(), &self.rtt_stats)
1206
+ .to_bytes_per_second(),
1207
+ ),
1208
+ delivery_rate: Some(self.delivery_rate().to_bytes_per_second()),
1209
+ send_rate: Some(self.send_rate().to_bytes_per_second()),
1210
+ ack_rate: Some(self.ack_rate().to_bytes_per_second()),
1211
+ lost_packets: Some(self.lost_count as u64),
1212
+ lost_bytes: Some(self.bytes_lost),
1213
+ pto_count: Some(self.pto_count),
1214
+ };
1215
+
1216
+ self.qlog_metrics.maybe_update(qlog_metrics)
1217
+ }
1218
+
1219
+ #[cfg(feature = "qlog")]
1220
+ fn get_updated_qlog_cc_state(
1221
+ &mut self, now: Instant,
1222
+ ) -> Option<&'static str> {
1223
+ let cc_state = self.state_str(now);
1224
+ if cc_state != self.qlog_prev_cc_state {
1225
+ self.qlog_prev_cc_state = cc_state;
1226
+ Some(cc_state)
1227
+ } else {
1228
+ None
1229
+ }
1230
+ }
1231
+
1232
+ fn send_quantum(&self) -> usize {
1233
+ let pacing_rate = self
1234
+ .pacer
1235
+ .pacing_rate(self.bytes_in_flight.get(), &self.rtt_stats);
1236
+
1237
+ let floor = if pacing_rate < Bandwidth::from_kbits_per_second(1200) {
1238
+ self.max_datagram_size
1239
+ } else {
1240
+ 2 * self.max_datagram_size
1241
+ };
1242
+
1243
+ pacing_rate
1244
+ .to_bytes_per_period(ReleaseDecision::EQUAL_THRESHOLD)
1245
+ .min(64 * 1024)
1246
+ .max(floor as u64) as usize
1247
+ }
1248
+ }
1249
+
1250
+ impl std::fmt::Debug for GRecovery {
1251
+ fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
1252
+ write!(f, "timer={:?} ", self.loss_detection_timer())?;
1253
+ write!(f, "rtt_stats={:?} ", self.rtt_stats)?;
1254
+ write!(f, "bytes_in_flight={} ", self.bytes_in_flight.get())?;
1255
+ write!(f, "{:?} ", self.pacer)?;
1256
+ Ok(())
1257
+ }
1258
+ }
1259
+
1260
+ #[cfg(test)]
1261
+ mod tests {
1262
+ use super::*;
1263
+ use crate::Config;
1264
+
1265
+ #[test]
1266
+ fn loss_threshold() {
1267
+ let config = Config::new(crate::PROTOCOL_VERSION).unwrap();
1268
+ let recovery_config = RecoveryConfig::from_config(&config);
1269
+ assert!(!recovery_config.enable_relaxed_loss_threshold);
1270
+
1271
+ let mut loss_thresh = LossThreshold::new(&recovery_config);
1272
+ assert_eq!(loss_thresh.time_thresh_overhead, None);
1273
+ assert_eq!(loss_thresh.pkt_thresh().unwrap(), INITIAL_PACKET_THRESHOLD);
1274
+ assert_eq!(loss_thresh.time_thresh(), INITIAL_TIME_THRESHOLD);
1275
+
1276
+ // First spurious loss.
1277
+ loss_thresh.on_spurious_loss(INITIAL_PACKET_THRESHOLD);
1278
+ assert_eq!(loss_thresh.pkt_thresh().unwrap(), INITIAL_PACKET_THRESHOLD);
1279
+ assert_eq!(loss_thresh.time_thresh(), PACKET_REORDER_TIME_THRESHOLD);
1280
+
1281
+ // Packet gaps < INITIAL_PACKET_THRESHOLD will NOT change packet
1282
+ // threshold.
1283
+ for packet_gap in 0..INITIAL_PACKET_THRESHOLD {
1284
+ loss_thresh.on_spurious_loss(packet_gap);
1285
+
1286
+ // Packet threshold only increases once the packet gap increases.
1287
+ assert_eq!(
1288
+ loss_thresh.pkt_thresh().unwrap(),
1289
+ INITIAL_PACKET_THRESHOLD
1290
+ );
1291
+ assert_eq!(loss_thresh.time_thresh(), PACKET_REORDER_TIME_THRESHOLD);
1292
+ }
1293
+
1294
+ // Subsequent spurious loss with packet_gaps > INITIAL_PACKET_THRESHOLD.
1295
+ // Test values much larger than MAX_PACKET_THRESHOLD, i.e.
1296
+ // `MAX_PACKET_THRESHOLD * 2`
1297
+ for packet_gap in INITIAL_PACKET_THRESHOLD + 1..MAX_PACKET_THRESHOLD * 2 {
1298
+ loss_thresh.on_spurious_loss(packet_gap);
1299
+
1300
+ // Packet threshold is equal to packet gap beyond
1301
+ // INITIAL_PACKET_THRESHOLD, but capped
1302
+ // at MAX_PACKET_THRESHOLD.
1303
+ let new_packet_threshold = if packet_gap < MAX_PACKET_THRESHOLD {
1304
+ packet_gap
1305
+ } else {
1306
+ MAX_PACKET_THRESHOLD
1307
+ };
1308
+ assert_eq!(loss_thresh.pkt_thresh().unwrap(), new_packet_threshold);
1309
+ assert_eq!(loss_thresh.time_thresh(), PACKET_REORDER_TIME_THRESHOLD);
1310
+ }
1311
+ // Packet threshold is capped at MAX_PACKET_THRESHOLD
1312
+ assert_eq!(loss_thresh.pkt_thresh().unwrap(), MAX_PACKET_THRESHOLD);
1313
+ assert_eq!(loss_thresh.time_thresh(), PACKET_REORDER_TIME_THRESHOLD);
1314
+
1315
+ // Packet threshold is monotonically increasing
1316
+ loss_thresh.on_spurious_loss(INITIAL_PACKET_THRESHOLD);
1317
+ assert_eq!(loss_thresh.pkt_thresh().unwrap(), MAX_PACKET_THRESHOLD);
1318
+ assert_eq!(loss_thresh.time_thresh(), PACKET_REORDER_TIME_THRESHOLD);
1319
+ }
1320
+
1321
+ #[test]
1322
+ fn relaxed_loss_threshold() {
1323
+ // The max time threshold when operating in relaxed loss mode.
1324
+ const MAX_TIME_THRESHOLD: f64 = 2.0;
1325
+
1326
+ let mut config = Config::new(crate::PROTOCOL_VERSION).unwrap();
1327
+ config.set_enable_relaxed_loss_threshold(true);
1328
+ let recovery_config = RecoveryConfig::from_config(&config);
1329
+ assert!(recovery_config.enable_relaxed_loss_threshold);
1330
+
1331
+ let mut loss_thresh = LossThreshold::new(&recovery_config);
1332
+ assert_eq!(
1333
+ loss_thresh.time_thresh_overhead,
1334
+ Some(INITIAL_TIME_THRESHOLD_OVERHEAD)
1335
+ );
1336
+ assert_eq!(loss_thresh.pkt_thresh().unwrap(), INITIAL_PACKET_THRESHOLD);
1337
+ assert_eq!(loss_thresh.time_thresh(), INITIAL_TIME_THRESHOLD);
1338
+
1339
+ // First spurious loss.
1340
+ loss_thresh.on_spurious_loss(INITIAL_PACKET_THRESHOLD);
1341
+ assert_eq!(loss_thresh.pkt_thresh(), None);
1342
+ assert_eq!(loss_thresh.time_thresh(), INITIAL_TIME_THRESHOLD);
1343
+
1344
+ // Subsequent spurious loss.
1345
+ for subsequent_loss_count in 1..100 {
1346
+ // Double the overhead until it caps at `2.0`.
1347
+ //
1348
+ // It takes `3` rounds of doubling for INITIAL_TIME_THRESHOLD_OVERHEAD
1349
+ // to equal `1.0`.
1350
+ let new_time_threshold = if subsequent_loss_count <= 3 {
1351
+ 1.0 + INITIAL_TIME_THRESHOLD_OVERHEAD *
1352
+ 2_f64.powi(subsequent_loss_count as i32)
1353
+ } else {
1354
+ 2.0
1355
+ };
1356
+
1357
+ loss_thresh.on_spurious_loss(subsequent_loss_count);
1358
+ assert_eq!(loss_thresh.pkt_thresh(), None);
1359
+ assert_eq!(loss_thresh.time_thresh(), new_time_threshold);
1360
+ }
1361
+ // Time threshold is capped at 2.0.
1362
+ assert_eq!(loss_thresh.pkt_thresh(), None);
1363
+ assert_eq!(loss_thresh.time_thresh(), MAX_TIME_THRESHOLD);
1364
+ }
1365
+
1366
+ #[test]
1367
+ fn test_high_pto_count_no_panic() {
1368
+ let mut config = Config::new(crate::PROTOCOL_VERSION).unwrap();
1369
+ config.set_cc_algorithm(CongestionControlAlgorithm::Bbr2Gcongestion);
1370
+ let recovery_config = RecoveryConfig::from_config(&config);
1371
+ let mut r = GRecovery::new(&recovery_config).unwrap();
1372
+
1373
+ r.pto_count = 99999;
1374
+
1375
+ let handshake_status = HandshakeStatus {
1376
+ completed: true,
1377
+ has_handshake_keys: true,
1378
+ peer_verified_address: true,
1379
+ };
1380
+ let now = Instant::now();
1381
+
1382
+ let _ = r.pto_time_and_space(handshake_status, now);
1383
+ }
1384
+ }