rwebtransport 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/Cargo.lock +666 -666
  2. package/Cargo.toml +31 -31
  3. package/LICENSE +202 -202
  4. package/NOTICE +41 -41
  5. package/README.md +550 -550
  6. package/crates/echo-server/Cargo.toml +20 -20
  7. package/crates/echo-server/src/main.rs +666 -666
  8. package/crates/native/Cargo.toml +23 -23
  9. package/crates/native/src/config.rs +129 -129
  10. package/crates/native/src/driver.rs +463 -463
  11. package/crates/native/src/lib.rs +821 -821
  12. package/crates/native/src/server.rs +718 -718
  13. package/crates/native/src/session.rs +1191 -1191
  14. package/crates/native/src/tls.rs +96 -96
  15. package/crates/wtcore/Cargo.toml +15 -15
  16. package/crates/wtcore/src/h3.rs +226 -226
  17. package/crates/wtcore/src/lib.rs +9 -9
  18. package/dist/index.js.map +1 -1
  19. package/dist/index.mjs.map +1 -1
  20. package/package.json +80 -79
  21. package/prebuilds/darwin-arm64/rwebtransport.node +0 -0
  22. package/prebuilds/linux-arm64/rwebtransport.node +0 -0
  23. package/prebuilds/linux-x64/rwebtransport.node +0 -0
  24. package/prebuilds/win32-x64/rwebtransport.node +0 -0
  25. package/scripts/bench.mjs +180 -180
  26. package/scripts/build.js +67 -67
  27. package/scripts/postinstall.js +67 -67
  28. package/scripts/setup-unix.sh +129 -129
  29. package/scripts/setup-windows.ps1 +229 -229
  30. package/vendor/quiche/AGENTS.md +75 -75
  31. package/vendor/quiche/COPYING +23 -23
  32. package/vendor/quiche/Cargo.lock +1212 -1212
  33. package/vendor/quiche/Cargo.toml +215 -215
  34. package/vendor/quiche/README.md +466 -466
  35. package/vendor/quiche/examples/Makefile +46 -46
  36. package/vendor/quiche/examples/README.md +20 -20
  37. package/vendor/quiche/examples/cert-big.crt +90 -90
  38. package/vendor/quiche/examples/cert.crt +18 -18
  39. package/vendor/quiche/examples/cert.key +28 -28
  40. package/vendor/quiche/examples/client.c +343 -343
  41. package/vendor/quiche/examples/client.rs +281 -281
  42. package/vendor/quiche/examples/gen-certs.sh +15 -15
  43. package/vendor/quiche/examples/http3-client.c +473 -473
  44. package/vendor/quiche/examples/http3-client.rs +349 -349
  45. package/vendor/quiche/examples/http3-server.c +614 -614
  46. package/vendor/quiche/examples/http3-server.rs +680 -680
  47. package/vendor/quiche/examples/qpack-decode.rs +87 -87
  48. package/vendor/quiche/examples/qpack-encode.rs +88 -88
  49. package/vendor/quiche/examples/rootca.crt +21 -21
  50. package/vendor/quiche/examples/server.c +529 -529
  51. package/vendor/quiche/examples/server.rs +529 -529
  52. package/vendor/quiche/include/quiche.h +1282 -1282
  53. package/vendor/quiche/src/buffers.rs +105 -105
  54. package/vendor/quiche/src/build.rs +66 -66
  55. package/vendor/quiche/src/cid.rs +1079 -1079
  56. package/vendor/quiche/src/crypto/boringssl.rs +373 -373
  57. package/vendor/quiche/src/crypto/mod.rs +671 -671
  58. package/vendor/quiche/src/dgram.rs +118 -118
  59. package/vendor/quiche/src/error.rs +263 -263
  60. package/vendor/quiche/src/ffi.rs +2290 -2290
  61. package/vendor/quiche/src/flowcontrol.rs +246 -246
  62. package/vendor/quiche/src/frame.rs +2172 -2172
  63. package/vendor/quiche/src/h3/AGENTS.md +54 -54
  64. package/vendor/quiche/src/h3/ffi.rs +442 -442
  65. package/vendor/quiche/src/h3/frame.rs +1404 -1404
  66. package/vendor/quiche/src/h3/mod.rs +8223 -8223
  67. package/vendor/quiche/src/h3/qpack/decoder.rs +288 -288
  68. package/vendor/quiche/src/h3/qpack/encoder.rs +235 -235
  69. package/vendor/quiche/src/h3/qpack/mod.rs +202 -202
  70. package/vendor/quiche/src/h3/qpack/static_table.rs +323 -323
  71. package/vendor/quiche/src/h3/stream.rs +1595 -1595
  72. package/vendor/quiche/src/lib.rs +9556 -9556
  73. package/vendor/quiche/src/minmax.rs +446 -446
  74. package/vendor/quiche/src/packet.rs +2334 -2334
  75. package/vendor/quiche/src/path.rs +1374 -1374
  76. package/vendor/quiche/src/pmtud.rs +652 -652
  77. package/vendor/quiche/src/rand.rs +64 -64
  78. package/vendor/quiche/src/range_buf.rs +186 -186
  79. package/vendor/quiche/src/ranges.rs +755 -755
  80. package/vendor/quiche/src/recovery/AGENTS.md +80 -80
  81. package/vendor/quiche/src/recovery/bandwidth.rs +565 -565
  82. package/vendor/quiche/src/recovery/bytes_in_flight.rs +179 -179
  83. package/vendor/quiche/src/recovery/congestion/cubic.rs +1038 -1038
  84. package/vendor/quiche/src/recovery/congestion/delivery_rate.rs +514 -514
  85. package/vendor/quiche/src/recovery/congestion/hystart.rs +241 -241
  86. package/vendor/quiche/src/recovery/congestion/mod.rs +361 -361
  87. package/vendor/quiche/src/recovery/congestion/prr.rs +238 -238
  88. package/vendor/quiche/src/recovery/congestion/recovery.rs +1170 -1170
  89. package/vendor/quiche/src/recovery/congestion/reno.rs +291 -291
  90. package/vendor/quiche/src/recovery/congestion/test_sender.rs +201 -201
  91. package/vendor/quiche/src/recovery/gcongestion/bbr/bandwidth_sampler.rs +1947 -1947
  92. package/vendor/quiche/src/recovery/gcongestion/bbr/windowed_filter.rs +158 -158
  93. package/vendor/quiche/src/recovery/gcongestion/bbr.rs +37 -37
  94. package/vendor/quiche/src/recovery/gcongestion/bbr2/drain.rs +120 -120
  95. package/vendor/quiche/src/recovery/gcongestion/bbr2/mode.rs +318 -318
  96. package/vendor/quiche/src/recovery/gcongestion/bbr2/network_model.rs +784 -784
  97. package/vendor/quiche/src/recovery/gcongestion/bbr2/probe_bw.rs +729 -729
  98. package/vendor/quiche/src/recovery/gcongestion/bbr2/probe_rtt.rs +175 -175
  99. package/vendor/quiche/src/recovery/gcongestion/bbr2/startup.rs +190 -190
  100. package/vendor/quiche/src/recovery/gcongestion/bbr2.rs +855 -855
  101. package/vendor/quiche/src/recovery/gcongestion/mod.rs +275 -275
  102. package/vendor/quiche/src/recovery/gcongestion/pacer.rs +294 -294
  103. package/vendor/quiche/src/recovery/gcongestion/recovery.rs +1384 -1384
  104. package/vendor/quiche/src/recovery/mod.rs +3086 -3086
  105. package/vendor/quiche/src/recovery/rtt.rs +152 -152
  106. package/vendor/quiche/src/stream/mod.rs +2210 -2210
  107. package/vendor/quiche/src/stream/recv_buf.rs +1236 -1236
  108. package/vendor/quiche/src/stream/send_buf.rs +849 -849
  109. package/vendor/quiche/src/test_utils.rs +681 -681
  110. package/vendor/quiche/src/tests.rs +12922 -12922
  111. package/vendor/quiche/src/tls/boringssl.rs +375 -375
  112. package/vendor/quiche/src/tls/mod.rs +1245 -1245
  113. package/vendor/quiche/src/transport_params.rs +628 -628
@@ -1,1170 +1,1170 @@
1
- // Copyright (C) 2018-2019, Cloudflare, Inc.
2
- // All rights reserved.
3
- //
4
- // Redistribution and use in source and binary forms, with or without
5
- // modification, are permitted provided that the following conditions are
6
- // met:
7
- //
8
- // * Redistributions of source code must retain the above copyright notice,
9
- // this list of conditions and the following disclaimer.
10
- //
11
- // * Redistributions in binary form must reproduce the above copyright
12
- // notice, this list of conditions and the following disclaimer in the
13
- // documentation and/or other materials provided with the distribution.
14
- //
15
- // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
16
- // IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
17
- // THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
18
- // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
19
- // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
20
- // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21
- // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
22
- // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
23
- // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
24
- // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25
- // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26
-
27
- use std::cmp;
28
-
29
- use std::time::Duration;
30
- use std::time::Instant;
31
-
32
- use std::collections::VecDeque;
33
-
34
- use super::RecoveryConfig;
35
- use super::Sent;
36
-
37
- use crate::packet::Epoch;
38
- use crate::ranges::RangeSet;
39
- use crate::recovery::Bandwidth;
40
- use crate::recovery::HandshakeStatus;
41
- use crate::recovery::OnLossDetectionTimeoutOutcome;
42
- use crate::recovery::RecoveryOps;
43
- use crate::recovery::StartupExit;
44
- use crate::Error;
45
- use crate::Result;
46
-
47
- #[cfg(feature = "qlog")]
48
- use crate::recovery::QlogMetrics;
49
-
50
- use crate::frame;
51
-
52
- #[cfg(feature = "qlog")]
53
- use qlog::events::EventData;
54
-
55
- use super::Congestion;
56
- use crate::recovery::bytes_in_flight::BytesInFlight;
57
- use crate::recovery::rtt::RttStats;
58
- use crate::recovery::LossDetectionTimer;
59
- use crate::recovery::OnAckReceivedOutcome;
60
- use crate::recovery::ReleaseDecision;
61
- use crate::recovery::ReleaseTime;
62
- use crate::recovery::GRANULARITY;
63
- use crate::recovery::INITIAL_PACKET_THRESHOLD;
64
- use crate::recovery::INITIAL_TIME_THRESHOLD;
65
- use crate::recovery::MAX_OUTSTANDING_NON_ACK_ELICITING;
66
- use crate::recovery::MAX_PACKET_THRESHOLD;
67
- use crate::recovery::MAX_PTO_EXPONENT;
68
- use crate::recovery::MAX_PTO_PROBES_COUNT;
69
- use crate::recovery::PACKET_REORDER_TIME_THRESHOLD;
70
-
71
- #[derive(Default)]
72
- struct RecoveryEpoch {
73
- /// The time the most recent ack-eliciting packet was sent.
74
- time_of_last_ack_eliciting_packet: Option<Instant>,
75
-
76
- /// The largest packet number acknowledged in the packet number space so
77
- /// far.
78
- largest_acked_packet: Option<u64>,
79
-
80
- /// The time at which the next packet in that packet number space can be
81
- /// considered lost based on exceeding the reordering window in time.
82
- loss_time: Option<Instant>,
83
-
84
- /// An association of packet numbers in a packet number space to information
85
- /// about them.
86
- sent_packets: VecDeque<Sent>,
87
-
88
- loss_probes: usize,
89
- in_flight_count: usize,
90
-
91
- acked_frames: Vec<frame::Frame>,
92
-
93
- // Frames scheduled for retransmission due to PTO are tracked
94
- // separately so we can check that frames were drained before
95
- // generating more PTO probes.
96
- lost_frames_ack: Vec<frame::Frame>,
97
- lost_frames_pto: Vec<frame::Frame>,
98
-
99
- /// The largest packet number sent in the packet number space so far.
100
- #[cfg(test)]
101
- test_largest_sent_pkt_num_on_path: Option<u64>,
102
- }
103
-
104
- struct AckedDetectionResult {
105
- acked_bytes: usize,
106
- spurious_losses: usize,
107
- spurious_pkt_thresh: Option<u64>,
108
- has_ack_eliciting: bool,
109
- has_in_flight_spurious_loss: bool,
110
- }
111
-
112
- struct LossDetectionResult {
113
- largest_lost_pkt: Option<Sent>,
114
- lost_packets: usize,
115
- lost_bytes: usize,
116
- pmtud_lost_bytes: usize,
117
- }
118
-
119
- impl RecoveryEpoch {
120
- // `peer_sent_ack_ranges` should not be used without validation.
121
- fn detect_and_remove_acked_packets(
122
- &mut self, now: Instant, peer_sent_ack_ranges: &RangeSet,
123
- newly_acked: &mut Vec<Acked>, rtt_stats: &RttStats, skip_pn: Option<u64>,
124
- trace_id: &str,
125
- ) -> Result<AckedDetectionResult> {
126
- newly_acked.clear();
127
-
128
- let mut acked_bytes = 0;
129
- let mut spurious_losses = 0;
130
- let mut spurious_pkt_thresh = None;
131
- let mut has_ack_eliciting = false;
132
- let mut has_in_flight_spurious_loss = false;
133
-
134
- let largest_ack_received = peer_sent_ack_ranges
135
- .last()
136
- .expect("ACK frames should always have at least one ack range");
137
- let largest_acked = self
138
- .largest_acked_packet
139
- .unwrap_or(0)
140
- .max(largest_ack_received);
141
-
142
- for peer_sent_range in peer_sent_ack_ranges.iter() {
143
- if skip_pn.is_some_and(|skip_pn| peer_sent_range.contains(&skip_pn)) {
144
- // https://www.rfc-editor.org/rfc/rfc9000#section-13.1
145
- // An endpoint SHOULD treat receipt of an acknowledgment
146
- // for a packet it did not send as
147
- // a connection error of type PROTOCOL_VIOLATION
148
- return Err(Error::OptimisticAckDetected);
149
- }
150
-
151
- // Because packets always have incrementing numbers, they are always
152
- // in sorted order.
153
- let start = if self
154
- .sent_packets
155
- .front()
156
- .filter(|e| e.pkt_num >= peer_sent_range.start)
157
- .is_some()
158
- {
159
- // Usually it will be the first packet.
160
- 0
161
- } else {
162
- self.sent_packets
163
- .binary_search_by_key(&peer_sent_range.start, |p| p.pkt_num)
164
- .unwrap_or_else(|e| e)
165
- };
166
-
167
- for unacked in self.sent_packets.range_mut(start..) {
168
- if unacked.pkt_num >= peer_sent_range.end {
169
- break;
170
- }
171
-
172
- if unacked.time_acked.is_some() {
173
- // Already acked.
174
- } else if unacked.time_lost.is_some() {
175
- // An acked packet was already declared lost.
176
- spurious_losses += 1;
177
- spurious_pkt_thresh
178
- .get_or_insert(largest_acked - unacked.pkt_num + 1);
179
- unacked.time_acked = Some(now);
180
-
181
- if unacked.in_flight {
182
- has_in_flight_spurious_loss = true;
183
- }
184
- } else {
185
- if unacked.in_flight {
186
- self.in_flight_count -= 1;
187
- acked_bytes += unacked.size;
188
- }
189
-
190
- newly_acked.push(Acked {
191
- pkt_num: unacked.pkt_num,
192
- time_sent: unacked.time_sent,
193
- size: unacked.size,
194
-
195
- rtt: now.saturating_duration_since(unacked.time_sent),
196
- delivered: unacked.delivered,
197
- delivered_time: unacked.delivered_time,
198
- first_sent_time: unacked.first_sent_time,
199
- is_app_limited: unacked.is_app_limited,
200
- });
201
-
202
- trace!("{} packet newly acked {}", trace_id, unacked.pkt_num);
203
-
204
- self.acked_frames
205
- .extend(std::mem::take(&mut unacked.frames));
206
-
207
- has_ack_eliciting |= unacked.ack_eliciting;
208
- unacked.time_acked = Some(now);
209
- }
210
- }
211
- }
212
-
213
- self.drain_acked_and_lost_packets(now - rtt_stats.rtt());
214
-
215
- Ok(AckedDetectionResult {
216
- acked_bytes,
217
- spurious_losses,
218
- spurious_pkt_thresh,
219
- has_ack_eliciting,
220
- has_in_flight_spurious_loss,
221
- })
222
- }
223
-
224
- fn detect_lost_packets(
225
- &mut self, loss_delay: Duration, pkt_thresh: u64, now: Instant,
226
- trace_id: &str, epoch: Epoch,
227
- ) -> LossDetectionResult {
228
- self.loss_time = None;
229
-
230
- // Minimum time of kGranularity before packets are deemed lost.
231
- let loss_delay = cmp::max(loss_delay, GRANULARITY);
232
- let largest_acked = self.largest_acked_packet.unwrap_or(0);
233
-
234
- // Packets sent before this time are deemed lost.
235
- let lost_send_time = now.checked_sub(loss_delay).unwrap();
236
-
237
- let mut lost_packets = 0;
238
- let mut lost_bytes = 0;
239
- let mut pmtud_lost_bytes = 0;
240
-
241
- let mut largest_lost_pkt = None;
242
-
243
- let unacked_iter = self.sent_packets
244
- .iter_mut()
245
- // Skip packets that follow the largest acked packet.
246
- .take_while(|p| p.pkt_num <= largest_acked)
247
- // Skip packets that have already been acked or lost.
248
- .filter(|p| p.time_acked.is_none() && p.time_lost.is_none());
249
-
250
- for unacked in unacked_iter {
251
- // Mark packet as lost, or set time when it should be marked.
252
- if unacked.time_sent <= lost_send_time ||
253
- largest_acked >= unacked.pkt_num + pkt_thresh
254
- {
255
- self.lost_frames_ack.extend(unacked.frames.drain(..));
256
-
257
- unacked.time_lost = Some(now);
258
-
259
- if unacked.is_pmtud_probe {
260
- pmtud_lost_bytes += unacked.size;
261
- self.in_flight_count -= 1;
262
-
263
- // Do not track PMTUD probes losses.
264
- continue;
265
- }
266
-
267
- if unacked.in_flight {
268
- lost_bytes += unacked.size;
269
-
270
- // Frames have already been removed from the packet, so
271
- // cloning the whole packet should be relatively cheap.
272
- largest_lost_pkt = Some(unacked.clone());
273
-
274
- self.in_flight_count -= 1;
275
-
276
- trace!(
277
- "{} packet {} lost on epoch {}",
278
- trace_id,
279
- unacked.pkt_num,
280
- epoch
281
- );
282
- }
283
-
284
- lost_packets += 1;
285
- } else {
286
- let loss_time = match self.loss_time {
287
- None => unacked.time_sent + loss_delay,
288
-
289
- Some(loss_time) =>
290
- cmp::min(loss_time, unacked.time_sent + loss_delay),
291
- };
292
-
293
- self.loss_time = Some(loss_time);
294
- break;
295
- }
296
- }
297
-
298
- LossDetectionResult {
299
- largest_lost_pkt,
300
- lost_packets,
301
- lost_bytes,
302
- pmtud_lost_bytes,
303
- }
304
- }
305
-
306
- fn drain_acked_and_lost_packets(&mut self, loss_thresh: Instant) {
307
- // In order to avoid removing elements from the middle of the list
308
- // (which would require copying other elements to compact the list),
309
- // we only remove a contiguous range of elements from the start of the
310
- // list.
311
- //
312
- // This means that acked or lost elements coming after this will not
313
- // be removed at this point, but their removal is delayed for a later
314
- // time, once the gaps have been filled.
315
- while let Some(pkt) = self.sent_packets.front() {
316
- if let Some(time_lost) = pkt.time_lost {
317
- if time_lost > loss_thresh {
318
- break;
319
- }
320
- }
321
-
322
- if pkt.time_acked.is_none() && pkt.time_lost.is_none() {
323
- break;
324
- }
325
-
326
- self.sent_packets.pop_front();
327
- }
328
- }
329
-
330
- /// Returns the next lost frame, trying ACK-based lost frames first,
331
- /// then PTO-based lost frames.
332
- fn next_lost_frame(&mut self) -> Option<frame::Frame> {
333
- self.lost_frames_ack
334
- .pop()
335
- .or_else(|| self.lost_frames_pto.pop())
336
- }
337
-
338
- /// Returns true if there are any lost frames (ACK or PTO).
339
- fn has_lost_frames(&self) -> bool {
340
- !self.lost_frames_ack.is_empty() || !self.lost_frames_pto.is_empty()
341
- }
342
-
343
- /// Returns the total count of lost frames (ACK + PTO).
344
- #[cfg(test)]
345
- fn lost_frames_count(&self) -> usize {
346
- self.lost_frames_ack.len() + self.lost_frames_pto.len()
347
- }
348
-
349
- /// Clears all lost frames (both ACK and PTO).
350
- fn clear_lost_frames(&mut self) {
351
- self.lost_frames_ack.clear();
352
- self.lost_frames_pto.clear();
353
- }
354
- }
355
-
356
- pub struct LegacyRecovery {
357
- epochs: [RecoveryEpoch; Epoch::count()],
358
-
359
- loss_timer: LossDetectionTimer,
360
-
361
- pto_count: u32,
362
-
363
- rtt_stats: RttStats,
364
-
365
- lost_spurious_count: usize,
366
-
367
- pkt_thresh: u64,
368
-
369
- time_thresh: f64,
370
-
371
- bytes_in_flight: BytesInFlight,
372
-
373
- bytes_sent: usize,
374
-
375
- bytes_lost: u64,
376
-
377
- pub max_datagram_size: usize,
378
-
379
- #[cfg(feature = "qlog")]
380
- qlog_metrics: QlogMetrics,
381
-
382
- #[cfg(feature = "qlog")]
383
- qlog_prev_cc_state: &'static str,
384
-
385
- /// How many non-ack-eliciting packets have been sent.
386
- outstanding_non_ack_eliciting: usize,
387
-
388
- pub congestion: Congestion,
389
-
390
- /// A resusable list of acks.
391
- newly_acked: Vec<Acked>,
392
- }
393
-
394
- impl LegacyRecovery {
395
- pub fn new_with_config(recovery_config: &RecoveryConfig) -> Self {
396
- Self {
397
- epochs: Default::default(),
398
-
399
- loss_timer: Default::default(),
400
-
401
- pto_count: 0,
402
-
403
- rtt_stats: RttStats::new(
404
- recovery_config.initial_rtt,
405
- recovery_config.max_ack_delay,
406
- ),
407
-
408
- lost_spurious_count: 0,
409
-
410
- pkt_thresh: INITIAL_PACKET_THRESHOLD,
411
-
412
- time_thresh: INITIAL_TIME_THRESHOLD,
413
-
414
- bytes_in_flight: Default::default(),
415
-
416
- bytes_sent: 0,
417
-
418
- bytes_lost: 0,
419
-
420
- max_datagram_size: recovery_config.max_send_udp_payload_size,
421
-
422
- #[cfg(feature = "qlog")]
423
- qlog_metrics: QlogMetrics::default(),
424
-
425
- #[cfg(feature = "qlog")]
426
- qlog_prev_cc_state: "",
427
-
428
- outstanding_non_ack_eliciting: 0,
429
-
430
- congestion: Congestion::from_config(recovery_config),
431
-
432
- newly_acked: Vec::new(),
433
- }
434
- }
435
-
436
- #[cfg(test)]
437
- pub fn new(config: &crate::Config) -> Self {
438
- Self::new_with_config(&RecoveryConfig::from_config(config))
439
- }
440
-
441
- fn loss_time_and_space(&self) -> (Option<Instant>, Epoch) {
442
- let mut epoch = Epoch::Initial;
443
- let mut time = self.epochs[epoch].loss_time;
444
-
445
- // Iterate over all packet number spaces starting from Handshake.
446
- for e in [Epoch::Handshake, Epoch::Application] {
447
- let new_time = self.epochs[e].loss_time;
448
- if time.is_none() || new_time < time {
449
- time = new_time;
450
- epoch = e;
451
- }
452
- }
453
-
454
- (time, epoch)
455
- }
456
-
457
- fn pto_time_and_space(
458
- &self, handshake_status: HandshakeStatus, now: Instant,
459
- ) -> (Option<Instant>, Epoch) {
460
- let mut duration =
461
- self.pto() * 2_u32.pow(self.pto_count.min(MAX_PTO_EXPONENT));
462
-
463
- // Arm PTO from now when there are no inflight packets.
464
- if self.bytes_in_flight.is_zero() {
465
- if handshake_status.has_handshake_keys {
466
- return (Some(now + duration), Epoch::Handshake);
467
- } else {
468
- return (Some(now + duration), Epoch::Initial);
469
- }
470
- }
471
-
472
- let mut pto_timeout = None;
473
- let mut pto_space = Epoch::Initial;
474
-
475
- // Iterate over all packet number spaces.
476
- for e in [Epoch::Initial, Epoch::Handshake, Epoch::Application] {
477
- let epoch = &self.epochs[e];
478
- if epoch.in_flight_count == 0 {
479
- continue;
480
- }
481
-
482
- if e == Epoch::Application {
483
- // Skip Application Data until handshake completes.
484
- if !handshake_status.completed {
485
- return (pto_timeout, pto_space);
486
- }
487
-
488
- // Include max_ack_delay and backoff for Application Data.
489
- duration += self.rtt_stats.max_ack_delay *
490
- 2_u32.pow(self.pto_count.min(MAX_PTO_EXPONENT));
491
- }
492
-
493
- let new_time = epoch
494
- .time_of_last_ack_eliciting_packet
495
- .map(|t| t + duration);
496
-
497
- if pto_timeout.is_none() || new_time < pto_timeout {
498
- pto_timeout = new_time;
499
- pto_space = e;
500
- }
501
- }
502
-
503
- (pto_timeout, pto_space)
504
- }
505
-
506
- fn set_loss_detection_timer(
507
- &mut self, handshake_status: HandshakeStatus, now: Instant,
508
- ) {
509
- let (earliest_loss_time, _) = self.loss_time_and_space();
510
-
511
- if let Some(to) = earliest_loss_time {
512
- // Time threshold loss detection.
513
- self.loss_timer.update(to);
514
- return;
515
- }
516
-
517
- if self.bytes_in_flight.is_zero() &&
518
- handshake_status.peer_verified_address
519
- {
520
- self.loss_timer.clear();
521
- return;
522
- }
523
-
524
- // PTO timer.
525
- if let (Some(timeout), _) = self.pto_time_and_space(handshake_status, now)
526
- {
527
- self.loss_timer.update(timeout);
528
- } else {
529
- self.loss_timer.clear();
530
- }
531
- }
532
-
533
- fn detect_lost_packets(
534
- &mut self, epoch: Epoch, now: Instant, trace_id: &str,
535
- ) -> (usize, usize) {
536
- let loss_delay = cmp::max(self.rtt_stats.latest_rtt, self.rtt())
537
- .mul_f64(self.time_thresh);
538
-
539
- let loss = self.epochs[epoch].detect_lost_packets(
540
- loss_delay,
541
- self.pkt_thresh,
542
- now,
543
- trace_id,
544
- epoch,
545
- );
546
-
547
- if let Some(pkt) = loss.largest_lost_pkt {
548
- if !self.congestion.in_congestion_recovery(pkt.time_sent) {
549
- (self.congestion.cc_ops.checkpoint)(&mut self.congestion);
550
- }
551
-
552
- (self.congestion.cc_ops.congestion_event)(
553
- &mut self.congestion,
554
- self.bytes_in_flight.get(),
555
- loss.lost_bytes,
556
- &pkt,
557
- now,
558
- );
559
-
560
- self.bytes_in_flight
561
- .saturating_subtract(loss.lost_bytes, now);
562
- };
563
-
564
- self.bytes_in_flight
565
- .saturating_subtract(loss.pmtud_lost_bytes, now);
566
-
567
- self.epochs[epoch]
568
- .drain_acked_and_lost_packets(now - self.rtt_stats.rtt());
569
-
570
- self.congestion.lost_count += loss.lost_packets;
571
-
572
- (loss.lost_packets, loss.lost_bytes)
573
- }
574
- }
575
-
576
- impl RecoveryOps for LegacyRecovery {
577
- /// Returns whether or not we should elicit an ACK even if we wouldn't
578
- /// otherwise have constructed an ACK eliciting packet.
579
- fn should_elicit_ack(&self, epoch: Epoch) -> bool {
580
- self.epochs[epoch].loss_probes > 0 ||
581
- self.outstanding_non_ack_eliciting >=
582
- MAX_OUTSTANDING_NON_ACK_ELICITING
583
- }
584
-
585
- fn next_acked_frame(&mut self, epoch: Epoch) -> Option<frame::Frame> {
586
- self.epochs[epoch].acked_frames.pop()
587
- }
588
-
589
- fn next_lost_frame(&mut self, epoch: Epoch) -> Option<frame::Frame> {
590
- self.epochs[epoch].next_lost_frame()
591
- }
592
-
593
- fn get_largest_acked_on_epoch(&self, epoch: Epoch) -> Option<u64> {
594
- self.epochs[epoch].largest_acked_packet
595
- }
596
-
597
- fn has_lost_frames(&self, epoch: Epoch) -> bool {
598
- self.epochs[epoch].has_lost_frames()
599
- }
600
-
601
- fn loss_probes(&self, epoch: Epoch) -> usize {
602
- self.epochs[epoch].loss_probes
603
- }
604
-
605
- #[cfg(test)]
606
- fn inc_loss_probes(&mut self, epoch: Epoch) {
607
- self.epochs[epoch].loss_probes += 1;
608
- }
609
-
610
- #[cfg(test)]
611
- fn lost_frames_count(&self, epoch: Epoch) -> usize {
612
- self.epochs[epoch].lost_frames_count()
613
- }
614
-
615
- fn ping_sent(&mut self, epoch: Epoch) {
616
- self.epochs[epoch].loss_probes =
617
- self.epochs[epoch].loss_probes.saturating_sub(1);
618
- }
619
-
620
- fn on_packet_sent(
621
- &mut self, mut pkt: Sent, epoch: Epoch,
622
- handshake_status: HandshakeStatus, now: Instant, trace_id: &str,
623
- ) {
624
- let ack_eliciting = pkt.ack_eliciting;
625
- let in_flight = pkt.in_flight;
626
- let sent_bytes = pkt.size;
627
-
628
- if ack_eliciting {
629
- self.outstanding_non_ack_eliciting = 0;
630
- } else {
631
- self.outstanding_non_ack_eliciting += 1;
632
- }
633
-
634
- if in_flight && ack_eliciting {
635
- self.epochs[epoch].time_of_last_ack_eliciting_packet = Some(now);
636
- }
637
-
638
- self.congestion.on_packet_sent(
639
- self.bytes_in_flight.get(),
640
- sent_bytes,
641
- now,
642
- &mut pkt,
643
- self.bytes_lost,
644
- in_flight,
645
- );
646
-
647
- if in_flight {
648
- self.epochs[epoch].in_flight_count += 1;
649
- self.bytes_in_flight.add(sent_bytes, now);
650
-
651
- self.set_loss_detection_timer(handshake_status, now);
652
- }
653
-
654
- self.bytes_sent += sent_bytes;
655
-
656
- #[cfg(test)]
657
- {
658
- self.epochs[epoch].test_largest_sent_pkt_num_on_path = self.epochs
659
- [epoch]
660
- .test_largest_sent_pkt_num_on_path
661
- .max(Some(pkt.pkt_num));
662
- }
663
-
664
- self.epochs[epoch].sent_packets.push_back(pkt);
665
-
666
- trace!("{trace_id} {self:?}");
667
- }
668
-
669
- fn get_packet_send_time(&self, now: Instant) -> Instant {
670
- now
671
- }
672
-
673
- // `peer_sent_ack_ranges` should not be used without validation.
674
- fn on_ack_received(
675
- &mut self, peer_sent_ack_ranges: &RangeSet, ack_delay: u64, epoch: Epoch,
676
- handshake_status: HandshakeStatus, now: Instant, skip_pn: Option<u64>,
677
- trace_id: &str,
678
- ) -> Result<OnAckReceivedOutcome> {
679
- let AckedDetectionResult {
680
- acked_bytes,
681
- spurious_losses,
682
- spurious_pkt_thresh,
683
- has_ack_eliciting,
684
- has_in_flight_spurious_loss,
685
- } = self.epochs[epoch].detect_and_remove_acked_packets(
686
- now,
687
- peer_sent_ack_ranges,
688
- &mut self.newly_acked,
689
- &self.rtt_stats,
690
- skip_pn,
691
- trace_id,
692
- )?;
693
-
694
- self.lost_spurious_count += spurious_losses;
695
- if let Some(thresh) = spurious_pkt_thresh {
696
- self.pkt_thresh =
697
- self.pkt_thresh.max(thresh.min(MAX_PACKET_THRESHOLD));
698
- self.time_thresh = PACKET_REORDER_TIME_THRESHOLD;
699
- }
700
-
701
- // Undo congestion window update.
702
- if has_in_flight_spurious_loss {
703
- (self.congestion.cc_ops.rollback)(&mut self.congestion);
704
- }
705
-
706
- if self.newly_acked.is_empty() {
707
- return Ok(OnAckReceivedOutcome::default());
708
- }
709
-
710
- let largest_newly_acked = self.newly_acked.last().unwrap();
711
-
712
- // Update `largest_acked_packet` based on the validated `newly_acked`
713
- // value.
714
- let largest_acked_pkt_num = self.epochs[epoch]
715
- .largest_acked_packet
716
- .unwrap_or(0)
717
- .max(largest_newly_acked.pkt_num);
718
- self.epochs[epoch].largest_acked_packet = Some(largest_acked_pkt_num);
719
-
720
- // Check if largest packet is newly acked.
721
- if largest_newly_acked.pkt_num == largest_acked_pkt_num &&
722
- has_ack_eliciting
723
- {
724
- let latest_rtt = now - largest_newly_acked.time_sent;
725
- self.rtt_stats.update_rtt(
726
- latest_rtt,
727
- Duration::from_micros(ack_delay),
728
- now,
729
- handshake_status.completed,
730
- );
731
- }
732
-
733
- // Detect and mark lost packets without removing them from the sent
734
- // packets list.
735
- let (lost_packets, lost_bytes) =
736
- self.detect_lost_packets(epoch, now, trace_id);
737
-
738
- self.congestion.on_packets_acked(
739
- self.bytes_in_flight.get(),
740
- &mut self.newly_acked,
741
- &self.rtt_stats,
742
- now,
743
- );
744
-
745
- self.bytes_in_flight.saturating_subtract(acked_bytes, now);
746
-
747
- self.pto_count = 0;
748
-
749
- self.set_loss_detection_timer(handshake_status, now);
750
-
751
- self.epochs[epoch]
752
- .drain_acked_and_lost_packets(now - self.rtt_stats.rtt());
753
-
754
- Ok(OnAckReceivedOutcome {
755
- lost_packets,
756
- lost_bytes,
757
- acked_bytes,
758
- spurious_losses,
759
- })
760
- }
761
-
762
- fn on_loss_detection_timeout(
763
- &mut self, handshake_status: HandshakeStatus, now: Instant,
764
- trace_id: &str,
765
- ) -> OnLossDetectionTimeoutOutcome {
766
- let (earliest_loss_time, epoch) = self.loss_time_and_space();
767
-
768
- if earliest_loss_time.is_some() {
769
- // Time threshold loss detection.
770
- let (lost_packets, lost_bytes) =
771
- self.detect_lost_packets(epoch, now, trace_id);
772
-
773
- self.set_loss_detection_timer(handshake_status, now);
774
-
775
- trace!("{trace_id} {self:?}");
776
- return OnLossDetectionTimeoutOutcome {
777
- lost_packets,
778
- lost_bytes,
779
- };
780
- }
781
-
782
- let epoch = if self.bytes_in_flight.get() > 0 {
783
- // Send new data if available, else retransmit old data. If neither
784
- // is available, send a single PING frame.
785
- let (_, e) = self.pto_time_and_space(handshake_status, now);
786
-
787
- e
788
- } else {
789
- // Client sends an anti-deadlock packet: Initial is padded to earn
790
- // more anti-amplification credit, a Handshake packet proves address
791
- // ownership.
792
- if handshake_status.has_handshake_keys {
793
- Epoch::Handshake
794
- } else {
795
- Epoch::Initial
796
- }
797
- };
798
-
799
- self.pto_count += 1;
800
-
801
- let epoch = &mut self.epochs[epoch];
802
-
803
- epoch.loss_probes =
804
- cmp::min(self.pto_count as usize, MAX_PTO_PROBES_COUNT);
805
-
806
- let sent_packets_iter_limit = if !epoch.lost_frames_pto.is_empty() {
807
- // Skip the search for frames to add to PTO probes if frames
808
- // added in a prior PTO haven't been processed yet.
809
- 0
810
- } else {
811
- usize::MAX
812
- };
813
-
814
- let unacked_iter = epoch.sent_packets
815
- .iter()
816
- .take(sent_packets_iter_limit)
817
- // Skip packets that have already been acked or lost, and packets
818
- // that don't contain either CRYPTO or STREAM frames.
819
- .filter(|p| p.has_data && p.time_acked.is_none() && p.time_lost.is_none())
820
- // Only return as many packets as the number of probe packets that
821
- // will be sent.
822
- .take(epoch.loss_probes);
823
-
824
- // Retransmit the frames from the oldest sent packets on PTO. However
825
- // the packets are not actually declared lost (so there is no effect to
826
- // congestion control), we just reschedule the data they carried.
827
- //
828
- // This will also trigger sending an ACK and retransmitting frames like
829
- // HANDSHAKE_DONE and MAX_DATA / MAX_STREAM_DATA as well, in addition
830
- // to CRYPTO and STREAM, if the original packet carried them.
831
- for unacked in unacked_iter {
832
- epoch.lost_frames_pto.extend_from_slice(&unacked.frames);
833
- }
834
-
835
- self.set_loss_detection_timer(handshake_status, now);
836
-
837
- trace!("{trace_id} {self:?}");
838
-
839
- OnLossDetectionTimeoutOutcome {
840
- lost_packets: 0,
841
- lost_bytes: 0,
842
- }
843
- }
844
-
845
- fn on_pkt_num_space_discarded(
846
- &mut self, epoch: Epoch, handshake_status: HandshakeStatus, now: Instant,
847
- ) {
848
- let epoch = &mut self.epochs[epoch];
849
-
850
- let unacked_bytes = epoch
851
- .sent_packets
852
- .iter()
853
- .filter(|p| {
854
- p.in_flight && p.time_acked.is_none() && p.time_lost.is_none()
855
- })
856
- .fold(0, |acc, p| acc + p.size);
857
-
858
- self.bytes_in_flight.saturating_subtract(unacked_bytes, now);
859
-
860
- epoch.sent_packets.clear();
861
- epoch.clear_lost_frames();
862
- epoch.acked_frames.clear();
863
-
864
- epoch.time_of_last_ack_eliciting_packet = None;
865
- epoch.loss_time = None;
866
- epoch.loss_probes = 0;
867
- epoch.in_flight_count = 0;
868
-
869
- self.set_loss_detection_timer(handshake_status, now);
870
- }
871
-
872
- fn on_path_change(
873
- &mut self, epoch: Epoch, now: Instant, trace_id: &str,
874
- ) -> (usize, usize) {
875
- // Time threshold loss detection.
876
- self.detect_lost_packets(epoch, now, trace_id)
877
- }
878
-
879
- fn loss_detection_timer(&self) -> Option<Instant> {
880
- self.loss_timer.time
881
- }
882
-
883
- fn cwnd(&self) -> usize {
884
- self.congestion.congestion_window()
885
- }
886
-
887
- fn cwnd_available(&self) -> usize {
888
- // Ignore cwnd when sending probe packets.
889
- if self.epochs.iter().any(|e| e.loss_probes > 0) {
890
- return usize::MAX;
891
- }
892
-
893
- // Open more space (snd_cnt) for PRR when allowed.
894
- self.cwnd().saturating_sub(self.bytes_in_flight.get()) +
895
- self.congestion.prr.snd_cnt
896
- }
897
-
898
- fn rtt(&self) -> Duration {
899
- self.rtt_stats.rtt()
900
- }
901
-
902
- fn min_rtt(&self) -> Option<Duration> {
903
- self.rtt_stats.min_rtt()
904
- }
905
-
906
- fn max_rtt(&self) -> Option<Duration> {
907
- self.rtt_stats.max_rtt()
908
- }
909
-
910
- fn rttvar(&self) -> Duration {
911
- self.rtt_stats.rttvar
912
- }
913
-
914
- fn pto(&self) -> Duration {
915
- self.rtt() + cmp::max(self.rtt_stats.rttvar * 4, GRANULARITY)
916
- }
917
-
918
- /// The most recent data delivery rate estimate.
919
- fn delivery_rate(&self) -> Bandwidth {
920
- self.congestion.delivery_rate()
921
- }
922
-
923
- fn max_bandwidth(&self) -> Option<Bandwidth> {
924
- // TODO implement
925
- None
926
- }
927
-
928
- /// Statistics from when a CCA first exited the startup phase.
929
- fn startup_exit(&self) -> Option<StartupExit> {
930
- self.congestion.ssthresh.startup_exit()
931
- }
932
-
933
- fn max_datagram_size(&self) -> usize {
934
- self.max_datagram_size
935
- }
936
-
937
- fn pmtud_update_max_datagram_size(&mut self, new_max_datagram_size: usize) {
938
- // Congestion Window is updated only when it's not updated already.
939
- // Update cwnd if it hasn't been updated yet.
940
- if self.cwnd() ==
941
- self.max_datagram_size *
942
- self.congestion.initial_congestion_window_packets
943
- {
944
- self.congestion.congestion_window = new_max_datagram_size *
945
- self.congestion.initial_congestion_window_packets;
946
- }
947
-
948
- self.max_datagram_size = new_max_datagram_size;
949
- }
950
-
951
- fn update_max_datagram_size(&mut self, new_max_datagram_size: usize) {
952
- self.pmtud_update_max_datagram_size(
953
- self.max_datagram_size.min(new_max_datagram_size),
954
- )
955
- }
956
-
957
- #[cfg(test)]
958
- fn sent_packets_len(&self, epoch: Epoch) -> usize {
959
- self.epochs[epoch].sent_packets.len()
960
- }
961
-
962
- #[cfg(test)]
963
- fn in_flight_count(&self, epoch: Epoch) -> usize {
964
- self.epochs[epoch].in_flight_count
965
- }
966
-
967
- fn bytes_in_flight(&self) -> usize {
968
- self.bytes_in_flight.get()
969
- }
970
-
971
- fn bytes_in_flight_duration(&self) -> Duration {
972
- self.bytes_in_flight.get_duration()
973
- }
974
-
975
- #[cfg(test)]
976
- fn pacing_rate(&self) -> u64 {
977
- 0
978
- }
979
-
980
- #[cfg(test)]
981
- fn pto_count(&self) -> u32 {
982
- self.pto_count
983
- }
984
-
985
- #[cfg(test)]
986
- fn pkt_thresh(&self) -> Option<u64> {
987
- Some(self.pkt_thresh)
988
- }
989
-
990
- #[cfg(test)]
991
- fn time_thresh(&self) -> f64 {
992
- self.time_thresh
993
- }
994
-
995
- #[cfg(test)]
996
- fn lost_spurious_count(&self) -> usize {
997
- self.lost_spurious_count
998
- }
999
-
1000
- #[cfg(test)]
1001
- fn detect_lost_packets_for_test(
1002
- &mut self, epoch: Epoch, now: Instant,
1003
- ) -> (usize, usize) {
1004
- self.detect_lost_packets(epoch, now, "")
1005
- }
1006
-
1007
- // FIXME only used by gcongestion
1008
- fn on_app_limited(&mut self) {
1009
- // Not implemented for legacy recovery, update_app_limited and
1010
- // delivery_rate_update_app_limited used instead.
1011
- }
1012
-
1013
- #[cfg(test)]
1014
- fn largest_sent_pkt_num_on_path(&self, epoch: Epoch) -> Option<u64> {
1015
- self.epochs[epoch].test_largest_sent_pkt_num_on_path
1016
- }
1017
-
1018
- #[cfg(test)]
1019
- fn app_limited(&self) -> bool {
1020
- self.congestion.app_limited
1021
- }
1022
-
1023
- fn update_app_limited(&mut self, v: bool) {
1024
- self.congestion.update_app_limited(v);
1025
- }
1026
-
1027
- // FIXME only used by congestion
1028
- fn delivery_rate_update_app_limited(&mut self, v: bool) {
1029
- self.congestion.delivery_rate.update_app_limited(v);
1030
- }
1031
-
1032
- // FIXME only used by congestion
1033
- fn update_max_ack_delay(&mut self, max_ack_delay: Duration) {
1034
- self.rtt_stats.max_ack_delay = max_ack_delay;
1035
- }
1036
-
1037
- #[cfg(feature = "qlog")]
1038
- fn state_str(&self, now: Instant) -> &'static str {
1039
- (self.congestion.cc_ops.state_str)(&self.congestion, now)
1040
- }
1041
-
1042
- #[cfg(feature = "qlog")]
1043
- fn get_updated_qlog_event_data(&mut self) -> Option<EventData> {
1044
- let qlog_metrics = QlogMetrics {
1045
- min_rtt: *self.rtt_stats.min_rtt,
1046
- smoothed_rtt: self.rtt(),
1047
- latest_rtt: self.rtt_stats.latest_rtt,
1048
- rttvar: self.rtt_stats.rttvar,
1049
- cwnd: self.cwnd() as u64,
1050
- bytes_in_flight: self.bytes_in_flight.get() as u64,
1051
- ssthresh: Some(self.congestion.ssthresh.get() as u64),
1052
- lost_packets: Some(self.congestion.lost_count as u64),
1053
- lost_bytes: Some(self.bytes_lost),
1054
- pto_count: Some(self.pto_count),
1055
- ..Default::default()
1056
- };
1057
-
1058
- self.qlog_metrics.maybe_update(qlog_metrics)
1059
- }
1060
-
1061
- #[cfg(feature = "qlog")]
1062
- fn get_updated_qlog_cc_state(
1063
- &mut self, now: Instant,
1064
- ) -> Option<&'static str> {
1065
- let cc_state = self.state_str(now);
1066
- if cc_state != self.qlog_prev_cc_state {
1067
- self.qlog_prev_cc_state = cc_state;
1068
- Some(cc_state)
1069
- } else {
1070
- None
1071
- }
1072
- }
1073
-
1074
- fn send_quantum(&self) -> usize {
1075
- self.congestion.send_quantum()
1076
- }
1077
-
1078
- fn get_next_release_time(&self) -> ReleaseDecision {
1079
- ReleaseDecision {
1080
- time: ReleaseTime::Immediate,
1081
- allow_burst: false,
1082
- }
1083
- }
1084
-
1085
- fn gcongestion_enabled(&self) -> bool {
1086
- false
1087
- }
1088
-
1089
- fn lost_count(&self) -> usize {
1090
- self.congestion.lost_count
1091
- }
1092
-
1093
- fn bytes_lost(&self) -> u64 {
1094
- self.bytes_lost
1095
- }
1096
- }
1097
-
1098
- impl std::fmt::Debug for LegacyRecovery {
1099
- fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
1100
- write!(f, "timer={:?} ", self.loss_timer)?;
1101
- write!(f, "latest_rtt={:?} ", self.rtt_stats.latest_rtt)?;
1102
- write!(f, "srtt={:?} ", self.rtt_stats.smoothed_rtt)?;
1103
- write!(f, "min_rtt={:?} ", *self.rtt_stats.min_rtt)?;
1104
- write!(f, "rttvar={:?} ", self.rtt_stats.rttvar)?;
1105
- write!(f, "cwnd={} ", self.cwnd())?;
1106
- write!(f, "ssthresh={} ", self.congestion.ssthresh.get())?;
1107
- write!(f, "bytes_in_flight={} ", self.bytes_in_flight.get())?;
1108
- write!(f, "app_limited={} ", self.congestion.app_limited)?;
1109
- write!(
1110
- f,
1111
- "congestion_recovery_start_time={:?} ",
1112
- self.congestion.congestion_recovery_start_time
1113
- )?;
1114
- write!(f, "{:?} ", self.congestion.delivery_rate)?;
1115
-
1116
- if self.congestion.hystart.enabled() {
1117
- write!(f, "hystart={:?} ", self.congestion.hystart)?;
1118
- }
1119
-
1120
- // CC-specific debug info
1121
- (self.congestion.cc_ops.debug_fmt)(&self.congestion, f)?;
1122
-
1123
- Ok(())
1124
- }
1125
- }
1126
-
1127
- #[derive(Clone)]
1128
- pub struct Acked {
1129
- pub pkt_num: u64,
1130
-
1131
- pub time_sent: Instant,
1132
-
1133
- pub size: usize,
1134
-
1135
- pub rtt: Duration,
1136
-
1137
- pub delivered: usize,
1138
-
1139
- pub delivered_time: Instant,
1140
-
1141
- pub first_sent_time: Instant,
1142
-
1143
- pub is_app_limited: bool,
1144
- }
1145
-
1146
- #[cfg(test)]
1147
- mod tests {
1148
- use super::*;
1149
- use crate::recovery::HandshakeStatus;
1150
- use crate::recovery::RecoveryConfig;
1151
- use std::time::Instant;
1152
-
1153
- #[test]
1154
- fn test_high_pto_count_no_panic() {
1155
- let config = crate::Config::new(crate::PROTOCOL_VERSION).unwrap();
1156
- let recovery_config = RecoveryConfig::from_config(&config);
1157
- let mut r = LegacyRecovery::new_with_config(&recovery_config);
1158
-
1159
- r.pto_count = 99999;
1160
-
1161
- let handshake_status = HandshakeStatus {
1162
- completed: true,
1163
- has_handshake_keys: true,
1164
- peer_verified_address: true,
1165
- };
1166
- let now = Instant::now();
1167
-
1168
- let _ = r.pto_time_and_space(handshake_status, now);
1169
- }
1170
- }
1
+ // Copyright (C) 2018-2019, Cloudflare, Inc.
2
+ // All rights reserved.
3
+ //
4
+ // Redistribution and use in source and binary forms, with or without
5
+ // modification, are permitted provided that the following conditions are
6
+ // met:
7
+ //
8
+ // * Redistributions of source code must retain the above copyright notice,
9
+ // this list of conditions and the following disclaimer.
10
+ //
11
+ // * Redistributions in binary form must reproduce the above copyright
12
+ // notice, this list of conditions and the following disclaimer in the
13
+ // documentation and/or other materials provided with the distribution.
14
+ //
15
+ // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
16
+ // IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
17
+ // THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
18
+ // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
19
+ // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
20
+ // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21
+ // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
22
+ // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
23
+ // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
24
+ // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25
+ // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26
+
27
+ use std::cmp;
28
+
29
+ use std::time::Duration;
30
+ use std::time::Instant;
31
+
32
+ use std::collections::VecDeque;
33
+
34
+ use super::RecoveryConfig;
35
+ use super::Sent;
36
+
37
+ use crate::packet::Epoch;
38
+ use crate::ranges::RangeSet;
39
+ use crate::recovery::Bandwidth;
40
+ use crate::recovery::HandshakeStatus;
41
+ use crate::recovery::OnLossDetectionTimeoutOutcome;
42
+ use crate::recovery::RecoveryOps;
43
+ use crate::recovery::StartupExit;
44
+ use crate::Error;
45
+ use crate::Result;
46
+
47
+ #[cfg(feature = "qlog")]
48
+ use crate::recovery::QlogMetrics;
49
+
50
+ use crate::frame;
51
+
52
+ #[cfg(feature = "qlog")]
53
+ use qlog::events::EventData;
54
+
55
+ use super::Congestion;
56
+ use crate::recovery::bytes_in_flight::BytesInFlight;
57
+ use crate::recovery::rtt::RttStats;
58
+ use crate::recovery::LossDetectionTimer;
59
+ use crate::recovery::OnAckReceivedOutcome;
60
+ use crate::recovery::ReleaseDecision;
61
+ use crate::recovery::ReleaseTime;
62
+ use crate::recovery::GRANULARITY;
63
+ use crate::recovery::INITIAL_PACKET_THRESHOLD;
64
+ use crate::recovery::INITIAL_TIME_THRESHOLD;
65
+ use crate::recovery::MAX_OUTSTANDING_NON_ACK_ELICITING;
66
+ use crate::recovery::MAX_PACKET_THRESHOLD;
67
+ use crate::recovery::MAX_PTO_EXPONENT;
68
+ use crate::recovery::MAX_PTO_PROBES_COUNT;
69
+ use crate::recovery::PACKET_REORDER_TIME_THRESHOLD;
70
+
71
+ #[derive(Default)]
72
+ struct RecoveryEpoch {
73
+ /// The time the most recent ack-eliciting packet was sent.
74
+ time_of_last_ack_eliciting_packet: Option<Instant>,
75
+
76
+ /// The largest packet number acknowledged in the packet number space so
77
+ /// far.
78
+ largest_acked_packet: Option<u64>,
79
+
80
+ /// The time at which the next packet in that packet number space can be
81
+ /// considered lost based on exceeding the reordering window in time.
82
+ loss_time: Option<Instant>,
83
+
84
+ /// An association of packet numbers in a packet number space to information
85
+ /// about them.
86
+ sent_packets: VecDeque<Sent>,
87
+
88
+ loss_probes: usize,
89
+ in_flight_count: usize,
90
+
91
+ acked_frames: Vec<frame::Frame>,
92
+
93
+ // Frames scheduled for retransmission due to PTO are tracked
94
+ // separately so we can check that frames were drained before
95
+ // generating more PTO probes.
96
+ lost_frames_ack: Vec<frame::Frame>,
97
+ lost_frames_pto: Vec<frame::Frame>,
98
+
99
+ /// The largest packet number sent in the packet number space so far.
100
+ #[cfg(test)]
101
+ test_largest_sent_pkt_num_on_path: Option<u64>,
102
+ }
103
+
104
+ struct AckedDetectionResult {
105
+ acked_bytes: usize,
106
+ spurious_losses: usize,
107
+ spurious_pkt_thresh: Option<u64>,
108
+ has_ack_eliciting: bool,
109
+ has_in_flight_spurious_loss: bool,
110
+ }
111
+
112
+ struct LossDetectionResult {
113
+ largest_lost_pkt: Option<Sent>,
114
+ lost_packets: usize,
115
+ lost_bytes: usize,
116
+ pmtud_lost_bytes: usize,
117
+ }
118
+
119
+ impl RecoveryEpoch {
120
+ // `peer_sent_ack_ranges` should not be used without validation.
121
+ fn detect_and_remove_acked_packets(
122
+ &mut self, now: Instant, peer_sent_ack_ranges: &RangeSet,
123
+ newly_acked: &mut Vec<Acked>, rtt_stats: &RttStats, skip_pn: Option<u64>,
124
+ trace_id: &str,
125
+ ) -> Result<AckedDetectionResult> {
126
+ newly_acked.clear();
127
+
128
+ let mut acked_bytes = 0;
129
+ let mut spurious_losses = 0;
130
+ let mut spurious_pkt_thresh = None;
131
+ let mut has_ack_eliciting = false;
132
+ let mut has_in_flight_spurious_loss = false;
133
+
134
+ let largest_ack_received = peer_sent_ack_ranges
135
+ .last()
136
+ .expect("ACK frames should always have at least one ack range");
137
+ let largest_acked = self
138
+ .largest_acked_packet
139
+ .unwrap_or(0)
140
+ .max(largest_ack_received);
141
+
142
+ for peer_sent_range in peer_sent_ack_ranges.iter() {
143
+ if skip_pn.is_some_and(|skip_pn| peer_sent_range.contains(&skip_pn)) {
144
+ // https://www.rfc-editor.org/rfc/rfc9000#section-13.1
145
+ // An endpoint SHOULD treat receipt of an acknowledgment
146
+ // for a packet it did not send as
147
+ // a connection error of type PROTOCOL_VIOLATION
148
+ return Err(Error::OptimisticAckDetected);
149
+ }
150
+
151
+ // Because packets always have incrementing numbers, they are always
152
+ // in sorted order.
153
+ let start = if self
154
+ .sent_packets
155
+ .front()
156
+ .filter(|e| e.pkt_num >= peer_sent_range.start)
157
+ .is_some()
158
+ {
159
+ // Usually it will be the first packet.
160
+ 0
161
+ } else {
162
+ self.sent_packets
163
+ .binary_search_by_key(&peer_sent_range.start, |p| p.pkt_num)
164
+ .unwrap_or_else(|e| e)
165
+ };
166
+
167
+ for unacked in self.sent_packets.range_mut(start..) {
168
+ if unacked.pkt_num >= peer_sent_range.end {
169
+ break;
170
+ }
171
+
172
+ if unacked.time_acked.is_some() {
173
+ // Already acked.
174
+ } else if unacked.time_lost.is_some() {
175
+ // An acked packet was already declared lost.
176
+ spurious_losses += 1;
177
+ spurious_pkt_thresh
178
+ .get_or_insert(largest_acked - unacked.pkt_num + 1);
179
+ unacked.time_acked = Some(now);
180
+
181
+ if unacked.in_flight {
182
+ has_in_flight_spurious_loss = true;
183
+ }
184
+ } else {
185
+ if unacked.in_flight {
186
+ self.in_flight_count -= 1;
187
+ acked_bytes += unacked.size;
188
+ }
189
+
190
+ newly_acked.push(Acked {
191
+ pkt_num: unacked.pkt_num,
192
+ time_sent: unacked.time_sent,
193
+ size: unacked.size,
194
+
195
+ rtt: now.saturating_duration_since(unacked.time_sent),
196
+ delivered: unacked.delivered,
197
+ delivered_time: unacked.delivered_time,
198
+ first_sent_time: unacked.first_sent_time,
199
+ is_app_limited: unacked.is_app_limited,
200
+ });
201
+
202
+ trace!("{} packet newly acked {}", trace_id, unacked.pkt_num);
203
+
204
+ self.acked_frames
205
+ .extend(std::mem::take(&mut unacked.frames));
206
+
207
+ has_ack_eliciting |= unacked.ack_eliciting;
208
+ unacked.time_acked = Some(now);
209
+ }
210
+ }
211
+ }
212
+
213
+ self.drain_acked_and_lost_packets(now - rtt_stats.rtt());
214
+
215
+ Ok(AckedDetectionResult {
216
+ acked_bytes,
217
+ spurious_losses,
218
+ spurious_pkt_thresh,
219
+ has_ack_eliciting,
220
+ has_in_flight_spurious_loss,
221
+ })
222
+ }
223
+
224
+ fn detect_lost_packets(
225
+ &mut self, loss_delay: Duration, pkt_thresh: u64, now: Instant,
226
+ trace_id: &str, epoch: Epoch,
227
+ ) -> LossDetectionResult {
228
+ self.loss_time = None;
229
+
230
+ // Minimum time of kGranularity before packets are deemed lost.
231
+ let loss_delay = cmp::max(loss_delay, GRANULARITY);
232
+ let largest_acked = self.largest_acked_packet.unwrap_or(0);
233
+
234
+ // Packets sent before this time are deemed lost.
235
+ let lost_send_time = now.checked_sub(loss_delay).unwrap();
236
+
237
+ let mut lost_packets = 0;
238
+ let mut lost_bytes = 0;
239
+ let mut pmtud_lost_bytes = 0;
240
+
241
+ let mut largest_lost_pkt = None;
242
+
243
+ let unacked_iter = self.sent_packets
244
+ .iter_mut()
245
+ // Skip packets that follow the largest acked packet.
246
+ .take_while(|p| p.pkt_num <= largest_acked)
247
+ // Skip packets that have already been acked or lost.
248
+ .filter(|p| p.time_acked.is_none() && p.time_lost.is_none());
249
+
250
+ for unacked in unacked_iter {
251
+ // Mark packet as lost, or set time when it should be marked.
252
+ if unacked.time_sent <= lost_send_time ||
253
+ largest_acked >= unacked.pkt_num + pkt_thresh
254
+ {
255
+ self.lost_frames_ack.extend(unacked.frames.drain(..));
256
+
257
+ unacked.time_lost = Some(now);
258
+
259
+ if unacked.is_pmtud_probe {
260
+ pmtud_lost_bytes += unacked.size;
261
+ self.in_flight_count -= 1;
262
+
263
+ // Do not track PMTUD probes losses.
264
+ continue;
265
+ }
266
+
267
+ if unacked.in_flight {
268
+ lost_bytes += unacked.size;
269
+
270
+ // Frames have already been removed from the packet, so
271
+ // cloning the whole packet should be relatively cheap.
272
+ largest_lost_pkt = Some(unacked.clone());
273
+
274
+ self.in_flight_count -= 1;
275
+
276
+ trace!(
277
+ "{} packet {} lost on epoch {}",
278
+ trace_id,
279
+ unacked.pkt_num,
280
+ epoch
281
+ );
282
+ }
283
+
284
+ lost_packets += 1;
285
+ } else {
286
+ let loss_time = match self.loss_time {
287
+ None => unacked.time_sent + loss_delay,
288
+
289
+ Some(loss_time) =>
290
+ cmp::min(loss_time, unacked.time_sent + loss_delay),
291
+ };
292
+
293
+ self.loss_time = Some(loss_time);
294
+ break;
295
+ }
296
+ }
297
+
298
+ LossDetectionResult {
299
+ largest_lost_pkt,
300
+ lost_packets,
301
+ lost_bytes,
302
+ pmtud_lost_bytes,
303
+ }
304
+ }
305
+
306
+ fn drain_acked_and_lost_packets(&mut self, loss_thresh: Instant) {
307
+ // In order to avoid removing elements from the middle of the list
308
+ // (which would require copying other elements to compact the list),
309
+ // we only remove a contiguous range of elements from the start of the
310
+ // list.
311
+ //
312
+ // This means that acked or lost elements coming after this will not
313
+ // be removed at this point, but their removal is delayed for a later
314
+ // time, once the gaps have been filled.
315
+ while let Some(pkt) = self.sent_packets.front() {
316
+ if let Some(time_lost) = pkt.time_lost {
317
+ if time_lost > loss_thresh {
318
+ break;
319
+ }
320
+ }
321
+
322
+ if pkt.time_acked.is_none() && pkt.time_lost.is_none() {
323
+ break;
324
+ }
325
+
326
+ self.sent_packets.pop_front();
327
+ }
328
+ }
329
+
330
+ /// Returns the next lost frame, trying ACK-based lost frames first,
331
+ /// then PTO-based lost frames.
332
+ fn next_lost_frame(&mut self) -> Option<frame::Frame> {
333
+ self.lost_frames_ack
334
+ .pop()
335
+ .or_else(|| self.lost_frames_pto.pop())
336
+ }
337
+
338
+ /// Returns true if there are any lost frames (ACK or PTO).
339
+ fn has_lost_frames(&self) -> bool {
340
+ !self.lost_frames_ack.is_empty() || !self.lost_frames_pto.is_empty()
341
+ }
342
+
343
+ /// Returns the total count of lost frames (ACK + PTO).
344
+ #[cfg(test)]
345
+ fn lost_frames_count(&self) -> usize {
346
+ self.lost_frames_ack.len() + self.lost_frames_pto.len()
347
+ }
348
+
349
+ /// Clears all lost frames (both ACK and PTO).
350
+ fn clear_lost_frames(&mut self) {
351
+ self.lost_frames_ack.clear();
352
+ self.lost_frames_pto.clear();
353
+ }
354
+ }
355
+
356
+ pub struct LegacyRecovery {
357
+ epochs: [RecoveryEpoch; Epoch::count()],
358
+
359
+ loss_timer: LossDetectionTimer,
360
+
361
+ pto_count: u32,
362
+
363
+ rtt_stats: RttStats,
364
+
365
+ lost_spurious_count: usize,
366
+
367
+ pkt_thresh: u64,
368
+
369
+ time_thresh: f64,
370
+
371
+ bytes_in_flight: BytesInFlight,
372
+
373
+ bytes_sent: usize,
374
+
375
+ bytes_lost: u64,
376
+
377
+ pub max_datagram_size: usize,
378
+
379
+ #[cfg(feature = "qlog")]
380
+ qlog_metrics: QlogMetrics,
381
+
382
+ #[cfg(feature = "qlog")]
383
+ qlog_prev_cc_state: &'static str,
384
+
385
+ /// How many non-ack-eliciting packets have been sent.
386
+ outstanding_non_ack_eliciting: usize,
387
+
388
+ pub congestion: Congestion,
389
+
390
+ /// A resusable list of acks.
391
+ newly_acked: Vec<Acked>,
392
+ }
393
+
394
+ impl LegacyRecovery {
395
+ pub fn new_with_config(recovery_config: &RecoveryConfig) -> Self {
396
+ Self {
397
+ epochs: Default::default(),
398
+
399
+ loss_timer: Default::default(),
400
+
401
+ pto_count: 0,
402
+
403
+ rtt_stats: RttStats::new(
404
+ recovery_config.initial_rtt,
405
+ recovery_config.max_ack_delay,
406
+ ),
407
+
408
+ lost_spurious_count: 0,
409
+
410
+ pkt_thresh: INITIAL_PACKET_THRESHOLD,
411
+
412
+ time_thresh: INITIAL_TIME_THRESHOLD,
413
+
414
+ bytes_in_flight: Default::default(),
415
+
416
+ bytes_sent: 0,
417
+
418
+ bytes_lost: 0,
419
+
420
+ max_datagram_size: recovery_config.max_send_udp_payload_size,
421
+
422
+ #[cfg(feature = "qlog")]
423
+ qlog_metrics: QlogMetrics::default(),
424
+
425
+ #[cfg(feature = "qlog")]
426
+ qlog_prev_cc_state: "",
427
+
428
+ outstanding_non_ack_eliciting: 0,
429
+
430
+ congestion: Congestion::from_config(recovery_config),
431
+
432
+ newly_acked: Vec::new(),
433
+ }
434
+ }
435
+
436
+ #[cfg(test)]
437
+ pub fn new(config: &crate::Config) -> Self {
438
+ Self::new_with_config(&RecoveryConfig::from_config(config))
439
+ }
440
+
441
+ fn loss_time_and_space(&self) -> (Option<Instant>, Epoch) {
442
+ let mut epoch = Epoch::Initial;
443
+ let mut time = self.epochs[epoch].loss_time;
444
+
445
+ // Iterate over all packet number spaces starting from Handshake.
446
+ for e in [Epoch::Handshake, Epoch::Application] {
447
+ let new_time = self.epochs[e].loss_time;
448
+ if time.is_none() || new_time < time {
449
+ time = new_time;
450
+ epoch = e;
451
+ }
452
+ }
453
+
454
+ (time, epoch)
455
+ }
456
+
457
+ fn pto_time_and_space(
458
+ &self, handshake_status: HandshakeStatus, now: Instant,
459
+ ) -> (Option<Instant>, Epoch) {
460
+ let mut duration =
461
+ self.pto() * 2_u32.pow(self.pto_count.min(MAX_PTO_EXPONENT));
462
+
463
+ // Arm PTO from now when there are no inflight packets.
464
+ if self.bytes_in_flight.is_zero() {
465
+ if handshake_status.has_handshake_keys {
466
+ return (Some(now + duration), Epoch::Handshake);
467
+ } else {
468
+ return (Some(now + duration), Epoch::Initial);
469
+ }
470
+ }
471
+
472
+ let mut pto_timeout = None;
473
+ let mut pto_space = Epoch::Initial;
474
+
475
+ // Iterate over all packet number spaces.
476
+ for e in [Epoch::Initial, Epoch::Handshake, Epoch::Application] {
477
+ let epoch = &self.epochs[e];
478
+ if epoch.in_flight_count == 0 {
479
+ continue;
480
+ }
481
+
482
+ if e == Epoch::Application {
483
+ // Skip Application Data until handshake completes.
484
+ if !handshake_status.completed {
485
+ return (pto_timeout, pto_space);
486
+ }
487
+
488
+ // Include max_ack_delay and backoff for Application Data.
489
+ duration += self.rtt_stats.max_ack_delay *
490
+ 2_u32.pow(self.pto_count.min(MAX_PTO_EXPONENT));
491
+ }
492
+
493
+ let new_time = epoch
494
+ .time_of_last_ack_eliciting_packet
495
+ .map(|t| t + duration);
496
+
497
+ if pto_timeout.is_none() || new_time < pto_timeout {
498
+ pto_timeout = new_time;
499
+ pto_space = e;
500
+ }
501
+ }
502
+
503
+ (pto_timeout, pto_space)
504
+ }
505
+
506
+ fn set_loss_detection_timer(
507
+ &mut self, handshake_status: HandshakeStatus, now: Instant,
508
+ ) {
509
+ let (earliest_loss_time, _) = self.loss_time_and_space();
510
+
511
+ if let Some(to) = earliest_loss_time {
512
+ // Time threshold loss detection.
513
+ self.loss_timer.update(to);
514
+ return;
515
+ }
516
+
517
+ if self.bytes_in_flight.is_zero() &&
518
+ handshake_status.peer_verified_address
519
+ {
520
+ self.loss_timer.clear();
521
+ return;
522
+ }
523
+
524
+ // PTO timer.
525
+ if let (Some(timeout), _) = self.pto_time_and_space(handshake_status, now)
526
+ {
527
+ self.loss_timer.update(timeout);
528
+ } else {
529
+ self.loss_timer.clear();
530
+ }
531
+ }
532
+
533
+ fn detect_lost_packets(
534
+ &mut self, epoch: Epoch, now: Instant, trace_id: &str,
535
+ ) -> (usize, usize) {
536
+ let loss_delay = cmp::max(self.rtt_stats.latest_rtt, self.rtt())
537
+ .mul_f64(self.time_thresh);
538
+
539
+ let loss = self.epochs[epoch].detect_lost_packets(
540
+ loss_delay,
541
+ self.pkt_thresh,
542
+ now,
543
+ trace_id,
544
+ epoch,
545
+ );
546
+
547
+ if let Some(pkt) = loss.largest_lost_pkt {
548
+ if !self.congestion.in_congestion_recovery(pkt.time_sent) {
549
+ (self.congestion.cc_ops.checkpoint)(&mut self.congestion);
550
+ }
551
+
552
+ (self.congestion.cc_ops.congestion_event)(
553
+ &mut self.congestion,
554
+ self.bytes_in_flight.get(),
555
+ loss.lost_bytes,
556
+ &pkt,
557
+ now,
558
+ );
559
+
560
+ self.bytes_in_flight
561
+ .saturating_subtract(loss.lost_bytes, now);
562
+ };
563
+
564
+ self.bytes_in_flight
565
+ .saturating_subtract(loss.pmtud_lost_bytes, now);
566
+
567
+ self.epochs[epoch]
568
+ .drain_acked_and_lost_packets(now - self.rtt_stats.rtt());
569
+
570
+ self.congestion.lost_count += loss.lost_packets;
571
+
572
+ (loss.lost_packets, loss.lost_bytes)
573
+ }
574
+ }
575
+
576
+ impl RecoveryOps for LegacyRecovery {
577
+ /// Returns whether or not we should elicit an ACK even if we wouldn't
578
+ /// otherwise have constructed an ACK eliciting packet.
579
+ fn should_elicit_ack(&self, epoch: Epoch) -> bool {
580
+ self.epochs[epoch].loss_probes > 0 ||
581
+ self.outstanding_non_ack_eliciting >=
582
+ MAX_OUTSTANDING_NON_ACK_ELICITING
583
+ }
584
+
585
+ fn next_acked_frame(&mut self, epoch: Epoch) -> Option<frame::Frame> {
586
+ self.epochs[epoch].acked_frames.pop()
587
+ }
588
+
589
+ fn next_lost_frame(&mut self, epoch: Epoch) -> Option<frame::Frame> {
590
+ self.epochs[epoch].next_lost_frame()
591
+ }
592
+
593
+ fn get_largest_acked_on_epoch(&self, epoch: Epoch) -> Option<u64> {
594
+ self.epochs[epoch].largest_acked_packet
595
+ }
596
+
597
+ fn has_lost_frames(&self, epoch: Epoch) -> bool {
598
+ self.epochs[epoch].has_lost_frames()
599
+ }
600
+
601
+ fn loss_probes(&self, epoch: Epoch) -> usize {
602
+ self.epochs[epoch].loss_probes
603
+ }
604
+
605
+ #[cfg(test)]
606
+ fn inc_loss_probes(&mut self, epoch: Epoch) {
607
+ self.epochs[epoch].loss_probes += 1;
608
+ }
609
+
610
+ #[cfg(test)]
611
+ fn lost_frames_count(&self, epoch: Epoch) -> usize {
612
+ self.epochs[epoch].lost_frames_count()
613
+ }
614
+
615
+ fn ping_sent(&mut self, epoch: Epoch) {
616
+ self.epochs[epoch].loss_probes =
617
+ self.epochs[epoch].loss_probes.saturating_sub(1);
618
+ }
619
+
620
+ fn on_packet_sent(
621
+ &mut self, mut pkt: Sent, epoch: Epoch,
622
+ handshake_status: HandshakeStatus, now: Instant, trace_id: &str,
623
+ ) {
624
+ let ack_eliciting = pkt.ack_eliciting;
625
+ let in_flight = pkt.in_flight;
626
+ let sent_bytes = pkt.size;
627
+
628
+ if ack_eliciting {
629
+ self.outstanding_non_ack_eliciting = 0;
630
+ } else {
631
+ self.outstanding_non_ack_eliciting += 1;
632
+ }
633
+
634
+ if in_flight && ack_eliciting {
635
+ self.epochs[epoch].time_of_last_ack_eliciting_packet = Some(now);
636
+ }
637
+
638
+ self.congestion.on_packet_sent(
639
+ self.bytes_in_flight.get(),
640
+ sent_bytes,
641
+ now,
642
+ &mut pkt,
643
+ self.bytes_lost,
644
+ in_flight,
645
+ );
646
+
647
+ if in_flight {
648
+ self.epochs[epoch].in_flight_count += 1;
649
+ self.bytes_in_flight.add(sent_bytes, now);
650
+
651
+ self.set_loss_detection_timer(handshake_status, now);
652
+ }
653
+
654
+ self.bytes_sent += sent_bytes;
655
+
656
+ #[cfg(test)]
657
+ {
658
+ self.epochs[epoch].test_largest_sent_pkt_num_on_path = self.epochs
659
+ [epoch]
660
+ .test_largest_sent_pkt_num_on_path
661
+ .max(Some(pkt.pkt_num));
662
+ }
663
+
664
+ self.epochs[epoch].sent_packets.push_back(pkt);
665
+
666
+ trace!("{trace_id} {self:?}");
667
+ }
668
+
669
+ fn get_packet_send_time(&self, now: Instant) -> Instant {
670
+ now
671
+ }
672
+
673
+ // `peer_sent_ack_ranges` should not be used without validation.
674
+ fn on_ack_received(
675
+ &mut self, peer_sent_ack_ranges: &RangeSet, ack_delay: u64, epoch: Epoch,
676
+ handshake_status: HandshakeStatus, now: Instant, skip_pn: Option<u64>,
677
+ trace_id: &str,
678
+ ) -> Result<OnAckReceivedOutcome> {
679
+ let AckedDetectionResult {
680
+ acked_bytes,
681
+ spurious_losses,
682
+ spurious_pkt_thresh,
683
+ has_ack_eliciting,
684
+ has_in_flight_spurious_loss,
685
+ } = self.epochs[epoch].detect_and_remove_acked_packets(
686
+ now,
687
+ peer_sent_ack_ranges,
688
+ &mut self.newly_acked,
689
+ &self.rtt_stats,
690
+ skip_pn,
691
+ trace_id,
692
+ )?;
693
+
694
+ self.lost_spurious_count += spurious_losses;
695
+ if let Some(thresh) = spurious_pkt_thresh {
696
+ self.pkt_thresh =
697
+ self.pkt_thresh.max(thresh.min(MAX_PACKET_THRESHOLD));
698
+ self.time_thresh = PACKET_REORDER_TIME_THRESHOLD;
699
+ }
700
+
701
+ // Undo congestion window update.
702
+ if has_in_flight_spurious_loss {
703
+ (self.congestion.cc_ops.rollback)(&mut self.congestion);
704
+ }
705
+
706
+ if self.newly_acked.is_empty() {
707
+ return Ok(OnAckReceivedOutcome::default());
708
+ }
709
+
710
+ let largest_newly_acked = self.newly_acked.last().unwrap();
711
+
712
+ // Update `largest_acked_packet` based on the validated `newly_acked`
713
+ // value.
714
+ let largest_acked_pkt_num = self.epochs[epoch]
715
+ .largest_acked_packet
716
+ .unwrap_or(0)
717
+ .max(largest_newly_acked.pkt_num);
718
+ self.epochs[epoch].largest_acked_packet = Some(largest_acked_pkt_num);
719
+
720
+ // Check if largest packet is newly acked.
721
+ if largest_newly_acked.pkt_num == largest_acked_pkt_num &&
722
+ has_ack_eliciting
723
+ {
724
+ let latest_rtt = now - largest_newly_acked.time_sent;
725
+ self.rtt_stats.update_rtt(
726
+ latest_rtt,
727
+ Duration::from_micros(ack_delay),
728
+ now,
729
+ handshake_status.completed,
730
+ );
731
+ }
732
+
733
+ // Detect and mark lost packets without removing them from the sent
734
+ // packets list.
735
+ let (lost_packets, lost_bytes) =
736
+ self.detect_lost_packets(epoch, now, trace_id);
737
+
738
+ self.congestion.on_packets_acked(
739
+ self.bytes_in_flight.get(),
740
+ &mut self.newly_acked,
741
+ &self.rtt_stats,
742
+ now,
743
+ );
744
+
745
+ self.bytes_in_flight.saturating_subtract(acked_bytes, now);
746
+
747
+ self.pto_count = 0;
748
+
749
+ self.set_loss_detection_timer(handshake_status, now);
750
+
751
+ self.epochs[epoch]
752
+ .drain_acked_and_lost_packets(now - self.rtt_stats.rtt());
753
+
754
+ Ok(OnAckReceivedOutcome {
755
+ lost_packets,
756
+ lost_bytes,
757
+ acked_bytes,
758
+ spurious_losses,
759
+ })
760
+ }
761
+
762
+ fn on_loss_detection_timeout(
763
+ &mut self, handshake_status: HandshakeStatus, now: Instant,
764
+ trace_id: &str,
765
+ ) -> OnLossDetectionTimeoutOutcome {
766
+ let (earliest_loss_time, epoch) = self.loss_time_and_space();
767
+
768
+ if earliest_loss_time.is_some() {
769
+ // Time threshold loss detection.
770
+ let (lost_packets, lost_bytes) =
771
+ self.detect_lost_packets(epoch, now, trace_id);
772
+
773
+ self.set_loss_detection_timer(handshake_status, now);
774
+
775
+ trace!("{trace_id} {self:?}");
776
+ return OnLossDetectionTimeoutOutcome {
777
+ lost_packets,
778
+ lost_bytes,
779
+ };
780
+ }
781
+
782
+ let epoch = if self.bytes_in_flight.get() > 0 {
783
+ // Send new data if available, else retransmit old data. If neither
784
+ // is available, send a single PING frame.
785
+ let (_, e) = self.pto_time_and_space(handshake_status, now);
786
+
787
+ e
788
+ } else {
789
+ // Client sends an anti-deadlock packet: Initial is padded to earn
790
+ // more anti-amplification credit, a Handshake packet proves address
791
+ // ownership.
792
+ if handshake_status.has_handshake_keys {
793
+ Epoch::Handshake
794
+ } else {
795
+ Epoch::Initial
796
+ }
797
+ };
798
+
799
+ self.pto_count += 1;
800
+
801
+ let epoch = &mut self.epochs[epoch];
802
+
803
+ epoch.loss_probes =
804
+ cmp::min(self.pto_count as usize, MAX_PTO_PROBES_COUNT);
805
+
806
+ let sent_packets_iter_limit = if !epoch.lost_frames_pto.is_empty() {
807
+ // Skip the search for frames to add to PTO probes if frames
808
+ // added in a prior PTO haven't been processed yet.
809
+ 0
810
+ } else {
811
+ usize::MAX
812
+ };
813
+
814
+ let unacked_iter = epoch.sent_packets
815
+ .iter()
816
+ .take(sent_packets_iter_limit)
817
+ // Skip packets that have already been acked or lost, and packets
818
+ // that don't contain either CRYPTO or STREAM frames.
819
+ .filter(|p| p.has_data && p.time_acked.is_none() && p.time_lost.is_none())
820
+ // Only return as many packets as the number of probe packets that
821
+ // will be sent.
822
+ .take(epoch.loss_probes);
823
+
824
+ // Retransmit the frames from the oldest sent packets on PTO. However
825
+ // the packets are not actually declared lost (so there is no effect to
826
+ // congestion control), we just reschedule the data they carried.
827
+ //
828
+ // This will also trigger sending an ACK and retransmitting frames like
829
+ // HANDSHAKE_DONE and MAX_DATA / MAX_STREAM_DATA as well, in addition
830
+ // to CRYPTO and STREAM, if the original packet carried them.
831
+ for unacked in unacked_iter {
832
+ epoch.lost_frames_pto.extend_from_slice(&unacked.frames);
833
+ }
834
+
835
+ self.set_loss_detection_timer(handshake_status, now);
836
+
837
+ trace!("{trace_id} {self:?}");
838
+
839
+ OnLossDetectionTimeoutOutcome {
840
+ lost_packets: 0,
841
+ lost_bytes: 0,
842
+ }
843
+ }
844
+
845
+ fn on_pkt_num_space_discarded(
846
+ &mut self, epoch: Epoch, handshake_status: HandshakeStatus, now: Instant,
847
+ ) {
848
+ let epoch = &mut self.epochs[epoch];
849
+
850
+ let unacked_bytes = epoch
851
+ .sent_packets
852
+ .iter()
853
+ .filter(|p| {
854
+ p.in_flight && p.time_acked.is_none() && p.time_lost.is_none()
855
+ })
856
+ .fold(0, |acc, p| acc + p.size);
857
+
858
+ self.bytes_in_flight.saturating_subtract(unacked_bytes, now);
859
+
860
+ epoch.sent_packets.clear();
861
+ epoch.clear_lost_frames();
862
+ epoch.acked_frames.clear();
863
+
864
+ epoch.time_of_last_ack_eliciting_packet = None;
865
+ epoch.loss_time = None;
866
+ epoch.loss_probes = 0;
867
+ epoch.in_flight_count = 0;
868
+
869
+ self.set_loss_detection_timer(handshake_status, now);
870
+ }
871
+
872
+ fn on_path_change(
873
+ &mut self, epoch: Epoch, now: Instant, trace_id: &str,
874
+ ) -> (usize, usize) {
875
+ // Time threshold loss detection.
876
+ self.detect_lost_packets(epoch, now, trace_id)
877
+ }
878
+
879
+ fn loss_detection_timer(&self) -> Option<Instant> {
880
+ self.loss_timer.time
881
+ }
882
+
883
+ fn cwnd(&self) -> usize {
884
+ self.congestion.congestion_window()
885
+ }
886
+
887
+ fn cwnd_available(&self) -> usize {
888
+ // Ignore cwnd when sending probe packets.
889
+ if self.epochs.iter().any(|e| e.loss_probes > 0) {
890
+ return usize::MAX;
891
+ }
892
+
893
+ // Open more space (snd_cnt) for PRR when allowed.
894
+ self.cwnd().saturating_sub(self.bytes_in_flight.get()) +
895
+ self.congestion.prr.snd_cnt
896
+ }
897
+
898
+ fn rtt(&self) -> Duration {
899
+ self.rtt_stats.rtt()
900
+ }
901
+
902
+ fn min_rtt(&self) -> Option<Duration> {
903
+ self.rtt_stats.min_rtt()
904
+ }
905
+
906
+ fn max_rtt(&self) -> Option<Duration> {
907
+ self.rtt_stats.max_rtt()
908
+ }
909
+
910
+ fn rttvar(&self) -> Duration {
911
+ self.rtt_stats.rttvar
912
+ }
913
+
914
+ fn pto(&self) -> Duration {
915
+ self.rtt() + cmp::max(self.rtt_stats.rttvar * 4, GRANULARITY)
916
+ }
917
+
918
+ /// The most recent data delivery rate estimate.
919
+ fn delivery_rate(&self) -> Bandwidth {
920
+ self.congestion.delivery_rate()
921
+ }
922
+
923
+ fn max_bandwidth(&self) -> Option<Bandwidth> {
924
+ // TODO implement
925
+ None
926
+ }
927
+
928
+ /// Statistics from when a CCA first exited the startup phase.
929
+ fn startup_exit(&self) -> Option<StartupExit> {
930
+ self.congestion.ssthresh.startup_exit()
931
+ }
932
+
933
+ fn max_datagram_size(&self) -> usize {
934
+ self.max_datagram_size
935
+ }
936
+
937
+ fn pmtud_update_max_datagram_size(&mut self, new_max_datagram_size: usize) {
938
+ // Congestion Window is updated only when it's not updated already.
939
+ // Update cwnd if it hasn't been updated yet.
940
+ if self.cwnd() ==
941
+ self.max_datagram_size *
942
+ self.congestion.initial_congestion_window_packets
943
+ {
944
+ self.congestion.congestion_window = new_max_datagram_size *
945
+ self.congestion.initial_congestion_window_packets;
946
+ }
947
+
948
+ self.max_datagram_size = new_max_datagram_size;
949
+ }
950
+
951
+ fn update_max_datagram_size(&mut self, new_max_datagram_size: usize) {
952
+ self.pmtud_update_max_datagram_size(
953
+ self.max_datagram_size.min(new_max_datagram_size),
954
+ )
955
+ }
956
+
957
+ #[cfg(test)]
958
+ fn sent_packets_len(&self, epoch: Epoch) -> usize {
959
+ self.epochs[epoch].sent_packets.len()
960
+ }
961
+
962
+ #[cfg(test)]
963
+ fn in_flight_count(&self, epoch: Epoch) -> usize {
964
+ self.epochs[epoch].in_flight_count
965
+ }
966
+
967
+ fn bytes_in_flight(&self) -> usize {
968
+ self.bytes_in_flight.get()
969
+ }
970
+
971
+ fn bytes_in_flight_duration(&self) -> Duration {
972
+ self.bytes_in_flight.get_duration()
973
+ }
974
+
975
+ #[cfg(test)]
976
+ fn pacing_rate(&self) -> u64 {
977
+ 0
978
+ }
979
+
980
+ #[cfg(test)]
981
+ fn pto_count(&self) -> u32 {
982
+ self.pto_count
983
+ }
984
+
985
+ #[cfg(test)]
986
+ fn pkt_thresh(&self) -> Option<u64> {
987
+ Some(self.pkt_thresh)
988
+ }
989
+
990
+ #[cfg(test)]
991
+ fn time_thresh(&self) -> f64 {
992
+ self.time_thresh
993
+ }
994
+
995
+ #[cfg(test)]
996
+ fn lost_spurious_count(&self) -> usize {
997
+ self.lost_spurious_count
998
+ }
999
+
1000
+ #[cfg(test)]
1001
+ fn detect_lost_packets_for_test(
1002
+ &mut self, epoch: Epoch, now: Instant,
1003
+ ) -> (usize, usize) {
1004
+ self.detect_lost_packets(epoch, now, "")
1005
+ }
1006
+
1007
+ // FIXME only used by gcongestion
1008
+ fn on_app_limited(&mut self) {
1009
+ // Not implemented for legacy recovery, update_app_limited and
1010
+ // delivery_rate_update_app_limited used instead.
1011
+ }
1012
+
1013
+ #[cfg(test)]
1014
+ fn largest_sent_pkt_num_on_path(&self, epoch: Epoch) -> Option<u64> {
1015
+ self.epochs[epoch].test_largest_sent_pkt_num_on_path
1016
+ }
1017
+
1018
+ #[cfg(test)]
1019
+ fn app_limited(&self) -> bool {
1020
+ self.congestion.app_limited
1021
+ }
1022
+
1023
+ fn update_app_limited(&mut self, v: bool) {
1024
+ self.congestion.update_app_limited(v);
1025
+ }
1026
+
1027
+ // FIXME only used by congestion
1028
+ fn delivery_rate_update_app_limited(&mut self, v: bool) {
1029
+ self.congestion.delivery_rate.update_app_limited(v);
1030
+ }
1031
+
1032
+ // FIXME only used by congestion
1033
+ fn update_max_ack_delay(&mut self, max_ack_delay: Duration) {
1034
+ self.rtt_stats.max_ack_delay = max_ack_delay;
1035
+ }
1036
+
1037
+ #[cfg(feature = "qlog")]
1038
+ fn state_str(&self, now: Instant) -> &'static str {
1039
+ (self.congestion.cc_ops.state_str)(&self.congestion, now)
1040
+ }
1041
+
1042
+ #[cfg(feature = "qlog")]
1043
+ fn get_updated_qlog_event_data(&mut self) -> Option<EventData> {
1044
+ let qlog_metrics = QlogMetrics {
1045
+ min_rtt: *self.rtt_stats.min_rtt,
1046
+ smoothed_rtt: self.rtt(),
1047
+ latest_rtt: self.rtt_stats.latest_rtt,
1048
+ rttvar: self.rtt_stats.rttvar,
1049
+ cwnd: self.cwnd() as u64,
1050
+ bytes_in_flight: self.bytes_in_flight.get() as u64,
1051
+ ssthresh: Some(self.congestion.ssthresh.get() as u64),
1052
+ lost_packets: Some(self.congestion.lost_count as u64),
1053
+ lost_bytes: Some(self.bytes_lost),
1054
+ pto_count: Some(self.pto_count),
1055
+ ..Default::default()
1056
+ };
1057
+
1058
+ self.qlog_metrics.maybe_update(qlog_metrics)
1059
+ }
1060
+
1061
+ #[cfg(feature = "qlog")]
1062
+ fn get_updated_qlog_cc_state(
1063
+ &mut self, now: Instant,
1064
+ ) -> Option<&'static str> {
1065
+ let cc_state = self.state_str(now);
1066
+ if cc_state != self.qlog_prev_cc_state {
1067
+ self.qlog_prev_cc_state = cc_state;
1068
+ Some(cc_state)
1069
+ } else {
1070
+ None
1071
+ }
1072
+ }
1073
+
1074
+ fn send_quantum(&self) -> usize {
1075
+ self.congestion.send_quantum()
1076
+ }
1077
+
1078
+ fn get_next_release_time(&self) -> ReleaseDecision {
1079
+ ReleaseDecision {
1080
+ time: ReleaseTime::Immediate,
1081
+ allow_burst: false,
1082
+ }
1083
+ }
1084
+
1085
+ fn gcongestion_enabled(&self) -> bool {
1086
+ false
1087
+ }
1088
+
1089
+ fn lost_count(&self) -> usize {
1090
+ self.congestion.lost_count
1091
+ }
1092
+
1093
+ fn bytes_lost(&self) -> u64 {
1094
+ self.bytes_lost
1095
+ }
1096
+ }
1097
+
1098
+ impl std::fmt::Debug for LegacyRecovery {
1099
+ fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
1100
+ write!(f, "timer={:?} ", self.loss_timer)?;
1101
+ write!(f, "latest_rtt={:?} ", self.rtt_stats.latest_rtt)?;
1102
+ write!(f, "srtt={:?} ", self.rtt_stats.smoothed_rtt)?;
1103
+ write!(f, "min_rtt={:?} ", *self.rtt_stats.min_rtt)?;
1104
+ write!(f, "rttvar={:?} ", self.rtt_stats.rttvar)?;
1105
+ write!(f, "cwnd={} ", self.cwnd())?;
1106
+ write!(f, "ssthresh={} ", self.congestion.ssthresh.get())?;
1107
+ write!(f, "bytes_in_flight={} ", self.bytes_in_flight.get())?;
1108
+ write!(f, "app_limited={} ", self.congestion.app_limited)?;
1109
+ write!(
1110
+ f,
1111
+ "congestion_recovery_start_time={:?} ",
1112
+ self.congestion.congestion_recovery_start_time
1113
+ )?;
1114
+ write!(f, "{:?} ", self.congestion.delivery_rate)?;
1115
+
1116
+ if self.congestion.hystart.enabled() {
1117
+ write!(f, "hystart={:?} ", self.congestion.hystart)?;
1118
+ }
1119
+
1120
+ // CC-specific debug info
1121
+ (self.congestion.cc_ops.debug_fmt)(&self.congestion, f)?;
1122
+
1123
+ Ok(())
1124
+ }
1125
+ }
1126
+
1127
+ #[derive(Clone)]
1128
+ pub struct Acked {
1129
+ pub pkt_num: u64,
1130
+
1131
+ pub time_sent: Instant,
1132
+
1133
+ pub size: usize,
1134
+
1135
+ pub rtt: Duration,
1136
+
1137
+ pub delivered: usize,
1138
+
1139
+ pub delivered_time: Instant,
1140
+
1141
+ pub first_sent_time: Instant,
1142
+
1143
+ pub is_app_limited: bool,
1144
+ }
1145
+
1146
+ #[cfg(test)]
1147
+ mod tests {
1148
+ use super::*;
1149
+ use crate::recovery::HandshakeStatus;
1150
+ use crate::recovery::RecoveryConfig;
1151
+ use std::time::Instant;
1152
+
1153
+ #[test]
1154
+ fn test_high_pto_count_no_panic() {
1155
+ let config = crate::Config::new(crate::PROTOCOL_VERSION).unwrap();
1156
+ let recovery_config = RecoveryConfig::from_config(&config);
1157
+ let mut r = LegacyRecovery::new_with_config(&recovery_config);
1158
+
1159
+ r.pto_count = 99999;
1160
+
1161
+ let handshake_status = HandshakeStatus {
1162
+ completed: true,
1163
+ has_handshake_keys: true,
1164
+ peer_verified_address: true,
1165
+ };
1166
+ let now = Instant::now();
1167
+
1168
+ let _ = r.pto_time_and_space(handshake_status, now);
1169
+ }
1170
+ }