rwebtransport 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.lock +666 -0
- package/Cargo.toml +31 -0
- package/LICENSE +202 -0
- package/NOTICE +41 -0
- package/README.md +550 -0
- package/crates/echo-server/Cargo.toml +20 -0
- package/crates/echo-server/src/main.rs +666 -0
- package/crates/native/Cargo.toml +23 -0
- package/crates/native/src/config.rs +129 -0
- package/crates/native/src/driver.rs +463 -0
- package/crates/native/src/lib.rs +821 -0
- package/crates/native/src/server.rs +718 -0
- package/crates/native/src/session.rs +1191 -0
- package/crates/native/src/tls.rs +96 -0
- package/crates/wtcore/Cargo.toml +15 -0
- package/crates/wtcore/src/h3.rs +226 -0
- package/crates/wtcore/src/lib.rs +9 -0
- package/dist/datagrams.d.ts +16 -0
- package/dist/errors.d.ts +10 -0
- package/dist/index.d.ts +7 -0
- package/dist/index.js +1512 -0
- package/dist/index.js.map +1 -0
- package/dist/index.mjs +1473 -0
- package/dist/index.mjs.map +1 -0
- package/dist/loader.d.ts +2 -0
- package/dist/native.d.ts +226 -0
- package/dist/server.d.ts +44 -0
- package/dist/streams.d.ts +14 -0
- package/dist/types.d.ts +24 -0
- package/dist/webtransport.d.ts +20 -0
- package/package.json +79 -0
- package/prebuilds/win32-x64/rwebtransport.node +0 -0
- package/scripts/bench.mjs +180 -0
- package/scripts/build.js +67 -0
- package/scripts/postinstall.js +67 -0
- package/scripts/setup-unix.sh +129 -0
- package/scripts/setup-windows.ps1 +229 -0
- package/vendor/quiche/AGENTS.md +75 -0
- package/vendor/quiche/COPYING +23 -0
- package/vendor/quiche/Cargo.lock +1212 -0
- package/vendor/quiche/Cargo.toml +215 -0
- package/vendor/quiche/README.md +466 -0
- package/vendor/quiche/examples/Makefile +46 -0
- package/vendor/quiche/examples/README.md +20 -0
- package/vendor/quiche/examples/cert-big.crt +90 -0
- package/vendor/quiche/examples/cert.crt +18 -0
- package/vendor/quiche/examples/cert.key +28 -0
- package/vendor/quiche/examples/client.c +343 -0
- package/vendor/quiche/examples/client.rs +281 -0
- package/vendor/quiche/examples/gen-certs.sh +15 -0
- package/vendor/quiche/examples/http3-client.c +473 -0
- package/vendor/quiche/examples/http3-client.rs +349 -0
- package/vendor/quiche/examples/http3-server.c +614 -0
- package/vendor/quiche/examples/http3-server.rs +680 -0
- package/vendor/quiche/examples/qpack-decode.rs +87 -0
- package/vendor/quiche/examples/qpack-encode.rs +88 -0
- package/vendor/quiche/examples/rootca.crt +21 -0
- package/vendor/quiche/examples/server.c +529 -0
- package/vendor/quiche/examples/server.rs +529 -0
- package/vendor/quiche/include/quiche.h +1282 -0
- package/vendor/quiche/quiche.svg +1 -0
- package/vendor/quiche/src/buffers.rs +105 -0
- package/vendor/quiche/src/build.rs +66 -0
- package/vendor/quiche/src/cid.rs +1079 -0
- package/vendor/quiche/src/crypto/boringssl.rs +373 -0
- package/vendor/quiche/src/crypto/mod.rs +671 -0
- package/vendor/quiche/src/dgram.rs +118 -0
- package/vendor/quiche/src/error.rs +263 -0
- package/vendor/quiche/src/ffi.rs +2290 -0
- package/vendor/quiche/src/flowcontrol.rs +246 -0
- package/vendor/quiche/src/frame.rs +2172 -0
- package/vendor/quiche/src/h3/AGENTS.md +54 -0
- package/vendor/quiche/src/h3/ffi.rs +442 -0
- package/vendor/quiche/src/h3/frame.rs +1404 -0
- package/vendor/quiche/src/h3/mod.rs +8223 -0
- package/vendor/quiche/src/h3/qpack/decoder.rs +288 -0
- package/vendor/quiche/src/h3/qpack/encoder.rs +235 -0
- package/vendor/quiche/src/h3/qpack/mod.rs +202 -0
- package/vendor/quiche/src/h3/qpack/static_table.rs +323 -0
- package/vendor/quiche/src/h3/stream.rs +1595 -0
- package/vendor/quiche/src/lib.rs +9556 -0
- package/vendor/quiche/src/minmax.rs +446 -0
- package/vendor/quiche/src/packet.rs +2334 -0
- package/vendor/quiche/src/path.rs +1374 -0
- package/vendor/quiche/src/pmtud.rs +652 -0
- package/vendor/quiche/src/rand.rs +64 -0
- package/vendor/quiche/src/range_buf.rs +186 -0
- package/vendor/quiche/src/ranges.rs +755 -0
- package/vendor/quiche/src/recovery/AGENTS.md +80 -0
- package/vendor/quiche/src/recovery/bandwidth.rs +565 -0
- package/vendor/quiche/src/recovery/bytes_in_flight.rs +179 -0
- package/vendor/quiche/src/recovery/congestion/cubic.rs +1038 -0
- package/vendor/quiche/src/recovery/congestion/delivery_rate.rs +514 -0
- package/vendor/quiche/src/recovery/congestion/hystart.rs +241 -0
- package/vendor/quiche/src/recovery/congestion/mod.rs +361 -0
- package/vendor/quiche/src/recovery/congestion/prr.rs +238 -0
- package/vendor/quiche/src/recovery/congestion/recovery.rs +1170 -0
- package/vendor/quiche/src/recovery/congestion/reno.rs +291 -0
- package/vendor/quiche/src/recovery/congestion/test_sender.rs +201 -0
- package/vendor/quiche/src/recovery/gcongestion/bbr/bandwidth_sampler.rs +1947 -0
- package/vendor/quiche/src/recovery/gcongestion/bbr/windowed_filter.rs +158 -0
- package/vendor/quiche/src/recovery/gcongestion/bbr.rs +37 -0
- package/vendor/quiche/src/recovery/gcongestion/bbr2/drain.rs +120 -0
- package/vendor/quiche/src/recovery/gcongestion/bbr2/mode.rs +318 -0
- package/vendor/quiche/src/recovery/gcongestion/bbr2/network_model.rs +784 -0
- package/vendor/quiche/src/recovery/gcongestion/bbr2/probe_bw.rs +729 -0
- package/vendor/quiche/src/recovery/gcongestion/bbr2/probe_rtt.rs +175 -0
- package/vendor/quiche/src/recovery/gcongestion/bbr2/startup.rs +190 -0
- package/vendor/quiche/src/recovery/gcongestion/bbr2.rs +855 -0
- package/vendor/quiche/src/recovery/gcongestion/mod.rs +275 -0
- package/vendor/quiche/src/recovery/gcongestion/pacer.rs +294 -0
- package/vendor/quiche/src/recovery/gcongestion/recovery.rs +1384 -0
- package/vendor/quiche/src/recovery/mod.rs +3086 -0
- package/vendor/quiche/src/recovery/rtt.rs +152 -0
- package/vendor/quiche/src/stream/mod.rs +2210 -0
- package/vendor/quiche/src/stream/recv_buf.rs +1236 -0
- package/vendor/quiche/src/stream/send_buf.rs +849 -0
- package/vendor/quiche/src/test_utils.rs +681 -0
- package/vendor/quiche/src/tests.rs +12922 -0
- package/vendor/quiche/src/tls/boringssl.rs +375 -0
- package/vendor/quiche/src/tls/mod.rs +1245 -0
- package/vendor/quiche/src/transport_params.rs +628 -0
|
@@ -0,0 +1,1170 @@
|
|
|
1
|
+
// Copyright (C) 2018-2019, Cloudflare, Inc.
|
|
2
|
+
// All rights reserved.
|
|
3
|
+
//
|
|
4
|
+
// Redistribution and use in source and binary forms, with or without
|
|
5
|
+
// modification, are permitted provided that the following conditions are
|
|
6
|
+
// met:
|
|
7
|
+
//
|
|
8
|
+
// * Redistributions of source code must retain the above copyright notice,
|
|
9
|
+
// this list of conditions and the following disclaimer.
|
|
10
|
+
//
|
|
11
|
+
// * Redistributions in binary form must reproduce the above copyright
|
|
12
|
+
// notice, this list of conditions and the following disclaimer in the
|
|
13
|
+
// documentation and/or other materials provided with the distribution.
|
|
14
|
+
//
|
|
15
|
+
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
|
|
16
|
+
// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
|
|
17
|
+
// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
|
18
|
+
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
|
|
19
|
+
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
|
|
20
|
+
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
|
21
|
+
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
|
22
|
+
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
|
23
|
+
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
|
|
24
|
+
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
|
25
|
+
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
26
|
+
|
|
27
|
+
use std::cmp;
|
|
28
|
+
|
|
29
|
+
use std::time::Duration;
|
|
30
|
+
use std::time::Instant;
|
|
31
|
+
|
|
32
|
+
use std::collections::VecDeque;
|
|
33
|
+
|
|
34
|
+
use super::RecoveryConfig;
|
|
35
|
+
use super::Sent;
|
|
36
|
+
|
|
37
|
+
use crate::packet::Epoch;
|
|
38
|
+
use crate::ranges::RangeSet;
|
|
39
|
+
use crate::recovery::Bandwidth;
|
|
40
|
+
use crate::recovery::HandshakeStatus;
|
|
41
|
+
use crate::recovery::OnLossDetectionTimeoutOutcome;
|
|
42
|
+
use crate::recovery::RecoveryOps;
|
|
43
|
+
use crate::recovery::StartupExit;
|
|
44
|
+
use crate::Error;
|
|
45
|
+
use crate::Result;
|
|
46
|
+
|
|
47
|
+
#[cfg(feature = "qlog")]
|
|
48
|
+
use crate::recovery::QlogMetrics;
|
|
49
|
+
|
|
50
|
+
use crate::frame;
|
|
51
|
+
|
|
52
|
+
#[cfg(feature = "qlog")]
|
|
53
|
+
use qlog::events::EventData;
|
|
54
|
+
|
|
55
|
+
use super::Congestion;
|
|
56
|
+
use crate::recovery::bytes_in_flight::BytesInFlight;
|
|
57
|
+
use crate::recovery::rtt::RttStats;
|
|
58
|
+
use crate::recovery::LossDetectionTimer;
|
|
59
|
+
use crate::recovery::OnAckReceivedOutcome;
|
|
60
|
+
use crate::recovery::ReleaseDecision;
|
|
61
|
+
use crate::recovery::ReleaseTime;
|
|
62
|
+
use crate::recovery::GRANULARITY;
|
|
63
|
+
use crate::recovery::INITIAL_PACKET_THRESHOLD;
|
|
64
|
+
use crate::recovery::INITIAL_TIME_THRESHOLD;
|
|
65
|
+
use crate::recovery::MAX_OUTSTANDING_NON_ACK_ELICITING;
|
|
66
|
+
use crate::recovery::MAX_PACKET_THRESHOLD;
|
|
67
|
+
use crate::recovery::MAX_PTO_EXPONENT;
|
|
68
|
+
use crate::recovery::MAX_PTO_PROBES_COUNT;
|
|
69
|
+
use crate::recovery::PACKET_REORDER_TIME_THRESHOLD;
|
|
70
|
+
|
|
71
|
+
#[derive(Default)]
|
|
72
|
+
struct RecoveryEpoch {
|
|
73
|
+
/// The time the most recent ack-eliciting packet was sent.
|
|
74
|
+
time_of_last_ack_eliciting_packet: Option<Instant>,
|
|
75
|
+
|
|
76
|
+
/// The largest packet number acknowledged in the packet number space so
|
|
77
|
+
/// far.
|
|
78
|
+
largest_acked_packet: Option<u64>,
|
|
79
|
+
|
|
80
|
+
/// The time at which the next packet in that packet number space can be
|
|
81
|
+
/// considered lost based on exceeding the reordering window in time.
|
|
82
|
+
loss_time: Option<Instant>,
|
|
83
|
+
|
|
84
|
+
/// An association of packet numbers in a packet number space to information
|
|
85
|
+
/// about them.
|
|
86
|
+
sent_packets: VecDeque<Sent>,
|
|
87
|
+
|
|
88
|
+
loss_probes: usize,
|
|
89
|
+
in_flight_count: usize,
|
|
90
|
+
|
|
91
|
+
acked_frames: Vec<frame::Frame>,
|
|
92
|
+
|
|
93
|
+
// Frames scheduled for retransmission due to PTO are tracked
|
|
94
|
+
// separately so we can check that frames were drained before
|
|
95
|
+
// generating more PTO probes.
|
|
96
|
+
lost_frames_ack: Vec<frame::Frame>,
|
|
97
|
+
lost_frames_pto: Vec<frame::Frame>,
|
|
98
|
+
|
|
99
|
+
/// The largest packet number sent in the packet number space so far.
|
|
100
|
+
#[cfg(test)]
|
|
101
|
+
test_largest_sent_pkt_num_on_path: Option<u64>,
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
struct AckedDetectionResult {
|
|
105
|
+
acked_bytes: usize,
|
|
106
|
+
spurious_losses: usize,
|
|
107
|
+
spurious_pkt_thresh: Option<u64>,
|
|
108
|
+
has_ack_eliciting: bool,
|
|
109
|
+
has_in_flight_spurious_loss: bool,
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
struct LossDetectionResult {
|
|
113
|
+
largest_lost_pkt: Option<Sent>,
|
|
114
|
+
lost_packets: usize,
|
|
115
|
+
lost_bytes: usize,
|
|
116
|
+
pmtud_lost_bytes: usize,
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
impl RecoveryEpoch {
|
|
120
|
+
// `peer_sent_ack_ranges` should not be used without validation.
|
|
121
|
+
fn detect_and_remove_acked_packets(
|
|
122
|
+
&mut self, now: Instant, peer_sent_ack_ranges: &RangeSet,
|
|
123
|
+
newly_acked: &mut Vec<Acked>, rtt_stats: &RttStats, skip_pn: Option<u64>,
|
|
124
|
+
trace_id: &str,
|
|
125
|
+
) -> Result<AckedDetectionResult> {
|
|
126
|
+
newly_acked.clear();
|
|
127
|
+
|
|
128
|
+
let mut acked_bytes = 0;
|
|
129
|
+
let mut spurious_losses = 0;
|
|
130
|
+
let mut spurious_pkt_thresh = None;
|
|
131
|
+
let mut has_ack_eliciting = false;
|
|
132
|
+
let mut has_in_flight_spurious_loss = false;
|
|
133
|
+
|
|
134
|
+
let largest_ack_received = peer_sent_ack_ranges
|
|
135
|
+
.last()
|
|
136
|
+
.expect("ACK frames should always have at least one ack range");
|
|
137
|
+
let largest_acked = self
|
|
138
|
+
.largest_acked_packet
|
|
139
|
+
.unwrap_or(0)
|
|
140
|
+
.max(largest_ack_received);
|
|
141
|
+
|
|
142
|
+
for peer_sent_range in peer_sent_ack_ranges.iter() {
|
|
143
|
+
if skip_pn.is_some_and(|skip_pn| peer_sent_range.contains(&skip_pn)) {
|
|
144
|
+
// https://www.rfc-editor.org/rfc/rfc9000#section-13.1
|
|
145
|
+
// An endpoint SHOULD treat receipt of an acknowledgment
|
|
146
|
+
// for a packet it did not send as
|
|
147
|
+
// a connection error of type PROTOCOL_VIOLATION
|
|
148
|
+
return Err(Error::OptimisticAckDetected);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
// Because packets always have incrementing numbers, they are always
|
|
152
|
+
// in sorted order.
|
|
153
|
+
let start = if self
|
|
154
|
+
.sent_packets
|
|
155
|
+
.front()
|
|
156
|
+
.filter(|e| e.pkt_num >= peer_sent_range.start)
|
|
157
|
+
.is_some()
|
|
158
|
+
{
|
|
159
|
+
// Usually it will be the first packet.
|
|
160
|
+
0
|
|
161
|
+
} else {
|
|
162
|
+
self.sent_packets
|
|
163
|
+
.binary_search_by_key(&peer_sent_range.start, |p| p.pkt_num)
|
|
164
|
+
.unwrap_or_else(|e| e)
|
|
165
|
+
};
|
|
166
|
+
|
|
167
|
+
for unacked in self.sent_packets.range_mut(start..) {
|
|
168
|
+
if unacked.pkt_num >= peer_sent_range.end {
|
|
169
|
+
break;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
if unacked.time_acked.is_some() {
|
|
173
|
+
// Already acked.
|
|
174
|
+
} else if unacked.time_lost.is_some() {
|
|
175
|
+
// An acked packet was already declared lost.
|
|
176
|
+
spurious_losses += 1;
|
|
177
|
+
spurious_pkt_thresh
|
|
178
|
+
.get_or_insert(largest_acked - unacked.pkt_num + 1);
|
|
179
|
+
unacked.time_acked = Some(now);
|
|
180
|
+
|
|
181
|
+
if unacked.in_flight {
|
|
182
|
+
has_in_flight_spurious_loss = true;
|
|
183
|
+
}
|
|
184
|
+
} else {
|
|
185
|
+
if unacked.in_flight {
|
|
186
|
+
self.in_flight_count -= 1;
|
|
187
|
+
acked_bytes += unacked.size;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
newly_acked.push(Acked {
|
|
191
|
+
pkt_num: unacked.pkt_num,
|
|
192
|
+
time_sent: unacked.time_sent,
|
|
193
|
+
size: unacked.size,
|
|
194
|
+
|
|
195
|
+
rtt: now.saturating_duration_since(unacked.time_sent),
|
|
196
|
+
delivered: unacked.delivered,
|
|
197
|
+
delivered_time: unacked.delivered_time,
|
|
198
|
+
first_sent_time: unacked.first_sent_time,
|
|
199
|
+
is_app_limited: unacked.is_app_limited,
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
trace!("{} packet newly acked {}", trace_id, unacked.pkt_num);
|
|
203
|
+
|
|
204
|
+
self.acked_frames
|
|
205
|
+
.extend(std::mem::take(&mut unacked.frames));
|
|
206
|
+
|
|
207
|
+
has_ack_eliciting |= unacked.ack_eliciting;
|
|
208
|
+
unacked.time_acked = Some(now);
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
self.drain_acked_and_lost_packets(now - rtt_stats.rtt());
|
|
214
|
+
|
|
215
|
+
Ok(AckedDetectionResult {
|
|
216
|
+
acked_bytes,
|
|
217
|
+
spurious_losses,
|
|
218
|
+
spurious_pkt_thresh,
|
|
219
|
+
has_ack_eliciting,
|
|
220
|
+
has_in_flight_spurious_loss,
|
|
221
|
+
})
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
fn detect_lost_packets(
|
|
225
|
+
&mut self, loss_delay: Duration, pkt_thresh: u64, now: Instant,
|
|
226
|
+
trace_id: &str, epoch: Epoch,
|
|
227
|
+
) -> LossDetectionResult {
|
|
228
|
+
self.loss_time = None;
|
|
229
|
+
|
|
230
|
+
// Minimum time of kGranularity before packets are deemed lost.
|
|
231
|
+
let loss_delay = cmp::max(loss_delay, GRANULARITY);
|
|
232
|
+
let largest_acked = self.largest_acked_packet.unwrap_or(0);
|
|
233
|
+
|
|
234
|
+
// Packets sent before this time are deemed lost.
|
|
235
|
+
let lost_send_time = now.checked_sub(loss_delay).unwrap();
|
|
236
|
+
|
|
237
|
+
let mut lost_packets = 0;
|
|
238
|
+
let mut lost_bytes = 0;
|
|
239
|
+
let mut pmtud_lost_bytes = 0;
|
|
240
|
+
|
|
241
|
+
let mut largest_lost_pkt = None;
|
|
242
|
+
|
|
243
|
+
let unacked_iter = self.sent_packets
|
|
244
|
+
.iter_mut()
|
|
245
|
+
// Skip packets that follow the largest acked packet.
|
|
246
|
+
.take_while(|p| p.pkt_num <= largest_acked)
|
|
247
|
+
// Skip packets that have already been acked or lost.
|
|
248
|
+
.filter(|p| p.time_acked.is_none() && p.time_lost.is_none());
|
|
249
|
+
|
|
250
|
+
for unacked in unacked_iter {
|
|
251
|
+
// Mark packet as lost, or set time when it should be marked.
|
|
252
|
+
if unacked.time_sent <= lost_send_time ||
|
|
253
|
+
largest_acked >= unacked.pkt_num + pkt_thresh
|
|
254
|
+
{
|
|
255
|
+
self.lost_frames_ack.extend(unacked.frames.drain(..));
|
|
256
|
+
|
|
257
|
+
unacked.time_lost = Some(now);
|
|
258
|
+
|
|
259
|
+
if unacked.is_pmtud_probe {
|
|
260
|
+
pmtud_lost_bytes += unacked.size;
|
|
261
|
+
self.in_flight_count -= 1;
|
|
262
|
+
|
|
263
|
+
// Do not track PMTUD probes losses.
|
|
264
|
+
continue;
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
if unacked.in_flight {
|
|
268
|
+
lost_bytes += unacked.size;
|
|
269
|
+
|
|
270
|
+
// Frames have already been removed from the packet, so
|
|
271
|
+
// cloning the whole packet should be relatively cheap.
|
|
272
|
+
largest_lost_pkt = Some(unacked.clone());
|
|
273
|
+
|
|
274
|
+
self.in_flight_count -= 1;
|
|
275
|
+
|
|
276
|
+
trace!(
|
|
277
|
+
"{} packet {} lost on epoch {}",
|
|
278
|
+
trace_id,
|
|
279
|
+
unacked.pkt_num,
|
|
280
|
+
epoch
|
|
281
|
+
);
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
lost_packets += 1;
|
|
285
|
+
} else {
|
|
286
|
+
let loss_time = match self.loss_time {
|
|
287
|
+
None => unacked.time_sent + loss_delay,
|
|
288
|
+
|
|
289
|
+
Some(loss_time) =>
|
|
290
|
+
cmp::min(loss_time, unacked.time_sent + loss_delay),
|
|
291
|
+
};
|
|
292
|
+
|
|
293
|
+
self.loss_time = Some(loss_time);
|
|
294
|
+
break;
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
LossDetectionResult {
|
|
299
|
+
largest_lost_pkt,
|
|
300
|
+
lost_packets,
|
|
301
|
+
lost_bytes,
|
|
302
|
+
pmtud_lost_bytes,
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
fn drain_acked_and_lost_packets(&mut self, loss_thresh: Instant) {
|
|
307
|
+
// In order to avoid removing elements from the middle of the list
|
|
308
|
+
// (which would require copying other elements to compact the list),
|
|
309
|
+
// we only remove a contiguous range of elements from the start of the
|
|
310
|
+
// list.
|
|
311
|
+
//
|
|
312
|
+
// This means that acked or lost elements coming after this will not
|
|
313
|
+
// be removed at this point, but their removal is delayed for a later
|
|
314
|
+
// time, once the gaps have been filled.
|
|
315
|
+
while let Some(pkt) = self.sent_packets.front() {
|
|
316
|
+
if let Some(time_lost) = pkt.time_lost {
|
|
317
|
+
if time_lost > loss_thresh {
|
|
318
|
+
break;
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
if pkt.time_acked.is_none() && pkt.time_lost.is_none() {
|
|
323
|
+
break;
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
self.sent_packets.pop_front();
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
/// Returns the next lost frame, trying ACK-based lost frames first,
|
|
331
|
+
/// then PTO-based lost frames.
|
|
332
|
+
fn next_lost_frame(&mut self) -> Option<frame::Frame> {
|
|
333
|
+
self.lost_frames_ack
|
|
334
|
+
.pop()
|
|
335
|
+
.or_else(|| self.lost_frames_pto.pop())
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
/// Returns true if there are any lost frames (ACK or PTO).
|
|
339
|
+
fn has_lost_frames(&self) -> bool {
|
|
340
|
+
!self.lost_frames_ack.is_empty() || !self.lost_frames_pto.is_empty()
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
/// Returns the total count of lost frames (ACK + PTO).
|
|
344
|
+
#[cfg(test)]
|
|
345
|
+
fn lost_frames_count(&self) -> usize {
|
|
346
|
+
self.lost_frames_ack.len() + self.lost_frames_pto.len()
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/// Clears all lost frames (both ACK and PTO).
|
|
350
|
+
fn clear_lost_frames(&mut self) {
|
|
351
|
+
self.lost_frames_ack.clear();
|
|
352
|
+
self.lost_frames_pto.clear();
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
pub struct LegacyRecovery {
|
|
357
|
+
epochs: [RecoveryEpoch; Epoch::count()],
|
|
358
|
+
|
|
359
|
+
loss_timer: LossDetectionTimer,
|
|
360
|
+
|
|
361
|
+
pto_count: u32,
|
|
362
|
+
|
|
363
|
+
rtt_stats: RttStats,
|
|
364
|
+
|
|
365
|
+
lost_spurious_count: usize,
|
|
366
|
+
|
|
367
|
+
pkt_thresh: u64,
|
|
368
|
+
|
|
369
|
+
time_thresh: f64,
|
|
370
|
+
|
|
371
|
+
bytes_in_flight: BytesInFlight,
|
|
372
|
+
|
|
373
|
+
bytes_sent: usize,
|
|
374
|
+
|
|
375
|
+
bytes_lost: u64,
|
|
376
|
+
|
|
377
|
+
pub max_datagram_size: usize,
|
|
378
|
+
|
|
379
|
+
#[cfg(feature = "qlog")]
|
|
380
|
+
qlog_metrics: QlogMetrics,
|
|
381
|
+
|
|
382
|
+
#[cfg(feature = "qlog")]
|
|
383
|
+
qlog_prev_cc_state: &'static str,
|
|
384
|
+
|
|
385
|
+
/// How many non-ack-eliciting packets have been sent.
|
|
386
|
+
outstanding_non_ack_eliciting: usize,
|
|
387
|
+
|
|
388
|
+
pub congestion: Congestion,
|
|
389
|
+
|
|
390
|
+
/// A resusable list of acks.
|
|
391
|
+
newly_acked: Vec<Acked>,
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
impl LegacyRecovery {
|
|
395
|
+
pub fn new_with_config(recovery_config: &RecoveryConfig) -> Self {
|
|
396
|
+
Self {
|
|
397
|
+
epochs: Default::default(),
|
|
398
|
+
|
|
399
|
+
loss_timer: Default::default(),
|
|
400
|
+
|
|
401
|
+
pto_count: 0,
|
|
402
|
+
|
|
403
|
+
rtt_stats: RttStats::new(
|
|
404
|
+
recovery_config.initial_rtt,
|
|
405
|
+
recovery_config.max_ack_delay,
|
|
406
|
+
),
|
|
407
|
+
|
|
408
|
+
lost_spurious_count: 0,
|
|
409
|
+
|
|
410
|
+
pkt_thresh: INITIAL_PACKET_THRESHOLD,
|
|
411
|
+
|
|
412
|
+
time_thresh: INITIAL_TIME_THRESHOLD,
|
|
413
|
+
|
|
414
|
+
bytes_in_flight: Default::default(),
|
|
415
|
+
|
|
416
|
+
bytes_sent: 0,
|
|
417
|
+
|
|
418
|
+
bytes_lost: 0,
|
|
419
|
+
|
|
420
|
+
max_datagram_size: recovery_config.max_send_udp_payload_size,
|
|
421
|
+
|
|
422
|
+
#[cfg(feature = "qlog")]
|
|
423
|
+
qlog_metrics: QlogMetrics::default(),
|
|
424
|
+
|
|
425
|
+
#[cfg(feature = "qlog")]
|
|
426
|
+
qlog_prev_cc_state: "",
|
|
427
|
+
|
|
428
|
+
outstanding_non_ack_eliciting: 0,
|
|
429
|
+
|
|
430
|
+
congestion: Congestion::from_config(recovery_config),
|
|
431
|
+
|
|
432
|
+
newly_acked: Vec::new(),
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
#[cfg(test)]
|
|
437
|
+
pub fn new(config: &crate::Config) -> Self {
|
|
438
|
+
Self::new_with_config(&RecoveryConfig::from_config(config))
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
fn loss_time_and_space(&self) -> (Option<Instant>, Epoch) {
|
|
442
|
+
let mut epoch = Epoch::Initial;
|
|
443
|
+
let mut time = self.epochs[epoch].loss_time;
|
|
444
|
+
|
|
445
|
+
// Iterate over all packet number spaces starting from Handshake.
|
|
446
|
+
for e in [Epoch::Handshake, Epoch::Application] {
|
|
447
|
+
let new_time = self.epochs[e].loss_time;
|
|
448
|
+
if time.is_none() || new_time < time {
|
|
449
|
+
time = new_time;
|
|
450
|
+
epoch = e;
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
(time, epoch)
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
fn pto_time_and_space(
|
|
458
|
+
&self, handshake_status: HandshakeStatus, now: Instant,
|
|
459
|
+
) -> (Option<Instant>, Epoch) {
|
|
460
|
+
let mut duration =
|
|
461
|
+
self.pto() * 2_u32.pow(self.pto_count.min(MAX_PTO_EXPONENT));
|
|
462
|
+
|
|
463
|
+
// Arm PTO from now when there are no inflight packets.
|
|
464
|
+
if self.bytes_in_flight.is_zero() {
|
|
465
|
+
if handshake_status.has_handshake_keys {
|
|
466
|
+
return (Some(now + duration), Epoch::Handshake);
|
|
467
|
+
} else {
|
|
468
|
+
return (Some(now + duration), Epoch::Initial);
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
let mut pto_timeout = None;
|
|
473
|
+
let mut pto_space = Epoch::Initial;
|
|
474
|
+
|
|
475
|
+
// Iterate over all packet number spaces.
|
|
476
|
+
for e in [Epoch::Initial, Epoch::Handshake, Epoch::Application] {
|
|
477
|
+
let epoch = &self.epochs[e];
|
|
478
|
+
if epoch.in_flight_count == 0 {
|
|
479
|
+
continue;
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
if e == Epoch::Application {
|
|
483
|
+
// Skip Application Data until handshake completes.
|
|
484
|
+
if !handshake_status.completed {
|
|
485
|
+
return (pto_timeout, pto_space);
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
// Include max_ack_delay and backoff for Application Data.
|
|
489
|
+
duration += self.rtt_stats.max_ack_delay *
|
|
490
|
+
2_u32.pow(self.pto_count.min(MAX_PTO_EXPONENT));
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
let new_time = epoch
|
|
494
|
+
.time_of_last_ack_eliciting_packet
|
|
495
|
+
.map(|t| t + duration);
|
|
496
|
+
|
|
497
|
+
if pto_timeout.is_none() || new_time < pto_timeout {
|
|
498
|
+
pto_timeout = new_time;
|
|
499
|
+
pto_space = e;
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
(pto_timeout, pto_space)
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
fn set_loss_detection_timer(
|
|
507
|
+
&mut self, handshake_status: HandshakeStatus, now: Instant,
|
|
508
|
+
) {
|
|
509
|
+
let (earliest_loss_time, _) = self.loss_time_and_space();
|
|
510
|
+
|
|
511
|
+
if let Some(to) = earliest_loss_time {
|
|
512
|
+
// Time threshold loss detection.
|
|
513
|
+
self.loss_timer.update(to);
|
|
514
|
+
return;
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
if self.bytes_in_flight.is_zero() &&
|
|
518
|
+
handshake_status.peer_verified_address
|
|
519
|
+
{
|
|
520
|
+
self.loss_timer.clear();
|
|
521
|
+
return;
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
// PTO timer.
|
|
525
|
+
if let (Some(timeout), _) = self.pto_time_and_space(handshake_status, now)
|
|
526
|
+
{
|
|
527
|
+
self.loss_timer.update(timeout);
|
|
528
|
+
} else {
|
|
529
|
+
self.loss_timer.clear();
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
fn detect_lost_packets(
|
|
534
|
+
&mut self, epoch: Epoch, now: Instant, trace_id: &str,
|
|
535
|
+
) -> (usize, usize) {
|
|
536
|
+
let loss_delay = cmp::max(self.rtt_stats.latest_rtt, self.rtt())
|
|
537
|
+
.mul_f64(self.time_thresh);
|
|
538
|
+
|
|
539
|
+
let loss = self.epochs[epoch].detect_lost_packets(
|
|
540
|
+
loss_delay,
|
|
541
|
+
self.pkt_thresh,
|
|
542
|
+
now,
|
|
543
|
+
trace_id,
|
|
544
|
+
epoch,
|
|
545
|
+
);
|
|
546
|
+
|
|
547
|
+
if let Some(pkt) = loss.largest_lost_pkt {
|
|
548
|
+
if !self.congestion.in_congestion_recovery(pkt.time_sent) {
|
|
549
|
+
(self.congestion.cc_ops.checkpoint)(&mut self.congestion);
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
(self.congestion.cc_ops.congestion_event)(
|
|
553
|
+
&mut self.congestion,
|
|
554
|
+
self.bytes_in_flight.get(),
|
|
555
|
+
loss.lost_bytes,
|
|
556
|
+
&pkt,
|
|
557
|
+
now,
|
|
558
|
+
);
|
|
559
|
+
|
|
560
|
+
self.bytes_in_flight
|
|
561
|
+
.saturating_subtract(loss.lost_bytes, now);
|
|
562
|
+
};
|
|
563
|
+
|
|
564
|
+
self.bytes_in_flight
|
|
565
|
+
.saturating_subtract(loss.pmtud_lost_bytes, now);
|
|
566
|
+
|
|
567
|
+
self.epochs[epoch]
|
|
568
|
+
.drain_acked_and_lost_packets(now - self.rtt_stats.rtt());
|
|
569
|
+
|
|
570
|
+
self.congestion.lost_count += loss.lost_packets;
|
|
571
|
+
|
|
572
|
+
(loss.lost_packets, loss.lost_bytes)
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
impl RecoveryOps for LegacyRecovery {
|
|
577
|
+
/// Returns whether or not we should elicit an ACK even if we wouldn't
|
|
578
|
+
/// otherwise have constructed an ACK eliciting packet.
|
|
579
|
+
fn should_elicit_ack(&self, epoch: Epoch) -> bool {
|
|
580
|
+
self.epochs[epoch].loss_probes > 0 ||
|
|
581
|
+
self.outstanding_non_ack_eliciting >=
|
|
582
|
+
MAX_OUTSTANDING_NON_ACK_ELICITING
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
fn next_acked_frame(&mut self, epoch: Epoch) -> Option<frame::Frame> {
|
|
586
|
+
self.epochs[epoch].acked_frames.pop()
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
fn next_lost_frame(&mut self, epoch: Epoch) -> Option<frame::Frame> {
|
|
590
|
+
self.epochs[epoch].next_lost_frame()
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
fn get_largest_acked_on_epoch(&self, epoch: Epoch) -> Option<u64> {
|
|
594
|
+
self.epochs[epoch].largest_acked_packet
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
fn has_lost_frames(&self, epoch: Epoch) -> bool {
|
|
598
|
+
self.epochs[epoch].has_lost_frames()
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
fn loss_probes(&self, epoch: Epoch) -> usize {
|
|
602
|
+
self.epochs[epoch].loss_probes
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
#[cfg(test)]
|
|
606
|
+
fn inc_loss_probes(&mut self, epoch: Epoch) {
|
|
607
|
+
self.epochs[epoch].loss_probes += 1;
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
#[cfg(test)]
|
|
611
|
+
fn lost_frames_count(&self, epoch: Epoch) -> usize {
|
|
612
|
+
self.epochs[epoch].lost_frames_count()
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
fn ping_sent(&mut self, epoch: Epoch) {
|
|
616
|
+
self.epochs[epoch].loss_probes =
|
|
617
|
+
self.epochs[epoch].loss_probes.saturating_sub(1);
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
fn on_packet_sent(
|
|
621
|
+
&mut self, mut pkt: Sent, epoch: Epoch,
|
|
622
|
+
handshake_status: HandshakeStatus, now: Instant, trace_id: &str,
|
|
623
|
+
) {
|
|
624
|
+
let ack_eliciting = pkt.ack_eliciting;
|
|
625
|
+
let in_flight = pkt.in_flight;
|
|
626
|
+
let sent_bytes = pkt.size;
|
|
627
|
+
|
|
628
|
+
if ack_eliciting {
|
|
629
|
+
self.outstanding_non_ack_eliciting = 0;
|
|
630
|
+
} else {
|
|
631
|
+
self.outstanding_non_ack_eliciting += 1;
|
|
632
|
+
}
|
|
633
|
+
|
|
634
|
+
if in_flight && ack_eliciting {
|
|
635
|
+
self.epochs[epoch].time_of_last_ack_eliciting_packet = Some(now);
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
self.congestion.on_packet_sent(
|
|
639
|
+
self.bytes_in_flight.get(),
|
|
640
|
+
sent_bytes,
|
|
641
|
+
now,
|
|
642
|
+
&mut pkt,
|
|
643
|
+
self.bytes_lost,
|
|
644
|
+
in_flight,
|
|
645
|
+
);
|
|
646
|
+
|
|
647
|
+
if in_flight {
|
|
648
|
+
self.epochs[epoch].in_flight_count += 1;
|
|
649
|
+
self.bytes_in_flight.add(sent_bytes, now);
|
|
650
|
+
|
|
651
|
+
self.set_loss_detection_timer(handshake_status, now);
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
self.bytes_sent += sent_bytes;
|
|
655
|
+
|
|
656
|
+
#[cfg(test)]
|
|
657
|
+
{
|
|
658
|
+
self.epochs[epoch].test_largest_sent_pkt_num_on_path = self.epochs
|
|
659
|
+
[epoch]
|
|
660
|
+
.test_largest_sent_pkt_num_on_path
|
|
661
|
+
.max(Some(pkt.pkt_num));
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
self.epochs[epoch].sent_packets.push_back(pkt);
|
|
665
|
+
|
|
666
|
+
trace!("{trace_id} {self:?}");
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
fn get_packet_send_time(&self, now: Instant) -> Instant {
|
|
670
|
+
now
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
// `peer_sent_ack_ranges` should not be used without validation.
|
|
674
|
+
fn on_ack_received(
|
|
675
|
+
&mut self, peer_sent_ack_ranges: &RangeSet, ack_delay: u64, epoch: Epoch,
|
|
676
|
+
handshake_status: HandshakeStatus, now: Instant, skip_pn: Option<u64>,
|
|
677
|
+
trace_id: &str,
|
|
678
|
+
) -> Result<OnAckReceivedOutcome> {
|
|
679
|
+
let AckedDetectionResult {
|
|
680
|
+
acked_bytes,
|
|
681
|
+
spurious_losses,
|
|
682
|
+
spurious_pkt_thresh,
|
|
683
|
+
has_ack_eliciting,
|
|
684
|
+
has_in_flight_spurious_loss,
|
|
685
|
+
} = self.epochs[epoch].detect_and_remove_acked_packets(
|
|
686
|
+
now,
|
|
687
|
+
peer_sent_ack_ranges,
|
|
688
|
+
&mut self.newly_acked,
|
|
689
|
+
&self.rtt_stats,
|
|
690
|
+
skip_pn,
|
|
691
|
+
trace_id,
|
|
692
|
+
)?;
|
|
693
|
+
|
|
694
|
+
self.lost_spurious_count += spurious_losses;
|
|
695
|
+
if let Some(thresh) = spurious_pkt_thresh {
|
|
696
|
+
self.pkt_thresh =
|
|
697
|
+
self.pkt_thresh.max(thresh.min(MAX_PACKET_THRESHOLD));
|
|
698
|
+
self.time_thresh = PACKET_REORDER_TIME_THRESHOLD;
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
// Undo congestion window update.
|
|
702
|
+
if has_in_flight_spurious_loss {
|
|
703
|
+
(self.congestion.cc_ops.rollback)(&mut self.congestion);
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
if self.newly_acked.is_empty() {
|
|
707
|
+
return Ok(OnAckReceivedOutcome::default());
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
let largest_newly_acked = self.newly_acked.last().unwrap();
|
|
711
|
+
|
|
712
|
+
// Update `largest_acked_packet` based on the validated `newly_acked`
|
|
713
|
+
// value.
|
|
714
|
+
let largest_acked_pkt_num = self.epochs[epoch]
|
|
715
|
+
.largest_acked_packet
|
|
716
|
+
.unwrap_or(0)
|
|
717
|
+
.max(largest_newly_acked.pkt_num);
|
|
718
|
+
self.epochs[epoch].largest_acked_packet = Some(largest_acked_pkt_num);
|
|
719
|
+
|
|
720
|
+
// Check if largest packet is newly acked.
|
|
721
|
+
if largest_newly_acked.pkt_num == largest_acked_pkt_num &&
|
|
722
|
+
has_ack_eliciting
|
|
723
|
+
{
|
|
724
|
+
let latest_rtt = now - largest_newly_acked.time_sent;
|
|
725
|
+
self.rtt_stats.update_rtt(
|
|
726
|
+
latest_rtt,
|
|
727
|
+
Duration::from_micros(ack_delay),
|
|
728
|
+
now,
|
|
729
|
+
handshake_status.completed,
|
|
730
|
+
);
|
|
731
|
+
}
|
|
732
|
+
|
|
733
|
+
// Detect and mark lost packets without removing them from the sent
|
|
734
|
+
// packets list.
|
|
735
|
+
let (lost_packets, lost_bytes) =
|
|
736
|
+
self.detect_lost_packets(epoch, now, trace_id);
|
|
737
|
+
|
|
738
|
+
self.congestion.on_packets_acked(
|
|
739
|
+
self.bytes_in_flight.get(),
|
|
740
|
+
&mut self.newly_acked,
|
|
741
|
+
&self.rtt_stats,
|
|
742
|
+
now,
|
|
743
|
+
);
|
|
744
|
+
|
|
745
|
+
self.bytes_in_flight.saturating_subtract(acked_bytes, now);
|
|
746
|
+
|
|
747
|
+
self.pto_count = 0;
|
|
748
|
+
|
|
749
|
+
self.set_loss_detection_timer(handshake_status, now);
|
|
750
|
+
|
|
751
|
+
self.epochs[epoch]
|
|
752
|
+
.drain_acked_and_lost_packets(now - self.rtt_stats.rtt());
|
|
753
|
+
|
|
754
|
+
Ok(OnAckReceivedOutcome {
|
|
755
|
+
lost_packets,
|
|
756
|
+
lost_bytes,
|
|
757
|
+
acked_bytes,
|
|
758
|
+
spurious_losses,
|
|
759
|
+
})
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
fn on_loss_detection_timeout(
|
|
763
|
+
&mut self, handshake_status: HandshakeStatus, now: Instant,
|
|
764
|
+
trace_id: &str,
|
|
765
|
+
) -> OnLossDetectionTimeoutOutcome {
|
|
766
|
+
let (earliest_loss_time, epoch) = self.loss_time_and_space();
|
|
767
|
+
|
|
768
|
+
if earliest_loss_time.is_some() {
|
|
769
|
+
// Time threshold loss detection.
|
|
770
|
+
let (lost_packets, lost_bytes) =
|
|
771
|
+
self.detect_lost_packets(epoch, now, trace_id);
|
|
772
|
+
|
|
773
|
+
self.set_loss_detection_timer(handshake_status, now);
|
|
774
|
+
|
|
775
|
+
trace!("{trace_id} {self:?}");
|
|
776
|
+
return OnLossDetectionTimeoutOutcome {
|
|
777
|
+
lost_packets,
|
|
778
|
+
lost_bytes,
|
|
779
|
+
};
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
let epoch = if self.bytes_in_flight.get() > 0 {
|
|
783
|
+
// Send new data if available, else retransmit old data. If neither
|
|
784
|
+
// is available, send a single PING frame.
|
|
785
|
+
let (_, e) = self.pto_time_and_space(handshake_status, now);
|
|
786
|
+
|
|
787
|
+
e
|
|
788
|
+
} else {
|
|
789
|
+
// Client sends an anti-deadlock packet: Initial is padded to earn
|
|
790
|
+
// more anti-amplification credit, a Handshake packet proves address
|
|
791
|
+
// ownership.
|
|
792
|
+
if handshake_status.has_handshake_keys {
|
|
793
|
+
Epoch::Handshake
|
|
794
|
+
} else {
|
|
795
|
+
Epoch::Initial
|
|
796
|
+
}
|
|
797
|
+
};
|
|
798
|
+
|
|
799
|
+
self.pto_count += 1;
|
|
800
|
+
|
|
801
|
+
let epoch = &mut self.epochs[epoch];
|
|
802
|
+
|
|
803
|
+
epoch.loss_probes =
|
|
804
|
+
cmp::min(self.pto_count as usize, MAX_PTO_PROBES_COUNT);
|
|
805
|
+
|
|
806
|
+
let sent_packets_iter_limit = if !epoch.lost_frames_pto.is_empty() {
|
|
807
|
+
// Skip the search for frames to add to PTO probes if frames
|
|
808
|
+
// added in a prior PTO haven't been processed yet.
|
|
809
|
+
0
|
|
810
|
+
} else {
|
|
811
|
+
usize::MAX
|
|
812
|
+
};
|
|
813
|
+
|
|
814
|
+
let unacked_iter = epoch.sent_packets
|
|
815
|
+
.iter()
|
|
816
|
+
.take(sent_packets_iter_limit)
|
|
817
|
+
// Skip packets that have already been acked or lost, and packets
|
|
818
|
+
// that don't contain either CRYPTO or STREAM frames.
|
|
819
|
+
.filter(|p| p.has_data && p.time_acked.is_none() && p.time_lost.is_none())
|
|
820
|
+
// Only return as many packets as the number of probe packets that
|
|
821
|
+
// will be sent.
|
|
822
|
+
.take(epoch.loss_probes);
|
|
823
|
+
|
|
824
|
+
// Retransmit the frames from the oldest sent packets on PTO. However
|
|
825
|
+
// the packets are not actually declared lost (so there is no effect to
|
|
826
|
+
// congestion control), we just reschedule the data they carried.
|
|
827
|
+
//
|
|
828
|
+
// This will also trigger sending an ACK and retransmitting frames like
|
|
829
|
+
// HANDSHAKE_DONE and MAX_DATA / MAX_STREAM_DATA as well, in addition
|
|
830
|
+
// to CRYPTO and STREAM, if the original packet carried them.
|
|
831
|
+
for unacked in unacked_iter {
|
|
832
|
+
epoch.lost_frames_pto.extend_from_slice(&unacked.frames);
|
|
833
|
+
}
|
|
834
|
+
|
|
835
|
+
self.set_loss_detection_timer(handshake_status, now);
|
|
836
|
+
|
|
837
|
+
trace!("{trace_id} {self:?}");
|
|
838
|
+
|
|
839
|
+
OnLossDetectionTimeoutOutcome {
|
|
840
|
+
lost_packets: 0,
|
|
841
|
+
lost_bytes: 0,
|
|
842
|
+
}
|
|
843
|
+
}
|
|
844
|
+
|
|
845
|
+
fn on_pkt_num_space_discarded(
|
|
846
|
+
&mut self, epoch: Epoch, handshake_status: HandshakeStatus, now: Instant,
|
|
847
|
+
) {
|
|
848
|
+
let epoch = &mut self.epochs[epoch];
|
|
849
|
+
|
|
850
|
+
let unacked_bytes = epoch
|
|
851
|
+
.sent_packets
|
|
852
|
+
.iter()
|
|
853
|
+
.filter(|p| {
|
|
854
|
+
p.in_flight && p.time_acked.is_none() && p.time_lost.is_none()
|
|
855
|
+
})
|
|
856
|
+
.fold(0, |acc, p| acc + p.size);
|
|
857
|
+
|
|
858
|
+
self.bytes_in_flight.saturating_subtract(unacked_bytes, now);
|
|
859
|
+
|
|
860
|
+
epoch.sent_packets.clear();
|
|
861
|
+
epoch.clear_lost_frames();
|
|
862
|
+
epoch.acked_frames.clear();
|
|
863
|
+
|
|
864
|
+
epoch.time_of_last_ack_eliciting_packet = None;
|
|
865
|
+
epoch.loss_time = None;
|
|
866
|
+
epoch.loss_probes = 0;
|
|
867
|
+
epoch.in_flight_count = 0;
|
|
868
|
+
|
|
869
|
+
self.set_loss_detection_timer(handshake_status, now);
|
|
870
|
+
}
|
|
871
|
+
|
|
872
|
+
fn on_path_change(
|
|
873
|
+
&mut self, epoch: Epoch, now: Instant, trace_id: &str,
|
|
874
|
+
) -> (usize, usize) {
|
|
875
|
+
// Time threshold loss detection.
|
|
876
|
+
self.detect_lost_packets(epoch, now, trace_id)
|
|
877
|
+
}
|
|
878
|
+
|
|
879
|
+
fn loss_detection_timer(&self) -> Option<Instant> {
|
|
880
|
+
self.loss_timer.time
|
|
881
|
+
}
|
|
882
|
+
|
|
883
|
+
fn cwnd(&self) -> usize {
|
|
884
|
+
self.congestion.congestion_window()
|
|
885
|
+
}
|
|
886
|
+
|
|
887
|
+
fn cwnd_available(&self) -> usize {
|
|
888
|
+
// Ignore cwnd when sending probe packets.
|
|
889
|
+
if self.epochs.iter().any(|e| e.loss_probes > 0) {
|
|
890
|
+
return usize::MAX;
|
|
891
|
+
}
|
|
892
|
+
|
|
893
|
+
// Open more space (snd_cnt) for PRR when allowed.
|
|
894
|
+
self.cwnd().saturating_sub(self.bytes_in_flight.get()) +
|
|
895
|
+
self.congestion.prr.snd_cnt
|
|
896
|
+
}
|
|
897
|
+
|
|
898
|
+
fn rtt(&self) -> Duration {
|
|
899
|
+
self.rtt_stats.rtt()
|
|
900
|
+
}
|
|
901
|
+
|
|
902
|
+
fn min_rtt(&self) -> Option<Duration> {
|
|
903
|
+
self.rtt_stats.min_rtt()
|
|
904
|
+
}
|
|
905
|
+
|
|
906
|
+
fn max_rtt(&self) -> Option<Duration> {
|
|
907
|
+
self.rtt_stats.max_rtt()
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
fn rttvar(&self) -> Duration {
|
|
911
|
+
self.rtt_stats.rttvar
|
|
912
|
+
}
|
|
913
|
+
|
|
914
|
+
fn pto(&self) -> Duration {
|
|
915
|
+
self.rtt() + cmp::max(self.rtt_stats.rttvar * 4, GRANULARITY)
|
|
916
|
+
}
|
|
917
|
+
|
|
918
|
+
/// The most recent data delivery rate estimate.
|
|
919
|
+
fn delivery_rate(&self) -> Bandwidth {
|
|
920
|
+
self.congestion.delivery_rate()
|
|
921
|
+
}
|
|
922
|
+
|
|
923
|
+
fn max_bandwidth(&self) -> Option<Bandwidth> {
|
|
924
|
+
// TODO implement
|
|
925
|
+
None
|
|
926
|
+
}
|
|
927
|
+
|
|
928
|
+
/// Statistics from when a CCA first exited the startup phase.
|
|
929
|
+
fn startup_exit(&self) -> Option<StartupExit> {
|
|
930
|
+
self.congestion.ssthresh.startup_exit()
|
|
931
|
+
}
|
|
932
|
+
|
|
933
|
+
fn max_datagram_size(&self) -> usize {
|
|
934
|
+
self.max_datagram_size
|
|
935
|
+
}
|
|
936
|
+
|
|
937
|
+
fn pmtud_update_max_datagram_size(&mut self, new_max_datagram_size: usize) {
|
|
938
|
+
// Congestion Window is updated only when it's not updated already.
|
|
939
|
+
// Update cwnd if it hasn't been updated yet.
|
|
940
|
+
if self.cwnd() ==
|
|
941
|
+
self.max_datagram_size *
|
|
942
|
+
self.congestion.initial_congestion_window_packets
|
|
943
|
+
{
|
|
944
|
+
self.congestion.congestion_window = new_max_datagram_size *
|
|
945
|
+
self.congestion.initial_congestion_window_packets;
|
|
946
|
+
}
|
|
947
|
+
|
|
948
|
+
self.max_datagram_size = new_max_datagram_size;
|
|
949
|
+
}
|
|
950
|
+
|
|
951
|
+
fn update_max_datagram_size(&mut self, new_max_datagram_size: usize) {
|
|
952
|
+
self.pmtud_update_max_datagram_size(
|
|
953
|
+
self.max_datagram_size.min(new_max_datagram_size),
|
|
954
|
+
)
|
|
955
|
+
}
|
|
956
|
+
|
|
957
|
+
#[cfg(test)]
|
|
958
|
+
fn sent_packets_len(&self, epoch: Epoch) -> usize {
|
|
959
|
+
self.epochs[epoch].sent_packets.len()
|
|
960
|
+
}
|
|
961
|
+
|
|
962
|
+
#[cfg(test)]
|
|
963
|
+
fn in_flight_count(&self, epoch: Epoch) -> usize {
|
|
964
|
+
self.epochs[epoch].in_flight_count
|
|
965
|
+
}
|
|
966
|
+
|
|
967
|
+
fn bytes_in_flight(&self) -> usize {
|
|
968
|
+
self.bytes_in_flight.get()
|
|
969
|
+
}
|
|
970
|
+
|
|
971
|
+
fn bytes_in_flight_duration(&self) -> Duration {
|
|
972
|
+
self.bytes_in_flight.get_duration()
|
|
973
|
+
}
|
|
974
|
+
|
|
975
|
+
#[cfg(test)]
|
|
976
|
+
fn pacing_rate(&self) -> u64 {
|
|
977
|
+
0
|
|
978
|
+
}
|
|
979
|
+
|
|
980
|
+
#[cfg(test)]
|
|
981
|
+
fn pto_count(&self) -> u32 {
|
|
982
|
+
self.pto_count
|
|
983
|
+
}
|
|
984
|
+
|
|
985
|
+
#[cfg(test)]
|
|
986
|
+
fn pkt_thresh(&self) -> Option<u64> {
|
|
987
|
+
Some(self.pkt_thresh)
|
|
988
|
+
}
|
|
989
|
+
|
|
990
|
+
#[cfg(test)]
|
|
991
|
+
fn time_thresh(&self) -> f64 {
|
|
992
|
+
self.time_thresh
|
|
993
|
+
}
|
|
994
|
+
|
|
995
|
+
#[cfg(test)]
|
|
996
|
+
fn lost_spurious_count(&self) -> usize {
|
|
997
|
+
self.lost_spurious_count
|
|
998
|
+
}
|
|
999
|
+
|
|
1000
|
+
#[cfg(test)]
|
|
1001
|
+
fn detect_lost_packets_for_test(
|
|
1002
|
+
&mut self, epoch: Epoch, now: Instant,
|
|
1003
|
+
) -> (usize, usize) {
|
|
1004
|
+
self.detect_lost_packets(epoch, now, "")
|
|
1005
|
+
}
|
|
1006
|
+
|
|
1007
|
+
// FIXME only used by gcongestion
|
|
1008
|
+
fn on_app_limited(&mut self) {
|
|
1009
|
+
// Not implemented for legacy recovery, update_app_limited and
|
|
1010
|
+
// delivery_rate_update_app_limited used instead.
|
|
1011
|
+
}
|
|
1012
|
+
|
|
1013
|
+
#[cfg(test)]
|
|
1014
|
+
fn largest_sent_pkt_num_on_path(&self, epoch: Epoch) -> Option<u64> {
|
|
1015
|
+
self.epochs[epoch].test_largest_sent_pkt_num_on_path
|
|
1016
|
+
}
|
|
1017
|
+
|
|
1018
|
+
#[cfg(test)]
|
|
1019
|
+
fn app_limited(&self) -> bool {
|
|
1020
|
+
self.congestion.app_limited
|
|
1021
|
+
}
|
|
1022
|
+
|
|
1023
|
+
fn update_app_limited(&mut self, v: bool) {
|
|
1024
|
+
self.congestion.update_app_limited(v);
|
|
1025
|
+
}
|
|
1026
|
+
|
|
1027
|
+
// FIXME only used by congestion
|
|
1028
|
+
fn delivery_rate_update_app_limited(&mut self, v: bool) {
|
|
1029
|
+
self.congestion.delivery_rate.update_app_limited(v);
|
|
1030
|
+
}
|
|
1031
|
+
|
|
1032
|
+
// FIXME only used by congestion
|
|
1033
|
+
fn update_max_ack_delay(&mut self, max_ack_delay: Duration) {
|
|
1034
|
+
self.rtt_stats.max_ack_delay = max_ack_delay;
|
|
1035
|
+
}
|
|
1036
|
+
|
|
1037
|
+
#[cfg(feature = "qlog")]
|
|
1038
|
+
fn state_str(&self, now: Instant) -> &'static str {
|
|
1039
|
+
(self.congestion.cc_ops.state_str)(&self.congestion, now)
|
|
1040
|
+
}
|
|
1041
|
+
|
|
1042
|
+
#[cfg(feature = "qlog")]
|
|
1043
|
+
fn get_updated_qlog_event_data(&mut self) -> Option<EventData> {
|
|
1044
|
+
let qlog_metrics = QlogMetrics {
|
|
1045
|
+
min_rtt: *self.rtt_stats.min_rtt,
|
|
1046
|
+
smoothed_rtt: self.rtt(),
|
|
1047
|
+
latest_rtt: self.rtt_stats.latest_rtt,
|
|
1048
|
+
rttvar: self.rtt_stats.rttvar,
|
|
1049
|
+
cwnd: self.cwnd() as u64,
|
|
1050
|
+
bytes_in_flight: self.bytes_in_flight.get() as u64,
|
|
1051
|
+
ssthresh: Some(self.congestion.ssthresh.get() as u64),
|
|
1052
|
+
lost_packets: Some(self.congestion.lost_count as u64),
|
|
1053
|
+
lost_bytes: Some(self.bytes_lost),
|
|
1054
|
+
pto_count: Some(self.pto_count),
|
|
1055
|
+
..Default::default()
|
|
1056
|
+
};
|
|
1057
|
+
|
|
1058
|
+
self.qlog_metrics.maybe_update(qlog_metrics)
|
|
1059
|
+
}
|
|
1060
|
+
|
|
1061
|
+
#[cfg(feature = "qlog")]
|
|
1062
|
+
fn get_updated_qlog_cc_state(
|
|
1063
|
+
&mut self, now: Instant,
|
|
1064
|
+
) -> Option<&'static str> {
|
|
1065
|
+
let cc_state = self.state_str(now);
|
|
1066
|
+
if cc_state != self.qlog_prev_cc_state {
|
|
1067
|
+
self.qlog_prev_cc_state = cc_state;
|
|
1068
|
+
Some(cc_state)
|
|
1069
|
+
} else {
|
|
1070
|
+
None
|
|
1071
|
+
}
|
|
1072
|
+
}
|
|
1073
|
+
|
|
1074
|
+
fn send_quantum(&self) -> usize {
|
|
1075
|
+
self.congestion.send_quantum()
|
|
1076
|
+
}
|
|
1077
|
+
|
|
1078
|
+
fn get_next_release_time(&self) -> ReleaseDecision {
|
|
1079
|
+
ReleaseDecision {
|
|
1080
|
+
time: ReleaseTime::Immediate,
|
|
1081
|
+
allow_burst: false,
|
|
1082
|
+
}
|
|
1083
|
+
}
|
|
1084
|
+
|
|
1085
|
+
fn gcongestion_enabled(&self) -> bool {
|
|
1086
|
+
false
|
|
1087
|
+
}
|
|
1088
|
+
|
|
1089
|
+
fn lost_count(&self) -> usize {
|
|
1090
|
+
self.congestion.lost_count
|
|
1091
|
+
}
|
|
1092
|
+
|
|
1093
|
+
fn bytes_lost(&self) -> u64 {
|
|
1094
|
+
self.bytes_lost
|
|
1095
|
+
}
|
|
1096
|
+
}
|
|
1097
|
+
|
|
1098
|
+
impl std::fmt::Debug for LegacyRecovery {
|
|
1099
|
+
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
|
|
1100
|
+
write!(f, "timer={:?} ", self.loss_timer)?;
|
|
1101
|
+
write!(f, "latest_rtt={:?} ", self.rtt_stats.latest_rtt)?;
|
|
1102
|
+
write!(f, "srtt={:?} ", self.rtt_stats.smoothed_rtt)?;
|
|
1103
|
+
write!(f, "min_rtt={:?} ", *self.rtt_stats.min_rtt)?;
|
|
1104
|
+
write!(f, "rttvar={:?} ", self.rtt_stats.rttvar)?;
|
|
1105
|
+
write!(f, "cwnd={} ", self.cwnd())?;
|
|
1106
|
+
write!(f, "ssthresh={} ", self.congestion.ssthresh.get())?;
|
|
1107
|
+
write!(f, "bytes_in_flight={} ", self.bytes_in_flight.get())?;
|
|
1108
|
+
write!(f, "app_limited={} ", self.congestion.app_limited)?;
|
|
1109
|
+
write!(
|
|
1110
|
+
f,
|
|
1111
|
+
"congestion_recovery_start_time={:?} ",
|
|
1112
|
+
self.congestion.congestion_recovery_start_time
|
|
1113
|
+
)?;
|
|
1114
|
+
write!(f, "{:?} ", self.congestion.delivery_rate)?;
|
|
1115
|
+
|
|
1116
|
+
if self.congestion.hystart.enabled() {
|
|
1117
|
+
write!(f, "hystart={:?} ", self.congestion.hystart)?;
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
// CC-specific debug info
|
|
1121
|
+
(self.congestion.cc_ops.debug_fmt)(&self.congestion, f)?;
|
|
1122
|
+
|
|
1123
|
+
Ok(())
|
|
1124
|
+
}
|
|
1125
|
+
}
|
|
1126
|
+
|
|
1127
|
+
#[derive(Clone)]
|
|
1128
|
+
pub struct Acked {
|
|
1129
|
+
pub pkt_num: u64,
|
|
1130
|
+
|
|
1131
|
+
pub time_sent: Instant,
|
|
1132
|
+
|
|
1133
|
+
pub size: usize,
|
|
1134
|
+
|
|
1135
|
+
pub rtt: Duration,
|
|
1136
|
+
|
|
1137
|
+
pub delivered: usize,
|
|
1138
|
+
|
|
1139
|
+
pub delivered_time: Instant,
|
|
1140
|
+
|
|
1141
|
+
pub first_sent_time: Instant,
|
|
1142
|
+
|
|
1143
|
+
pub is_app_limited: bool,
|
|
1144
|
+
}
|
|
1145
|
+
|
|
1146
|
+
#[cfg(test)]
|
|
1147
|
+
mod tests {
|
|
1148
|
+
use super::*;
|
|
1149
|
+
use crate::recovery::HandshakeStatus;
|
|
1150
|
+
use crate::recovery::RecoveryConfig;
|
|
1151
|
+
use std::time::Instant;
|
|
1152
|
+
|
|
1153
|
+
#[test]
|
|
1154
|
+
fn test_high_pto_count_no_panic() {
|
|
1155
|
+
let config = crate::Config::new(crate::PROTOCOL_VERSION).unwrap();
|
|
1156
|
+
let recovery_config = RecoveryConfig::from_config(&config);
|
|
1157
|
+
let mut r = LegacyRecovery::new_with_config(&recovery_config);
|
|
1158
|
+
|
|
1159
|
+
r.pto_count = 99999;
|
|
1160
|
+
|
|
1161
|
+
let handshake_status = HandshakeStatus {
|
|
1162
|
+
completed: true,
|
|
1163
|
+
has_handshake_keys: true,
|
|
1164
|
+
peer_verified_address: true,
|
|
1165
|
+
};
|
|
1166
|
+
let now = Instant::now();
|
|
1167
|
+
|
|
1168
|
+
let _ = r.pto_time_and_space(handshake_status, now);
|
|
1169
|
+
}
|
|
1170
|
+
}
|