rwebtransport 0.2.3 → 0.2.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. package/Cargo.lock +153 -226
  2. package/Cargo.toml +5 -5
  3. package/README.md +4 -4
  4. package/crates/echo-server/src/bin/gen-test-cert.rs +1 -1
  5. package/crates/native/Cargo.toml +1 -1
  6. package/crates/native/src/config.rs +13 -1
  7. package/crates/native/src/driver.rs +249 -91
  8. package/crates/native/src/lib.rs +9 -10
  9. package/crates/native/src/parser_memory.rs +1 -0
  10. package/crates/native/src/server.rs +197 -182
  11. package/crates/native/src/session.rs +329 -177
  12. package/crates/native/src/test_support.rs +2 -2
  13. package/crates/wtcore/src/h3.rs +166 -10
  14. package/dist/index.js +92 -45
  15. package/dist/index.js.map +2 -2
  16. package/dist/index.mjs +92 -45
  17. package/dist/index.mjs.map +2 -2
  18. package/dist/native.d.ts +9 -4
  19. package/package.json +14 -11
  20. package/prebuilds/darwin-arm64/rwebtransport.node +0 -0
  21. package/prebuilds/linux-arm64/rwebtransport.node +0 -0
  22. package/prebuilds/linux-x64/rwebtransport.node +0 -0
  23. package/prebuilds/win32-x64/rwebtransport.node +0 -0
  24. package/scripts/build-linux-prebuilt.sh +46 -0
  25. package/scripts/build.js +10 -3
  26. package/vendor/quiche/AGENTS.md +10 -0
  27. package/vendor/quiche/Cargo.lock +239 -248
  28. package/vendor/quiche/Cargo.toml +23 -10
  29. package/vendor/quiche/examples/Makefile +4 -4
  30. package/vendor/quiche/examples/client.rs +10 -28
  31. package/vendor/quiche/examples/http3-client.rs +20 -34
  32. package/vendor/quiche/examples/http3-server.rs +47 -78
  33. package/vendor/quiche/examples/server.rs +19 -49
  34. package/vendor/quiche/include/quiche.h +35 -4
  35. package/vendor/quiche/src/build.rs +17 -0
  36. package/vendor/quiche/src/cid.rs +129 -61
  37. package/vendor/quiche/src/crypto/boringssl.rs +98 -86
  38. package/vendor/quiche/src/crypto/mod.rs +64 -77
  39. package/vendor/quiche/src/dgram.rs +2 -3
  40. package/vendor/quiche/src/error.rs +22 -20
  41. package/vendor/quiche/src/ffi.rs +558 -280
  42. package/vendor/quiche/src/flowcontrol.rs +0 -7
  43. package/vendor/quiche/src/frame.rs +134 -135
  44. package/vendor/quiche/src/h3/ffi.rs +73 -60
  45. package/vendor/quiche/src/h3/frame.rs +70 -90
  46. package/vendor/quiche/src/h3/mod.rs +1024 -593
  47. package/vendor/quiche/src/h3/qpack/decoder.rs +248 -34
  48. package/vendor/quiche/src/h3/qpack/dynamic_table.rs +859 -0
  49. package/vendor/quiche/src/h3/qpack/encoder.rs +14 -17
  50. package/vendor/quiche/src/h3/qpack/mod.rs +48 -0
  51. package/vendor/quiche/src/h3/qpack/static_table.rs +86 -73
  52. package/vendor/quiche/src/h3/stream.rs +1009 -194
  53. package/vendor/quiche/src/lib.rs +1346 -1195
  54. package/vendor/quiche/src/minmax.rs +2 -6
  55. package/vendor/quiche/src/packet.rs +420 -552
  56. package/vendor/quiche/src/path.rs +202 -72
  57. package/vendor/quiche/src/pmtud.rs +18 -16
  58. package/vendor/quiche/src/ranges.rs +184 -82
  59. package/vendor/quiche/src/recovery/AGENTS.md +5 -2
  60. package/vendor/quiche/src/recovery/bandwidth.rs +59 -40
  61. package/vendor/quiche/src/recovery/bytes_in_flight.rs +2 -2
  62. package/vendor/quiche/src/recovery/congestion/cubic.rs +38 -47
  63. package/vendor/quiche/src/recovery/congestion/delivery_rate.rs +54 -60
  64. package/vendor/quiche/src/recovery/congestion/hystart.rs +6 -10
  65. package/vendor/quiche/src/recovery/congestion/mod.rs +110 -42
  66. package/vendor/quiche/src/recovery/congestion/prr.rs +4 -1
  67. package/vendor/quiche/src/recovery/congestion/recovery.rs +91 -95
  68. package/vendor/quiche/src/recovery/congestion/reno.rs +17 -12
  69. package/vendor/quiche/src/recovery/congestion/test_sender.rs +8 -27
  70. package/vendor/quiche/src/recovery/gcongestion/bbr/bandwidth_sampler.rs +159 -247
  71. package/vendor/quiche/src/recovery/gcongestion/bbr/windowed_filter.rs +9 -14
  72. package/vendor/quiche/src/recovery/gcongestion/bbr2/drain.rs +18 -15
  73. package/vendor/quiche/src/recovery/gcongestion/bbr2/mode.rs +43 -28
  74. package/vendor/quiche/src/recovery/gcongestion/bbr2/network_model.rs +349 -76
  75. package/vendor/quiche/src/recovery/gcongestion/bbr2/probe_bw.rs +147 -195
  76. package/vendor/quiche/src/recovery/gcongestion/bbr2/probe_rtt.rs +29 -27
  77. package/vendor/quiche/src/recovery/gcongestion/bbr2/rtt_jump_detector/global_min.rs +180 -0
  78. package/vendor/quiche/src/recovery/gcongestion/bbr2/rtt_jump_detector/global_min_tests.rs +221 -0
  79. package/vendor/quiche/src/recovery/gcongestion/bbr2/rtt_jump_detector/hmm.rs +395 -0
  80. package/vendor/quiche/src/recovery/gcongestion/bbr2/rtt_jump_detector/hmm_tests.rs +473 -0
  81. package/vendor/quiche/src/recovery/gcongestion/bbr2/rtt_jump_detector/mod.rs +175 -0
  82. package/vendor/quiche/src/recovery/gcongestion/bbr2/startup.rs +33 -26
  83. package/vendor/quiche/src/recovery/gcongestion/bbr2.rs +107 -60
  84. package/vendor/quiche/src/recovery/gcongestion/mod.rs +64 -11
  85. package/vendor/quiche/src/recovery/gcongestion/pacer.rs +34 -25
  86. package/vendor/quiche/src/recovery/gcongestion/recovery.rs +81 -86
  87. package/vendor/quiche/src/recovery/mod.rs +414 -173
  88. package/vendor/quiche/src/recovery/rtt.rs +8 -5
  89. package/vendor/quiche/src/stream/mod.rs +864 -133
  90. package/vendor/quiche/src/stream/recv_buf.rs +471 -172
  91. package/vendor/quiche/src/stream/send_buf.rs +338 -81
  92. package/vendor/quiche/src/test_utils.rs +78 -64
  93. package/vendor/quiche/src/tests.rs +2559 -1335
  94. package/vendor/quiche/src/tls/boringssl.rs +60 -63
  95. package/vendor/quiche/src/tls/mod.rs +146 -193
  96. package/vendor/quiche/src/transport_params.rs +87 -103
@@ -372,6 +372,9 @@
372
372
  //! initiating a connection. Dangerous if the DCID does not meet QUIC's
373
373
  //! unpredictability and length requirements.
374
374
  //!
375
+ //! * `congestion_window_unchecked_available`: Expose a congestion control
376
+ //! algorithm with a fixed, configurable window that defaults to `usize::MAX`.
377
+ //!
375
378
  //! [feature flags]: https://doc.rust-lang.org/cargo/reference/manifest.html#the-features-section
376
379
  //! [boring]: https://crates.io/crates/boring
377
380
  //! [qlog]: https://datatracker.ietf.org/doc/html/draft-ietf-quic-qlog-main-schema
@@ -392,6 +395,8 @@ use debug_panic::debug_panic;
392
395
 
393
396
  use std::net::SocketAddr;
394
397
 
398
+ use std::num::NonZeroUsize;
399
+
395
400
  use std::str::FromStr;
396
401
 
397
402
  use std::sync::Arc;
@@ -427,6 +432,7 @@ use crate::recovery::OnLossDetectionTimeoutOutcome;
427
432
  use crate::recovery::RecoveryOps;
428
433
  use crate::recovery::ReleaseDecision;
429
434
 
435
+ use crate::stream::ReceiveMemory;
430
436
  use crate::stream::RecvAction;
431
437
  use crate::stream::StreamPriorityKey;
432
438
 
@@ -481,9 +487,6 @@ const MAX_UNDECRYPTABLE_PACKETS: usize = 10;
481
487
 
482
488
  const RESERVED_VERSION_MASK: u32 = 0xfafafafa;
483
489
 
484
- // The default size of the receiver connection flow control window.
485
- const DEFAULT_CONNECTION_WINDOW: u64 = 48 * 1024;
486
-
487
490
  // The maximum size of the receiver connection flow control window.
488
491
  const MAX_CONNECTION_WINDOW: u64 = 24 * 1024 * 1024;
489
492
 
@@ -498,9 +501,18 @@ const MAX_PROBING_TIMEOUTS: usize = 3;
498
501
  // The default initial congestion window size in terms of packet count.
499
502
  const DEFAULT_INITIAL_CONGESTION_WINDOW_PACKETS: usize = 10;
500
503
 
504
+ /// The default number of sent packets that can be tracked for loss detection
505
+ /// before the connection is closed.
506
+ pub const DEFAULT_MAX_OUTSTANDING_SENT_PACKETS: NonZeroUsize = NonZeroUsize::new(22_000).unwrap();
507
+
501
508
  // The maximum data offset that can be stored in a crypto stream.
502
509
  const MAX_CRYPTO_STREAM_OFFSET: u64 = 1 << 16;
503
510
 
511
+ // TODO: Remove once https://github.com/cloudflare/quiche/pull/2453 is done.
512
+ // Preserve the previous app-limited threshold independently of STREAM frame
513
+ // header sizing.
514
+ const LEGACY_APP_LIMITED_BYTE_THRESHOLD: usize = 12;
515
+
504
516
  // The send capacity factor.
505
517
  const TX_CAP_FACTOR: f64 = 1.0;
506
518
 
@@ -540,7 +552,7 @@ pub struct SendInfo {
540
552
  #[derive(PartialEq, Eq)]
541
553
  pub enum Shutdown {
542
554
  /// Stop receiving stream data.
543
- Read = 0,
555
+ Read = 0,
544
556
 
545
557
  /// Stop sending stream data.
546
558
  Write = 1,
@@ -552,10 +564,10 @@ pub enum Shutdown {
552
564
  #[cfg_attr(docsrs, doc(cfg(feature = "qlog")))]
553
565
  pub enum QlogLevel {
554
566
  /// Logs any events of Core importance.
555
- Core = 0,
567
+ Core = 0,
556
568
 
557
569
  /// Logs any events of Core and Base importance.
558
- Base = 1,
570
+ Base = 1,
559
571
 
560
572
  /// Logs any events of Core, Base and Extra importance
561
573
  Extra = 2,
@@ -576,6 +588,7 @@ pub struct Config {
576
588
  cc_algorithm: CongestionControlAlgorithm,
577
589
  custom_bbr_params: Option<BbrParams>,
578
590
  initial_congestion_window_packets: usize,
591
+ unchecked_congestion_window: usize,
579
592
  enable_relaxed_loss_threshold: bool,
580
593
  enable_cubic_idle_restart_fix: bool,
581
594
  enable_send_streams_blocked: bool,
@@ -600,6 +613,7 @@ pub struct Config {
600
613
 
601
614
  max_connection_window: u64,
602
615
  max_stream_window: u64,
616
+ stream_recv_memory: Option<(usize, Arc<ReceiveMemory>)>,
603
617
 
604
618
  max_amplification_factor: usize,
605
619
 
@@ -609,9 +623,7 @@ pub struct Config {
609
623
 
610
624
  initial_rtt: Duration,
611
625
 
612
- /// When true, uses the initial max data (for connection
613
- /// and stream) as the initial flow control window.
614
- use_initial_max_data_as_flow_control_win: bool,
626
+ max_outstanding_sent_packets: Option<NonZeroUsize>,
615
627
  }
616
628
 
617
629
  // See https://quicwg.org/base-drafts/rfc9000.html#section-15
@@ -642,9 +654,10 @@ impl Config {
642
654
  #[cfg(feature = "boringssl-boring-crate")]
643
655
  #[cfg_attr(docsrs, doc(cfg(feature = "boringssl-boring-crate")))]
644
656
  pub fn with_boring_ssl_ctx_builder(
645
- version: u32, tls_ctx_builder: boring::ssl::SslContextBuilder,
657
+ version: u32,
658
+ tls_ctx_builder: boring::ssl::SslContextBuilder,
646
659
  ) -> Result<Config> {
647
- Self::with_tls_ctx(version, tls::Context::from_boring(tls_ctx_builder))
660
+ Self::with_tls_ctx(version, tls::Context::from_boring(tls_ctx_builder)?)
648
661
  }
649
662
 
650
663
  fn with_tls_ctx(version: u32, tls_ctx: tls::Context) -> Result<Config> {
@@ -660,8 +673,8 @@ impl Config {
660
673
  grease: true,
661
674
  cc_algorithm: CongestionControlAlgorithm::CUBIC,
662
675
  custom_bbr_params: None,
663
- initial_congestion_window_packets:
664
- DEFAULT_INITIAL_CONGESTION_WINDOW_PACKETS,
676
+ initial_congestion_window_packets: DEFAULT_INITIAL_CONGESTION_WINDOW_PACKETS,
677
+ unchecked_congestion_window: usize::MAX,
665
678
  enable_relaxed_loss_threshold: false,
666
679
  enable_cubic_idle_restart_fix: true,
667
680
  enable_send_streams_blocked: false,
@@ -676,13 +689,13 @@ impl Config {
676
689
  dgram_recv_max_queue_len: DEFAULT_MAX_DGRAM_QUEUE_LEN,
677
690
  dgram_send_max_queue_len: DEFAULT_MAX_DGRAM_QUEUE_LEN,
678
691
 
679
- path_challenge_recv_max_queue_len:
680
- DEFAULT_MAX_PATH_CHALLENGE_RX_QUEUE_LEN,
692
+ path_challenge_recv_max_queue_len: DEFAULT_MAX_PATH_CHALLENGE_RX_QUEUE_LEN,
681
693
 
682
694
  max_send_udp_payload_size: MAX_SEND_UDP_PAYLOAD_SIZE,
683
695
 
684
696
  max_connection_window: MAX_CONNECTION_WINDOW,
685
697
  max_stream_window: stream::MAX_STREAM_WINDOW,
698
+ stream_recv_memory: None,
686
699
 
687
700
  max_amplification_factor: MAX_AMPLIFICATION_FACTOR,
688
701
 
@@ -691,7 +704,7 @@ impl Config {
691
704
  track_unknown_transport_params: None,
692
705
  initial_rtt: DEFAULT_INITIAL_RTT,
693
706
 
694
- use_initial_max_data_as_flow_control_win: false,
707
+ max_outstanding_sent_packets: Some(DEFAULT_MAX_OUTSTANDING_SENT_PACKETS),
695
708
  })
696
709
  }
697
710
 
@@ -754,12 +767,20 @@ impl Config {
754
767
  /// config.load_verify_locations_from_directory("/path/to/certs")?;
755
768
  /// # Ok::<(), quiche::Error>(())
756
769
  /// ```
757
- pub fn load_verify_locations_from_directory(
758
- &mut self, dir: &str,
759
- ) -> Result<()> {
770
+ pub fn load_verify_locations_from_directory(&mut self, dir: &str) -> Result<()> {
760
771
  self.tls_ctx.load_verify_locations_from_directory(dir)
761
772
  }
762
773
 
774
+ /// Configures the TLS curve preference list.
775
+ ///
776
+ /// `curves` is a colon-separated list of curve (a.k.a. group) names, in
777
+ /// order of preference, e.g. `"X25519MLKEM768:X25519:P-256:P-384"`.
778
+ /// Corresponds to `SSL_CTX_set1_curves_list` (a.k.a.
779
+ /// `SSL_CTX_set1_groups_list`).
780
+ pub fn set_curves_list(&mut self, curves: &str) -> Result<()> {
781
+ self.tls_ctx.set_curves_list(curves)
782
+ }
783
+
763
784
  /// Configures whether to verify the peer's certificate.
764
785
  ///
765
786
  /// This should usually be `true` for client-side connections and `false`
@@ -781,6 +802,9 @@ impl Config {
781
802
 
782
803
  /// Configures whether to do path MTU discovery.
783
804
  ///
805
+ /// PMTUD-driven packet limit updates are reported to the application
806
+ /// through [`PathEvent::PmtuUpdated`].
807
+ ///
784
808
  /// The default value is `false`.
785
809
  pub fn discover_pmtu(&mut self, discover: bool) {
786
810
  self.pmtud = discover;
@@ -850,11 +874,8 @@ impl Config {
850
874
  /// config.set_application_protos(&[b"http/1.1", b"http/0.9"]);
851
875
  /// # Ok::<(), quiche::Error>(())
852
876
  /// ```
853
- pub fn set_application_protos(
854
- &mut self, protos_list: &[&[u8]],
855
- ) -> Result<()> {
856
- self.application_protos =
857
- protos_list.iter().map(|s| s.to_vec()).collect();
877
+ pub fn set_application_protos(&mut self, protos_list: &[&[u8]]) -> Result<()> {
878
+ self.application_protos = protos_list.iter().map(|s| s.to_vec()).collect();
858
879
 
859
880
  self.tls_ctx.set_alpn(protos_list)
860
881
  }
@@ -875,9 +896,7 @@ impl Config {
875
896
  /// config.set_application_protos_wire_format(b"\x08http/1.1\x08http/0.9")?;
876
897
  /// # Ok::<(), quiche::Error>(())
877
898
  /// ```
878
- pub fn set_application_protos_wire_format(
879
- &mut self, protos: &[u8],
880
- ) -> Result<()> {
899
+ pub fn set_application_protos_wire_format(&mut self, protos: &[u8]) -> Result<()> {
881
900
  let mut b = octets::Octets::with_slice(protos);
882
901
 
883
902
  let mut protos_list = Vec::new();
@@ -910,20 +929,33 @@ impl Config {
910
929
  self.initial_rtt = v;
911
930
  }
912
931
 
932
+ /// Sets the number of sent packets that can be tracked for loss detection
933
+ /// before the connection is closed.
934
+ ///
935
+ /// This limits the memory used to track sent packets. Packets that only
936
+ /// carry ACK frames are counted too. The limit should be larger than the
937
+ /// expected maximum congestion window, in packets, otherwise connections
938
+ /// that make progress can be closed.
939
+ ///
940
+ /// The connection is closed with a `PROTOCOL_VIOLATION` transport error.
941
+ ///
942
+ /// `None` disables this limit. The default value is `22,000`.
943
+ pub fn set_max_outstanding_sent_packets(&mut self, max_packets: Option<NonZeroUsize>) {
944
+ self.max_outstanding_sent_packets = max_packets;
945
+ }
946
+
913
947
  /// Sets the `max_idle_timeout` transport parameter, in milliseconds.
914
948
  ///
915
949
  /// The default value is infinite, that is, no timeout is used.
916
950
  pub fn set_max_idle_timeout(&mut self, v: u64) {
917
- self.local_transport_params.max_idle_timeout =
918
- cmp::min(v, octets::MAX_VAR_INT);
951
+ self.local_transport_params.max_idle_timeout = cmp::min(v, octets::MAX_VAR_INT);
919
952
  }
920
953
 
921
954
  /// Sets the `max_udp_payload_size transport` parameter.
922
955
  ///
923
956
  /// The default value is `65527`.
924
957
  pub fn set_max_recv_udp_payload_size(&mut self, v: usize) {
925
- self.local_transport_params.max_udp_payload_size =
926
- cmp::min(v as u64, octets::MAX_VAR_INT);
958
+ self.local_transport_params.max_udp_payload_size = cmp::min(v as u64, octets::MAX_VAR_INT);
927
959
  }
928
960
 
929
961
  /// Sets the maximum outgoing UDP payload size.
@@ -946,8 +978,7 @@ impl Config {
946
978
  ///
947
979
  /// The default value is `0`.
948
980
  pub fn set_initial_max_data(&mut self, v: u64) {
949
- self.local_transport_params.initial_max_data =
950
- cmp::min(v, octets::MAX_VAR_INT);
981
+ self.local_transport_params.initial_max_data = cmp::min(v, octets::MAX_VAR_INT);
951
982
  }
952
983
 
953
984
  /// Sets the `initial_max_stream_data_bidi_local` transport parameter.
@@ -965,8 +996,7 @@ impl Config {
965
996
  /// The default value is `0`.
966
997
  pub fn set_initial_max_stream_data_bidi_local(&mut self, v: u64) {
967
998
  self.local_transport_params
968
- .initial_max_stream_data_bidi_local =
969
- cmp::min(v, octets::MAX_VAR_INT);
999
+ .initial_max_stream_data_bidi_local = cmp::min(v, octets::MAX_VAR_INT);
970
1000
  }
971
1001
 
972
1002
  /// Sets the `initial_max_stream_data_bidi_remote` transport parameter.
@@ -984,8 +1014,7 @@ impl Config {
984
1014
  /// The default value is `0`.
985
1015
  pub fn set_initial_max_stream_data_bidi_remote(&mut self, v: u64) {
986
1016
  self.local_transport_params
987
- .initial_max_stream_data_bidi_remote =
988
- cmp::min(v, octets::MAX_VAR_INT);
1017
+ .initial_max_stream_data_bidi_remote = cmp::min(v, octets::MAX_VAR_INT);
989
1018
  }
990
1019
 
991
1020
  /// Sets the `initial_max_stream_data_uni` transport parameter.
@@ -1001,8 +1030,7 @@ impl Config {
1001
1030
  ///
1002
1031
  /// The default value is `0`.
1003
1032
  pub fn set_initial_max_stream_data_uni(&mut self, v: u64) {
1004
- self.local_transport_params.initial_max_stream_data_uni =
1005
- cmp::min(v, octets::MAX_VAR_INT);
1033
+ self.local_transport_params.initial_max_stream_data_uni = cmp::min(v, octets::MAX_VAR_INT);
1006
1034
  }
1007
1035
 
1008
1036
  /// Sets the `initial_max_streams_bidi` transport parameter.
@@ -1019,12 +1047,12 @@ impl Config {
1019
1047
  /// has been read by the application (up to the `fin` offset) or the
1020
1048
  /// stream's read direction has been shutdown, and all outgoing data has
1021
1049
  /// been acked by the peer (up to the `fin` offset) or the stream's write
1022
- /// direction has been shutdown.
1050
+ /// direction has been shutdown and the resulting `RESET_STREAM` frame has
1051
+ /// been acked by the peer.
1023
1052
  ///
1024
1053
  /// The default value is `0`.
1025
1054
  pub fn set_initial_max_streams_bidi(&mut self, v: u64) {
1026
- self.local_transport_params.initial_max_streams_bidi =
1027
- cmp::min(v, octets::MAX_VAR_INT);
1055
+ self.local_transport_params.initial_max_streams_bidi = cmp::min(v, octets::MAX_VAR_INT);
1028
1056
  }
1029
1057
 
1030
1058
  /// Sets the `initial_max_streams_uni` transport parameter.
@@ -1043,8 +1071,7 @@ impl Config {
1043
1071
  ///
1044
1072
  /// The default value is `0`.
1045
1073
  pub fn set_initial_max_streams_uni(&mut self, v: u64) {
1046
- self.local_transport_params.initial_max_streams_uni =
1047
- cmp::min(v, octets::MAX_VAR_INT);
1074
+ self.local_transport_params.initial_max_streams_uni = cmp::min(v, octets::MAX_VAR_INT);
1048
1075
  }
1049
1076
 
1050
1077
  /// Sets the `ack_delay_exponent` transport parameter.
@@ -1055,16 +1082,14 @@ impl Config {
1055
1082
  ///
1056
1083
  /// The default value is `3`.
1057
1084
  pub fn set_ack_delay_exponent(&mut self, v: u64) {
1058
- self.local_transport_params.ack_delay_exponent =
1059
- cmp::min(v, MAX_ACK_DELAY_EXPONENT);
1085
+ self.local_transport_params.ack_delay_exponent = cmp::min(v, MAX_ACK_DELAY_EXPONENT);
1060
1086
  }
1061
1087
 
1062
1088
  /// Sets the `max_ack_delay` transport parameter.
1063
1089
  ///
1064
1090
  /// The default value is `25`.
1065
1091
  pub fn set_max_ack_delay(&mut self, v: u64) {
1066
- self.local_transport_params.max_ack_delay =
1067
- cmp::min(v, octets::MAX_VAR_INT);
1092
+ self.local_transport_params.max_ack_delay = cmp::min(v, octets::MAX_VAR_INT);
1068
1093
  }
1069
1094
 
1070
1095
  /// Sets the `active_connection_id_limit` transport parameter.
@@ -1072,8 +1097,7 @@ impl Config {
1072
1097
  /// The default value is `2`. Lower values will be ignored.
1073
1098
  pub fn set_active_connection_id_limit(&mut self, v: u64) {
1074
1099
  if v >= 2 {
1075
- self.local_transport_params.active_conn_id_limit =
1076
- cmp::min(v, octets::MAX_VAR_INT);
1100
+ self.local_transport_params.active_conn_id_limit = cmp::min(v, octets::MAX_VAR_INT);
1077
1101
  }
1078
1102
  }
1079
1103
 
@@ -1130,6 +1154,23 @@ impl Config {
1130
1154
  self.initial_congestion_window_packets = packets;
1131
1155
  }
1132
1156
 
1157
+ /// Sets the fixed congestion window in bytes for the
1158
+ /// `congestion_window_unchecked` algorithm.
1159
+ ///
1160
+ /// The default is `usize::MAX`. Other congestion control algorithms ignore
1161
+ /// this setting. Recovery probes can temporarily bypass this window.
1162
+ ///
1163
+ /// Values are not clamped. Zero can prevent connection establishment, and
1164
+ /// windows too small for outgoing packets or PMTU probes can stall
1165
+ /// transmission. At least two maximum-sized datagrams are recommended.
1166
+ ///
1167
+ /// This API is available even when the
1168
+ /// `congestion_window_unchecked_available` feature is disabled, in which
1169
+ /// case the setting has no effect.
1170
+ pub fn set_unchecked_congestion_window(&mut self, window: usize) {
1171
+ self.unchecked_congestion_window = window;
1172
+ }
1173
+
1133
1174
  /// Configure whether to enable relaxed loss detection on spurious loss.
1134
1175
  ///
1135
1176
  /// The default value is false.
@@ -1186,9 +1227,7 @@ impl Config {
1186
1227
  /// to 65536 as recommended by draft-ietf-quic-datagram-01.
1187
1228
  ///
1188
1229
  /// The default is `false`.
1189
- pub fn enable_dgram(
1190
- &mut self, enabled: bool, recv_queue_len: usize, send_queue_len: usize,
1191
- ) {
1230
+ pub fn enable_dgram(&mut self, enabled: bool, recv_queue_len: usize, send_queue_len: usize) {
1192
1231
  self.local_transport_params.max_datagram_frame_size = if enabled {
1193
1232
  Some(MAX_DGRAM_FRAME_SIZE)
1194
1233
  } else {
@@ -1198,6 +1237,11 @@ impl Config {
1198
1237
  self.dgram_send_max_queue_len = send_queue_len;
1199
1238
  }
1200
1239
 
1240
+ /// Advertises support for QUIC RESET_STREAM_AT.
1241
+ pub fn enable_reliable_stream_reset(&mut self) {
1242
+ self.local_transport_params.reset_stream_at = true;
1243
+ }
1244
+
1201
1245
  /// Configures the max number of queued received PATH_CHALLENGE frames.
1202
1246
  ///
1203
1247
  /// When an endpoint receives a PATH_CHALLENGE frame and the queue is full,
@@ -1215,6 +1259,13 @@ impl Config {
1215
1259
  self.max_connection_window = v;
1216
1260
  }
1217
1261
 
1262
+ /// Sets budgets for retained stream and handshake receive allocations.
1263
+ /// The second limit is shared by all connections created from this config.
1264
+ /// Also bounds fragmented completed-stream history.
1265
+ pub fn set_stream_recv_memory_limits(&mut self, per_connection: usize, shared: usize) {
1266
+ self.stream_recv_memory = Some((per_connection, ReceiveMemory::new(shared, None)));
1267
+ }
1268
+
1218
1269
  /// Sets the maximum size of the stream window.
1219
1270
  ///
1220
1271
  /// The default value is MAX_STREAM_WINDOW (16MBytes).
@@ -1261,16 +1312,10 @@ impl Config {
1261
1312
  /// Sets whether the initial max data value should be used as the initial
1262
1313
  /// flow control window.
1263
1314
  ///
1264
- /// If set to true, the initial flow control window for streams and the
1265
- /// connection itself will be set to the initial max data value for streams
1266
- /// and the connection respectively. If false, the window is set to the
1267
- /// minimum of initial max data and `DEFAULT_STREAM_WINDOW` or
1268
- /// `DEFAULT_CONNECTION_WINDOW`
1269
- ///
1270
- /// The default is false.
1271
- pub fn set_use_initial_max_data_as_flow_control_win(&mut self, v: bool) {
1272
- self.use_initial_max_data_as_flow_control_win = v;
1273
- }
1315
+ /// This is now always enabled and this method is a no-op. It will be
1316
+ /// removed in a future release.
1317
+ #[deprecated(note = "This is now always enabled. This method is a no-op.")]
1318
+ pub fn set_use_initial_max_data_as_flow_control_win(&mut self, _v: bool) {}
1274
1319
  }
1275
1320
 
1276
1321
  /// Tracks the health of the tx_buffered value.
@@ -1425,12 +1470,6 @@ where
1425
1470
  /// The send capacity factor.
1426
1471
  tx_cap_factor: f64,
1427
1472
 
1428
- /// Number of bytes buffered in the send buffer.
1429
- tx_buffered: usize,
1430
-
1431
- /// Tracks the health of tx_buffered.
1432
- tx_buffered_state: TxBufferTrackingState,
1433
-
1434
1473
  /// Total number of bytes sent to the peer.
1435
1474
  tx_data: u64,
1436
1475
 
@@ -1457,7 +1496,7 @@ where
1457
1496
  lost_bytes: u64,
1458
1497
 
1459
1498
  /// Streams map, indexed by stream ID.
1460
- streams: stream::StreamMap<F>,
1499
+ pub(crate) streams: stream::StreamMap<F>,
1461
1500
 
1462
1501
  /// Peer's original destination connection ID. Used by the client to
1463
1502
  /// validate the server's transport parameter.
@@ -1610,6 +1649,10 @@ where
1610
1649
 
1611
1650
  /// The anti-amplification limit factor.
1612
1651
  max_amplification_factor: usize,
1652
+
1653
+ /// The number of tracked sent packets after which the connection is
1654
+ /// closed.
1655
+ max_outstanding_sent_packets: Option<NonZeroUsize>,
1613
1656
  }
1614
1657
 
1615
1658
  /// Creates a new server-side connection.
@@ -1634,24 +1677,29 @@ where
1634
1677
  /// ```
1635
1678
  #[inline(always)]
1636
1679
  pub fn accept(
1637
- scid: &ConnectionId, odcid: Option<&ConnectionId>, local: SocketAddr,
1638
- peer: SocketAddr, config: &mut Config,
1680
+ scid: &ConnectionId,
1681
+ odcid: Option<&ConnectionId>,
1682
+ local: SocketAddr,
1683
+ peer: SocketAddr,
1684
+ config: &mut Config,
1639
1685
  ) -> Result<Connection> {
1640
1686
  accept_with_buf_factory(scid, odcid, local, peer, config)
1641
1687
  }
1642
1688
 
1643
- /// Creates a new server-side connection, with a custom buffer generation
1644
- /// method.
1689
+ /// Creates a server-side connection with custom buffer generation.
1645
1690
  ///
1646
- /// The buffers generated can be anything that can be drereferenced as a byte
1647
- /// slice. See [`accept`] and [`BufFactory`] for more info.
1691
+ /// The buffers generated can be anything that can be dereferenced as a byte
1692
+ /// slice. See [`accept`] and [`BufFactory`] for more information.
1648
1693
  #[inline]
1649
1694
  pub fn accept_with_buf_factory<F: BufFactory>(
1650
- scid: &ConnectionId, odcid: Option<&ConnectionId>, local: SocketAddr,
1651
- peer: SocketAddr, config: &mut Config,
1695
+ scid: &ConnectionId,
1696
+ odcid: Option<&ConnectionId>,
1697
+ local: SocketAddr,
1698
+ peer: SocketAddr,
1699
+ config: &mut Config,
1652
1700
  ) -> Result<Connection<F>> {
1653
- // For connections with `odcid` set, we historically used `retry_source_cid =
1654
- // scid`. Keep this behavior to preserve backwards compatibility.
1701
+ // Connections with `odcid` historically used `scid` as the retry source
1702
+ // CID. Preserve this behavior for backwards compatibility.
1655
1703
  // `accept_with_retry` allows the SCIDs to be specified separately.
1656
1704
  let retry_cids = odcid.map(|odcid| RetryConnectionIds {
1657
1705
  original_destination_cid: odcid,
@@ -1688,8 +1736,11 @@ pub struct RetryConnectionIds<'a> {
1688
1736
  /// manner.
1689
1737
  #[inline]
1690
1738
  pub fn accept_with_retry<F: BufFactory>(
1691
- scid: &ConnectionId, retry_cids: RetryConnectionIds, local: SocketAddr,
1692
- peer: SocketAddr, config: &mut Config,
1739
+ scid: &ConnectionId,
1740
+ retry_cids: RetryConnectionIds,
1741
+ local: SocketAddr,
1742
+ peer: SocketAddr,
1743
+ config: &mut Config,
1693
1744
  ) -> Result<Connection<F>> {
1694
1745
  Connection::new(scid, Some(retry_cids), None, local, peer, config, true)
1695
1746
  }
@@ -1714,8 +1765,11 @@ pub fn accept_with_retry<F: BufFactory>(
1714
1765
  /// ```
1715
1766
  #[inline]
1716
1767
  pub fn connect(
1717
- server_name: Option<&str>, scid: &ConnectionId, local: SocketAddr,
1718
- peer: SocketAddr, config: &mut Config,
1768
+ server_name: Option<&str>,
1769
+ scid: &ConnectionId,
1770
+ local: SocketAddr,
1771
+ peer: SocketAddr,
1772
+ config: &mut Config,
1719
1773
  ) -> Result<Connection> {
1720
1774
  let mut conn = Connection::new(scid, None, None, local, peer, config, false)?;
1721
1775
 
@@ -1740,11 +1794,14 @@ pub fn connect(
1740
1794
  #[cfg(feature = "custom-client-dcid")]
1741
1795
  #[cfg_attr(docsrs, doc(cfg(feature = "custom-client-dcid")))]
1742
1796
  pub fn connect_with_dcid(
1743
- server_name: Option<&str>, scid: &ConnectionId, dcid: &ConnectionId,
1744
- local: SocketAddr, peer: SocketAddr, config: &mut Config,
1797
+ server_name: Option<&str>,
1798
+ scid: &ConnectionId,
1799
+ dcid: &ConnectionId,
1800
+ local: SocketAddr,
1801
+ peer: SocketAddr,
1802
+ config: &mut Config,
1745
1803
  ) -> Result<Connection> {
1746
- let mut conn =
1747
- Connection::new(scid, None, Some(dcid), local, peer, config, false)?;
1804
+ let mut conn = Connection::new(scid, None, Some(dcid), local, peer, config, false)?;
1748
1805
 
1749
1806
  if let Some(server_name) = server_name {
1750
1807
  conn.handshake.set_host_name(server_name)?;
@@ -1760,8 +1817,11 @@ pub fn connect_with_dcid(
1760
1817
  /// slice. See [`connect`] and [`BufFactory`] for more info.
1761
1818
  #[inline]
1762
1819
  pub fn connect_with_buffer_factory<F: BufFactory>(
1763
- server_name: Option<&str>, scid: &ConnectionId, local: SocketAddr,
1764
- peer: SocketAddr, config: &mut Config,
1820
+ server_name: Option<&str>,
1821
+ scid: &ConnectionId,
1822
+ local: SocketAddr,
1823
+ peer: SocketAddr,
1824
+ config: &mut Config,
1765
1825
  ) -> Result<Connection<F>> {
1766
1826
  let mut conn = Connection::new(scid, None, None, local, peer, config, false)?;
1767
1827
 
@@ -1783,11 +1843,14 @@ pub fn connect_with_buffer_factory<F: BufFactory>(
1783
1843
  #[cfg(feature = "custom-client-dcid")]
1784
1844
  #[cfg_attr(docsrs, doc(cfg(feature = "custom-client-dcid")))]
1785
1845
  pub fn connect_with_dcid_and_buffer_factory<F: BufFactory>(
1786
- server_name: Option<&str>, scid: &ConnectionId, dcid: &ConnectionId,
1787
- local: SocketAddr, peer: SocketAddr, config: &mut Config,
1846
+ server_name: Option<&str>,
1847
+ scid: &ConnectionId,
1848
+ dcid: &ConnectionId,
1849
+ local: SocketAddr,
1850
+ peer: SocketAddr,
1851
+ config: &mut Config,
1788
1852
  ) -> Result<Connection<F>> {
1789
- let mut conn =
1790
- Connection::new(scid, None, Some(dcid), local, peer, config, false)?;
1853
+ let mut conn = Connection::new(scid, None, Some(dcid), local, peer, config, false)?;
1791
1854
 
1792
1855
  if let Some(server_name) = server_name {
1793
1856
  conn.handshake.set_host_name(server_name)?;
@@ -1821,7 +1884,9 @@ pub fn connect_with_dcid_and_buffer_factory<F: BufFactory>(
1821
1884
  /// ```
1822
1885
  #[inline]
1823
1886
  pub fn negotiate_version(
1824
- scid: &ConnectionId, dcid: &ConnectionId, out: &mut [u8],
1887
+ scid: &ConnectionId,
1888
+ dcid: &ConnectionId,
1889
+ out: &mut [u8],
1825
1890
  ) -> Result<usize> {
1826
1891
  packet::negotiate_version(scid, dcid, out)
1827
1892
  }
@@ -1887,8 +1952,12 @@ pub fn negotiate_version(
1887
1952
  /// ```
1888
1953
  #[inline]
1889
1954
  pub fn retry(
1890
- scid: &ConnectionId, dcid: &ConnectionId, new_scid: &ConnectionId,
1891
- token: &[u8], version: u32, out: &mut [u8],
1955
+ scid: &ConnectionId,
1956
+ dcid: &ConnectionId,
1957
+ new_scid: &ConnectionId,
1958
+ token: &[u8],
1959
+ version: u32,
1960
+ out: &mut [u8],
1892
1961
  ) -> Result<usize> {
1893
1962
  packet::retry(scid, dcid, new_scid, token, version, out)
1894
1963
  }
@@ -1937,28 +2006,22 @@ macro_rules! qlog_with_type {
1937
2006
  }
1938
2007
 
1939
2008
  #[cfg(feature = "qlog")]
1940
- const QLOG_PARAMS_SET: EventType =
1941
- EventType::QuicEventType(QuicEventType::ParametersSet);
2009
+ const QLOG_PARAMS_SET: EventType = EventType::QuicEventType(QuicEventType::ParametersSet);
1942
2010
 
1943
2011
  #[cfg(feature = "qlog")]
1944
- const QLOG_PACKET_RX: EventType =
1945
- EventType::QuicEventType(QuicEventType::PacketReceived);
2012
+ const QLOG_PACKET_RX: EventType = EventType::QuicEventType(QuicEventType::PacketReceived);
1946
2013
 
1947
2014
  #[cfg(feature = "qlog")]
1948
- const QLOG_PACKET_TX: EventType =
1949
- EventType::QuicEventType(QuicEventType::PacketSent);
2015
+ const QLOG_PACKET_TX: EventType = EventType::QuicEventType(QuicEventType::PacketSent);
1950
2016
 
1951
2017
  #[cfg(feature = "qlog")]
1952
- const QLOG_DATA_MV: EventType =
1953
- EventType::QuicEventType(QuicEventType::StreamDataMoved);
2018
+ const QLOG_DATA_MV: EventType = EventType::QuicEventType(QuicEventType::StreamDataMoved);
1954
2019
 
1955
2020
  #[cfg(feature = "qlog")]
1956
- const QLOG_METRICS: EventType =
1957
- EventType::QuicEventType(QuicEventType::RecoveryMetricsUpdated);
2021
+ const QLOG_METRICS: EventType = EventType::QuicEventType(QuicEventType::RecoveryMetricsUpdated);
1958
2022
 
1959
2023
  #[cfg(feature = "qlog")]
1960
- const QLOG_CONNECTION_CLOSED: EventType =
1961
- EventType::QuicEventType(QuicEventType::ConnectionClosed);
2024
+ const QLOG_CONNECTION_CLOSED: EventType = EventType::QuicEventType(QuicEventType::ConnectionClosed);
1962
2025
 
1963
2026
  #[cfg(feature = "qlog")]
1964
2027
  struct QlogInfo {
@@ -1980,9 +2043,13 @@ impl Default for QlogInfo {
1980
2043
 
1981
2044
  impl<F: BufFactory> Connection<F> {
1982
2045
  fn new(
1983
- scid: &ConnectionId, retry_cids: Option<RetryConnectionIds>,
1984
- client_dcid: Option<&ConnectionId>, local: SocketAddr, peer: SocketAddr,
1985
- config: &mut Config, is_server: bool,
2046
+ scid: &ConnectionId,
2047
+ retry_cids: Option<RetryConnectionIds>,
2048
+ client_dcid: Option<&ConnectionId>,
2049
+ local: SocketAddr,
2050
+ peer: SocketAddr,
2051
+ config: &mut Config,
2052
+ is_server: bool,
1986
2053
  ) -> Result<Connection<F>> {
1987
2054
  let tls = config.tls_ctx.new_handshake()?;
1988
2055
  Connection::with_tls(
@@ -1999,9 +2066,14 @@ impl<F: BufFactory> Connection<F> {
1999
2066
 
2000
2067
  #[allow(clippy::too_many_arguments)]
2001
2068
  fn with_tls(
2002
- scid: &ConnectionId, retry_cids: Option<RetryConnectionIds>,
2003
- client_dcid: Option<&ConnectionId>, local: SocketAddr, peer: SocketAddr,
2004
- config: &Config, tls: tls::Handshake, is_server: bool,
2069
+ scid: &ConnectionId,
2070
+ retry_cids: Option<RetryConnectionIds>,
2071
+ client_dcid: Option<&ConnectionId>,
2072
+ local: SocketAddr,
2073
+ peer: SocketAddr,
2074
+ config: &Config,
2075
+ tls: tls::Handshake,
2076
+ is_server: bool,
2005
2077
  ) -> Result<Connection<F>> {
2006
2078
  if retry_cids.is_some() && client_dcid.is_some() {
2007
2079
  // These are exclusive, the caller should only specify one or the
@@ -2023,8 +2095,7 @@ impl<F: BufFactory> Connection<F> {
2023
2095
 
2024
2096
  let max_rx_data = config.local_transport_params.initial_max_data;
2025
2097
 
2026
- let scid_as_hex: Vec<String> =
2027
- scid.iter().map(|b| format!("{b:02x}")).collect();
2098
+ let scid_as_hex: Vec<String> = scid.iter().map(|b| format!("{b:02x}")).collect();
2028
2099
 
2029
2100
  let reset_token = if is_server {
2030
2101
  config.local_transport_params.stateless_reset_token
@@ -2064,12 +2135,15 @@ impl<F: BufFactory> Connection<F> {
2064
2135
  reset_token,
2065
2136
  );
2066
2137
 
2067
- let initial_flow_control_window =
2068
- if config.use_initial_max_data_as_flow_control_win {
2069
- max_rx_data
2070
- } else {
2071
- cmp::min(max_rx_data / 2 * 3, DEFAULT_CONNECTION_WINDOW)
2072
- };
2138
+ let initial_flow_control_window = max_rx_data;
2139
+ let mut streams = stream::StreamMap::new(
2140
+ config.local_transport_params.initial_max_streams_bidi,
2141
+ config.local_transport_params.initial_max_streams_uni,
2142
+ config.max_stream_window,
2143
+ );
2144
+ if let Some((limit, shared)) = &config.stream_recv_memory {
2145
+ streams.receive_memory = Some(ReceiveMemory::new(*limit, Some(shared.clone())));
2146
+ }
2073
2147
  let mut conn = Connection {
2074
2148
  version: config.version,
2075
2149
 
@@ -2095,8 +2169,7 @@ impl<F: BufFactory> Connection<F> {
2095
2169
 
2096
2170
  peer_transport_params: TransportParams::default(),
2097
2171
 
2098
- peer_transport_params_track_unknown: config
2099
- .track_unknown_transport_params,
2172
+ peer_transport_params_track_unknown: config.track_unknown_transport_params,
2100
2173
 
2101
2174
  local_transport_params: config.local_transport_params.clone(),
2102
2175
 
@@ -2107,8 +2180,7 @@ impl<F: BufFactory> Connection<F> {
2107
2180
  recovery_config,
2108
2181
 
2109
2182
  paths,
2110
- path_challenge_recv_max_queue_len: config
2111
- .path_challenge_recv_max_queue_len,
2183
+ path_challenge_recv_max_queue_len: config.path_challenge_recv_max_queue_len,
2112
2184
  path_challenge_rx_count: 0,
2113
2185
 
2114
2186
  application_protos: config.application_protos.clone(),
@@ -2138,20 +2210,13 @@ impl<F: BufFactory> Connection<F> {
2138
2210
  tx_cap: 0,
2139
2211
  tx_cap_factor: config.tx_cap_factor,
2140
2212
 
2141
- tx_buffered: 0,
2142
- tx_buffered_state: TxBufferTrackingState::Ok,
2143
-
2144
2213
  tx_data: 0,
2145
2214
  max_tx_data: 0,
2146
2215
  last_tx_data: 0,
2147
2216
 
2148
2217
  stream_retrans_bytes: 0,
2149
2218
 
2150
- streams: stream::StreamMap::new(
2151
- config.local_transport_params.initial_max_streams_bidi,
2152
- config.local_transport_params.initial_max_streams_uni,
2153
- config.max_stream_window,
2154
- ),
2219
+ streams,
2155
2220
 
2156
2221
  odcid: None,
2157
2222
 
@@ -2212,13 +2277,9 @@ impl<F: BufFactory> Connection<F> {
2212
2277
  #[cfg(feature = "qlog")]
2213
2278
  qlog: Default::default(),
2214
2279
 
2215
- dgram_recv_queue: dgram::DatagramQueue::new(
2216
- config.dgram_recv_max_queue_len,
2217
- ),
2280
+ dgram_recv_queue: dgram::DatagramQueue::new(config.dgram_recv_max_queue_len),
2218
2281
 
2219
- dgram_send_queue: dgram::DatagramQueue::new(
2220
- config.dgram_send_max_queue_len,
2221
- ),
2282
+ dgram_send_queue: dgram::DatagramQueue::new(config.dgram_send_max_queue_len),
2222
2283
 
2223
2284
  emit_dgram: true,
2224
2285
 
@@ -2243,11 +2304,9 @@ impl<F: BufFactory> Connection<F> {
2243
2304
  streams_blocked_uni_state: Default::default(),
2244
2305
 
2245
2306
  max_amplification_factor: config.max_amplification_factor,
2246
- };
2247
- conn.streams.set_use_initial_max_data_as_flow_control_win(
2248
- config.use_initial_max_data_as_flow_control_win,
2249
- );
2250
2307
 
2308
+ max_outstanding_sent_packets: config.max_outstanding_sent_packets,
2309
+ };
2251
2310
  if let Some(retry_cids) = retry_cids {
2252
2311
  conn.local_transport_params
2253
2312
  .original_destination_connection_id =
@@ -2282,19 +2341,11 @@ impl<F: BufFactory> Connection<F> {
2282
2341
  dcid.to_vec()
2283
2342
  };
2284
2343
 
2285
- let (aead_open, aead_seal) = crypto::derive_initial_key_material(
2286
- &dcid,
2287
- conn.version,
2288
- conn.is_server,
2289
- false,
2290
- )?;
2344
+ let (aead_open, aead_seal) =
2345
+ crypto::derive_initial_key_material(&dcid, conn.version, conn.is_server, false)?;
2291
2346
 
2292
2347
  let reset_token = conn.peer_transport_params.stateless_reset_token;
2293
- conn.set_initial_dcid(
2294
- dcid.to_vec().into(),
2295
- reset_token,
2296
- active_path_id,
2297
- )?;
2348
+ conn.set_initial_dcid(dcid.to_vec().into(), reset_token, active_path_id)?;
2298
2349
 
2299
2350
  conn.crypto_ctx[packet::Epoch::Initial].crypto_open = Some(aead_open);
2300
2351
  conn.crypto_ctx[packet::Epoch::Initial].crypto_seal = Some(aead_seal);
@@ -2328,7 +2379,9 @@ impl<F: BufFactory> Connection<F> {
2328
2379
  #[cfg(feature = "qlog")]
2329
2380
  #[cfg_attr(docsrs, doc(cfg(feature = "qlog")))]
2330
2381
  pub fn set_qlog(
2331
- &mut self, writer: Box<dyn std::io::Write + Send + Sync>, title: String,
2382
+ &mut self,
2383
+ writer: Box<dyn std::io::Write + Send + Sync>,
2384
+ title: String,
2332
2385
  description: String,
2333
2386
  ) {
2334
2387
  self.set_qlog_with_level(writer, title, description, QlogLevel::Base)
@@ -2346,8 +2399,11 @@ impl<F: BufFactory> Connection<F> {
2346
2399
  #[cfg(feature = "qlog")]
2347
2400
  #[cfg_attr(docsrs, doc(cfg(feature = "qlog")))]
2348
2401
  pub fn set_qlog_with_level(
2349
- &mut self, writer: Box<dyn std::io::Write + Send + Sync>, title: String,
2350
- description: String, qlog_level: QlogLevel,
2402
+ &mut self,
2403
+ writer: Box<dyn std::io::Write + Send + Sync>,
2404
+ title: String,
2405
+ description: String,
2406
+ qlog_level: QlogLevel,
2351
2407
  ) {
2352
2408
  use qlog::events::quic::TransportInitiator;
2353
2409
  use qlog::events::HTTP3_URI;
@@ -2460,8 +2516,7 @@ impl<F: BufFactory> Connection<F> {
2460
2516
  /// The default value is infinite, that is, no timeout is used unless
2461
2517
  /// already configured when creating the connection.
2462
2518
  pub fn set_max_idle_timeout(&mut self, v: u64) -> Result<()> {
2463
- self.local_transport_params.max_idle_timeout =
2464
- cmp::min(v, octets::MAX_VAR_INT);
2519
+ self.local_transport_params.max_idle_timeout = cmp::min(v, octets::MAX_VAR_INT);
2465
2520
 
2466
2521
  self.encode_transport_params()
2467
2522
  }
@@ -2478,7 +2533,8 @@ impl<F: BufFactory> Connection<F> {
2478
2533
  #[cfg(feature = "boringssl-boring-crate")]
2479
2534
  #[cfg_attr(docsrs, doc(cfg(feature = "boringssl-boring-crate")))]
2480
2535
  pub fn set_cc_algorithm_in_handshake(
2481
- ssl: &mut boring::ssl::SslRef, algo: CongestionControlAlgorithm,
2536
+ ssl: &mut boring::ssl::SslRef,
2537
+ algo: CongestionControlAlgorithm,
2482
2538
  ) -> Result<()> {
2483
2539
  let ex_data = tls::ExData::from_ssl_ref(ssl).ok_or(Error::TlsFail)?;
2484
2540
 
@@ -2505,7 +2561,8 @@ impl<F: BufFactory> Connection<F> {
2505
2561
  #[cfg_attr(docsrs, doc(cfg(feature = "boringssl-boring-crate")))]
2506
2562
  #[doc(hidden)]
2507
2563
  pub fn set_custom_bbr_settings_in_handshake(
2508
- ssl: &mut boring::ssl::SslRef, custom_bbr_params: BbrParams,
2564
+ ssl: &mut boring::ssl::SslRef,
2565
+ custom_bbr_params: BbrParams,
2509
2566
  ) -> Result<()> {
2510
2567
  let ex_data = tls::ExData::from_ssl_ref(ssl).ok_or(Error::TlsFail)?;
2511
2568
 
@@ -2526,7 +2583,8 @@ impl<F: BufFactory> Connection<F> {
2526
2583
  #[cfg(feature = "boringssl-boring-crate")]
2527
2584
  #[cfg_attr(docsrs, doc(cfg(feature = "boringssl-boring-crate")))]
2528
2585
  pub fn set_cc_algorithm_name_in_handshake(
2529
- ssl: &mut boring::ssl::SslRef, name: &str,
2586
+ ssl: &mut boring::ssl::SslRef,
2587
+ name: &str,
2530
2588
  ) -> Result<()> {
2531
2589
  let cc_algo = CongestionControlAlgorithm::from_str(name)?;
2532
2590
  Self::set_cc_algorithm_in_handshake(ssl, cc_algo)
@@ -2544,7 +2602,8 @@ impl<F: BufFactory> Connection<F> {
2544
2602
  #[cfg(feature = "boringssl-boring-crate")]
2545
2603
  #[cfg_attr(docsrs, doc(cfg(feature = "boringssl-boring-crate")))]
2546
2604
  pub fn set_initial_congestion_window_packets_in_handshake(
2547
- ssl: &mut boring::ssl::SslRef, packets: usize,
2605
+ ssl: &mut boring::ssl::SslRef,
2606
+ packets: usize,
2548
2607
  ) -> Result<()> {
2549
2608
  let ex_data = tls::ExData::from_ssl_ref(ssl).ok_or(Error::TlsFail)?;
2550
2609
 
@@ -2565,7 +2624,8 @@ impl<F: BufFactory> Connection<F> {
2565
2624
  #[cfg(feature = "boringssl-boring-crate")]
2566
2625
  #[cfg_attr(docsrs, doc(cfg(feature = "boringssl-boring-crate")))]
2567
2626
  pub fn set_enable_relaxed_loss_threshold_in_handshake(
2568
- ssl: &mut boring::ssl::SslRef, enable: bool,
2627
+ ssl: &mut boring::ssl::SslRef,
2628
+ enable: bool,
2569
2629
  ) -> Result<()> {
2570
2630
  let ex_data = tls::ExData::from_ssl_ref(ssl).ok_or(Error::TlsFail)?;
2571
2631
 
@@ -2586,7 +2646,8 @@ impl<F: BufFactory> Connection<F> {
2586
2646
  #[cfg(feature = "boringssl-boring-crate")]
2587
2647
  #[cfg_attr(docsrs, doc(cfg(feature = "boringssl-boring-crate")))]
2588
2648
  pub fn set_enable_cubic_idle_restart_fix_in_handshake(
2589
- ssl: &mut boring::ssl::SslRef, enable: bool,
2649
+ ssl: &mut boring::ssl::SslRef,
2650
+ enable: bool,
2590
2651
  ) -> Result<()> {
2591
2652
  let ex_data = tls::ExData::from_ssl_ref(ssl).ok_or(Error::TlsFail)?;
2592
2653
 
@@ -2606,9 +2667,7 @@ impl<F: BufFactory> Connection<F> {
2606
2667
  /// [`Config::enable_hystart()`]: struct.Config.html#method.enable_hystart
2607
2668
  #[cfg(feature = "boringssl-boring-crate")]
2608
2669
  #[cfg_attr(docsrs, doc(cfg(feature = "boringssl-boring-crate")))]
2609
- pub fn set_hystart_in_handshake(
2610
- ssl: &mut boring::ssl::SslRef, v: bool,
2611
- ) -> Result<()> {
2670
+ pub fn set_hystart_in_handshake(ssl: &mut boring::ssl::SslRef, v: bool) -> Result<()> {
2612
2671
  let ex_data = tls::ExData::from_ssl_ref(ssl).ok_or(Error::TlsFail)?;
2613
2672
 
2614
2673
  ex_data.recovery_config.hystart = v;
@@ -2627,9 +2686,7 @@ impl<F: BufFactory> Connection<F> {
2627
2686
  /// [`Config::enable_pacing()`]: struct.Config.html#method.enable_pacing
2628
2687
  #[cfg(feature = "boringssl-boring-crate")]
2629
2688
  #[cfg_attr(docsrs, doc(cfg(feature = "boringssl-boring-crate")))]
2630
- pub fn set_pacing_in_handshake(
2631
- ssl: &mut boring::ssl::SslRef, v: bool,
2632
- ) -> Result<()> {
2689
+ pub fn set_pacing_in_handshake(ssl: &mut boring::ssl::SslRef, v: bool) -> Result<()> {
2633
2690
  let ex_data = tls::ExData::from_ssl_ref(ssl).ok_or(Error::TlsFail)?;
2634
2691
 
2635
2692
  ex_data.recovery_config.pacing = v;
@@ -2649,7 +2706,8 @@ impl<F: BufFactory> Connection<F> {
2649
2706
  #[cfg(feature = "boringssl-boring-crate")]
2650
2707
  #[cfg_attr(docsrs, doc(cfg(feature = "boringssl-boring-crate")))]
2651
2708
  pub fn set_max_pacing_rate_in_handshake(
2652
- ssl: &mut boring::ssl::SslRef, v: Option<u64>,
2709
+ ssl: &mut boring::ssl::SslRef,
2710
+ v: Option<u64>,
2653
2711
  ) -> Result<()> {
2654
2712
  let ex_data = tls::ExData::from_ssl_ref(ssl).ok_or(Error::TlsFail)?;
2655
2713
 
@@ -2670,7 +2728,8 @@ impl<F: BufFactory> Connection<F> {
2670
2728
  #[cfg(feature = "boringssl-boring-crate")]
2671
2729
  #[cfg_attr(docsrs, doc(cfg(feature = "boringssl-boring-crate")))]
2672
2730
  pub fn set_max_send_udp_payload_size_in_handshake(
2673
- ssl: &mut boring::ssl::SslRef, v: usize,
2731
+ ssl: &mut boring::ssl::SslRef,
2732
+ v: usize,
2674
2733
  ) -> Result<()> {
2675
2734
  let ex_data = tls::ExData::from_ssl_ref(ssl).ok_or(Error::TlsFail)?;
2676
2735
 
@@ -2691,7 +2750,8 @@ impl<F: BufFactory> Connection<F> {
2691
2750
  #[cfg(feature = "boringssl-boring-crate")]
2692
2751
  #[cfg_attr(docsrs, doc(cfg(feature = "boringssl-boring-crate")))]
2693
2752
  pub fn set_send_capacity_factor_in_handshake(
2694
- ssl: &mut boring::ssl::SslRef, v: f64,
2753
+ ssl: &mut boring::ssl::SslRef,
2754
+ v: f64,
2695
2755
  ) -> Result<()> {
2696
2756
  let ex_data = tls::ExData::from_ssl_ref(ssl).ok_or(Error::TlsFail)?;
2697
2757
 
@@ -2712,7 +2772,9 @@ impl<F: BufFactory> Connection<F> {
2712
2772
  #[cfg(feature = "boringssl-boring-crate")]
2713
2773
  #[cfg_attr(docsrs, doc(cfg(feature = "boringssl-boring-crate")))]
2714
2774
  pub fn set_discover_pmtu_in_handshake(
2715
- ssl: &mut boring::ssl::SslRef, discover: bool, max_probes: u8,
2775
+ ssl: &mut boring::ssl::SslRef,
2776
+ discover: bool,
2777
+ max_probes: u8,
2716
2778
  ) -> Result<()> {
2717
2779
  let ex_data = tls::ExData::from_ssl_ref(ssl).ok_or(Error::TlsFail)?;
2718
2780
 
@@ -2732,9 +2794,7 @@ impl<F: BufFactory> Connection<F> {
2732
2794
  /// [`Config::set_max_idle_timeout()`]: struct.Config.html#method.set_max_idle_timeout
2733
2795
  #[cfg(feature = "boringssl-boring-crate")]
2734
2796
  #[cfg_attr(docsrs, doc(cfg(feature = "boringssl-boring-crate")))]
2735
- pub fn set_max_idle_timeout_in_handshake(
2736
- ssl: &mut boring::ssl::SslRef, v: u64,
2737
- ) -> Result<()> {
2797
+ pub fn set_max_idle_timeout_in_handshake(ssl: &mut boring::ssl::SslRef, v: u64) -> Result<()> {
2738
2798
  let ex_data = tls::ExData::from_ssl_ref(ssl).ok_or(Error::TlsFail)?;
2739
2799
 
2740
2800
  ex_data.local_transport_params.max_idle_timeout = v;
@@ -2746,6 +2806,27 @@ impl<F: BufFactory> Connection<F> {
2746
2806
  )
2747
2807
  }
2748
2808
 
2809
+ /// Sets the number of sent packets that can be tracked for loss detection
2810
+ /// before the connection is closed.
2811
+ ///
2812
+ /// This function can only be called inside one of BoringSSL's handshake
2813
+ /// callbacks, before any packet has been sent. Calling this function any
2814
+ /// other time will have no effect.
2815
+ ///
2816
+ /// See [`Config::set_max_outstanding_sent_packets()`].
2817
+ #[cfg(feature = "boringssl-boring-crate")]
2818
+ #[cfg_attr(docsrs, doc(cfg(feature = "boringssl-boring-crate")))]
2819
+ pub fn set_max_outstanding_sent_packets_in_handshake(
2820
+ ssl: &mut boring::ssl::SslRef,
2821
+ max_packets: Option<NonZeroUsize>,
2822
+ ) -> Result<()> {
2823
+ let ex_data = tls::ExData::from_ssl_ref(ssl).ok_or(Error::TlsFail)?;
2824
+
2825
+ ex_data.max_outstanding_sent_packets = max_packets;
2826
+
2827
+ Ok(())
2828
+ }
2829
+
2749
2830
  /// Sets the `initial_max_streams_bidi` transport parameter.
2750
2831
  ///
2751
2832
  /// This function can only be called inside one of BoringSSL's handshake
@@ -2758,7 +2839,8 @@ impl<F: BufFactory> Connection<F> {
2758
2839
  #[cfg(feature = "boringssl-boring-crate")]
2759
2840
  #[cfg_attr(docsrs, doc(cfg(feature = "boringssl-boring-crate")))]
2760
2841
  pub fn set_initial_max_streams_bidi_in_handshake(
2761
- ssl: &mut boring::ssl::SslRef, v: u64,
2842
+ ssl: &mut boring::ssl::SslRef,
2843
+ v: u64,
2762
2844
  ) -> Result<()> {
2763
2845
  let ex_data = tls::ExData::from_ssl_ref(ssl).ok_or(Error::TlsFail)?;
2764
2846
 
@@ -2773,7 +2855,9 @@ impl<F: BufFactory> Connection<F> {
2773
2855
 
2774
2856
  #[cfg(feature = "boringssl-boring-crate")]
2775
2857
  fn set_transport_parameters_in_hanshake(
2776
- params: TransportParams, is_server: bool, ssl: &mut boring::ssl::SslRef,
2858
+ params: TransportParams,
2859
+ is_server: bool,
2860
+ ssl: &mut boring::ssl::SslRef,
2777
2861
  ) -> Result<()> {
2778
2862
  use foreign_types_shared::ForeignTypeRef;
2779
2863
  use std::mem::ManuallyDrop;
@@ -2787,9 +2871,8 @@ impl<F: BufFactory> Connection<F> {
2787
2871
  // Wrap the temporary `Handshake` in `ManuallyDrop` because this is only
2788
2872
  // a borrowed view of `ssl`. The caller retains ownership of the
2789
2873
  // underlying BoringSSL object.
2790
- let mut handshake = ManuallyDrop::new(unsafe {
2791
- tls::Handshake::from_ptr(ssl.as_ptr() as _)
2792
- });
2874
+ let mut handshake =
2875
+ ManuallyDrop::new(unsafe { tls::Handshake::from_ptr(ssl.as_ptr() as _)? });
2793
2876
 
2794
2877
  handshake.set_quic_transport_params(&params, is_server)
2795
2878
  }
@@ -2797,19 +2880,14 @@ impl<F: BufFactory> Connection<F> {
2797
2880
  /// Sets the `use_initial_max_data_as_flow_control_win` flag during SSL
2798
2881
  /// handshake.
2799
2882
  ///
2800
- /// This function can only be called inside one of BoringSSL's handshake
2801
- /// callbacks, before any packet has been sent. Calling this function any
2802
- /// other time will have no effect.
2803
- ///
2804
- /// See [`Connection::enable_use_initial_max_data_as_flow_control_win()`].
2883
+ /// This is now always enabled and this method is a no-op. It will be
2884
+ /// removed in a future release.
2805
2885
  #[cfg(feature = "boringssl-boring-crate")]
2806
2886
  #[cfg_attr(docsrs, doc(cfg(feature = "boringssl-boring-crate")))]
2887
+ #[deprecated(note = "This is now always enabled. This method is a no-op.")]
2807
2888
  pub fn set_use_initial_max_data_as_flow_control_win_in_handshake(
2808
- ssl: &mut boring::ssl::SslRef,
2889
+ _ssl: &mut boring::ssl::SslRef,
2809
2890
  ) -> Result<()> {
2810
- let ex_data = tls::ExData::from_ssl_ref(ssl).ok_or(Error::TlsFail)?;
2811
-
2812
- ex_data.use_initial_max_data_as_flow_control_win = true;
2813
2891
  Ok(())
2814
2892
  }
2815
2893
 
@@ -2898,11 +2976,7 @@ impl<F: BufFactory> Connection<F> {
2898
2976
 
2899
2977
  // Process coalesced packets.
2900
2978
  while left > 0 {
2901
- let read = match self.recv_single(
2902
- &mut buf[len - left..len],
2903
- &info,
2904
- recv_pid,
2905
- ) {
2979
+ let read = match self.recv_single(&mut buf[len - left..len], &info, recv_pid) {
2906
2980
  Ok(v) => v,
2907
2981
 
2908
2982
  Err(Error::Done) => {
@@ -2915,14 +2989,19 @@ impl<F: BufFactory> Connection<F> {
2915
2989
  }
2916
2990
 
2917
2991
  left
2918
- },
2992
+ }
2919
2993
 
2920
2994
  Err(e) => {
2921
2995
  // In case of error processing the incoming packet, close
2922
2996
  // the connection.
2923
- self.close(false, e.to_wire(), b"").ok();
2997
+ let reason: &[u8] = if e == Error::StreamMemoryLimit {
2998
+ b"QUIC receive memory limit"
2999
+ } else {
3000
+ b""
3001
+ };
3002
+ self.close(false, e.to_wire(), reason).ok();
2924
3003
  return Err(e);
2925
- },
3004
+ }
2926
3005
  };
2927
3006
 
2928
3007
  done += read;
@@ -2944,8 +3023,7 @@ impl<F: BufFactory> Connection<F> {
2944
3023
  .crypto_0rtt_open
2945
3024
  .is_some()
2946
3025
  {
2947
- while let Some((mut pkt, info)) = self.undecryptable_pkts.pop_front()
2948
- {
3026
+ while let Some((mut pkt, info)) = self.undecryptable_pkts.pop_front() {
2949
3027
  if let Err(e) = self.recv(&mut pkt, info) {
2950
3028
  self.undecryptable_pkts.clear();
2951
3029
 
@@ -2975,7 +3053,7 @@ impl<F: BufFactory> Connection<F> {
2975
3053
  &buf[buf_len - token_len..buf_len],
2976
3054
  )
2977
3055
  .is_ok()
2978
- },
3056
+ }
2979
3057
 
2980
3058
  None => false,
2981
3059
  }
@@ -2996,7 +3074,10 @@ impl<F: BufFactory> Connection<F> {
2996
3074
  ///
2997
3075
  /// [`Done`]: enum.Error.html#variant.Done
2998
3076
  fn recv_single(
2999
- &mut self, buf: &mut [u8], info: &RecvInfo, recv_pid: Option<usize>,
3077
+ &mut self,
3078
+ buf: &mut [u8],
3079
+ info: &RecvInfo,
3080
+ recv_pid: Option<usize>,
3000
3081
  ) -> Result<usize> {
3001
3082
  let now = Instant::now();
3002
3083
 
@@ -3019,14 +3100,7 @@ impl<F: BufFactory> Connection<F> {
3019
3100
  let mut b = octets::OctetsMut::with_slice(buf);
3020
3101
 
3021
3102
  let mut hdr = Header::from_bytes(&mut b, self.source_id().len())
3022
- .map_err(|e| {
3023
- drop_pkt_on_err(
3024
- e,
3025
- self.recv_count,
3026
- self.is_server,
3027
- &self.trace_id,
3028
- )
3029
- })?;
3103
+ .map_err(|e| drop_pkt_on_err(e, self.recv_count, self.is_server, &self.trace_id))?;
3030
3104
 
3031
3105
  if hdr.ty == Type::VersionNegotiation {
3032
3106
  // Version negotiation packets can only be sent by the server.
@@ -3063,8 +3137,7 @@ impl<F: BufFactory> Connection<F> {
3063
3137
  return Err(Error::Done);
3064
3138
  }
3065
3139
 
3066
- let supported_versions =
3067
- versions.iter().filter(|&&v| version_is_supported(v));
3140
+ let supported_versions = versions.iter().filter(|&&v| version_is_supported(v));
3068
3141
 
3069
3142
  let mut found_version = false;
3070
3143
 
@@ -3131,13 +3204,7 @@ impl<F: BufFactory> Connection<F> {
3131
3204
  }
3132
3205
 
3133
3206
  // Check if Retry packet is valid.
3134
- if packet::verify_retry_integrity(
3135
- &b,
3136
- &self.destination_id(),
3137
- self.version,
3138
- )
3139
- .is_err()
3140
- {
3207
+ if packet::verify_retry_integrity(&b, &self.destination_id(), self.version).is_err() {
3141
3208
  return Err(Error::Done);
3142
3209
  }
3143
3210
 
@@ -3149,21 +3216,13 @@ impl<F: BufFactory> Connection<F> {
3149
3216
  // Remember peer's new connection ID.
3150
3217
  self.odcid = Some(self.destination_id().into_owned());
3151
3218
 
3152
- self.set_initial_dcid(
3153
- hdr.scid.clone(),
3154
- None,
3155
- self.paths.get_active_path_id()?,
3156
- )?;
3219
+ self.set_initial_dcid(hdr.scid.clone(), None, self.paths.get_active_path_id()?)?;
3157
3220
 
3158
3221
  self.rscid = Some(self.destination_id().into_owned());
3159
3222
 
3160
3223
  // Derive Initial secrets using the new connection ID.
3161
- let (aead_open, aead_seal) = crypto::derive_initial_key_material(
3162
- &hdr.scid,
3163
- self.version,
3164
- self.is_server,
3165
- true,
3166
- )?;
3224
+ let (aead_open, aead_seal) =
3225
+ crypto::derive_initial_key_material(&hdr.scid, self.version, self.is_server, true)?;
3167
3226
 
3168
3227
  // Reset connection state to force sending another Initial packet.
3169
3228
  self.drop_epoch_state(packet::Epoch::Initial, now);
@@ -3204,12 +3263,7 @@ impl<F: BufFactory> Connection<F> {
3204
3263
  b.cap()
3205
3264
  } else {
3206
3265
  b.get_varint().map_err(|e| {
3207
- drop_pkt_on_err(
3208
- e.into(),
3209
- self.recv_count,
3210
- self.is_server,
3211
- &self.trace_id,
3212
- )
3266
+ drop_pkt_on_err(e.into(), self.recv_count, self.is_server, &self.trace_id)
3213
3267
  })? as usize
3214
3268
  };
3215
3269
 
@@ -3256,9 +3310,9 @@ impl<F: BufFactory> Connection<F> {
3256
3310
  Some(v) => v,
3257
3311
 
3258
3312
  None => {
3259
- if hdr.ty == Type::ZeroRTT &&
3260
- self.undecryptable_pkts.len() < MAX_UNDECRYPTABLE_PACKETS &&
3261
- !self.is_established()
3313
+ if hdr.ty == Type::ZeroRTT
3314
+ && self.undecryptable_pkts.len() < MAX_UNDECRYPTABLE_PACKETS
3315
+ && !self.is_established()
3262
3316
  {
3263
3317
  // Buffer 0-RTT packets when the required read key is not
3264
3318
  // available yet, and process them later.
@@ -3280,14 +3334,13 @@ impl<F: BufFactory> Connection<F> {
3280
3334
  );
3281
3335
 
3282
3336
  return Err(e);
3283
- },
3337
+ }
3284
3338
  };
3285
3339
 
3286
3340
  let aead_tag_len = aead.alg().tag_len();
3287
3341
 
3288
- packet::decrypt_hdr(&mut b, &mut hdr, aead).map_err(|e| {
3289
- drop_pkt_on_err(e, self.recv_count, self.is_server, &self.trace_id)
3290
- })?;
3342
+ packet::decrypt_hdr(&mut b, &mut hdr, aead)
3343
+ .map_err(|e| drop_pkt_on_err(e, self.recv_count, self.is_server, &self.trace_id))?;
3291
3344
 
3292
3345
  let pn = packet::decode_pkt_num(
3293
3346
  self.pkt_num_spaces[epoch].largest_rx_pkt_num,
@@ -3312,17 +3365,12 @@ impl<F: BufFactory> Connection<F> {
3312
3365
  // Check for key update.
3313
3366
  let mut aead_next = None;
3314
3367
 
3315
- if self.handshake_confirmed &&
3316
- hdr.ty != Type::ZeroRTT &&
3317
- hdr.key_phase != self.key_phase
3318
- {
3368
+ if self.handshake_confirmed && hdr.ty != Type::ZeroRTT && hdr.key_phase != self.key_phase {
3319
3369
  // Check if this packet arrived before key update.
3320
3370
  if let Some(key_update) = self.crypto_ctx[epoch]
3321
3371
  .key_update
3322
3372
  .as_ref()
3323
- .and_then(|key_update| {
3324
- (pn < key_update.pn_on_update).then_some(key_update)
3325
- })
3373
+ .and_then(|key_update| (pn < key_update.pn_on_update).then_some(key_update))
3326
3374
  {
3327
3375
  aead = &key_update.crypto_open;
3328
3376
  } else {
@@ -3341,22 +3389,14 @@ impl<F: BufFactory> Connection<F> {
3341
3389
  .derive_next_packet_key()?,
3342
3390
  ));
3343
3391
 
3344
- // `aead_next` is always `Some()` at this point, so the `unwrap()`
3345
- // will never fail.
3392
+ // `aead_next` is always `Some` at this point, so the
3393
+ // `unwrap()` will never fail.
3346
3394
  aead = &aead_next.as_ref().unwrap().0;
3347
3395
  }
3348
3396
  }
3349
3397
 
3350
- let mut payload = packet::decrypt_pkt(
3351
- &mut b,
3352
- pn,
3353
- pn_len,
3354
- payload_len,
3355
- aead,
3356
- )
3357
- .map_err(|e| {
3358
- drop_pkt_on_err(e, self.recv_count, self.is_server, &self.trace_id)
3359
- })?;
3398
+ let mut payload = packet::decrypt_pkt(&mut b, pn, pn_len, payload_len, aead)
3399
+ .map_err(|e| drop_pkt_on_err(e, self.recv_count, self.is_server, &self.trace_id))?;
3360
3400
 
3361
3401
  if self.pkt_num_spaces[epoch].recv_pkt_num.contains(pn) {
3362
3402
  trace!("{} ignored duplicate packet {}", self.trace_id, pn);
@@ -3411,25 +3451,21 @@ impl<F: BufFactory> Connection<F> {
3411
3451
  self.key_phase = !self.key_phase;
3412
3452
 
3413
3453
  qlog_with_type!(QLOG_PACKET_RX, self.qlog, q, {
3414
- let trigger = Some(
3415
- qlog::events::quic::KeyUpdateOrRetiredTrigger::RemoteUpdate,
3416
- );
3454
+ let trigger = Some(qlog::events::quic::KeyUpdateOrRetiredTrigger::RemoteUpdate);
3417
3455
 
3418
- let ev_data_client =
3419
- EventData::QuicKeyUpdated(qlog::events::quic::KeyUpdated {
3420
- key_type: qlog::events::quic::KeyType::Client1RttSecret,
3421
- trigger: trigger.clone(),
3422
- ..Default::default()
3423
- });
3456
+ let ev_data_client = EventData::QuicKeyUpdated(qlog::events::quic::KeyUpdated {
3457
+ key_type: qlog::events::quic::KeyType::Client1RttSecret,
3458
+ trigger: trigger.clone(),
3459
+ ..Default::default()
3460
+ });
3424
3461
 
3425
3462
  q.add_event_data_with_instant(ev_data_client, now).ok();
3426
3463
 
3427
- let ev_data_server =
3428
- EventData::QuicKeyUpdated(qlog::events::quic::KeyUpdated {
3429
- key_type: qlog::events::quic::KeyType::Server1RttSecret,
3430
- trigger,
3431
- ..Default::default()
3432
- });
3464
+ let ev_data_server = EventData::QuicKeyUpdated(qlog::events::quic::KeyUpdated {
3465
+ key_type: qlog::events::quic::KeyType::Server1RttSecret,
3466
+ trigger,
3467
+ ..Default::default()
3468
+ });
3433
3469
 
3434
3470
  q.add_event_data_with_instant(ev_data_server, now).ok();
3435
3471
  });
@@ -3456,8 +3492,7 @@ impl<F: BufFactory> Connection<F> {
3456
3492
 
3457
3493
  if !self.did_retry {
3458
3494
  self.local_transport_params
3459
- .original_destination_connection_id =
3460
- Some(hdr.dcid.to_vec().into());
3495
+ .original_destination_connection_id = Some(hdr.dcid.to_vec().into());
3461
3496
 
3462
3497
  self.encode_transport_params()?;
3463
3498
  }
@@ -3494,8 +3529,7 @@ impl<F: BufFactory> Connection<F> {
3494
3529
  probing = false;
3495
3530
  }
3496
3531
 
3497
- if let Err(e) = self.process_frame(frame, &hdr, recv_pid, epoch, now)
3498
- {
3532
+ if let Err(e) = self.process_frame(frame, &hdr, recv_pid, epoch, now) {
3499
3533
  frame_processing_err = Some(e);
3500
3534
  break;
3501
3535
  }
@@ -3518,14 +3552,12 @@ impl<F: BufFactory> Connection<F> {
3518
3552
  data: None,
3519
3553
  };
3520
3554
 
3521
- let ev_data = EventData::QuicPacketReceived(
3522
- qlog::events::quic::PacketReceived {
3523
- header: qlog_pkt_hdr,
3524
- frames: Some(qlog_frames),
3525
- raw: Some(qlog_raw_info),
3526
- ..Default::default()
3527
- },
3528
- );
3555
+ let ev_data = EventData::QuicPacketReceived(qlog::events::quic::PacketReceived {
3556
+ header: qlog_pkt_hdr,
3557
+ frames: Some(qlog_frames),
3558
+ raw: Some(qlog_raw_info),
3559
+ ..Default::default()
3560
+ });
3529
3561
 
3530
3562
  q.add_event_data_with_instant(ev_data, now).ok();
3531
3563
  });
@@ -3546,10 +3578,9 @@ impl<F: BufFactory> Connection<F> {
3546
3578
  if self.is_established() {
3547
3579
  qlog_with_type!(QLOG_PARAMS_SET, self.qlog, q, {
3548
3580
  if !self.qlog.logged_peer_params {
3549
- let ev_data = self.peer_transport_params.to_qlog(
3550
- TransportInitiator::Remote,
3551
- self.handshake.cipher(),
3552
- );
3581
+ let ev_data = self
3582
+ .peer_transport_params
3583
+ .to_qlog(TransportInitiator::Remote, self.handshake.cipher());
3553
3584
 
3554
3585
  q.add_event_data_with_instant(ev_data, now).ok();
3555
3586
 
@@ -3560,54 +3591,54 @@ impl<F: BufFactory> Connection<F> {
3560
3591
 
3561
3592
  // Process acked frames. Note that several packets from several paths
3562
3593
  // might have been acked by the received packet.
3563
- for (_, p) in self.paths.iter_mut() {
3594
+ let (paths, path_events) = self.paths.iter_mut_and_events();
3595
+ for (_, p) in paths {
3564
3596
  while let Some(acked) = p.recovery.next_acked_frame(epoch) {
3565
3597
  match acked {
3566
3598
  frame::Frame::Ping {
3567
3599
  mtu_probe: Some(mtu_probe),
3568
3600
  } => {
3601
+ trace!(
3602
+ "{} pmtud probe acked; probe size {:?}",
3603
+ self.trace_id,
3604
+ mtu_probe
3605
+ );
3606
+
3607
+ let local = p.local_addr();
3608
+ let peer = p.peer_addr();
3569
3609
  if let Some(pmtud) = p.pmtud.as_mut() {
3570
- trace!(
3571
- "{} pmtud probe acked; probe size {:?}",
3572
- self.trace_id,
3573
- mtu_probe
3574
- );
3610
+ let old_pmtu = pmtud.get_current_mtu();
3611
+ let current_mtu = pmtud.successful_probe(mtu_probe);
3612
+ let new_pmtu = pmtud.get_current_mtu();
3575
3613
 
3576
- // Ensure the probe is within the supported MTU range
3577
- // before updating the max datagram size
3578
- if let Some(current_mtu) =
3579
- pmtud.successful_probe(mtu_probe)
3580
- {
3614
+ // Update the datagram size only after validating
3615
+ // the MTU.
3616
+ if let Some(current_mtu) = current_mtu {
3581
3617
  qlog_with_type!(
3582
- EventType::QuicEventType(
3583
- QuicEventType::MtuUpdated
3584
- ),
3618
+ EventType::QuicEventType(QuicEventType::MtuUpdated),
3585
3619
  self.qlog,
3586
3620
  q,
3587
3621
  {
3588
3622
  let pmtu_data = EventData::QuicMtuUpdated(
3589
3623
  qlog::events::quic::MtuUpdated {
3590
- old: Some(
3591
- p.recovery.max_datagram_size()
3592
- as u32,
3593
- ),
3624
+ old: Some(p.recovery.max_datagram_size() as u32),
3594
3625
  new: current_mtu as u32,
3595
3626
  done: Some(true),
3596
3627
  },
3597
3628
  );
3598
3629
 
3599
- q.add_event_data_with_instant(
3600
- pmtu_data, now,
3601
- )
3602
- .ok();
3630
+ q.add_event_data_with_instant(pmtu_data, now).ok();
3603
3631
  }
3604
3632
  );
3605
3633
 
3606
- p.recovery
3607
- .pmtud_update_max_datagram_size(current_mtu);
3634
+ p.recovery.pmtud_update_max_datagram_size(current_mtu);
3635
+ }
3636
+
3637
+ if let Some(event) = path::pmtu_event(local, peer, old_pmtu, new_pmtu) {
3638
+ path_events.push_back(event);
3608
3639
  }
3609
3640
  }
3610
- },
3641
+ }
3611
3642
 
3612
3643
  frame::Frame::ACK { ranges, .. } => {
3613
3644
  // Stop acknowledging packets less than or equal to the
@@ -3618,27 +3649,24 @@ impl<F: BufFactory> Connection<F> {
3618
3649
  .recv_pkt_need_ack
3619
3650
  .remove_until(largest_acked);
3620
3651
  }
3621
- },
3652
+ }
3622
3653
 
3623
3654
  frame::Frame::CryptoHeader { offset, length } => {
3624
3655
  self.crypto_ctx[epoch]
3625
3656
  .crypto_stream
3626
3657
  .send
3627
3658
  .ack_and_drop(offset, length);
3628
- },
3659
+ }
3629
3660
 
3630
3661
  frame::Frame::StreamHeader {
3631
3662
  stream_id,
3632
3663
  offset,
3633
3664
  length,
3634
- ..
3665
+ fin,
3635
3666
  } => {
3636
- // Update tx_buffered and emit qlog before checking if the
3637
- // stream still exists. The client does need to ACK
3638
- // frames that were received after the client sends a
3639
- // ResetStream.
3640
- self.tx_buffered =
3641
- self.tx_buffered.saturating_sub(length);
3667
+ // Emit qlog before checking if the stream still exists.
3668
+ // The client does need to ACK frames that were received
3669
+ // after the client sends a ResetStream.
3642
3670
 
3643
3671
  qlog_with_type!(QLOG_DATA_MV, self.qlog, q, {
3644
3672
  let ev_data = EventData::QuicStreamDataMoved(
@@ -3664,41 +3692,21 @@ impl<F: BufFactory> Connection<F> {
3664
3692
  None => continue,
3665
3693
  };
3666
3694
 
3667
- stream.send.ack_and_drop(offset, length);
3695
+ let dropped = stream.send.ack_and_drop(offset, length);
3668
3696
 
3669
- let priority_key = Arc::clone(&stream.priority_key);
3697
+ if fin {
3698
+ stream.send.ack_fin();
3699
+ }
3670
3700
 
3671
- // Only collect the stream if it is complete and not
3672
- // readable or writable.
3673
- //
3674
- // If it is readable, it will get collected when
3675
- // stream_recv() is next used.
3676
- //
3677
- // If it is writable, it might mean that the stream
3678
- // has been stopped by the peer (i.e. a STOP_SENDING
3679
- // frame is received), in which case before collecting
3680
- // the stream we will need to propagate the
3681
- // `StreamStopped` error to the application. It will
3682
- // instead get collected when one of stream_capacity(),
3683
- // stream_writable(), stream_send(), ... is next called.
3684
- //
3685
- // Note that we can't use `is_writable()` here because
3686
- // it returns false if the stream is stopped. Instead,
3687
- // since the stream is marked as writable when a
3688
- // STOP_SENDING frame is received, we check the writable
3689
- // queue directly instead.
3690
- let is_writable = priority_key.writable.is_linked() &&
3691
- // Ensure that the stream is actually stopped.
3692
- stream.send.is_stopped();
3693
-
3694
- let is_complete = stream.is_complete();
3695
- let is_readable = stream.is_readable();
3696
-
3697
- if is_complete && !is_readable && !is_writable {
3698
- let local = stream.local;
3699
- self.streams.collect(stream_id, local);
3701
+ self.streams.try_collect(stream_id);
3702
+
3703
+ // Update `tx_buffered` for data dropped from stream
3704
+ // buffers, such as retransmission data acknowledged
3705
+ // before it could be resent.
3706
+ if dropped > 0 {
3707
+ self.streams.sub_tx_buffered(dropped);
3700
3708
  }
3701
- },
3709
+ }
3702
3710
 
3703
3711
  frame::Frame::HandshakeDone => {
3704
3712
  // Explicitly set this to true, so that if the frame was
@@ -3706,48 +3714,15 @@ impl<F: BufFactory> Connection<F> {
3706
3714
  self.handshake_done_sent = true;
3707
3715
 
3708
3716
  self.handshake_done_acked = true;
3709
- },
3717
+ }
3710
3718
 
3711
3719
  frame::Frame::ResetStream { stream_id, .. } => {
3712
- let stream = match self.streams.get_mut(stream_id) {
3713
- Some(v) => v,
3714
-
3715
- None => continue,
3716
- };
3717
-
3718
- let priority_key = Arc::clone(&stream.priority_key);
3719
-
3720
- // Only collect the stream if it is complete and not
3721
- // readable or writable.
3722
- //
3723
- // If it is readable, it will get collected when
3724
- // stream_recv() is next used.
3725
- //
3726
- // If it is writable, it might mean that the stream
3727
- // has been stopped by the peer (i.e. a STOP_SENDING
3728
- // frame is received), in which case before collecting
3729
- // the stream we will need to propagate the
3730
- // `StreamStopped` error to the application. It will
3731
- // instead get collected when one of stream_capacity(),
3732
- // stream_writable(), stream_send(), ... is next called.
3733
- //
3734
- // Note that we can't use `is_writable()` here because
3735
- // it returns false if the stream is stopped. Instead,
3736
- // since the stream is marked as writable when a
3737
- // STOP_SENDING frame is received, we check the writable
3738
- // queue directly instead.
3739
- let is_writable = priority_key.writable.is_linked() &&
3740
- // Ensure that the stream is actually stopped.
3741
- stream.send.is_stopped();
3742
-
3743
- let is_complete = stream.is_complete();
3744
- let is_readable = stream.is_readable();
3745
-
3746
- if is_complete && !is_readable && !is_writable {
3747
- let local = stream.local;
3748
- self.streams.collect(stream_id, local);
3720
+ if let Some(stream) = self.streams.get_mut(stream_id) {
3721
+ stream.send.ack_reset_stream();
3749
3722
  }
3750
- },
3723
+ self.streams.remove_reset(stream_id);
3724
+ self.streams.try_collect(stream_id);
3725
+ }
3751
3726
 
3752
3727
  _ => (),
3753
3728
  }
@@ -3802,9 +3777,9 @@ impl<F: BufFactory> Connection<F> {
3802
3777
  // Did the peer migrated to another path?
3803
3778
  let active_path_id = self.paths.get_active_path_id()?;
3804
3779
 
3805
- if self.is_server &&
3806
- recv_pid != active_path_id &&
3807
- self.pkt_num_spaces[epoch].largest_rx_non_probing_pkt_num == pn
3780
+ if self.is_server
3781
+ && recv_pid != active_path_id
3782
+ && self.pkt_num_spaces[epoch].largest_rx_non_probing_pkt_num == pn
3808
3783
  {
3809
3784
  self.on_peer_migrated(recv_pid, self.disable_dcid_reuse, now)?;
3810
3785
  }
@@ -3988,7 +3963,9 @@ impl<F: BufFactory> Connection<F> {
3988
3963
  /// # Ok::<(), quiche::Error>(())
3989
3964
  /// ```
3990
3965
  pub fn send_on_path(
3991
- &mut self, out: &mut [u8], from: Option<SocketAddr>,
3966
+ &mut self,
3967
+ out: &mut [u8],
3968
+ from: Option<SocketAddr>,
3992
3969
  to: Option<SocketAddr>,
3993
3970
  ) -> Result<(usize, SendInfo)> {
3994
3971
  if out.is_empty() {
@@ -4001,6 +3978,14 @@ impl<F: BufFactory> Connection<F> {
4001
3978
 
4002
3979
  let now = Instant::now();
4003
3980
 
3981
+ if self.streams.collected_history_exhausted() {
3982
+ let _ = self.close(
3983
+ false,
3984
+ Error::StreamMemoryLimit.to_wire(),
3985
+ b"stream history limit",
3986
+ );
3987
+ }
3988
+
4004
3989
  if self.local_error.is_none() {
4005
3990
  self.do_handshake(now)?;
4006
3991
  }
@@ -4038,11 +4023,10 @@ impl<F: BufFactory> Connection<F> {
4038
4023
 
4039
4024
  let send_path = self.paths.get_mut(send_pid)?;
4040
4025
 
4041
- // Update max datagram size to allow path MTU discovery probe to be sent.
4026
+ // Increase the maximum datagram size for a PMTUD probe.
4042
4027
  if let Some(pmtud) = send_path.pmtud.as_mut() {
4043
4028
  if pmtud.should_probe() {
4044
- let size = if self.handshake_confirmed || self.handshake_completed
4045
- {
4029
+ let size = if self.handshake_confirmed || self.handshake_completed {
4046
4030
  pmtud.get_probe_size()
4047
4031
  } else {
4048
4032
  pmtud.get_current_mtu()
@@ -4050,8 +4034,7 @@ impl<F: BufFactory> Connection<F> {
4050
4034
 
4051
4035
  send_path.recovery.pmtud_update_max_datagram_size(size);
4052
4036
 
4053
- left =
4054
- cmp::min(out.len(), send_path.recovery.max_datagram_size());
4037
+ left = cmp::min(out.len(), send_path.recovery.max_datagram_size());
4055
4038
  }
4056
4039
  }
4057
4040
 
@@ -4063,18 +4046,14 @@ impl<F: BufFactory> Connection<F> {
4063
4046
 
4064
4047
  // Generate coalesced packets.
4065
4048
  while left > 0 {
4066
- let (ty, written) = match self.send_single(
4067
- &mut out[done..done + left],
4068
- send_pid,
4069
- has_initial,
4070
- now,
4071
- ) {
4072
- Ok(v) => v,
4049
+ let (ty, written) =
4050
+ match self.send_single(&mut out[done..done + left], send_pid, has_initial, now) {
4051
+ Ok(v) => v,
4073
4052
 
4074
- Err(Error::BufferTooShort) | Err(Error::Done) => break,
4053
+ Err(Error::BufferTooShort) | Err(Error::Done) => break,
4075
4054
 
4076
- Err(e) => return Err(e),
4077
- };
4055
+ Err(e) => return Err(e),
4056
+ };
4078
4057
 
4079
4058
  done += written;
4080
4059
  left -= written;
@@ -4097,9 +4076,7 @@ impl<F: BufFactory> Connection<F> {
4097
4076
  }
4098
4077
 
4099
4078
  // Don't coalesce packets that must go on different paths.
4100
- if !(from.is_some() && to.is_some()) &&
4101
- self.get_send_path_id(from, to)? != send_pid
4102
- {
4079
+ if !(from.is_some() && to.is_some()) && self.get_send_path_id(from, to)? != send_pid {
4103
4080
  break;
4104
4081
  }
4105
4082
  }
@@ -4133,7 +4110,10 @@ impl<F: BufFactory> Connection<F> {
4133
4110
  }
4134
4111
 
4135
4112
  fn send_single(
4136
- &mut self, out: &mut [u8], send_pid: usize, has_initial: bool,
4113
+ &mut self,
4114
+ out: &mut [u8],
4115
+ send_pid: usize,
4116
+ has_initial: bool,
4137
4117
  now: Instant,
4138
4118
  ) -> Result<(Type, usize)> {
4139
4119
  if out.is_empty() {
@@ -4144,14 +4124,61 @@ impl<F: BufFactory> Connection<F> {
4144
4124
  return Err(Error::Done);
4145
4125
  }
4146
4126
 
4127
+ if self.local_error.is_none() {
4128
+ for &epoch in packet::Epoch::epochs(packet::Epoch::Initial..=packet::Epoch::Application)
4129
+ {
4130
+ let has_probe = self.crypto_ctx[epoch].crypto_seal.is_some()
4131
+ && self.paths.get(send_pid)?.recovery.loss_probes(epoch) > 0;
4132
+
4133
+ if !has_probe {
4134
+ continue;
4135
+ }
4136
+
4137
+ match self.send_single_for_type(
4138
+ out,
4139
+ send_pid,
4140
+ has_initial,
4141
+ now,
4142
+ Type::from_epoch(epoch),
4143
+ true,
4144
+ ) {
4145
+ Err(Error::Done) => (),
4146
+
4147
+ result => return result,
4148
+ }
4149
+ }
4150
+ }
4151
+
4152
+ let pkt_type = self.write_pkt_type(send_pid)?;
4153
+ let epoch = pkt_type.to_epoch()?;
4154
+ let send_path = self.paths.get(send_pid)?;
4155
+
4156
+ // A closing connection sends a single packet with a CONNECTION_CLOSE
4157
+ // frame before draining, which must not be delayed when the congestion
4158
+ // window is full of unacknowledged packets.
4159
+ let sends_close =
4160
+ self.local_error.is_some() && (send_path.active() || self.paths.len() == 1);
4161
+
4162
+ let bypass_cwnd = send_path.recovery.loss_probes(epoch) > 0 || sends_close;
4163
+
4164
+ self.send_single_for_type(out, send_pid, has_initial, now, pkt_type, bypass_cwnd)
4165
+ }
4166
+
4167
+ fn send_single_for_type(
4168
+ &mut self,
4169
+ out: &mut [u8],
4170
+ send_pid: usize,
4171
+ has_initial: bool,
4172
+ now: Instant,
4173
+ pkt_type: Type,
4174
+ bypass_cwnd: bool,
4175
+ ) -> Result<(Type, usize)> {
4147
4176
  let is_closing = self.local_error.is_some();
4148
4177
 
4149
4178
  let out_len = out.len();
4150
4179
 
4151
4180
  let mut b = octets::OctetsMut::with_slice(out);
4152
4181
 
4153
- let pkt_type = self.write_pkt_type(send_pid)?;
4154
-
4155
4182
  let max_dgram_len = if !self.dgram_send_queue.is_empty() {
4156
4183
  self.dgram_max_writable_len()
4157
4184
  } else {
@@ -4163,7 +4190,8 @@ impl<F: BufFactory> Connection<F> {
4163
4190
  let crypto_ctx = &mut self.crypto_ctx[epoch];
4164
4191
 
4165
4192
  // Process lost frames. There might be several paths having lost frames.
4166
- for (_, p) in self.paths.iter_mut() {
4193
+ let (paths, path_events) = self.paths.iter_mut_and_events();
4194
+ for (_, p) in paths {
4167
4195
  while let Some(lost) = p.recovery.next_lost_frame(epoch) {
4168
4196
  match lost {
4169
4197
  frame::Frame::CryptoHeader { offset, length } => {
@@ -4174,7 +4202,7 @@ impl<F: BufFactory> Connection<F> {
4174
4202
 
4175
4203
  self.retrans_count += 1;
4176
4204
  p.retrans_count += 1;
4177
- },
4205
+ }
4178
4206
 
4179
4207
  frame::Frame::StreamHeader {
4180
4208
  stream_id,
@@ -4184,16 +4212,13 @@ impl<F: BufFactory> Connection<F> {
4184
4212
  } => {
4185
4213
  let stream = match self.streams.get_mut(stream_id) {
4186
4214
  // Only retransmit data if the stream is not closed
4187
- // or stopped.
4188
- Some(v) if !v.send.is_stopped() => v,
4215
+ // or reset.
4216
+ Some(v) if v.send.can_send_data() => v,
4189
4217
 
4190
4218
  // Data on a closed stream will not be retransmitted
4191
- // or acked after it is declared lost, so update
4192
- // tx_buffered and qlog.
4219
+ // or acked after it is declared lost, so just drop
4220
+ // it.
4193
4221
  _ => {
4194
- self.tx_buffered =
4195
- self.tx_buffered.saturating_sub(length);
4196
-
4197
4222
  qlog_with_type!(QLOG_DATA_MV, self.qlog, q, {
4198
4223
  let ev_data = EventData::QuicStreamDataMoved(
4199
4224
  qlog::events::quic::StreamDataMoved {
@@ -4209,19 +4234,18 @@ impl<F: BufFactory> Connection<F> {
4209
4234
  },
4210
4235
  );
4211
4236
 
4212
- q.add_event_data_with_instant(ev_data, now)
4213
- .ok();
4237
+ q.add_event_data_with_instant(ev_data, now).ok();
4214
4238
  });
4215
4239
 
4216
4240
  continue;
4217
- },
4241
+ }
4218
4242
  };
4219
4243
 
4220
4244
  let was_flushable = stream.is_flushable();
4221
4245
 
4222
4246
  let empty_fin = length == 0 && fin;
4223
4247
 
4224
- stream.send.retransmit(offset, length);
4248
+ let retransmitted = stream.send.retransmit(offset, length);
4225
4249
 
4226
4250
  // If the stream is now flushable push it to the
4227
4251
  // flushable queue, but only if it wasn't already
@@ -4230,31 +4254,37 @@ impl<F: BufFactory> Connection<F> {
4230
4254
  // Consider the stream flushable also when we are
4231
4255
  // sending a zero-length frame that has the fin flag
4232
4256
  // set.
4233
- if (stream.is_flushable() || empty_fin) && !was_flushable
4234
- {
4257
+ if (stream.is_flushable() || empty_fin) && !was_flushable {
4235
4258
  let priority_key = Arc::clone(&stream.priority_key);
4236
4259
  self.streams.insert_flushable(&priority_key);
4237
4260
  }
4238
4261
 
4262
+ // Update tx_buffered when data is marked for
4263
+ // retransmission (it was decremented when emitted).
4264
+ // Only increment by the actual amount retransmitted,
4265
+ // which may be less than `length` if some data was
4266
+ // already acked.
4267
+ self.streams.add_tx_buffered(retransmitted);
4268
+
4239
4269
  self.stream_retrans_bytes += length as u64;
4240
4270
  p.stream_retrans_bytes += length as u64;
4241
4271
 
4242
4272
  self.retrans_count += 1;
4243
4273
  p.retrans_count += 1;
4244
- },
4274
+ }
4245
4275
 
4246
4276
  frame::Frame::ACK { .. } => {
4247
4277
  pkt_space.ack_elicited = true;
4248
- },
4278
+ }
4249
4279
 
4250
4280
  frame::Frame::ResetStream {
4251
4281
  stream_id,
4252
4282
  error_code,
4253
4283
  final_size,
4284
+ ..
4254
4285
  } => {
4255
- self.streams
4256
- .insert_reset(stream_id, error_code, final_size);
4257
- },
4286
+ self.streams.insert_reset(stream_id, error_code, final_size);
4287
+ }
4258
4288
 
4259
4289
  frame::Frame::StopSending {
4260
4290
  stream_id,
@@ -4264,37 +4294,39 @@ impl<F: BufFactory> Connection<F> {
4264
4294
  // the stream is still active and not FIN'd. Even if the
4265
4295
  // packet was lost, if the application has the final
4266
4296
  // size at this point there is no need to retransmit.
4297
+ {
4267
4298
  if let Some(stream) = self.streams.get(stream_id) {
4268
4299
  if !stream.recv.is_fin() {
4269
- self.streams
4270
- .insert_stopped(stream_id, error_code);
4300
+ self.streams.insert_stopped(stream_id, error_code);
4271
4301
  }
4272
- },
4302
+ }
4303
+ }
4273
4304
 
4274
4305
  // Retransmit HANDSHAKE_DONE only if it hasn't been acked at
4275
4306
  // least once already.
4276
- frame::Frame::HandshakeDone =>
4307
+ frame::Frame::HandshakeDone => {
4277
4308
  if !self.handshake_done_acked {
4278
4309
  self.handshake_done_sent = false;
4279
- },
4310
+ }
4311
+ }
4280
4312
 
4281
4313
  frame::Frame::MaxStreamData { stream_id, .. } => {
4282
4314
  if self.streams.get(stream_id).is_some() {
4283
4315
  self.streams.insert_almost_full(stream_id);
4284
4316
  }
4285
- },
4317
+ }
4286
4318
 
4287
4319
  frame::Frame::MaxData { .. } => {
4288
4320
  self.should_send_max_data = true;
4289
- },
4321
+ }
4290
4322
 
4291
4323
  frame::Frame::MaxStreamsUni { .. } => {
4292
4324
  self.should_send_max_streams_uni = true;
4293
- },
4325
+ }
4294
4326
 
4295
4327
  frame::Frame::MaxStreamsBidi { .. } => {
4296
4328
  self.should_send_max_streams_bidi = true;
4297
- },
4329
+ }
4298
4330
 
4299
4331
  // Retransmit STREAMS_BLOCKED frames if the frame with the
4300
4332
  // most recent limit is lost. These are informational
@@ -4304,30 +4336,42 @@ impl<F: BufFactory> Connection<F> {
4304
4336
  frame::Frame::StreamsBlockedBidi { limit } => {
4305
4337
  self.streams_blocked_bidi_state
4306
4338
  .force_retransmit_sent_limit_eq(limit);
4307
- },
4339
+ }
4308
4340
 
4309
4341
  frame::Frame::StreamsBlockedUni { limit } => {
4310
4342
  self.streams_blocked_uni_state
4311
4343
  .force_retransmit_sent_limit_eq(limit);
4312
- },
4344
+ }
4313
4345
 
4314
4346
  frame::Frame::NewConnectionId { seq_num, .. } => {
4315
4347
  self.ids.mark_advertise_new_scid_seq(seq_num, true);
4316
- },
4348
+ }
4317
4349
 
4318
4350
  frame::Frame::RetireConnectionId { seq_num } => {
4319
4351
  self.ids.mark_retire_dcid_seq(seq_num, true)?;
4320
- },
4352
+ }
4321
4353
 
4322
4354
  frame::Frame::Ping { mtu_probe } => {
4323
4355
  // Ping frames are not retransmitted.
4324
4356
  if let Some(failed_probe) = mtu_probe {
4357
+ trace!("pmtud probe dropped: {failed_probe}");
4358
+
4359
+ let local = p.local_addr();
4360
+ let peer = p.peer_addr();
4325
4361
  if let Some(pmtud) = p.pmtud.as_mut() {
4326
- trace!("pmtud probe dropped: {failed_probe}");
4362
+ let old_pmtu = pmtud.get_current_mtu();
4327
4363
  pmtud.failed_probe(failed_probe);
4364
+ let new_pmtu = pmtud.get_current_mtu();
4365
+
4366
+ if let Some(event) =
4367
+ path::pmtu_event(local, peer, old_pmtu, new_pmtu)
4368
+ {
4369
+ p.recovery.pmtud_update_max_datagram_size(new_pmtu);
4370
+ path_events.push_back(event);
4371
+ }
4328
4372
  }
4329
4373
  }
4330
- },
4374
+ }
4331
4375
 
4332
4376
  // Sent as StreamHeader frames. Stream frames are never
4333
4377
  // generated by quiche.
@@ -4337,7 +4381,7 @@ impl<F: BufFactory> Connection<F> {
4337
4381
  have tracked retransmittable stream data as \
4338
4382
  StreamHeader frames."
4339
4383
  );
4340
- },
4384
+ }
4341
4385
 
4342
4386
  // Sent as CryptoHeader frames. Crypto frames are never
4343
4387
  // generated by quiche.
@@ -4347,7 +4391,7 @@ impl<F: BufFactory> Connection<F> {
4347
4391
  have tracked retransmittable crypto data as \
4348
4392
  CryptoHeader frames."
4349
4393
  );
4350
- },
4394
+ }
4351
4395
 
4352
4396
  // NewToken frames are never sent by quiche; they are not
4353
4397
  // implemented.
@@ -4357,13 +4401,12 @@ impl<F: BufFactory> Connection<F> {
4357
4401
  not implement NewToken frames, retransmission of \
4358
4402
  these frames is not implemented."
4359
4403
  );
4360
- },
4404
+ }
4361
4405
 
4362
4406
  // Data blocked frames are an optional advisory
4363
4407
  // signal. We choose to not retransmit them to
4364
4408
  // avoid unnecessary network usage.
4365
- frame::Frame::DataBlocked { .. } |
4366
- frame::Frame::StreamDataBlocked { .. } => (),
4409
+ frame::Frame::DataBlocked { .. } | frame::Frame::StreamDataBlocked { .. } => (),
4367
4410
 
4368
4411
  // Path challenge and response have their own
4369
4412
  // retry logic. They should not be retransmitted
@@ -4371,32 +4414,31 @@ impl<F: BufFactory> Connection<F> {
4371
4414
  // 8.2.2: "An endpoint MUST NOT send more than one
4372
4415
  // PATH_RESPONSE frame in response to one
4373
4416
  // PATH_CHALLENGE frame".
4374
- frame::Frame::PathChallenge { .. } |
4375
- frame::Frame::PathResponse { .. } => (),
4417
+ frame::Frame::PathChallenge { .. } | frame::Frame::PathResponse { .. } => (),
4376
4418
 
4377
4419
  // From RFC 9000 Section 13.3: CONNECTION_CLOSE
4378
4420
  // frames, are not sent again when packet loss is
4379
4421
  // detected. Resending these signals is described
4380
4422
  // in Section 10.
4381
- frame::Frame::ConnectionClose { .. } |
4382
- frame::Frame::ApplicationClose { .. } => (),
4423
+ frame::Frame::ConnectionClose { .. }
4424
+ | frame::Frame::ApplicationClose { .. } => (),
4383
4425
 
4384
4426
  // Padding doesn't require retransmission.
4385
4427
  frame::Frame::Padding { .. } => (),
4386
4428
 
4387
- frame::Frame::DatagramHeader { .. } |
4388
- frame::Frame::Datagram { .. } => {
4429
+ frame::Frame::DatagramHeader { .. } | frame::Frame::Datagram { .. } => {
4389
4430
  // Datagrams do not require retransmission. Just update
4390
4431
  // stats.
4391
4432
  p.dgram_lost_count = p.dgram_lost_count.saturating_add(1);
4392
- },
4393
- // IMPORTANT: Do not add an exhaustive catch
4394
- // all. We want to add explicit handling for frame
4395
- // types that can be safely ignored when lost.
4433
+ } // IMPORTANT: Do not add an exhaustive catch
4434
+ // all. We want to add explicit handling for frame
4435
+ // types that can be safely ignored when lost.
4396
4436
  }
4397
4437
  }
4398
4438
  }
4399
- self.check_tx_buffered_invariant();
4439
+
4440
+ #[cfg(debug_assertions)]
4441
+ self.streams.debug_check_tx_buffered_consistency();
4400
4442
 
4401
4443
  let is_app_limited = self.delivery_rate_check_if_app_limited();
4402
4444
  let n_paths = self.paths.len();
@@ -4419,8 +4461,7 @@ impl<F: BufFactory> Connection<F> {
4419
4461
  };
4420
4462
  let pn = self.next_pkt_num;
4421
4463
 
4422
- let largest_acked_pkt =
4423
- path.recovery.get_largest_acked_on_epoch(epoch).unwrap_or(0);
4464
+ let largest_acked_pkt = path.recovery.get_largest_acked_on_epoch(epoch).unwrap_or(0);
4424
4465
  let pn_len = packet::pkt_num_len(pn, largest_acked_pkt);
4425
4466
 
4426
4467
  // The AEAD overhead at the current encryption level.
@@ -4428,8 +4469,7 @@ impl<F: BufFactory> Connection<F> {
4428
4469
 
4429
4470
  let dcid_seq = path.active_dcid_seq.ok_or(Error::OutOfIdentifiers)?;
4430
4471
 
4431
- let dcid =
4432
- ConnectionId::from_ref(self.ids.get_dcid(dcid_seq)?.cid.as_ref());
4472
+ let dcid = ConnectionId::from_ref(self.ids.get_dcid(dcid_seq)?.cid.as_ref());
4433
4473
 
4434
4474
  let scid = if let Some(scid_seq) = path.active_scid_seq {
4435
4475
  ConnectionId::from_ref(self.ids.get_scid(scid_seq)?.cid.as_ref())
@@ -4507,7 +4547,7 @@ impl<F: BufFactory> Connection<F> {
4507
4547
  // is set to false here to make cwnd grow when ACK is received.
4508
4548
  path.recovery.update_app_limited(false);
4509
4549
  return Err(Error::Done);
4510
- },
4550
+ }
4511
4551
  }
4512
4552
 
4513
4553
  // Make sure there is enough space for the minimum payload length.
@@ -4522,10 +4562,11 @@ impl<F: BufFactory> Connection<F> {
4522
4562
  let mut in_flight = false;
4523
4563
  let mut is_pmtud_probe = false;
4524
4564
  let mut has_data = false;
4565
+ let mut stream_data_skipped = false;
4525
4566
 
4526
- // Whether or not we should explicitly elicit an ACK via PING frame if we
4527
- // implicitly elicit one otherwise.
4528
- let ack_elicit_required = path.recovery.should_elicit_ack(epoch);
4567
+ // Whether a PING frame must explicitly elicit an ACK when no other
4568
+ // frame does so implicitly.
4569
+ let ack_elicit_required = !is_closing && path.recovery.should_elicit_ack(epoch);
4529
4570
 
4530
4571
  let header_offset = b.off();
4531
4572
 
@@ -4541,8 +4582,15 @@ impl<F: BufFactory> Connection<F> {
4541
4582
 
4542
4583
  let payload_offset = b.off();
4543
4584
 
4544
- let cwnd_available =
4545
- path.recovery.cwnd_available().saturating_sub(overhead);
4585
+ // `usize::MAX` bypasses cwnd when authorized by packet selection;
4586
+ // `left` still bounds the packet. Keep this local so other cwnd users
4587
+ // see the actual window.
4588
+ let send_limit_from_cwnd = if bypass_cwnd {
4589
+ usize::MAX
4590
+ } else {
4591
+ path.recovery.cwnd_available()
4592
+ }
4593
+ .saturating_sub(overhead);
4546
4594
 
4547
4595
  let left_before_packing_ack_frame = left;
4548
4596
 
@@ -4552,22 +4600,22 @@ impl<F: BufFactory> Connection<F> {
4552
4600
  // generate an ACK (if there's anything to ACK) since we're going to
4553
4601
  // send a packet with PING anyways, even if we haven't received anything
4554
4602
  // ACK eliciting.
4555
- if pkt_space.recv_pkt_need_ack.len() > 0 &&
4556
- (pkt_space.ack_elicited || ack_elicit_required) &&
4557
- (!is_closing ||
4558
- (pkt_type == Type::Handshake &&
4559
- self.local_error
4560
- .as_ref()
4561
- .is_some_and(|le| le.is_app))) &&
4562
- path.active()
4603
+ //
4604
+ // While closing, PING frames are suppressed, so ACK elicitation must
4605
+ // not generate ACK-only packets.
4606
+ if pkt_space.recv_pkt_need_ack.len() > 0
4607
+ && (pkt_space.ack_elicited || ack_elicit_required)
4608
+ && (!is_closing
4609
+ || (pkt_type == Type::Handshake
4610
+ && self.local_error.as_ref().is_some_and(|le| le.is_app)))
4611
+ && path.active()
4563
4612
  {
4564
4613
  #[cfg(not(feature = "fuzzing"))]
4565
4614
  let ack_delay = pkt_space.largest_rx_pkt_time.elapsed();
4566
4615
 
4567
4616
  #[cfg(not(feature = "fuzzing"))]
4568
- let ack_delay = ack_delay.as_micros() as u64 /
4569
- 2_u64
4570
- .pow(self.local_transport_params.ack_delay_exponent as u32);
4617
+ let ack_delay = ack_delay.as_micros() as u64
4618
+ / 2_u64.pow(self.local_transport_params.ack_delay_exponent as u32);
4571
4619
 
4572
4620
  // pseudo-random reproducible ack delays when fuzzing
4573
4621
  #[cfg(feature = "fuzzing")]
@@ -4583,7 +4631,7 @@ impl<F: BufFactory> Connection<F> {
4583
4631
  // there is not enough cwnd available for both (note that PING
4584
4632
  // frames are always 1 byte, so we just need to check that the
4585
4633
  // ACK's length is lower than cwnd).
4586
- if pkt_space.ack_elicited || frame.wire_len() < cwnd_available {
4634
+ if pkt_space.ack_elicited || frame.wire_len() < send_limit_from_cwnd {
4587
4635
  // ACK-only packets are not congestion controlled so ACKs must
4588
4636
  // be bundled considering the buffer capacity only, and not the
4589
4637
  // available cwnd.
@@ -4597,7 +4645,7 @@ impl<F: BufFactory> Connection<F> {
4597
4645
  left = cmp::min(
4598
4646
  left,
4599
4647
  // Bytes consumed by ACK frames.
4600
- cwnd_available.saturating_sub(left_before_packing_ack_frame - left),
4648
+ send_limit_from_cwnd.saturating_sub(left_before_packing_ack_frame - left),
4601
4649
  );
4602
4650
 
4603
4651
  let mut challenge_data = None;
@@ -4605,15 +4653,12 @@ impl<F: BufFactory> Connection<F> {
4605
4653
  if pkt_type == Type::Short {
4606
4654
  // Create PMTUD probe.
4607
4655
  //
4608
- // In order to send a PMTUD probe the current `left` value, which was
4609
- // already limited by the current PMTU measure, needs to be ignored,
4610
- // but the outgoing packet still needs to be limited by
4611
- // the output buffer size, as well as the congestion
4612
- // window.
4656
+ // A PMTUD probe must ignore `left`, which is already limited by the
4657
+ // current PMTU. The probe remains limited by the output buffer and
4658
+ // congestion window.
4613
4659
  //
4614
- // In addition, the PMTUD probe is only generated when the handshake
4615
- // is confirmed, to avoid interfering with the handshake
4616
- // (e.g. due to the anti-amplification limits).
4660
+ // Generate PMTUD probes only after handshake confirmation to avoid
4661
+ // interference from anti-amplification limits.
4617
4662
  if let Ok(active_path) = self.paths.get_active_mut() {
4618
4663
  let should_probe_pmtu = active_path.should_send_pmtu_probe(
4619
4664
  self.handshake_confirmed,
@@ -4627,11 +4672,11 @@ impl<F: BufFactory> Connection<F> {
4627
4672
  if let Some(pmtud) = active_path.pmtud.as_mut() {
4628
4673
  let probe_size = pmtud.get_probe_size();
4629
4674
  trace!(
4630
- "{} sending pmtud probe pmtu_probe={} estimated_pmtu={}",
4631
- self.trace_id,
4632
- probe_size,
4633
- pmtud.get_current_mtu(),
4634
- );
4675
+ "{} sending pmtud probe pmtu_probe={} estimated_pmtu={}",
4676
+ self.trace_id,
4677
+ probe_size,
4678
+ pmtud.get_current_mtu(),
4679
+ );
4635
4680
 
4636
4681
  left = probe_size;
4637
4682
 
@@ -4642,14 +4687,14 @@ impl<F: BufFactory> Connection<F> {
4642
4687
  // We can't send more because there isn't enough
4643
4688
  // space available in the output buffer.
4644
4689
  //
4645
- // This usually happens when we try to send a new
4646
- // packet but failed because cwnd is almost full.
4690
+ // The congestion window is nearly full. A new
4691
+ // packet does not fit.
4647
4692
  //
4648
- // In such case app_limited is set to false here
4649
- // to make cwnd grow when ACK is received.
4693
+ // Clear the app-limited state so ACKs can grow
4694
+ // the congestion window.
4650
4695
  active_path.recovery.update_app_limited(false);
4651
4696
  return Err(Error::Done);
4652
- },
4697
+ }
4653
4698
  }
4654
4699
 
4655
4700
  let frame = frame::Frame::Padding {
@@ -4733,10 +4778,7 @@ impl<F: BufFactory> Connection<F> {
4733
4778
  if pkt_type == Type::Short && !is_closing && path.active() {
4734
4779
  // Create HANDSHAKE_DONE frame.
4735
4780
  // self.should_send_handshake_done() but without the need to borrow
4736
- if self.handshake_completed &&
4737
- !self.handshake_done_sent &&
4738
- self.is_server
4739
- {
4781
+ if self.handshake_completed && !self.handshake_done_sent && self.is_server {
4740
4782
  let frame = frame::Frame::HandshakeDone;
4741
4783
 
4742
4784
  if push_frame_to_pkt!(b, frames, frame, left) {
@@ -4748,9 +4790,7 @@ impl<F: BufFactory> Connection<F> {
4748
4790
  }
4749
4791
 
4750
4792
  // Create MAX_STREAMS_BIDI frame.
4751
- if self.streams.should_update_max_streams_bidi() ||
4752
- self.should_send_max_streams_bidi
4753
- {
4793
+ if self.streams.should_update_max_streams_bidi() || self.should_send_max_streams_bidi {
4754
4794
  let frame = frame::Frame::MaxStreamsBidi {
4755
4795
  max: self.streams.max_streams_bidi_next(),
4756
4796
  };
@@ -4765,9 +4805,7 @@ impl<F: BufFactory> Connection<F> {
4765
4805
  }
4766
4806
 
4767
4807
  // Create MAX_STREAMS_UNI frame.
4768
- if self.streams.should_update_max_streams_uni() ||
4769
- self.should_send_max_streams_uni
4770
- {
4808
+ if self.streams.should_update_max_streams_uni() || self.should_send_max_streams_uni {
4771
4809
  let frame = frame::Frame::MaxStreamsUni {
4772
4810
  max: self.streams.max_streams_uni_next(),
4773
4811
  };
@@ -4787,8 +4825,7 @@ impl<F: BufFactory> Connection<F> {
4787
4825
 
4788
4826
  if push_frame_to_pkt!(b, frames, frame, left) {
4789
4827
  self.blocked_limit = None;
4790
- self.data_blocked_sent_count =
4791
- self.data_blocked_sent_count.saturating_add(1);
4828
+ self.data_blocked_sent_count = self.data_blocked_sent_count.saturating_add(1);
4792
4829
 
4793
4830
  ack_eliciting = true;
4794
4831
  in_flight = true;
@@ -4808,8 +4845,7 @@ impl<F: BufFactory> Connection<F> {
4808
4845
  // Record the limit we just notified the peer about so
4809
4846
  // that redundant frames for the same limit are
4810
4847
  // suppressed.
4811
- self.streams_blocked_bidi_state.blocked_sent =
4812
- Some(limit);
4848
+ self.streams_blocked_bidi_state.blocked_sent = Some(limit);
4813
4849
 
4814
4850
  ack_eliciting = true;
4815
4851
  in_flight = true;
@@ -4848,7 +4884,7 @@ impl<F: BufFactory> Connection<F> {
4848
4884
  // the almost full set.
4849
4885
  self.streams.remove_almost_full(stream_id);
4850
4886
  continue;
4851
- },
4887
+ }
4852
4888
  };
4853
4889
 
4854
4890
  // Autotune the stream window size, but only if this is not a
@@ -4883,8 +4919,8 @@ impl<F: BufFactory> Connection<F> {
4883
4919
  }
4884
4920
 
4885
4921
  // Create MAX_DATA frame as needed.
4886
- if flow_control.should_update_max_data() &&
4887
- flow_control.max_data() < flow_control.max_data_next()
4922
+ if flow_control.should_update_max_data()
4923
+ && flow_control.max_data() < flow_control.max_data_next()
4888
4924
  {
4889
4925
  // Autotune the connection window size. We only tune the window
4890
4926
  // if we are sending an "organic" update, not on retransmits.
@@ -4929,16 +4965,13 @@ impl<F: BufFactory> Connection<F> {
4929
4965
  }
4930
4966
 
4931
4967
  // Create RESET_STREAM frames as needed.
4932
- for (stream_id, (error_code, final_size)) in self
4933
- .streams
4934
- .reset()
4935
- .map(|(&k, &v)| (k, v))
4936
- .collect::<Vec<(u64, (u64, u64))>>()
4937
- {
4968
+ let resets: Vec<_> = self.streams.reset().map(|(&k, &v)| (k, v)).collect();
4969
+ for (stream_id, (error_code, final_size, reliable_size)) in resets {
4938
4970
  let frame = frame::Frame::ResetStream {
4939
4971
  stream_id,
4940
4972
  error_code,
4941
4973
  final_size,
4974
+ reliable_size,
4942
4975
  };
4943
4976
 
4944
4977
  if push_frame_to_pkt!(b, frames, frame, left) {
@@ -5034,10 +5067,10 @@ impl<F: BufFactory> Connection<F> {
5034
5067
  }
5035
5068
 
5036
5069
  // Create CRYPTO frame.
5037
- if crypto_ctx.crypto_stream.is_flushable() &&
5038
- left > frame::MAX_CRYPTO_OVERHEAD &&
5039
- !is_closing &&
5040
- path.active()
5070
+ if crypto_ctx.crypto_stream.is_flushable()
5071
+ && left > frame::MAX_CRYPTO_OVERHEAD
5072
+ && !is_closing
5073
+ && path.active()
5041
5074
  {
5042
5075
  let crypto_off = crypto_ctx.crypto_stream.send.off_front();
5043
5076
 
@@ -5060,8 +5093,7 @@ impl<F: BufFactory> Connection<F> {
5060
5093
  2; // length, always encode as 2-byte varint
5061
5094
 
5062
5095
  if let Some(max_len) = left.checked_sub(hdr_len) {
5063
- let (mut crypto_hdr, mut crypto_payload) =
5064
- b.split_at(hdr_off + hdr_len)?;
5096
+ let (mut crypto_hdr, mut crypto_payload) = b.split_at(hdr_off + hdr_len)?;
5065
5097
 
5066
5098
  // Write stream data into the packet buffer.
5067
5099
  let (len, _) = crypto_ctx
@@ -5077,11 +5109,7 @@ impl<F: BufFactory> Connection<F> {
5077
5109
  // initial frame offset.
5078
5110
  crypto_hdr.skip(hdr_off)?;
5079
5111
 
5080
- frame::encode_crypto_header(
5081
- crypto_off,
5082
- len as u64,
5083
- &mut crypto_hdr,
5084
- )?;
5112
+ frame::encode_crypto_header(crypto_off, len as u64, &mut crypto_hdr)?;
5085
5113
 
5086
5114
  // Advance the packet buffer's offset.
5087
5115
  b.skip(hdr_len + len)?;
@@ -5116,11 +5144,11 @@ impl<F: BufFactory> Connection<F> {
5116
5144
  }
5117
5145
 
5118
5146
  // Create DATAGRAM frame.
5119
- if (pkt_type == Type::Short || pkt_type == Type::ZeroRTT) &&
5120
- left > frame::MAX_DGRAM_OVERHEAD &&
5121
- !is_closing &&
5122
- path.active() &&
5123
- do_dgram
5147
+ if (pkt_type == Type::Short || pkt_type == Type::ZeroRTT)
5148
+ && left > frame::MAX_DGRAM_OVERHEAD
5149
+ && !is_closing
5150
+ && path.active()
5151
+ && do_dgram
5124
5152
  {
5125
5153
  if let Some(max_dgram_payload) = max_dgram_len {
5126
5154
  while let Some(len) = self.dgram_send_queue.peek_front_len() {
@@ -5151,8 +5179,7 @@ impl<F: BufFactory> Connection<F> {
5151
5179
  let (mut dgram_hdr, mut dgram_payload) =
5152
5180
  b.split_at(hdr_off + hdr_len)?;
5153
5181
 
5154
- dgram_payload.as_mut()[..len]
5155
- .copy_from_slice(data.as_ref());
5182
+ dgram_payload.as_mut()[..len].copy_from_slice(data.as_ref());
5156
5183
 
5157
5184
  // Encode the frame's header.
5158
5185
  //
@@ -5163,27 +5190,21 @@ impl<F: BufFactory> Connection<F> {
5163
5190
  // offset.
5164
5191
  dgram_hdr.skip(hdr_off)?;
5165
5192
 
5166
- frame::encode_dgram_header(
5167
- len as u64,
5168
- &mut dgram_hdr,
5169
- )?;
5193
+ frame::encode_dgram_header(len as u64, &mut dgram_hdr)?;
5170
5194
 
5171
5195
  // Advance the packet buffer's offset.
5172
5196
  b.skip(hdr_len + len)?;
5173
5197
 
5174
- let frame =
5175
- frame::Frame::DatagramHeader { length: len };
5198
+ let frame = frame::Frame::DatagramHeader { length: len };
5176
5199
 
5177
5200
  if push_frame_to_pkt!(b, frames, frame, left) {
5178
5201
  ack_eliciting = true;
5179
5202
  in_flight = true;
5180
5203
  dgram_emitted = true;
5181
- self.dgram_sent_count =
5182
- self.dgram_sent_count.saturating_add(1);
5183
- path.dgram_sent_count =
5184
- path.dgram_sent_count.saturating_add(1);
5204
+ self.dgram_sent_count = self.dgram_sent_count.saturating_add(1);
5205
+ path.dgram_sent_count = path.dgram_sent_count.saturating_add(1);
5185
5206
  }
5186
- },
5207
+ }
5187
5208
 
5188
5209
  None => continue,
5189
5210
  };
@@ -5198,24 +5219,21 @@ impl<F: BufFactory> Connection<F> {
5198
5219
  }
5199
5220
 
5200
5221
  // Create a single STREAM frame for the first stream that is flushable.
5201
- if (pkt_type == Type::Short || pkt_type == Type::ZeroRTT) &&
5202
- left > frame::MAX_STREAM_OVERHEAD &&
5203
- !is_closing &&
5204
- path.active() &&
5205
- !dgram_emitted
5222
+ if (pkt_type == Type::Short || pkt_type == Type::ZeroRTT)
5223
+ && !is_closing
5224
+ && path.active()
5225
+ && !dgram_emitted
5206
5226
  {
5207
5227
  while let Some(priority_key) = self.streams.peek_flushable() {
5208
5228
  let stream_id = priority_key.id;
5209
5229
  let stream = match self.streams.get_mut(stream_id) {
5210
- // Avoid sending frames for streams that were already stopped.
5211
- //
5212
- // This might happen if stream data was buffered but not yet
5213
- // flushed on the wire when a STOP_SENDING frame is received.
5214
- Some(v) if !v.send.is_stopped() => v,
5230
+ // A reset can abandon queued data or FINs while the stream
5231
+ // remains alive waiting for the reset to be acknowledged.
5232
+ Some(stream) if stream.send.can_send_data() => stream,
5215
5233
  _ => {
5216
5234
  self.streams.remove_flushable(&priority_key);
5217
5235
  continue;
5218
- },
5236
+ }
5219
5237
  };
5220
5238
 
5221
5239
  let stream_off = stream.send.off_front();
@@ -5239,22 +5257,38 @@ impl<F: BufFactory> Connection<F> {
5239
5257
  octets::varint_len(stream_off) + // offset
5240
5258
  2; // length, always encode as 2-byte varint
5241
5259
 
5260
+ // A non-FIN STREAM frame must carry at least one byte. If the
5261
+ // header and payload don't fit, leave the stream flushable for
5262
+ // a later packet. Empty FIN frames only need the header.
5242
5263
  let max_len = match left.checked_sub(hdr_len) {
5243
- Some(v) => v,
5244
- None => {
5245
- let priority_key = Arc::clone(&stream.priority_key);
5246
- self.streams.remove_flushable(&priority_key);
5264
+ Some(v) if v > 0 || stream.send.empty_fin_next() => v,
5265
+ _ => {
5266
+ stream_data_skipped = true;
5247
5267
 
5248
- continue;
5249
- },
5268
+ if stream.incremental {
5269
+ let priority_key = Arc::clone(&stream.priority_key);
5270
+ self.streams.remove_flushable(&priority_key);
5271
+ self.streams.insert_flushable(&priority_key);
5272
+ }
5273
+
5274
+ break;
5275
+ }
5250
5276
  };
5251
5277
 
5252
- let (mut stream_hdr, mut stream_payload) =
5253
- b.split_at(hdr_off + hdr_len)?;
5278
+ let (mut stream_hdr, mut stream_payload) = b.split_at(hdr_off + hdr_len)?;
5254
5279
 
5255
5280
  // Write stream data into the packet buffer.
5256
- let (len, fin) =
5257
- stream.send.emit(&mut stream_payload.as_mut()[..max_len])?;
5281
+ let (len, fin) = stream.send.emit(&mut stream_payload.as_mut()[..max_len])?;
5282
+
5283
+ // ACK processing can leave a stream in the flushable queue
5284
+ // after removing all of its retransmission data. Unlink the
5285
+ // stale entry and continue with the next stream;
5286
+ // future retransmits or writedata will queue the stream again.
5287
+ if len == 0 && !fin {
5288
+ let priority_key = Arc::clone(&stream.priority_key);
5289
+ self.streams.remove_flushable(&priority_key);
5290
+ continue;
5291
+ }
5258
5292
 
5259
5293
  // Encode the frame's header.
5260
5294
  //
@@ -5289,7 +5323,7 @@ impl<F: BufFactory> Connection<F> {
5289
5323
  }
5290
5324
 
5291
5325
  let priority_key = Arc::clone(&stream.priority_key);
5292
- // If the stream is no longer flushable, remove it from the queue
5326
+ // Remove the stream when it is no longer flushable.
5293
5327
  if !stream.is_flushable() {
5294
5328
  self.streams.remove_flushable(&priority_key);
5295
5329
  } else if stream.incremental {
@@ -5299,12 +5333,14 @@ impl<F: BufFactory> Connection<F> {
5299
5333
  self.streams.insert_flushable(&priority_key);
5300
5334
  }
5301
5335
 
5336
+ // Update tx_buffered when data is emitted.
5337
+ self.streams.sub_tx_buffered(len);
5338
+
5302
5339
  #[cfg(feature = "fuzzing")]
5303
5340
  // Coalesce STREAM frames when fuzzing.
5304
- if left > frame::MAX_STREAM_OVERHEAD {
5305
- continue;
5306
- }
5341
+ continue;
5307
5342
 
5343
+ #[cfg(not(feature = "fuzzing"))]
5308
5344
  break;
5309
5345
  }
5310
5346
  }
@@ -5317,10 +5353,10 @@ impl<F: BufFactory> Connection<F> {
5317
5353
  // - if we've sent too many non ack-eliciting packets without having
5318
5354
  // sent an ACK eliciting one; OR
5319
5355
  // - the application requested an ack-eliciting frame be sent.
5320
- if (ack_elicit_required || path.needs_ack_eliciting) &&
5321
- !ack_eliciting &&
5322
- left >= 1 &&
5323
- !is_closing
5356
+ if (ack_elicit_required || path.needs_ack_eliciting)
5357
+ && !ack_eliciting
5358
+ && left >= 1
5359
+ && !is_closing
5324
5360
  {
5325
5361
  let frame = frame::Frame::Ping { mtu_probe: None };
5326
5362
 
@@ -5335,9 +5371,11 @@ impl<F: BufFactory> Connection<F> {
5335
5371
  path.recovery.ping_sent(epoch);
5336
5372
  }
5337
5373
 
5338
- if !has_data &&
5339
- !dgram_emitted &&
5340
- cwnd_available > frame::MAX_STREAM_OVERHEAD
5374
+ // Pending stream data means the sender is size-, not app-limited.
5375
+ if !has_data
5376
+ && !stream_data_skipped
5377
+ && !dgram_emitted
5378
+ && send_limit_from_cwnd > LEGACY_APP_LIMITED_BYTE_THRESHOLD
5341
5379
  {
5342
5380
  path.recovery.on_app_limited();
5343
5381
  }
@@ -5357,10 +5395,7 @@ impl<F: BufFactory> Connection<F> {
5357
5395
  // as Initial always requires padding.
5358
5396
  //
5359
5397
  // 2) this is a probing packet towards an unvalidated peer address.
5360
- if (has_initial || !path.validated()) &&
5361
- pkt_type == Type::Short &&
5362
- left >= 1
5363
- {
5398
+ if (has_initial || !path.validated()) && pkt_type == Type::Short && left >= 1 {
5364
5399
  let frame = frame::Frame::Padding { len: left };
5365
5400
 
5366
5401
  if push_frame_to_pkt!(b, frames, frame, left) {
@@ -5389,8 +5424,7 @@ impl<F: BufFactory> Connection<F> {
5389
5424
  let len = pn_len + payload_len + crypto_overhead;
5390
5425
 
5391
5426
  let (_, mut payload_with_len) = b.split_at(header_offset)?;
5392
- payload_with_len
5393
- .put_varint_with_len(len as u64, PAYLOAD_LENGTH_LEN)?;
5427
+ payload_with_len.put_varint_with_len(len as u64, PAYLOAD_LENGTH_LEN)?;
5394
5428
  }
5395
5429
 
5396
5430
  trace!(
@@ -5403,8 +5437,7 @@ impl<F: BufFactory> Connection<F> {
5403
5437
  );
5404
5438
 
5405
5439
  #[cfg(feature = "qlog")]
5406
- let mut qlog_frames: Vec<qlog::events::quic::QuicFrame> =
5407
- Vec::with_capacity(frames.len());
5440
+ let mut qlog_frames: Vec<qlog::events::quic::QuicFrame> = Vec::with_capacity(frames.len());
5408
5441
 
5409
5442
  for frame in &mut frames {
5410
5443
  trace!("{} tx frm {:?}", self.trace_id, frame);
@@ -5427,17 +5460,15 @@ impl<F: BufFactory> Connection<F> {
5427
5460
  data: None,
5428
5461
  };
5429
5462
 
5430
- let send_at_time =
5431
- now.duration_since(q.start_time()).as_secs_f64() * 1000.0;
5463
+ let send_at_time = now.duration_since(q.start_time()).as_secs_f64() * 1000.0;
5432
5464
 
5433
- let ev_data =
5434
- EventData::QuicPacketSent(qlog::events::quic::PacketSent {
5435
- header,
5436
- frames: Some(qlog_frames),
5437
- raw: Some(qlog_raw_info),
5438
- send_at_time: Some(send_at_time),
5439
- ..Default::default()
5440
- });
5465
+ let ev_data = EventData::QuicPacketSent(qlog::events::quic::PacketSent {
5466
+ header,
5467
+ frames: Some(qlog_frames),
5468
+ raw: Some(qlog_raw_info),
5469
+ send_at_time: Some(send_at_time),
5470
+ ..Default::default()
5471
+ });
5441
5472
 
5442
5473
  q.add_event_data_with_instant(ev_data, now).ok();
5443
5474
  }
@@ -5448,15 +5479,8 @@ impl<F: BufFactory> Connection<F> {
5448
5479
  None => return Err(Error::InvalidState),
5449
5480
  };
5450
5481
 
5451
- let written = packet::encrypt_pkt(
5452
- &mut b,
5453
- pn,
5454
- pn_len,
5455
- payload_len,
5456
- payload_offset,
5457
- None,
5458
- aead,
5459
- )?;
5482
+ let written =
5483
+ packet::encrypt_pkt(&mut b, pn, pn_len, payload_len, payload_offset, None, aead)?;
5460
5484
 
5461
5485
  let sent_pkt_has_data = if path.recovery.gcongestion_enabled() {
5462
5486
  has_data || dgram_emitted
@@ -5490,8 +5514,7 @@ impl<F: BufFactory> Connection<F> {
5490
5514
  self.next_pkt_num += 1;
5491
5515
 
5492
5516
  let handshake_status = recovery::HandshakeStatus {
5493
- has_handshake_keys: self.crypto_ctx[packet::Epoch::Handshake]
5494
- .has_keys(),
5517
+ has_handshake_keys: self.crypto_ctx[packet::Epoch::Handshake].has_keys(),
5495
5518
  peer_verified_address: self.peer_verified_initial_address,
5496
5519
  completed: self.handshake_completed,
5497
5520
  };
@@ -5519,13 +5542,12 @@ impl<F: BufFactory> Connection<F> {
5519
5542
 
5520
5543
  let had_send_budget = path.max_send_bytes > 0;
5521
5544
  path.max_send_bytes = path.max_send_bytes.saturating_sub(written);
5522
- if self.is_server &&
5523
- !path.verified_peer_address &&
5524
- had_send_budget &&
5525
- path.max_send_bytes == 0
5545
+ if self.is_server
5546
+ && !path.verified_peer_address
5547
+ && had_send_budget
5548
+ && path.max_send_bytes == 0
5526
5549
  {
5527
- self.amplification_limited_count =
5528
- self.amplification_limited_count.saturating_add(1);
5550
+ self.amplification_limited_count = self.amplification_limited_count.saturating_add(1);
5529
5551
  }
5530
5552
 
5531
5553
  // On the client, drop initial state after sending an Handshake packet.
@@ -5549,29 +5571,54 @@ impl<F: BufFactory> Connection<F> {
5549
5571
  }
5550
5572
 
5551
5573
  fn on_packet_sent(
5552
- &mut self, send_pid: usize, sent_pkt: recovery::Sent,
5553
- epoch: packet::Epoch, handshake_status: recovery::HandshakeStatus,
5574
+ &mut self,
5575
+ send_pid: usize,
5576
+ sent_pkt: recovery::Sent,
5577
+ epoch: packet::Epoch,
5578
+ handshake_status: recovery::HandshakeStatus,
5554
5579
  now: Instant,
5555
5580
  ) -> Result<()> {
5556
5581
  let path = self.paths.get_mut(send_pid)?;
5557
5582
 
5558
- // It's fine to set the skip counter based on a non-active path's values.
5583
+ // The skip counter may use values from an inactive path.
5559
5584
  let cwnd = path.recovery.cwnd();
5560
5585
  let max_datagram_size = path.recovery.max_datagram_size();
5561
5586
  self.pkt_num_spaces[epoch].on_packet_sent(&sent_pkt);
5562
- self.pkt_num_manager.on_packet_sent(
5563
- cwnd,
5564
- max_datagram_size,
5565
- self.handshake_completed,
5566
- );
5587
+ self.pkt_num_manager
5588
+ .on_packet_sent(cwnd, max_datagram_size, self.handshake_completed);
5567
5589
 
5568
- path.recovery.on_packet_sent(
5569
- sent_pkt,
5570
- epoch,
5571
- handshake_status,
5572
- now,
5573
- &self.trace_id,
5574
- );
5590
+ path.recovery
5591
+ .on_packet_sent(sent_pkt, epoch, handshake_status, now, &self.trace_id);
5592
+
5593
+ // Bound recovery metadata when sent packet records accumulate.
5594
+ if self.local_error.is_none() {
5595
+ if let Some(max_outstanding_sent_packets) = self.max_outstanding_sent_packets {
5596
+ let tracked_sent_packets: usize = self
5597
+ .paths
5598
+ .iter()
5599
+ .flat_map(|(_, p)| {
5600
+ packet::Epoch::epochs(packet::Epoch::Initial..=packet::Epoch::Application)
5601
+ .iter()
5602
+ .map(|&e| p.recovery.sent_packets_len(e))
5603
+ })
5604
+ .sum();
5605
+
5606
+ if tracked_sent_packets > max_outstanding_sent_packets.get() {
5607
+ trace!(
5608
+ "{} too many outstanding sent packets: {}",
5609
+ self.trace_id,
5610
+ tracked_sent_packets
5611
+ );
5612
+
5613
+ self.close(
5614
+ false,
5615
+ WireErrorCode::ProtocolViolation as u64,
5616
+ b"too many outstanding sent packets",
5617
+ )
5618
+ .ok();
5619
+ }
5620
+ }
5621
+ }
5575
5622
 
5576
5623
  Ok(())
5577
5624
  }
@@ -5640,9 +5687,7 @@ impl<F: BufFactory> Connection<F> {
5640
5687
  ///
5641
5688
  /// If the (`local_addr`, peer_addr`) 4-tuple relates to a non-existing
5642
5689
  /// path, this method returns 0.
5643
- pub fn send_quantum_on_path(
5644
- &self, local_addr: SocketAddr, peer_addr: SocketAddr,
5645
- ) -> usize {
5690
+ pub fn send_quantum_on_path(&self, local_addr: SocketAddr, peer_addr: SocketAddr) -> usize {
5646
5691
  self.paths
5647
5692
  .path_id_from_addrs(&(local_addr, peer_addr))
5648
5693
  .and_then(|pid| self.paths.get(pid).ok())
@@ -5681,9 +5726,7 @@ impl<F: BufFactory> Connection<F> {
5681
5726
  /// # Ok::<(), quiche::Error>(())
5682
5727
  /// ```
5683
5728
  #[inline]
5684
- pub fn stream_recv(
5685
- &mut self, stream_id: u64, out: &mut [u8],
5686
- ) -> Result<(usize, bool)> {
5729
+ pub fn stream_recv(&mut self, stream_id: u64, out: &mut [u8]) -> Result<(usize, bool)> {
5687
5730
  self.stream_recv_buf(stream_id, out)
5688
5731
  }
5689
5732
 
@@ -5744,7 +5787,9 @@ impl<F: BufFactory> Connection<F> {
5744
5787
  /// # Ok::<(), quiche::Error>(())
5745
5788
  /// ```
5746
5789
  pub fn stream_recv_buf<B: bytes::BufMut>(
5747
- &mut self, stream_id: u64, out: B,
5790
+ &mut self,
5791
+ stream_id: u64,
5792
+ out: B,
5748
5793
  ) -> Result<(usize, bool)> {
5749
5794
  self.do_stream_recv(stream_id, RecvAction::Emit { out })
5750
5795
  }
@@ -5776,9 +5821,7 @@ impl<F: BufFactory> Connection<F> {
5776
5821
  /// }
5777
5822
  /// # Ok::<(), quiche::Error>(())
5778
5823
  /// ```
5779
- pub fn stream_discard(
5780
- &mut self, stream_id: u64, len: usize,
5781
- ) -> Result<(usize, bool)> {
5824
+ pub fn stream_discard(&mut self, stream_id: u64, len: usize) -> Result<(usize, bool)> {
5782
5825
  // `do_stream_recv()` is generic on the kind of `BufMut` in RecvAction.
5783
5826
  // Since we are discarding, it doesn't matter, but the compiler still
5784
5827
  // wants to know, so we say `&mut [u8]`.
@@ -5804,12 +5847,12 @@ impl<F: BufFactory> Connection<F> {
5804
5847
  // [`Done`]: enum.Error.html#variant.Done
5805
5848
  // [`send()`]: struct.Connection.html#method.send
5806
5849
  fn do_stream_recv<B: bytes::BufMut>(
5807
- &mut self, stream_id: u64, action: RecvAction<B>,
5850
+ &mut self,
5851
+ stream_id: u64,
5852
+ action: RecvAction<B>,
5808
5853
  ) -> Result<(usize, bool)> {
5809
5854
  // We can't read on our own unidirectional streams.
5810
- if !stream::is_bidi(stream_id) &&
5811
- stream::is_local(stream_id, self.is_server)
5812
- {
5855
+ if !stream::is_bidi(stream_id) && stream::is_local(stream_id, self.is_server) {
5813
5856
  return Err(Error::InvalidStreamState(stream_id));
5814
5857
  }
5815
5858
 
@@ -5839,24 +5882,22 @@ impl<F: BufFactory> Connection<F> {
5839
5882
  Ok(v) => v,
5840
5883
 
5841
5884
  Err(e) => {
5842
- // Collect the stream if it is now complete. This can happen if
5843
- // we got a `StreamReset` error which will now be propagated to
5844
- // the application, so we don't need to keep the stream's state
5845
- // anymore.
5846
- if stream.is_complete() {
5885
+ // A StreamReset read completes the receive side, but an
5886
+ // unreported STOP error still needs the send-side state.
5887
+ if stream.state() == stream::StreamState::Collectable {
5847
5888
  self.streams.collect(stream_id, local);
5848
5889
  }
5849
5890
 
5850
5891
  self.streams.remove_readable(&priority_key);
5851
5892
  return Err(e);
5852
- },
5893
+ }
5853
5894
  };
5854
5895
 
5855
5896
  self.flow_control.add_consumed(read as u64);
5856
5897
 
5857
5898
  let readable = stream.is_readable();
5858
5899
 
5859
- let complete = stream.is_complete();
5900
+ let state = stream.state();
5860
5901
 
5861
5902
  if stream.recv.almost_full() {
5862
5903
  self.streams.insert_almost_full(stream_id);
@@ -5866,25 +5907,22 @@ impl<F: BufFactory> Connection<F> {
5866
5907
  self.streams.remove_readable(&priority_key);
5867
5908
  }
5868
5909
 
5869
- if complete {
5910
+ if state == stream::StreamState::Collectable {
5870
5911
  self.streams.collect(stream_id, local);
5871
5912
  }
5872
5913
 
5873
5914
  qlog_with_type!(QLOG_DATA_MV, self.qlog, q, {
5874
- let ev_data = EventData::QuicStreamDataMoved(
5875
- qlog::events::quic::StreamDataMoved {
5876
- stream_id: Some(stream_id),
5877
- offset: Some(offset),
5878
- raw: Some(RawInfo {
5879
- length: Some(read as u64),
5880
- ..Default::default()
5881
- }),
5882
- from: Some(DataRecipient::Transport),
5883
- to,
5884
- additional_info: fin
5885
- .then_some(DataMovedAdditionalInfo::FinSet),
5886
- },
5887
- );
5915
+ let ev_data = EventData::QuicStreamDataMoved(qlog::events::quic::StreamDataMoved {
5916
+ stream_id: Some(stream_id),
5917
+ offset: Some(offset),
5918
+ raw: Some(RawInfo {
5919
+ length: Some(read as u64),
5920
+ ..Default::default()
5921
+ }),
5922
+ from: Some(DataRecipient::Transport),
5923
+ to,
5924
+ additional_info: fin.then_some(DataMovedAdditionalInfo::FinSet),
5925
+ });
5888
5926
 
5889
5927
  let now = Instant::now();
5890
5928
  q.add_event_data_with_instant(ev_data, now).ok();
@@ -5946,17 +5984,12 @@ impl<F: BufFactory> Connection<F> {
5946
5984
  /// conn.stream_send(stream_id, b"hello", true)?;
5947
5985
  /// # Ok::<(), quiche::Error>(())
5948
5986
  /// ```
5949
- pub fn stream_send(
5950
- &mut self, stream_id: u64, buf: &[u8], fin: bool,
5951
- ) -> Result<usize> {
5987
+ pub fn stream_send(&mut self, stream_id: u64, buf: &[u8], fin: bool) -> Result<usize> {
5952
5988
  self.stream_do_send(
5953
5989
  stream_id,
5954
5990
  buf,
5955
5991
  fin,
5956
- |stream: &mut stream::Stream<F>,
5957
- buf: &[u8],
5958
- cap: usize,
5959
- fin: bool| {
5992
+ |stream: &mut stream::Stream<F>, buf: &[u8], cap: usize, fin: bool| {
5960
5993
  stream.send.write(&buf[..cap], fin).map(|v| (v, v))
5961
5994
  },
5962
5995
  )
@@ -5971,7 +6004,10 @@ impl<F: BufFactory> Connection<F> {
5971
6004
  /// The application should retry the operation once the stream is
5972
6005
  /// reported as writable again.
5973
6006
  pub fn stream_send_zc(
5974
- &mut self, stream_id: u64, buf: F::Buf, fin: bool,
6007
+ &mut self,
6008
+ stream_id: u64,
6009
+ buf: F::Buf,
6010
+ fin: bool,
5975
6011
  ) -> Result<(usize, Option<F::Buf>)>
5976
6012
  where
5977
6013
  F::Buf: BufSplit,
@@ -5980,10 +6016,7 @@ impl<F: BufFactory> Connection<F> {
5980
6016
  stream_id,
5981
6017
  buf,
5982
6018
  fin,
5983
- |stream: &mut stream::Stream<F>,
5984
- buf: F::Buf,
5985
- cap: usize,
5986
- fin: bool| {
6019
+ |stream: &mut stream::Stream<F>, buf: F::Buf, cap: usize, fin: bool| {
5987
6020
  let (sent, remaining) = stream.send.append_buf(buf, cap, fin)?;
5988
6021
  Ok((sent, (sent, remaining)))
5989
6022
  },
@@ -5991,16 +6024,18 @@ impl<F: BufFactory> Connection<F> {
5991
6024
  }
5992
6025
 
5993
6026
  fn stream_do_send<B, R, SND>(
5994
- &mut self, stream_id: u64, buf: B, fin: bool, write_fn: SND,
6027
+ &mut self,
6028
+ stream_id: u64,
6029
+ buf: B,
6030
+ fin: bool,
6031
+ write_fn: SND,
5995
6032
  ) -> Result<R>
5996
6033
  where
5997
6034
  B: AsRef<[u8]>,
5998
6035
  SND: FnOnce(&mut stream::Stream<F>, B, usize, bool) -> Result<(usize, R)>,
5999
6036
  {
6000
6037
  // We can't write on the peer's unidirectional streams.
6001
- if !stream::is_bidi(stream_id) &&
6002
- !stream::is_local(stream_id, self.is_server)
6003
- {
6038
+ if !stream::is_bidi(stream_id) && !stream::is_local(stream_id, self.is_server) {
6004
6039
  return Err(Error::InvalidStreamState(stream_id));
6005
6040
  }
6006
6041
 
@@ -6025,9 +6060,7 @@ impl<F: BufFactory> Connection<F> {
6025
6060
  // If the local endpoint has exhausted the peer's stream count
6026
6061
  // limit, record the current limit so that a STREAMS_BLOCKED
6027
6062
  // frame can be sent.
6028
- if self.enable_send_streams_blocked &&
6029
- stream::is_local(stream_id, self.is_server)
6030
- {
6063
+ if self.enable_send_streams_blocked && stream::is_local(stream_id, self.is_server) {
6031
6064
  if stream::is_bidi(stream_id) {
6032
6065
  let limit = self.streams.peer_max_streams_bidi();
6033
6066
  self.streams_blocked_bidi_state.update_at(limit);
@@ -6038,7 +6071,7 @@ impl<F: BufFactory> Connection<F> {
6038
6071
  }
6039
6072
 
6040
6073
  return Err(Error::StreamLimit);
6041
- },
6074
+ }
6042
6075
 
6043
6076
  Err(e) => return Err(e),
6044
6077
  };
@@ -6050,27 +6083,14 @@ impl<F: BufFactory> Connection<F> {
6050
6083
 
6051
6084
  let was_flushable = stream.is_flushable();
6052
6085
 
6053
- let is_complete = stream.is_complete();
6054
- let is_readable = stream.is_readable();
6055
-
6056
- let priority_key = Arc::clone(&stream.priority_key);
6057
-
6058
- // Return early if the stream has been stopped, and collect its state
6059
- // if complete.
6060
- if let Err(Error::StreamStopped(e)) = stream.send.cap() {
6061
- // Only collect the stream if it is complete and not readable.
6062
- // If it is readable, it will get collected when stream_recv()
6063
- // is used.
6064
- //
6065
- // The stream can't be writable if it has been stopped.
6066
- if is_complete && !is_readable {
6067
- let local = stream.local;
6068
- self.streams.collect(stream_id, local);
6069
- }
6070
-
6086
+ // Return the stop before collecting a completed stream.
6087
+ if let Err(Error::StreamStopped(e)) = stream.send.cap_and_report_stop_sending() {
6088
+ self.streams.on_stop_sending_reported(stream_id);
6071
6089
  return Err(Error::StreamStopped(e));
6072
6090
  };
6073
6091
 
6092
+ let priority_key = Arc::clone(&stream.priority_key);
6093
+
6074
6094
  // Truncate the input buffer based on the connection's send capacity if
6075
6095
  // necessary.
6076
6096
  //
@@ -6102,7 +6122,7 @@ impl<F: BufFactory> Connection<F> {
6102
6122
  Err(e) => {
6103
6123
  self.streams.remove_writable(&priority_key);
6104
6124
  return Err(e);
6105
- },
6125
+ }
6106
6126
  };
6107
6127
 
6108
6128
  let incremental = stream.incremental;
@@ -6151,24 +6171,20 @@ impl<F: BufFactory> Connection<F> {
6151
6171
 
6152
6172
  self.tx_data += sent as u64;
6153
6173
 
6154
- self.tx_buffered += sent;
6155
- self.check_tx_buffered_invariant();
6174
+ self.streams.add_tx_buffered(sent);
6156
6175
 
6157
6176
  qlog_with_type!(QLOG_DATA_MV, self.qlog, q, {
6158
- let ev_data = EventData::QuicStreamDataMoved(
6159
- qlog::events::quic::StreamDataMoved {
6160
- stream_id: Some(stream_id),
6161
- offset: Some(offset),
6162
- raw: Some(RawInfo {
6163
- length: Some(sent as u64),
6164
- ..Default::default()
6165
- }),
6166
- from: Some(DataRecipient::Application),
6167
- to: Some(DataRecipient::Transport),
6168
- additional_info: fin
6169
- .then_some(DataMovedAdditionalInfo::FinSet),
6170
- },
6171
- );
6177
+ let ev_data = EventData::QuicStreamDataMoved(qlog::events::quic::StreamDataMoved {
6178
+ stream_id: Some(stream_id),
6179
+ offset: Some(offset),
6180
+ raw: Some(RawInfo {
6181
+ length: Some(sent as u64),
6182
+ ..Default::default()
6183
+ }),
6184
+ from: Some(DataRecipient::Application),
6185
+ to: Some(DataRecipient::Transport),
6186
+ additional_info: fin.then_some(DataMovedAdditionalInfo::FinSet),
6187
+ });
6172
6188
 
6173
6189
  let now = Instant::now();
6174
6190
  q.add_event_data_with_instant(ev_data, now).ok();
@@ -6196,7 +6212,10 @@ impl<F: BufFactory> Connection<F> {
6196
6212
  /// The target stream is created if it did not exist before calling this
6197
6213
  /// method.
6198
6214
  pub fn stream_priority(
6199
- &mut self, stream_id: u64, urgency: u8, incremental: bool,
6215
+ &mut self,
6216
+ stream_id: u64,
6217
+ urgency: u8,
6218
+ incremental: bool,
6200
6219
  ) -> Result<()> {
6201
6220
  // Get existing stream or create a new one, but if the stream
6202
6221
  // has already been closed and collected, ignore the prioritization.
@@ -6244,7 +6263,8 @@ impl<F: BufFactory> Connection<F> {
6244
6263
  /// data in the stream's send buffer is dropped, and no additional data is
6245
6264
  /// added to it. Data passed to [`stream_send()`] after calling this method
6246
6265
  /// will be ignored. In addition, a `RESET_STREAM` frame will be sent to the
6247
- /// peer to signal the reset.
6266
+ /// peer to signal the reset, unless all outgoing data has already been
6267
+ /// acked by the peer (up to the `fin` offset).
6248
6268
  ///
6249
6269
  /// Locally-initiated unidirectional streams can only be closed in the
6250
6270
  /// [`Shutdown::Write`] direction. Remotely-initiated unidirectional streams
@@ -6256,21 +6276,19 @@ impl<F: BufFactory> Connection<F> {
6256
6276
  /// [`stream_recv()`]: struct.Connection.html#method.stream_recv
6257
6277
  /// [`stream_send()`]: struct.Connection.html#method.stream_send
6258
6278
  /// [`InvalidStreamState`]: enum.Error.html#variant.InvalidStreamState
6259
- pub fn stream_shutdown(
6260
- &mut self, stream_id: u64, direction: Shutdown, err: u64,
6261
- ) -> Result<()> {
6279
+ pub fn stream_shutdown(&mut self, stream_id: u64, direction: Shutdown, err: u64) -> Result<()> {
6262
6280
  // Don't try to stop a local unidirectional stream.
6263
- if direction == Shutdown::Read &&
6264
- stream::is_local(stream_id, self.is_server) &&
6265
- !stream::is_bidi(stream_id)
6281
+ if direction == Shutdown::Read
6282
+ && stream::is_local(stream_id, self.is_server)
6283
+ && !stream::is_bidi(stream_id)
6266
6284
  {
6267
6285
  return Err(Error::InvalidStreamState(stream_id));
6268
6286
  }
6269
6287
 
6270
6288
  // Don't try to reset a remote unidirectional stream.
6271
- if direction == Shutdown::Write &&
6272
- !stream::is_local(stream_id, self.is_server) &&
6273
- !stream::is_bidi(stream_id)
6289
+ if direction == Shutdown::Write
6290
+ && !stream::is_local(stream_id, self.is_server)
6291
+ && !stream::is_bidi(stream_id)
6274
6292
  {
6275
6293
  return Err(Error::InvalidStreamState(stream_id));
6276
6294
  }
@@ -6285,6 +6303,11 @@ impl<F: BufFactory> Connection<F> {
6285
6303
  let consumed = stream.recv.shutdown()?;
6286
6304
  self.flow_control.add_consumed(consumed);
6287
6305
 
6306
+ // Discarding unread data can complete the receive side, so no
6307
+ // later read would collect the stream.
6308
+ let state = stream.state();
6309
+ let local = stream.local;
6310
+
6288
6311
  if !stream.recv.is_fin() {
6289
6312
  self.streams.insert_stopped(stream_id, err);
6290
6313
  }
@@ -6292,29 +6315,36 @@ impl<F: BufFactory> Connection<F> {
6292
6315
  // Once shutdown, the stream is guaranteed to be non-readable.
6293
6316
  self.streams.remove_readable(&priority_key);
6294
6317
 
6295
- self.stopped_stream_local_count =
6296
- self.stopped_stream_local_count.saturating_add(1);
6297
- },
6318
+ self.stopped_stream_local_count = self.stopped_stream_local_count.saturating_add(1);
6319
+
6320
+ if state == stream::StreamState::Collectable {
6321
+ self.streams.collect(stream_id, local);
6322
+ }
6323
+ }
6298
6324
 
6299
6325
  Shutdown::Write => {
6326
+ // Save the buffered length before shutdown (shutdown clears the
6327
+ // buffer).
6328
+ let buffered_len = stream.send.buffered_bytes() as usize;
6329
+
6300
6330
  let (final_size, unsent) = stream.send.shutdown()?;
6331
+ let dropped = buffered_len.saturating_sub(stream.send.buffered_bytes() as usize);
6332
+ let flushable = stream.is_flushable();
6301
6333
 
6302
6334
  // Claw back some flow control allowance from data that was
6303
6335
  // buffered but not actually sent before the stream was reset.
6304
6336
  self.tx_data = self.tx_data.saturating_sub(unsent);
6305
6337
 
6306
- self.tx_buffered =
6307
- self.tx_buffered.saturating_sub(unsent as usize);
6338
+ // Update tx_buffered: subtract only the buffered data, not
6339
+ // inflight data.
6340
+ self.streams.sub_tx_buffered(dropped);
6308
6341
 
6309
- // These drops in qlog are a bit weird, but the only way to ensure
6310
- // that all bytes that are moved from App to Transport in
6311
- // stream_do_send are eventually moved from Transport to Dropped.
6312
- // Ideally we would add a Transport to Network transition also as
6313
- // a way to indicate when bytes were transmitted vs dropped
6314
- // without ever being sent.
6342
+ // Match App-to-Transport moves with Transport-to-Dropped moves.
6343
+ // A Network transition would distinguish sent bytes from drops
6344
+ // before transmission.
6315
6345
  qlog_with_type!(QLOG_DATA_MV, self.qlog, q, {
6316
- let ev_data = EventData::QuicStreamDataMoved(
6317
- qlog::events::quic::StreamDataMoved {
6346
+ let ev_data =
6347
+ EventData::QuicStreamDataMoved(qlog::events::quic::StreamDataMoved {
6318
6348
  stream_id: Some(stream_id),
6319
6349
  offset: Some(final_size),
6320
6350
  raw: Some(RawInfo {
@@ -6324,8 +6354,7 @@ impl<F: BufFactory> Connection<F> {
6324
6354
  from: Some(DataRecipient::Transport),
6325
6355
  to: Some(DataRecipient::Dropped),
6326
6356
  ..Default::default()
6327
- },
6328
- );
6357
+ });
6329
6358
 
6330
6359
  q.add_event_data_with_instant(ev_data, Instant::now()).ok();
6331
6360
  });
@@ -6335,17 +6364,40 @@ impl<F: BufFactory> Connection<F> {
6335
6364
 
6336
6365
  self.streams.insert_reset(stream_id, err, final_size);
6337
6366
 
6338
- // Once shutdown, the stream is guaranteed to be non-writable.
6339
6367
  self.streams.remove_writable(&priority_key);
6368
+ if !flushable {
6369
+ self.streams.remove_flushable(&priority_key);
6370
+ }
6340
6371
 
6341
- self.reset_stream_local_count =
6342
- self.reset_stream_local_count.saturating_add(1);
6343
- },
6372
+ self.reset_stream_local_count = self.reset_stream_local_count.saturating_add(1);
6373
+
6374
+ self.streams.try_collect(stream_id);
6375
+ }
6344
6376
  }
6345
6377
 
6346
6378
  Ok(())
6347
6379
  }
6348
6380
 
6381
+ /// Returns whether both endpoints advertised QUIC RESET_STREAM_AT.
6382
+ pub fn reliable_stream_reset_negotiated(&self) -> bool {
6383
+ self.local_transport_params.reset_stream_at && self.peer_transport_params.reset_stream_at
6384
+ }
6385
+
6386
+ /// Use RESET_STREAM_AT for this stream and retain the given prefix on a
6387
+ /// local write shutdown or a peer STOP_SENDING. The prefix must already be
6388
+ /// queued with stream_send.
6389
+ pub fn stream_set_reliable_reset(&mut self, id: u64, size: u64) -> Result<()> {
6390
+ if !self.reliable_stream_reset_negotiated() {
6391
+ return Err(Error::InvalidState);
6392
+ }
6393
+ let stream = self.streams.get_mut(id).ok_or(Error::Done)?;
6394
+ if stream.send.is_reset() || size > stream.send.off_back() {
6395
+ return Err(Error::InvalidStreamState(id));
6396
+ }
6397
+ stream.send.reliable_reset_size = Some(size);
6398
+ Ok(())
6399
+ }
6400
+
6349
6401
  /// Returns the stream's send capacity in bytes.
6350
6402
  ///
6351
6403
  /// The returned capacity takes into account the stream's flow control limit
@@ -6363,21 +6415,14 @@ impl<F: BufFactory> Connection<F> {
6363
6415
  /// [`StreamStopped`]: enum.Error.html#variant.StreamStopped
6364
6416
  #[inline]
6365
6417
  pub fn stream_capacity(&mut self, stream_id: u64) -> Result<usize> {
6366
- if let Some(stream) = self.streams.get(stream_id) {
6367
- let stream_cap = match stream.send.cap() {
6418
+ if let Some(stream) = self.streams.get_mut(stream_id) {
6419
+ let stream_cap = match stream.send.cap_and_report_stop_sending() {
6368
6420
  Ok(v) => v,
6369
6421
 
6370
6422
  Err(Error::StreamStopped(e)) => {
6371
- // Only collect the stream if it is complete and not
6372
- // readable. If it is readable, it will get collected when
6373
- // stream_recv() is used.
6374
- if stream.is_complete() && !stream.is_readable() {
6375
- let local = stream.local;
6376
- self.streams.collect(stream_id, local);
6377
- }
6378
-
6423
+ self.streams.on_stop_sending_reported(stream_id);
6379
6424
  return Err(Error::StreamStopped(e));
6380
- },
6425
+ }
6381
6426
 
6382
6427
  Err(e) => return Err(e),
6383
6428
  };
@@ -6420,6 +6465,33 @@ impl<F: BufFactory> Connection<F> {
6420
6465
  stream.is_readable()
6421
6466
  }
6422
6467
 
6468
+ /// Returns the number of contiguous bytes buffered for a stream, up to
6469
+ /// `max_len`.
6470
+ ///
6471
+ /// This is the length of the contiguous, in-order data buffered at the
6472
+ /// stream's current read offset, i.e. the bytes a call to [`stream_recv`]
6473
+ /// would return right now. Data received out of order that sits behind a
6474
+ /// gap is not counted, so this never reports bytes that are not yet
6475
+ /// readable.
6476
+ ///
6477
+ /// This is a companion to [`stream_readable`], which only reports *whether*
6478
+ /// data is available; this reports *how much*. It is intended for sizing a
6479
+ /// receive buffer. The cost is proportional to the number of contiguous
6480
+ /// buffered chunks at the front of the stream, up to `max_len`, and no data
6481
+ /// is copied.
6482
+ ///
6483
+ /// Returns 0 if the stream does not exist.
6484
+ ///
6485
+ /// [`stream_recv`]: struct.Connection.html#method.stream_recv
6486
+ /// [`stream_readable`]: struct.Connection.html#method.stream_readable
6487
+ pub fn stream_readable_len(&self, stream_id: u64, max_len: usize) -> usize {
6488
+ match self.streams.get(stream_id) {
6489
+ Some(s) => s.recv.readable_len(max_len),
6490
+
6491
+ None => 0,
6492
+ }
6493
+ }
6494
+
6423
6495
  /// Returns the next stream that can be written to.
6424
6496
  ///
6425
6497
  /// Note that once returned by this method, a stream ID will not be returned
@@ -6433,13 +6505,15 @@ impl<F: BufFactory> Connection<F> {
6433
6505
  /// The [`stream_writable()`] method can also be used to fine-tune when a
6434
6506
  /// stream is reported as writable again.
6435
6507
  ///
6508
+ /// A stopped stream is returned even without send capacity. Querying its
6509
+ /// capacity or writing to it returns [`StreamStopped`].
6510
+ ///
6436
6511
  /// [`stream_writable()`]: struct.Connection.html#method.stream_writable
6437
6512
  /// [`writable()`]: struct.Connection.html#method.writable
6513
+ /// [`StreamStopped`]: enum.Error.html#variant.StreamStopped
6438
6514
  pub fn stream_writable_next(&mut self) -> Option<u64> {
6439
- // If there is not enough connection-level send capacity, none of the
6440
- // streams are writable.
6441
6515
  if self.tx_cap == 0 {
6442
- return None;
6516
+ return self.streams.pop_stopped_writable();
6443
6517
  }
6444
6518
 
6445
6519
  let mut cursor = self.streams.writable.front();
@@ -6451,12 +6525,13 @@ impl<F: BufFactory> Connection<F> {
6451
6525
 
6452
6526
  // Return the stream to the application immediately if it's
6453
6527
  // stopped.
6454
- Err(_) =>
6528
+ Err(_) => {
6455
6529
  return {
6456
6530
  self.streams.remove_writable(&priority_key);
6457
6531
 
6458
6532
  Some(priority_key.id)
6459
- },
6533
+ }
6534
+ }
6460
6535
  };
6461
6536
 
6462
6537
  if cmp::min(self.tx_cap, cap) >= stream.send_lowat {
@@ -6494,9 +6569,7 @@ impl<F: BufFactory> Connection<F> {
6494
6569
  /// [`InvalidStreamState`]: enum.Error.html#variant.InvalidStreamState
6495
6570
  /// [`StreamStopped`]: enum.Error.html#variant.StreamStopped
6496
6571
  #[inline]
6497
- pub fn stream_writable(
6498
- &mut self, stream_id: u64, len: usize,
6499
- ) -> Result<bool> {
6572
+ pub fn stream_writable(&mut self, stream_id: u64, len: usize) -> Result<bool> {
6500
6573
  if self.stream_capacity(stream_id)? >= len {
6501
6574
  return Ok(true);
6502
6575
  }
@@ -6649,6 +6722,9 @@ impl<F: BufFactory> Connection<F> {
6649
6722
  /// streams are only allowed to buffer outgoing data up to the amount that
6650
6723
  /// the peer allows to send.
6651
6724
  ///
6725
+ /// Stopped streams are also included so that a write or capacity query can
6726
+ /// return [`StreamStopped`], even without connection send capacity.
6727
+ ///
6652
6728
  /// Note that the iterator will only include streams that were writable at
6653
6729
  /// the time the iterator itself was created (i.e. when `writable()` was
6654
6730
  /// called). To account for newly writable streams, the iterator needs to be
@@ -6674,12 +6750,12 @@ impl<F: BufFactory> Connection<F> {
6674
6750
  /// # Ok::<(), quiche::Error>(())
6675
6751
  /// ```
6676
6752
  /// [`stream_priority()`]: struct.Connection.html#method.stream_priority
6753
+ /// [`StreamStopped`]: enum.Error.html#variant.StreamStopped
6677
6754
  #[inline]
6678
6755
  pub fn writable(&self) -> StreamIter {
6679
- // If there is not enough connection-level send capacity, none of the
6680
- // streams are writable, so return an empty iterator.
6756
+ // Stopped streams must be reported even without send capacity.
6681
6757
  if self.tx_cap == 0 {
6682
- return StreamIter::default();
6758
+ return self.streams.stopped_writable();
6683
6759
  }
6684
6760
 
6685
6761
  self.streams.writable()
@@ -6707,8 +6783,8 @@ impl<F: BufFactory> Connection<F> {
6707
6783
 
6708
6784
  if let Some(max_datagram_size) = max_datagram_size {
6709
6785
  if self.is_established() {
6710
- // We cap the maximum packet size to 16KB or so, so that it can be
6711
- // always encoded with a 2-byte varint.
6786
+ // Cap the packet size at 16,383 bytes so a two-byte varint can
6787
+ // always encode it.
6712
6788
  return cmp::min(16383, max_datagram_size);
6713
6789
  }
6714
6790
  }
@@ -6745,7 +6821,9 @@ impl<F: BufFactory> Connection<F> {
6745
6821
  /// [`send_ack_eliciting()`]: struct.Connection.html#method.send_ack_eliciting
6746
6822
  /// [`InvalidState`]: enum.Error.html#variant.InvalidState
6747
6823
  pub fn send_ack_eliciting_on_path(
6748
- &mut self, local: SocketAddr, peer: SocketAddr,
6824
+ &mut self,
6825
+ local: SocketAddr,
6826
+ peer: SocketAddr,
6749
6827
  ) -> Result<()> {
6750
6828
  if self.is_closed() || self.is_draining() {
6751
6829
  return Ok(());
@@ -6797,7 +6875,7 @@ impl<F: BufFactory> Connection<F> {
6797
6875
 
6798
6876
  buf[..len].copy_from_slice(d.as_ref());
6799
6877
  Ok(len)
6800
- },
6878
+ }
6801
6879
 
6802
6880
  None => Err(Error::Done),
6803
6881
  }
@@ -6931,9 +7009,7 @@ impl<F: BufFactory> Connection<F> {
6931
7009
 
6932
7010
  let active_path = self.paths.get_active_mut()?;
6933
7011
 
6934
- if self.dgram_send_queue.byte_size() >
6935
- active_path.recovery.cwnd_available()
6936
- {
7012
+ if self.dgram_send_queue.byte_size() > active_path.recovery.cwnd_available() {
6937
7013
  active_path.recovery.update_app_limited(false);
6938
7014
  }
6939
7015
 
@@ -6997,16 +7073,14 @@ impl<F: BufFactory> Connection<F> {
6997
7073
  // ...subtract the packet number (max len)...
6998
7074
  max_len = max_len.saturating_sub(packet::MAX_PKT_NUM_LEN);
6999
7075
  // ...subtract the crypto overhead...
7000
- max_len = max_len.saturating_sub(
7001
- self.crypto_ctx[packet::Epoch::Application]
7002
- .crypto_overhead()?,
7003
- );
7076
+ max_len = max_len
7077
+ .saturating_sub(self.crypto_ctx[packet::Epoch::Application].crypto_overhead()?);
7004
7078
  // ...clamp to what peer can support...
7005
7079
  max_len = cmp::min(peer_frame_len as usize, max_len);
7006
7080
  // ...subtract frame overhead, checked for underflow.
7007
7081
  // (1 byte of frame type + len of length )
7008
7082
  max_len.checked_sub(1 + frame::MAX_DGRAM_OVERHEAD)
7009
- },
7083
+ }
7010
7084
  }
7011
7085
  }
7012
7086
 
@@ -7153,12 +7227,12 @@ impl<F: BufFactory> Connection<F> {
7153
7227
  // The connection cannot continue.
7154
7228
  self.mark_closed();
7155
7229
  }
7156
- },
7230
+ }
7157
7231
 
7158
7232
  // The connection cannot continue.
7159
7233
  None => {
7160
7234
  self.mark_closed();
7161
- },
7235
+ }
7162
7236
  }
7163
7237
  }
7164
7238
  }
@@ -7193,9 +7267,7 @@ impl<F: BufFactory> Connection<F> {
7193
7267
  /// [`InvalidState`]: enum.Error.html#InvalidState
7194
7268
  /// [`send()`]: struct.Connection.html#method.send
7195
7269
  /// [`send_on_path()`]: struct.Connection.html#method.send_on_path
7196
- pub fn probe_path(
7197
- &mut self, local_addr: SocketAddr, peer_addr: SocketAddr,
7198
- ) -> Result<u64> {
7270
+ pub fn probe_path(&mut self, local_addr: SocketAddr, peer_addr: SocketAddr) -> Result<u64> {
7199
7271
  // We may want to probe an existing path.
7200
7272
  let pid = match self.paths.path_id_from_addrs(&(local_addr, peer_addr)) {
7201
7273
  Some(pid) => pid,
@@ -7235,61 +7307,58 @@ impl<F: BufFactory> Connection<F> {
7235
7307
  ///
7236
7308
  /// [`OutOfIdentifiers`]: enum.Error.html#OutOfIdentifiers
7237
7309
  /// [`InvalidState`]: enum.Error.html#InvalidState
7238
- pub fn migrate(
7239
- &mut self, local_addr: SocketAddr, peer_addr: SocketAddr,
7240
- ) -> Result<u64> {
7310
+ pub fn migrate(&mut self, local_addr: SocketAddr, peer_addr: SocketAddr) -> Result<u64> {
7241
7311
  if self.is_server {
7242
7312
  return Err(Error::InvalidState);
7243
7313
  }
7244
7314
 
7245
7315
  // If the path already exists, mark it as the active one.
7246
- let (pid, dcid_seq) = if let Some(pid) =
7247
- self.paths.path_id_from_addrs(&(local_addr, peer_addr))
7248
- {
7249
- let path = self.paths.get_mut(pid)?;
7316
+ let (pid, dcid_seq) =
7317
+ if let Some(pid) = self.paths.path_id_from_addrs(&(local_addr, peer_addr)) {
7318
+ let path = self.paths.get_mut(pid)?;
7250
7319
 
7251
- // If it is already active, do nothing.
7252
- if path.active() {
7253
- return path.active_dcid_seq.ok_or(Error::OutOfIdentifiers);
7254
- }
7320
+ // If it is already active, do nothing.
7321
+ if path.active() {
7322
+ return path.active_dcid_seq.ok_or(Error::OutOfIdentifiers);
7323
+ }
7255
7324
 
7256
- // Ensures that a Source Connection ID has been dedicated to this
7257
- // path, or a free one is available. This is only required if the
7258
- // host uses non-zero length Source Connection IDs.
7259
- if !self.ids.zero_length_scid() &&
7260
- path.active_scid_seq.is_none() &&
7261
- self.ids.available_scids() == 0
7262
- {
7263
- return Err(Error::OutOfIdentifiers);
7264
- }
7325
+ // Ensures that a Source Connection ID has been dedicated to this
7326
+ // path, or a free one is available. This is only required if the
7327
+ // host uses non-zero length Source Connection IDs.
7328
+ if !self.ids.zero_length_scid()
7329
+ && path.active_scid_seq.is_none()
7330
+ && self.ids.available_scids() == 0
7331
+ {
7332
+ return Err(Error::OutOfIdentifiers);
7333
+ }
7265
7334
 
7266
- // Ensures that the migrated path has a Destination Connection ID.
7267
- let dcid_seq = if let Some(dcid_seq) = path.active_dcid_seq {
7268
- dcid_seq
7269
- } else {
7270
- let dcid_seq = self
7271
- .ids
7272
- .lowest_available_dcid_seq()
7273
- .ok_or(Error::OutOfIdentifiers)?;
7335
+ // Ensures that the migrated path has a Destination Connection ID.
7336
+ let dcid_seq = if let Some(dcid_seq) = path.active_dcid_seq {
7337
+ dcid_seq
7338
+ } else {
7339
+ let dcid_seq = self
7340
+ .ids
7341
+ .lowest_available_dcid_seq()
7342
+ .ok_or(Error::OutOfIdentifiers)?;
7274
7343
 
7275
- self.ids.link_dcid_to_path_id(dcid_seq, pid)?;
7276
- path.active_dcid_seq = Some(dcid_seq);
7344
+ self.ids.link_dcid_to_path_id(dcid_seq, pid)?;
7345
+ path.active_dcid_seq = Some(dcid_seq);
7277
7346
 
7278
- dcid_seq
7279
- };
7347
+ dcid_seq
7348
+ };
7280
7349
 
7281
- (pid, dcid_seq)
7282
- } else {
7283
- let pid = self.create_path_on_client(local_addr, peer_addr)?;
7350
+ (pid, dcid_seq)
7351
+ } else {
7352
+ let pid = self.create_path_on_client(local_addr, peer_addr)?;
7284
7353
 
7285
- let dcid_seq = self
7286
- .paths
7287
- .get(pid)?
7288
- .active_dcid_seq
7289
- .ok_or(Error::InvalidState)?;
7354
+ let dcid_seq = self
7355
+ .paths
7356
+ .get(pid)?
7357
+ .active_dcid_seq
7358
+ .ok_or(Error::InvalidState)?;
7290
7359
 
7291
- (pid, dcid_seq)
7292
- };
7360
+ (pid, dcid_seq)
7361
+ };
7293
7362
 
7294
7363
  // Change the active path.
7295
7364
  self.set_active_path(pid, Instant::now())?;
@@ -7330,7 +7399,10 @@ impl<F: BufFactory> Connection<F> {
7330
7399
  /// [`IdLimit`]: enum.Error.html#IdLimit
7331
7400
  /// [`InvalidState`]: enum.Error.html#InvalidState
7332
7401
  pub fn new_scid(
7333
- &mut self, scid: &ConnectionId, reset_token: u128, retire_if_needed: bool,
7402
+ &mut self,
7403
+ scid: &ConnectionId,
7404
+ reset_token: u128,
7405
+ retire_if_needed: bool,
7334
7406
  ) -> Result<u64> {
7335
7407
  self.ids.new_scid(
7336
7408
  scid.to_vec().into(),
@@ -7404,9 +7476,9 @@ impl<F: BufFactory> Connection<F> {
7404
7476
 
7405
7477
  let active_path_id = self.paths.get_active_path_id()?;
7406
7478
 
7407
- if active_path_dcid_seq == dcid_seq &&
7408
- self.ids.lowest_available_dcid_seq().is_none() &&
7409
- !self
7479
+ if active_path_dcid_seq == dcid_seq
7480
+ && self.ids.lowest_available_dcid_seq().is_none()
7481
+ && !self
7410
7482
  .paths
7411
7483
  .iter()
7412
7484
  .any(|(pid, p)| pid != active_path_id && p.usable())
@@ -7567,8 +7639,7 @@ impl<F: BufFactory> Connection<F> {
7567
7639
  return Err(Error::Done);
7568
7640
  }
7569
7641
 
7570
- let is_safe_to_send_app_data =
7571
- self.is_established() || self.is_in_early_data();
7642
+ let is_safe_to_send_app_data = self.is_established() || self.is_in_early_data();
7572
7643
 
7573
7644
  if app && !is_safe_to_send_app_data {
7574
7645
  // Clear error information.
@@ -7585,7 +7656,7 @@ impl<F: BufFactory> Connection<F> {
7585
7656
  });
7586
7657
  }
7587
7658
 
7588
- // When no packet was successfully processed close connection immediately.
7659
+ // Close immediately if no packet was processed successfully.
7589
7660
  if self.recv_count == 0 {
7590
7661
  self.mark_closed();
7591
7662
  }
@@ -7616,7 +7687,10 @@ impl<F: BufFactory> Connection<F> {
7616
7687
  /// optional `context`, into `out`. Only available once the handshake has
7617
7688
  /// completed; calling it earlier returns [`Error::TlsFail`].
7618
7689
  pub fn export_keying_material(
7619
- &mut self, out: &mut [u8], label: &[u8], context: Option<&[u8]>,
7690
+ &mut self,
7691
+ out: &mut [u8],
7692
+ label: &[u8],
7693
+ context: Option<&[u8]>,
7620
7694
  ) -> Result<()> {
7621
7695
  self.handshake.export_keying_material(out, label, context)
7622
7696
  }
@@ -7719,13 +7793,30 @@ impl<F: BufFactory> Connection<F> {
7719
7793
  /// Revalidates the PMTU for the active path by sending a new probe packet
7720
7794
  /// of PMTU size. If the probe is dropped PMTUD will restart and find a new
7721
7795
  /// valid PMTU.
7796
+ ///
7797
+ /// If revalidation invalidates a previously discovered larger size, a
7798
+ /// [`PathEvent::PmtuUpdated`] event is queued with QUIC's minimum packet
7799
+ /// size. Further events report larger sizes as probes validate them.
7722
7800
  #[inline]
7723
7801
  pub fn revalidate_pmtu(&mut self) {
7724
- if let Ok(active_path) = self.paths.get_active_mut() {
7725
- if let Some(pmtud) = active_path.pmtud.as_mut() {
7726
- pmtud.revalidate_pmtu();
7727
- }
7728
- }
7802
+ let Ok(active_path) = self.paths.get_active_mut() else {
7803
+ return;
7804
+ };
7805
+
7806
+ let local = active_path.local_addr();
7807
+ let peer = active_path.peer_addr();
7808
+ let Some(pmtud) = active_path.pmtud.as_mut() else {
7809
+ return;
7810
+ };
7811
+
7812
+ let old_pmtu = pmtud.get_current_mtu();
7813
+ pmtud.revalidate_pmtu();
7814
+
7815
+ let Some(event) = path::pmtu_event(local, peer, old_pmtu, pmtud.get_current_mtu()) else {
7816
+ return;
7817
+ };
7818
+
7819
+ self.paths.notify_event(event);
7729
7820
  }
7730
7821
 
7731
7822
  /// Returns true if the connection handshake is complete.
@@ -7771,9 +7862,7 @@ impl<F: BufFactory> Connection<F> {
7771
7862
  /// [`InvalidState`].
7772
7863
  ///
7773
7864
  /// [`InvalidState`]: enum.Error.html#variant.InvalidState
7774
- pub fn is_path_validated(
7775
- &self, from: SocketAddr, to: SocketAddr,
7776
- ) -> Result<bool> {
7865
+ pub fn is_path_validated(&self, from: SocketAddr, to: SocketAddr) -> Result<bool> {
7777
7866
  let pid = self
7778
7867
  .paths
7779
7868
  .path_id_from_addrs(&(from, to))
@@ -7872,7 +7961,11 @@ impl<F: BufFactory> Connection<F> {
7872
7961
  path_challenge_rx_count: self.path_challenge_rx_count,
7873
7962
  amplification_limited_count: self.amplification_limited_count,
7874
7963
  bytes_in_flight_duration: self.bytes_in_flight_duration(),
7875
- tx_buffered_state: self.tx_buffered_state,
7964
+ tx_buffered_state: if self.streams.tx_buffered_is_consistent() {
7965
+ TxBufferTrackingState::Ok
7966
+ } else {
7967
+ TxBufferTrackingState::Inconsistent
7968
+ },
7876
7969
  }
7877
7970
  }
7878
7971
 
@@ -7912,19 +8005,14 @@ impl<F: BufFactory> Connection<F> {
7912
8005
  }
7913
8006
 
7914
8007
  fn encode_transport_params(&mut self) -> Result<()> {
7915
- self.handshake.set_quic_transport_params(
7916
- &self.local_transport_params,
7917
- self.is_server,
7918
- )
8008
+ self.handshake
8009
+ .set_quic_transport_params(&self.local_transport_params, self.is_server)
7919
8010
  }
7920
8011
 
7921
- fn parse_peer_transport_params(
7922
- &mut self, peer_params: TransportParams,
7923
- ) -> Result<()> {
8012
+ fn parse_peer_transport_params(&mut self, peer_params: TransportParams) -> Result<()> {
7924
8013
  // Validate initial_source_connection_id.
7925
8014
  match &peer_params.initial_source_connection_id {
7926
- Some(v) if v != &self.destination_id() =>
7927
- return Err(Error::InvalidTransportParam),
8015
+ Some(v) if v != &self.destination_id() => return Err(Error::InvalidTransportParam),
7928
8016
 
7929
8017
  Some(_) => (),
7930
8018
 
@@ -7936,15 +8024,13 @@ impl<F: BufFactory> Connection<F> {
7936
8024
  // Validate original_destination_connection_id.
7937
8025
  if let Some(odcid) = &self.odcid {
7938
8026
  match &peer_params.original_destination_connection_id {
7939
- Some(v) if v != odcid =>
7940
- return Err(Error::InvalidTransportParam),
8027
+ Some(v) if v != odcid => return Err(Error::InvalidTransportParam),
7941
8028
 
7942
8029
  Some(_) => (),
7943
8030
 
7944
8031
  // original_destination_connection_id must be
7945
8032
  // sent by the server.
7946
- None if !self.is_server =>
7947
- return Err(Error::InvalidTransportParam),
8033
+ None if !self.is_server => return Err(Error::InvalidTransportParam),
7948
8034
 
7949
8035
  None => (),
7950
8036
  }
@@ -7953,8 +8039,7 @@ impl<F: BufFactory> Connection<F> {
7953
8039
  // Validate retry_source_connection_id.
7954
8040
  if let Some(rscid) = &self.rscid {
7955
8041
  match &peer_params.retry_source_connection_id {
7956
- Some(v) if v != rscid =>
7957
- return Err(Error::InvalidTransportParam),
8042
+ Some(v) if v != rscid => return Err(Error::InvalidTransportParam),
7958
8043
 
7959
8044
  Some(_) => (),
7960
8045
 
@@ -7971,9 +8056,7 @@ impl<F: BufFactory> Connection<F> {
7971
8056
  Ok(())
7972
8057
  }
7973
8058
 
7974
- fn process_peer_transport_params(
7975
- &mut self, peer_params: TransportParams,
7976
- ) -> Result<()> {
8059
+ fn process_peer_transport_params(&mut self, peer_params: TransportParams) -> Result<()> {
7977
8060
  self.max_tx_data = peer_params.initial_max_data;
7978
8061
 
7979
8062
  // Update send capacity.
@@ -7985,8 +8068,12 @@ impl<F: BufFactory> Connection<F> {
7985
8068
  .update_peer_max_streams_uni(peer_params.initial_max_streams_uni);
7986
8069
 
7987
8070
  let max_ack_delay = Duration::from_millis(peer_params.max_ack_delay);
8071
+ let max_udp_payload_size =
8072
+ usize::try_from(peer_params.max_udp_payload_size).unwrap_or(usize::MAX);
7988
8073
 
7989
8074
  self.recovery_config.max_ack_delay = max_ack_delay;
8075
+ self.recovery_config
8076
+ .update_max_send_udp_payload_size(max_udp_payload_size);
7990
8077
 
7991
8078
  let active_path = self.paths.get_active_mut()?;
7992
8079
 
@@ -8004,12 +8091,12 @@ impl<F: BufFactory> Connection<F> {
8004
8091
  .as_mut()
8005
8092
  .expect("PMTUD existence verified above")
8006
8093
  .get_probe_size()
8007
- .min(peer_params.max_udp_payload_size as usize),
8094
+ .min(max_udp_payload_size),
8008
8095
  );
8009
8096
  } else {
8010
- active_path.recovery.update_max_datagram_size(
8011
- peer_params.max_udp_payload_size as usize,
8012
- );
8097
+ active_path
8098
+ .recovery
8099
+ .update_max_datagram_size(max_udp_payload_size);
8013
8100
  }
8014
8101
 
8015
8102
  // Record the max_active_conn_id parameter advertised by the peer.
@@ -8044,89 +8131,93 @@ impl<F: BufFactory> Connection<F> {
8044
8131
 
8045
8132
  tx_cap_factor: self.tx_cap_factor,
8046
8133
 
8134
+ max_outstanding_sent_packets: self.max_outstanding_sent_packets,
8135
+
8047
8136
  pmtud: None,
8048
8137
 
8049
8138
  is_server: self.is_server,
8050
-
8051
- use_initial_max_data_as_flow_control_win: false,
8052
8139
  };
8053
8140
 
8054
8141
  if self.handshake_completed {
8055
8142
  return self.handshake.process_post_handshake(&mut ex_data);
8056
8143
  }
8057
8144
 
8058
- match self.handshake.do_handshake(&mut ex_data) {
8059
- Ok(_) => (),
8145
+ let handshake_needs_retry = match self.handshake.do_handshake(&mut ex_data) {
8146
+ Ok(_) => false,
8147
+ Err(Error::Done) => true,
8148
+ Err(e) => return Err(e),
8149
+ };
8060
8150
 
8061
- Err(Error::Done) => {
8062
- // Apply in-handshake configuration from callbacks if the path's
8063
- // Recovery module can still be reinitilized.
8064
- if self
8065
- .paths
8066
- .get_active()
8067
- .map(|p| p.can_reinit_recovery())
8068
- .unwrap_or(false)
8069
- {
8070
- if ex_data.recovery_config != self.recovery_config {
8071
- if let Ok(path) = self.paths.get_active_mut() {
8072
- self.recovery_config = ex_data.recovery_config;
8073
- path.reinit_recovery(&self.recovery_config);
8074
- }
8075
- }
8151
+ // BoringSSL reports success when entering early data before the
8152
+ // handshake completes. Apply callback configuration after either
8153
+ // non-fatal outcome so it is not lost on that path.
8154
+ if self
8155
+ .paths
8156
+ .get_active()
8157
+ .map(|p| p.can_reinit_recovery())
8158
+ .unwrap_or(false)
8159
+ {
8160
+ if ex_data.recovery_config != self.recovery_config {
8161
+ // If we received peer transport parameters already, respect the
8162
+ // peer's `max_udp_payload_size`.
8163
+ if self.parsed_peer_transport_params {
8164
+ let max_udp_payload_size =
8165
+ usize::try_from(self.peer_transport_params.max_udp_payload_size)
8166
+ .unwrap_or(usize::MAX);
8167
+ ex_data
8168
+ .recovery_config
8169
+ .update_max_send_udp_payload_size(max_udp_payload_size);
8170
+ }
8076
8171
 
8077
- if ex_data.tx_cap_factor != self.tx_cap_factor {
8078
- self.tx_cap_factor = ex_data.tx_cap_factor;
8079
- }
8172
+ if let Ok(path) = self.paths.get_active_mut() {
8173
+ self.recovery_config = ex_data.recovery_config;
8174
+ path.reinit_recovery(&self.recovery_config);
8175
+ }
8176
+ }
8080
8177
 
8081
- if let Some((discover, max_probes)) = ex_data.pmtud {
8082
- self.paths.set_discover_pmtu_on_existing_paths(
8083
- discover,
8084
- self.recovery_config.max_send_udp_payload_size,
8085
- max_probes,
8086
- );
8087
- }
8178
+ if ex_data.tx_cap_factor != self.tx_cap_factor {
8179
+ self.tx_cap_factor = ex_data.tx_cap_factor;
8180
+ }
8088
8181
 
8089
- if ex_data.local_transport_params !=
8090
- self.local_transport_params
8091
- {
8092
- self.streams.set_max_streams_bidi(
8093
- ex_data
8094
- .local_transport_params
8095
- .initial_max_streams_bidi,
8096
- );
8182
+ self.max_outstanding_sent_packets = ex_data.max_outstanding_sent_packets;
8097
8183
 
8098
- self.local_transport_params =
8099
- ex_data.local_transport_params;
8100
- }
8101
- }
8184
+ if let Some((discover, max_probes)) = ex_data.pmtud {
8185
+ self.paths.set_discover_pmtu_on_existing_paths(
8186
+ discover,
8187
+ self.recovery_config.max_send_udp_payload_size,
8188
+ max_probes,
8189
+ );
8190
+ }
8102
8191
 
8103
- if ex_data.use_initial_max_data_as_flow_control_win {
8104
- self.enable_use_initial_max_data_as_flow_control_win();
8105
- }
8192
+ if ex_data.local_transport_params != self.local_transport_params {
8193
+ self.streams
8194
+ .set_max_streams_bidi(ex_data.local_transport_params.initial_max_streams_bidi);
8106
8195
 
8107
- // Try to parse transport parameters as soon as the first flight
8108
- // of handshake data is processed.
8109
- //
8110
- // This is potentially dangerous as the handshake hasn't been
8111
- // completed yet, though it's required to be able to send data
8112
- // in 0.5 RTT.
8113
- let raw_params = self.handshake.quic_transport_params();
8114
-
8115
- if !self.parsed_peer_transport_params && !raw_params.is_empty() {
8116
- let peer_params = TransportParams::decode(
8117
- raw_params,
8118
- self.is_server,
8119
- self.peer_transport_params_track_unknown,
8120
- )?;
8196
+ self.local_transport_params = ex_data.local_transport_params;
8197
+ }
8198
+ }
8121
8199
 
8122
- self.parse_peer_transport_params(peer_params)?;
8123
- }
8200
+ if handshake_needs_retry {
8201
+ // Try to parse transport parameters as soon as the first flight of
8202
+ // handshake data is processed.
8203
+ //
8204
+ // This is potentially dangerous as the handshake hasn't been
8205
+ // completed yet, though it's required to be able to send data in
8206
+ // 0.5 RTT.
8207
+ let raw_params = self.handshake.quic_transport_params();
8208
+
8209
+ if !self.parsed_peer_transport_params && !raw_params.is_empty() {
8210
+ let peer_params = TransportParams::decode(
8211
+ raw_params,
8212
+ self.is_server,
8213
+ self.peer_transport_params_track_unknown,
8214
+ )?;
8124
8215
 
8125
- return Ok(());
8126
- },
8216
+ self.parse_peer_transport_params(peer_params)?;
8217
+ }
8127
8218
 
8128
- Err(e) => return Err(e),
8129
- };
8219
+ return Ok(());
8220
+ }
8130
8221
 
8131
8222
  self.handshake_completed = self.handshake.is_completed();
8132
8223
 
@@ -8171,19 +8262,6 @@ impl<F: BufFactory> Connection<F> {
8171
8262
  Ok(())
8172
8263
  }
8173
8264
 
8174
- /// Use the value of the intial max_data / initial stream max_data setting
8175
- /// as the initial flow control window for the connection and streams.
8176
- /// The connection-level flow control window will only be changed if it
8177
- /// hasn't been auto tuned yet. For streams: only newly created streams
8178
- /// receive the new setting.
8179
- fn enable_use_initial_max_data_as_flow_control_win(&mut self) {
8180
- self.flow_control.set_window_if_not_tuned_yet(
8181
- self.local_transport_params.initial_max_data,
8182
- );
8183
- self.streams
8184
- .set_use_initial_max_data_as_flow_control_win(true);
8185
- }
8186
-
8187
8265
  /// Selects the packet type for the next outgoing packet.
8188
8266
  fn write_pkt_type(&self, send_pid: usize) -> Result<Type> {
8189
8267
  // On error send packet in the latest epoch available, but only send
@@ -8210,7 +8288,9 @@ impl<F: BufFactory> Connection<F> {
8210
8288
  // not be able to decrypt handshake packets yet.
8211
8289
  packet::Epoch::Handshake
8212
8290
  if self.crypto_ctx[packet::Epoch::Initial].has_keys() =>
8213
- return Ok(Type::Initial),
8291
+ {
8292
+ return Ok(Type::Initial)
8293
+ }
8214
8294
 
8215
8295
  _ => (),
8216
8296
  };
@@ -8219,9 +8299,19 @@ impl<F: BufFactory> Connection<F> {
8219
8299
  return Ok(Type::from_epoch(epoch));
8220
8300
  }
8221
8301
 
8222
- for &epoch in packet::Epoch::epochs(
8223
- packet::Epoch::Initial..=packet::Epoch::Application,
8224
- ) {
8302
+ // APPLICATION_CLOSE can only be sent in a 1-RTT packet. Prioritize it
8303
+ // over obsolete lower-epoch PTO probes, which cannot be consumed while
8304
+ // closing because PING frames are suppressed.
8305
+ if self.is_established()
8306
+ && self
8307
+ .local_error
8308
+ .as_ref()
8309
+ .is_some_and(|conn_err| conn_err.is_app)
8310
+ {
8311
+ return Ok(Type::Short);
8312
+ }
8313
+
8314
+ for &epoch in packet::Epoch::epochs(packet::Epoch::Initial..=packet::Epoch::Application) {
8225
8315
  let crypto_ctx = &self.crypto_ctx[epoch];
8226
8316
  let pkt_space = &self.pkt_num_spaces[epoch];
8227
8317
 
@@ -8251,36 +8341,39 @@ impl<F: BufFactory> Connection<F> {
8251
8341
  // If there are flushable, almost full or blocked streams, use the
8252
8342
  // Application epoch.
8253
8343
  let send_path = self.paths.get(send_pid)?;
8254
- if (self.is_established() || self.is_in_early_data()) &&
8255
- (self.should_send_handshake_done() ||
8256
- self.flow_control.should_update_max_data() ||
8257
- self.should_send_max_data ||
8258
- self.blocked_limit.is_some() ||
8259
- self.streams_blocked_bidi_state
8260
- .has_pending_stream_blocked_frame() ||
8261
- self.streams_blocked_uni_state
8262
- .has_pending_stream_blocked_frame() ||
8263
- self.dgram_send_queue.has_pending() ||
8264
- self.local_error
8344
+ if (self.is_established() || self.is_in_early_data())
8345
+ && (self.should_send_handshake_done()
8346
+ || self.flow_control.should_update_max_data()
8347
+ || self.should_send_max_data
8348
+ || self.blocked_limit.is_some()
8349
+ || self
8350
+ .streams_blocked_bidi_state
8351
+ .has_pending_stream_blocked_frame()
8352
+ || self
8353
+ .streams_blocked_uni_state
8354
+ .has_pending_stream_blocked_frame()
8355
+ || self.dgram_send_queue.has_pending()
8356
+ || self
8357
+ .local_error
8265
8358
  .as_ref()
8266
- .is_some_and(|conn_err| conn_err.is_app) ||
8267
- self.should_send_max_streams_bidi ||
8268
- self.streams.should_update_max_streams_bidi() ||
8269
- self.should_send_max_streams_uni ||
8270
- self.streams.should_update_max_streams_uni() ||
8271
- self.streams.has_flushable() ||
8272
- self.streams.has_almost_full() ||
8273
- self.streams.has_blocked() ||
8274
- self.streams.has_reset() ||
8275
- self.streams.has_stopped() ||
8276
- self.ids.has_new_scids() ||
8277
- self.ids.has_retire_dcids() ||
8278
- send_path
8359
+ .is_some_and(|conn_err| conn_err.is_app)
8360
+ || self.should_send_max_streams_bidi
8361
+ || self.streams.should_update_max_streams_bidi()
8362
+ || self.should_send_max_streams_uni
8363
+ || self.streams.should_update_max_streams_uni()
8364
+ || self.streams.has_flushable()
8365
+ || self.streams.has_almost_full()
8366
+ || self.streams.has_blocked()
8367
+ || self.streams.has_reset()
8368
+ || self.streams.has_stopped()
8369
+ || self.ids.has_new_scids()
8370
+ || self.ids.has_retire_dcids()
8371
+ || send_path
8279
8372
  .pmtud
8280
8373
  .as_ref()
8281
- .is_some_and(|pmtud| pmtud.should_probe()) ||
8282
- send_path.needs_ack_eliciting ||
8283
- send_path.probing_required())
8374
+ .is_some_and(|pmtud| pmtud.should_probe())
8375
+ || send_path.needs_ack_eliciting
8376
+ || send_path.probing_required())
8284
8377
  {
8285
8378
  // Only clients can send 0-RTT packets.
8286
8379
  if !self.is_server && self.is_in_early_data() {
@@ -8295,9 +8388,7 @@ impl<F: BufFactory> Connection<F> {
8295
8388
 
8296
8389
  /// Returns the mutable stream with the given ID if it exists, or creates
8297
8390
  /// a new one otherwise.
8298
- fn get_or_create_stream(
8299
- &mut self, id: u64, local: bool,
8300
- ) -> Result<&mut stream::Stream<F>> {
8391
+ fn get_or_create_stream(&mut self, id: u64, local: bool) -> Result<&mut stream::Stream<F>> {
8301
8392
  self.streams.get_or_create(
8302
8393
  id,
8303
8394
  &self.local_transport_params,
@@ -8307,10 +8398,30 @@ impl<F: BufFactory> Connection<F> {
8307
8398
  )
8308
8399
  }
8309
8400
 
8401
+ /// Gets or creates a stream referenced by a received frame.
8402
+ fn get_or_create_stream_for_received_frame(
8403
+ &mut self,
8404
+ id: u64,
8405
+ ) -> Result<&mut stream::Stream<F>> {
8406
+ let local = stream::is_local(id, self.is_server);
8407
+
8408
+ // Opening a higher-numbered stream also opens lower streams of the
8409
+ // same type, even if they do not have Stream objects yet.
8410
+ if local && !self.streams.local_stream_opened(id) {
8411
+ return Err(Error::InvalidStreamState(id));
8412
+ }
8413
+
8414
+ self.get_or_create_stream(id, local)
8415
+ }
8416
+
8310
8417
  /// Processes an incoming frame.
8311
8418
  fn process_frame(
8312
- &mut self, frame: frame::Frame, hdr: &Header, recv_path_id: usize,
8313
- epoch: packet::Epoch, now: Instant,
8419
+ &mut self,
8420
+ frame: frame::Frame,
8421
+ hdr: &Header,
8422
+ recv_path_id: usize,
8423
+ epoch: packet::Epoch,
8424
+ now: Instant,
8314
8425
  ) -> Result<()> {
8315
8426
  trace!("{} rx frm {:?}", self.trace_id, frame);
8316
8427
 
@@ -8323,14 +8434,11 @@ impl<F: BufFactory> Connection<F> {
8323
8434
  ranges, ack_delay, ..
8324
8435
  } => {
8325
8436
  let ack_delay = ack_delay
8326
- .checked_mul(2_u64.pow(
8327
- self.peer_transport_params.ack_delay_exponent as u32,
8328
- ))
8437
+ .checked_mul(2_u64.pow(self.peer_transport_params.ack_delay_exponent as u32))
8329
8438
  .ok_or(Error::InvalidFrame)?;
8330
8439
 
8331
- if epoch == packet::Epoch::Handshake ||
8332
- (epoch == packet::Epoch::Application &&
8333
- self.is_established())
8440
+ if epoch == packet::Epoch::Handshake
8441
+ || (epoch == packet::Epoch::Application && self.is_established())
8334
8442
  {
8335
8443
  self.peer_verified_initial_address = true;
8336
8444
  }
@@ -8339,9 +8447,9 @@ impl<F: BufFactory> Connection<F> {
8339
8447
 
8340
8448
  let is_app_limited = self.delivery_rate_check_if_app_limited();
8341
8449
 
8342
- let largest_acked = ranges.last().expect(
8343
- "ACK frames should always have at least one ack range",
8344
- );
8450
+ let largest_acked = ranges
8451
+ .last()
8452
+ .expect("ACK frames should always have at least one ack range");
8345
8453
 
8346
8454
  for (_, p) in self.paths.iter_mut() {
8347
8455
  if self.pkt_num_spaces[epoch]
@@ -8375,14 +8483,10 @@ impl<F: BufFactory> Connection<F> {
8375
8483
  )?;
8376
8484
 
8377
8485
  let skip_pn = self.pkt_num_manager.skip_pn();
8378
- let largest_acked =
8379
- p.recovery.get_largest_acked_on_epoch(epoch);
8486
+ let largest_acked = p.recovery.get_largest_acked_on_epoch(epoch);
8380
8487
 
8381
- // Consider the skip_pn validated if the peer has sent an ack
8382
- // for a larger pkt number.
8383
- if let Some((largest_acked, skip_pn)) =
8384
- largest_acked.zip(skip_pn)
8385
- {
8488
+ // A higher ACK validates `skip_pn`.
8489
+ if let Some((largest_acked, skip_pn)) = largest_acked.zip(skip_pn) {
8386
8490
  if largest_acked > skip_pn {
8387
8491
  self.pkt_num_manager.set_skip_pn(None);
8388
8492
  }
@@ -8393,17 +8497,23 @@ impl<F: BufFactory> Connection<F> {
8393
8497
  self.acked_bytes += acked_bytes as u64;
8394
8498
  self.spurious_lost_count += spurious_losses;
8395
8499
  }
8396
- },
8500
+ }
8397
8501
 
8398
8502
  frame::Frame::ResetStream {
8399
8503
  stream_id,
8400
8504
  error_code,
8401
8505
  final_size,
8506
+ reliable_size,
8402
8507
  } => {
8508
+ if reliable_size.is_some() && !self.local_transport_params.reset_stream_at {
8509
+ return Err(Error::InvalidState);
8510
+ }
8511
+ if reliable_size.is_some_and(|size| size > final_size) {
8512
+ return Err(Error::InvalidFrame);
8513
+ }
8514
+ let check_error_code = self.local_transport_params.reset_stream_at;
8403
8515
  // Peer can't send on our unidirectional streams.
8404
- if !stream::is_bidi(stream_id) &&
8405
- stream::is_local(stream_id, self.is_server)
8406
- {
8516
+ if !stream::is_bidi(stream_id) && stream::is_local(stream_id, self.is_server) {
8407
8517
  return Err(Error::InvalidStreamState(stream_id));
8408
8518
  }
8409
8519
 
@@ -8419,7 +8529,7 @@ impl<F: BufFactory> Connection<F> {
8419
8529
  // Note that it makes it impossible to check if the frame is
8420
8530
  // illegal, since we have no state, but since we ignore the
8421
8531
  // frame, it should be fine.
8422
- let stream = match self.get_or_create_stream(stream_id, false) {
8532
+ let stream = match self.get_or_create_stream_for_received_frame(stream_id) {
8423
8533
  Ok(v) => v,
8424
8534
 
8425
8535
  Err(Error::Done) => return Ok(()),
@@ -8430,15 +8540,31 @@ impl<F: BufFactory> Connection<F> {
8430
8540
  let was_readable = stream.is_readable();
8431
8541
  let priority_key = Arc::clone(&stream.priority_key);
8432
8542
 
8543
+ if check_error_code
8544
+ && stream
8545
+ .recv
8546
+ .reset_error()
8547
+ .is_some_and(|old| old != error_code)
8548
+ {
8549
+ return Err(Error::InvalidStreamState(stream_id));
8550
+ }
8433
8551
  let stream::RecvBufResetReturn {
8434
8552
  max_data_delta,
8435
8553
  consumed_flowcontrol,
8436
- } = stream.recv.reset(error_code, final_size)?;
8554
+ } = match reliable_size {
8555
+ Some(size) => stream.recv.reset_at(error_code, final_size, size)?,
8556
+ None => stream.recv.reset(error_code, final_size)?,
8557
+ };
8437
8558
 
8438
8559
  if max_data_delta > max_rx_data_left {
8439
8560
  return Err(Error::FlowControl);
8440
8561
  }
8441
8562
 
8563
+ // The receive side can complete without the stream becoming
8564
+ // readable, so no later read would collect it.
8565
+ let state = stream.state();
8566
+ let local = stream.local;
8567
+
8442
8568
  if !was_readable && stream.is_readable() {
8443
8569
  self.streams.insert_readable(&priority_key);
8444
8570
  }
@@ -8448,18 +8574,19 @@ impl<F: BufFactory> Connection<F> {
8448
8574
  // flow-control
8449
8575
  self.flow_control.add_consumed(consumed_flowcontrol);
8450
8576
 
8451
- self.reset_stream_remote_count =
8452
- self.reset_stream_remote_count.saturating_add(1);
8453
- },
8577
+ self.reset_stream_remote_count = self.reset_stream_remote_count.saturating_add(1);
8578
+
8579
+ if state == stream::StreamState::Collectable {
8580
+ self.streams.collect(stream_id, local);
8581
+ }
8582
+ }
8454
8583
 
8455
8584
  frame::Frame::StopSending {
8456
8585
  stream_id,
8457
8586
  error_code,
8458
8587
  } => {
8459
8588
  // STOP_SENDING on a receive-only stream is a fatal error.
8460
- if !stream::is_local(stream_id, self.is_server) &&
8461
- !stream::is_bidi(stream_id)
8462
- {
8589
+ if !stream::is_local(stream_id, self.is_server) && !stream::is_bidi(stream_id) {
8463
8590
  return Err(Error::InvalidStreamState(stream_id));
8464
8591
  }
8465
8592
 
@@ -8473,7 +8600,7 @@ impl<F: BufFactory> Connection<F> {
8473
8600
  // Note that it makes it impossible to check if the frame is
8474
8601
  // illegal, since we have no state, but since we ignore the
8475
8602
  // frame, it should be fine.
8476
- let stream = match self.get_or_create_stream(stream_id, false) {
8603
+ let stream = match self.get_or_create_stream_for_received_frame(stream_id) {
8477
8604
  Ok(v) => v,
8478
8605
 
8479
8606
  Err(Error::Done) => return Ok(()),
@@ -8481,12 +8608,17 @@ impl<F: BufFactory> Connection<F> {
8481
8608
  Err(e) => return Err(e),
8482
8609
  };
8483
8610
 
8484
- let was_writable = stream.is_writable();
8485
-
8486
8611
  let priority_key = Arc::clone(&stream.priority_key);
8487
8612
 
8613
+ // Save the buffered length before stopping (stop clears the
8614
+ // buffer).
8615
+ let buffered_len = stream.send.buffered_bytes() as usize;
8616
+
8488
8617
  // Try stopping the stream.
8489
8618
  if let Ok((final_size, unsent)) = stream.send.stop(error_code) {
8619
+ let dropped =
8620
+ buffered_len.saturating_sub(stream.send.buffered_bytes() as usize);
8621
+ let flushable = stream.is_flushable();
8490
8622
  // Claw back some flow control allowance from data that was
8491
8623
  // buffered but not actually sent before the stream was
8492
8624
  // reset.
@@ -8495,18 +8627,17 @@ impl<F: BufFactory> Connection<F> {
8495
8627
  // to touch it here.
8496
8628
  self.tx_data = self.tx_data.saturating_sub(unsent);
8497
8629
 
8498
- self.tx_buffered =
8499
- self.tx_buffered.saturating_sub(unsent as usize);
8630
+ // Update tx_buffered: subtract only the buffered data, not
8631
+ // inflight data.
8632
+ self.streams.sub_tx_buffered(dropped);
8500
8633
 
8501
- // These drops in qlog are a bit weird, but the only way to
8502
- // ensure that all bytes that are moved from App to Transport
8503
- // in stream_do_send are eventually moved from Transport to
8504
- // Dropped. Ideally we would add a Transport to Network
8505
- // transition also as a way to indicate when bytes were
8506
- // transmitted vs dropped without ever being sent.
8634
+ // Match moves from App to Transport with moves from
8635
+ // Transport to Dropped.
8636
+ // A Network transition would distinguish sent bytes from
8637
+ // drops before transmission.
8507
8638
  qlog_with_type!(QLOG_DATA_MV, self.qlog, q, {
8508
- let ev_data = EventData::QuicStreamDataMoved(
8509
- qlog::events::quic::StreamDataMoved {
8639
+ let ev_data =
8640
+ EventData::QuicStreamDataMoved(qlog::events::quic::StreamDataMoved {
8510
8641
  stream_id: Some(stream_id),
8511
8642
  offset: Some(final_size),
8512
8643
  raw: Some(RawInfo {
@@ -8516,24 +8647,24 @@ impl<F: BufFactory> Connection<F> {
8516
8647
  from: Some(DataRecipient::Transport),
8517
8648
  to: Some(DataRecipient::Dropped),
8518
8649
  ..Default::default()
8519
- },
8520
- );
8650
+ });
8521
8651
 
8522
8652
  q.add_event_data_with_instant(ev_data, now).ok();
8523
8653
  });
8524
8654
 
8525
8655
  self.streams.insert_reset(stream_id, error_code, final_size);
8526
8656
 
8527
- if !was_writable {
8528
- self.streams.insert_writable(&priority_key);
8657
+ self.streams.insert_stopped_writable(&priority_key);
8658
+
8659
+ if !flushable {
8660
+ self.streams.remove_flushable(&priority_key);
8529
8661
  }
8530
8662
 
8531
8663
  self.stopped_stream_remote_count =
8532
8664
  self.stopped_stream_remote_count.saturating_add(1);
8533
- self.reset_stream_local_count =
8534
- self.reset_stream_local_count.saturating_add(1);
8665
+ self.reset_stream_local_count = self.reset_stream_local_count.saturating_add(1);
8535
8666
  }
8536
- },
8667
+ }
8537
8668
 
8538
8669
  frame::Frame::Crypto { data } => {
8539
8670
  if data.max_off() >= MAX_CRYPTO_STREAM_OFFSET {
@@ -8541,7 +8672,11 @@ impl<F: BufFactory> Connection<F> {
8541
8672
  }
8542
8673
 
8543
8674
  // Push the data to the stream so it can be re-ordered.
8544
- self.crypto_ctx[epoch].crypto_stream.recv.write(data)?;
8675
+ let stream = &mut self.crypto_ctx[epoch].crypto_stream;
8676
+ stream
8677
+ .recv
8678
+ .set_memory_budget(self.streams.receive_memory.clone());
8679
+ stream.recv.write(data)?;
8545
8680
 
8546
8681
  // Feed crypto data to the TLS state, if there's data
8547
8682
  // available at the expected offset.
@@ -8549,29 +8684,26 @@ impl<F: BufFactory> Connection<F> {
8549
8684
 
8550
8685
  let level = crypto::Level::from_epoch(epoch);
8551
8686
 
8552
- let stream = &mut self.crypto_ctx[epoch].crypto_stream;
8553
-
8554
8687
  while let Ok((read, _)) = stream.recv.emit(&mut crypto_buf) {
8555
8688
  let recv_buf = &crypto_buf[..read];
8556
8689
  self.handshake.provide_data(level, recv_buf)?;
8557
8690
  }
8558
8691
 
8559
8692
  self.do_handshake(now)?;
8560
- },
8693
+ }
8561
8694
 
8562
8695
  frame::Frame::CryptoHeader { .. } => unreachable!(),
8563
8696
 
8564
8697
  // TODO: implement stateless retry
8565
- frame::Frame::NewToken { .. } =>
8698
+ frame::Frame::NewToken { .. } => {
8566
8699
  if self.is_server {
8567
8700
  return Err(Error::InvalidPacket);
8568
- },
8701
+ }
8702
+ }
8569
8703
 
8570
8704
  frame::Frame::Stream { stream_id, data } => {
8571
8705
  // Peer can't send on our unidirectional streams.
8572
- if !stream::is_bidi(stream_id) &&
8573
- stream::is_local(stream_id, self.is_server)
8574
- {
8706
+ if !stream::is_bidi(stream_id) && stream::is_local(stream_id, self.is_server) {
8575
8707
  return Err(Error::InvalidStreamState(stream_id));
8576
8708
  }
8577
8709
 
@@ -8587,7 +8719,7 @@ impl<F: BufFactory> Connection<F> {
8587
8719
  // Note that it makes it impossible to check if the frame is
8588
8720
  // illegal, since we have no state, but since we ignore the
8589
8721
  // frame, it should be fine.
8590
- let stream = match self.get_or_create_stream(stream_id, false) {
8722
+ let stream = match self.get_or_create_stream_for_received_frame(stream_id) {
8591
8723
  Ok(v) => v,
8592
8724
 
8593
8725
  Err(Error::Done) => return Ok(()),
@@ -8596,8 +8728,7 @@ impl<F: BufFactory> Connection<F> {
8596
8728
  };
8597
8729
 
8598
8730
  // Check for the connection-level flow control limit.
8599
- let max_off_delta =
8600
- data.max_off().saturating_sub(stream.recv.max_off());
8731
+ let max_off_delta = data.max_off().saturating_sub(stream.recv.max_off());
8601
8732
 
8602
8733
  if max_off_delta > max_rx_data_left {
8603
8734
  return Err(Error::FlowControl);
@@ -8610,6 +8741,11 @@ impl<F: BufFactory> Connection<F> {
8610
8741
 
8611
8742
  stream.recv.write(data)?;
8612
8743
 
8744
+ // The receive side can complete without the stream becoming
8745
+ // readable, so no later read would collect it.
8746
+ let state = stream.state();
8747
+ let local = stream.local;
8748
+
8613
8749
  if !was_readable && stream.is_readable() {
8614
8750
  self.streams.insert_readable(&priority_key);
8615
8751
  }
@@ -8623,19 +8759,21 @@ impl<F: BufFactory> Connection<F> {
8623
8759
  // control update.
8624
8760
  self.flow_control.add_consumed(max_off_delta);
8625
8761
  }
8626
- },
8762
+
8763
+ if state == stream::StreamState::Collectable {
8764
+ self.streams.collect(stream_id, local);
8765
+ }
8766
+ }
8627
8767
 
8628
8768
  frame::Frame::StreamHeader { .. } => unreachable!(),
8629
8769
 
8630
8770
  frame::Frame::MaxData { max } => {
8631
8771
  self.max_tx_data = cmp::max(self.max_tx_data, max);
8632
- },
8772
+ }
8633
8773
 
8634
8774
  frame::Frame::MaxStreamData { stream_id, max } => {
8635
8775
  // Peer can't receive on its own unidirectional streams.
8636
- if !stream::is_bidi(stream_id) &&
8637
- !stream::is_local(stream_id, self.is_server)
8638
- {
8776
+ if !stream::is_bidi(stream_id) && !stream::is_local(stream_id, self.is_server) {
8639
8777
  return Err(Error::InvalidStreamState(stream_id));
8640
8778
  }
8641
8779
 
@@ -8649,7 +8787,7 @@ impl<F: BufFactory> Connection<F> {
8649
8787
  // Note that it makes it impossible to check if the frame is
8650
8788
  // illegal, since we have no state, but since we ignore the
8651
8789
  // frame, it should be fine.
8652
- let stream = match self.get_or_create_stream(stream_id, false) {
8790
+ let stream = match self.get_or_create_stream_for_received_frame(stream_id) {
8653
8791
  Ok(v) => v,
8654
8792
 
8655
8793
  Err(Error::Done) => return Ok(()),
@@ -8665,8 +8803,8 @@ impl<F: BufFactory> Connection<F> {
8665
8803
 
8666
8804
  let priority_key = Arc::clone(&stream.priority_key);
8667
8805
 
8668
- // If the stream is now flushable push it to the flushable queue,
8669
- // but only if it wasn't already queued.
8806
+ // If the stream became flushable, add it to the queue unless it
8807
+ // is already present.
8670
8808
  if stream.is_flushable() && !was_flushable {
8671
8809
  let priority_key = Arc::clone(&stream.priority_key);
8672
8810
  self.streams.insert_flushable(&priority_key);
@@ -8675,7 +8813,7 @@ impl<F: BufFactory> Connection<F> {
8675
8813
  if writable {
8676
8814
  self.streams.insert_writable(&priority_key);
8677
8815
  }
8678
- },
8816
+ }
8679
8817
 
8680
8818
  frame::Frame::MaxStreamsBidi { max } => {
8681
8819
  if max > MAX_STREAM_ID {
@@ -8683,7 +8821,7 @@ impl<F: BufFactory> Connection<F> {
8683
8821
  }
8684
8822
 
8685
8823
  self.streams.update_peer_max_streams_bidi(max);
8686
- },
8824
+ }
8687
8825
 
8688
8826
  frame::Frame::MaxStreamsUni { max } => {
8689
8827
  if max > MAX_STREAM_ID {
@@ -8691,17 +8829,16 @@ impl<F: BufFactory> Connection<F> {
8691
8829
  }
8692
8830
 
8693
8831
  self.streams.update_peer_max_streams_uni(max);
8694
- },
8832
+ }
8695
8833
 
8696
8834
  frame::Frame::DataBlocked { .. } => {
8697
- self.data_blocked_recv_count =
8698
- self.data_blocked_recv_count.saturating_add(1);
8699
- },
8835
+ self.data_blocked_recv_count = self.data_blocked_recv_count.saturating_add(1);
8836
+ }
8700
8837
 
8701
8838
  frame::Frame::StreamDataBlocked { .. } => {
8702
8839
  self.stream_data_blocked_recv_count =
8703
8840
  self.stream_data_blocked_recv_count.saturating_add(1);
8704
- },
8841
+ }
8705
8842
 
8706
8843
  frame::Frame::StreamsBlockedBidi { limit } => {
8707
8844
  if limit > MAX_STREAM_ID {
@@ -8710,7 +8847,7 @@ impl<F: BufFactory> Connection<F> {
8710
8847
 
8711
8848
  self.streams_blocked_bidi_recv_count =
8712
8849
  self.streams_blocked_bidi_recv_count.saturating_add(1);
8713
- },
8850
+ }
8714
8851
 
8715
8852
  frame::Frame::StreamsBlockedUni { limit } => {
8716
8853
  if limit > MAX_STREAM_ID {
@@ -8719,7 +8856,7 @@ impl<F: BufFactory> Connection<F> {
8719
8856
 
8720
8857
  self.streams_blocked_uni_recv_count =
8721
8858
  self.streams_blocked_uni_recv_count.saturating_add(1);
8722
- },
8859
+ }
8723
8860
 
8724
8861
  frame::Frame::NewConnectionId {
8725
8862
  seq_num,
@@ -8751,16 +8888,17 @@ impl<F: BufFactory> Connection<F> {
8751
8888
  continue;
8752
8889
  }
8753
8890
 
8754
- if let Some(new_dcid_seq) =
8755
- self.ids.lowest_available_dcid_seq()
8756
- {
8891
+ if let Some(new_dcid_seq) = self.ids.lowest_available_dcid_seq() {
8757
8892
  path.active_dcid_seq = Some(new_dcid_seq);
8758
8893
 
8759
8894
  self.ids.link_dcid_to_path_id(new_dcid_seq, pid)?;
8760
8895
 
8761
8896
  trace!(
8762
8897
  "{} path ID {} changed DCID: old seq num {} new seq num {}",
8763
- self.trace_id, pid, dcid_seq, new_dcid_seq,
8898
+ self.trace_id,
8899
+ pid,
8900
+ dcid_seq,
8901
+ new_dcid_seq,
8764
8902
  );
8765
8903
  } else {
8766
8904
  // We cannot use this path anymore for now.
@@ -8768,14 +8906,16 @@ impl<F: BufFactory> Connection<F> {
8768
8906
 
8769
8907
  trace!(
8770
8908
  "{} path ID {} cannot be used; DCID seq num {} has been retired",
8771
- self.trace_id, pid, dcid_seq,
8909
+ self.trace_id,
8910
+ pid,
8911
+ dcid_seq,
8772
8912
  );
8773
8913
  }
8774
8914
  }
8775
8915
 
8776
8916
  // Propagate error (if any) now...
8777
8917
  new_dcid_res?;
8778
- },
8918
+ }
8779
8919
 
8780
8920
  frame::Frame::RetireConnectionId { seq_num } => {
8781
8921
  if self.ids.zero_length_scid() {
@@ -8793,7 +8933,7 @@ impl<F: BufFactory> Connection<F> {
8793
8933
  path.active_scid_seq = None;
8794
8934
  }
8795
8935
  }
8796
- },
8936
+ }
8797
8937
 
8798
8938
  frame::Frame::PathChallenge { data } => {
8799
8939
  self.path_challenge_rx_count += 1;
@@ -8801,11 +8941,11 @@ impl<F: BufFactory> Connection<F> {
8801
8941
  self.paths
8802
8942
  .get_mut(recv_path_id)?
8803
8943
  .on_challenge_received(data);
8804
- },
8944
+ }
8805
8945
 
8806
8946
  frame::Frame::PathResponse { data } => {
8807
8947
  self.paths.on_response_received(data)?;
8808
- },
8948
+ }
8809
8949
 
8810
8950
  frame::Frame::ConnectionClose {
8811
8951
  error_code, reason, ..
@@ -8818,7 +8958,7 @@ impl<F: BufFactory> Connection<F> {
8818
8958
 
8819
8959
  let path = self.paths.get_active()?;
8820
8960
  self.draining_timer = Some(now + (path.recovery.pto() * 3));
8821
- },
8961
+ }
8822
8962
 
8823
8963
  frame::Frame::ApplicationClose { error_code, reason } => {
8824
8964
  self.peer_error = Some(ConnectionError {
@@ -8829,7 +8969,7 @@ impl<F: BufFactory> Connection<F> {
8829
8969
 
8830
8970
  let path = self.paths.get_active()?;
8831
8971
  self.draining_timer = Some(now + (path.recovery.pto() * 3));
8832
- },
8972
+ }
8833
8973
 
8834
8974
  frame::Frame::HandshakeDone => {
8835
8975
  if self.is_server {
@@ -8842,7 +8982,7 @@ impl<F: BufFactory> Connection<F> {
8842
8982
 
8843
8983
  // Once the handshake is confirmed, we can drop Handshake keys.
8844
8984
  self.drop_epoch_state(packet::Epoch::Handshake, now);
8845
- },
8985
+ }
8846
8986
 
8847
8987
  frame::Frame::Datagram { data } => {
8848
8988
  // Close the connection if DATAGRAMs are not enabled.
@@ -8864,7 +9004,7 @@ impl<F: BufFactory> Connection<F> {
8864
9004
 
8865
9005
  let path = self.paths.get_mut(recv_path_id)?;
8866
9006
  path.dgram_recv_count = path.dgram_recv_count.saturating_add(1);
8867
- },
9007
+ }
8868
9008
 
8869
9009
  frame::Frame::DatagramHeader { .. } => unreachable!(),
8870
9010
  }
@@ -8907,8 +9047,8 @@ impl<F: BufFactory> Connection<F> {
8907
9047
  // If the transport parameter is set to 0, then the respective endpoint
8908
9048
  // decided to disable the idle timeout. If both are disabled we should
8909
9049
  // not set any timeout.
8910
- if self.local_transport_params.max_idle_timeout == 0 &&
8911
- self.peer_transport_params.max_idle_timeout == 0
9050
+ if self.local_transport_params.max_idle_timeout == 0
9051
+ && self.peer_transport_params.max_idle_timeout == 0
8912
9052
  {
8913
9053
  return None;
8914
9054
  }
@@ -8940,8 +9080,7 @@ impl<F: BufFactory> Connection<F> {
8940
9080
  /// Returns the connection's handshake status for use in loss recovery.
8941
9081
  fn handshake_status(&self) -> recovery::HandshakeStatus {
8942
9082
  recovery::HandshakeStatus {
8943
- has_handshake_keys: self.crypto_ctx[packet::Epoch::Handshake]
8944
- .has_keys(),
9083
+ has_handshake_keys: self.crypto_ctx[packet::Epoch::Handshake].has_keys(),
8945
9084
 
8946
9085
  peer_verified_address: self.peer_verified_initial_address,
8947
9086
 
@@ -8956,8 +9095,7 @@ impl<F: BufFactory> Connection<F> {
8956
9095
  Err(_) => 0,
8957
9096
  };
8958
9097
 
8959
- let cap =
8960
- cmp::min(cwin_available, self.max_tx_data - self.tx_data) as usize;
9098
+ let cap = cmp::min(cwin_available, self.max_tx_data - self.tx_data) as usize;
8961
9099
  self.tx_cap = (cap as f64 * self.tx_cap_factor).ceil() as usize;
8962
9100
  }
8963
9101
 
@@ -8965,18 +9103,18 @@ impl<F: BufFactory> Connection<F> {
8965
9103
  // Enter the app-limited phase of delivery rate when these conditions
8966
9104
  // are met:
8967
9105
  //
8968
- // - The remaining capacity is higher than available bytes in cwnd (there
9106
+ // - The remaining capacity exceeds the available bytes in CWND (there
8969
9107
  // is more room to send).
8970
- // - New data since the last send() is smaller than available bytes in
8971
- // cwnd (we queued less than what we can send).
8972
- // - There is room to send more data in cwnd.
9108
+ // - New data since the last `send()` is smaller than available bytes in
9109
+ // CWND (we queued less than what we can send).
9110
+ // - CWND has room for more data.
8973
9111
  //
8974
9112
  // In application-limited phases the transmission rate is limited by the
8975
9113
  // application rather than the congestion control algorithm.
8976
9114
  //
8977
- // Note that this is equivalent to CheckIfApplicationLimited() from the
8978
- // delivery rate draft. This is also separate from `recovery.app_limited`
8979
- // and only applies to delivery rate calculation.
9115
+ // This mirrors `CheckIfApplicationLimited()` from the delivery-rate
9116
+ // draft but affects only delivery-rate calculation, not
9117
+ // `recovery.app_limited`.
8980
9118
  let cwin_available = self
8981
9119
  .paths
8982
9120
  .iter()
@@ -8984,34 +9122,15 @@ impl<F: BufFactory> Connection<F> {
8984
9122
  .map(|(_, p)| p.recovery.cwnd_available())
8985
9123
  .sum();
8986
9124
 
8987
- ((self.tx_buffered + self.dgram_send_queue_byte_size()) < cwin_available) &&
8988
- (self.tx_data.saturating_sub(self.last_tx_data)) <
8989
- cwin_available as u64 &&
8990
- cwin_available > 0
8991
- }
8992
-
8993
- fn check_tx_buffered_invariant(&mut self) {
8994
- // tx_buffered should track bytes queued in the stream buffers
8995
- // and unacked retransmitable bytes in the network.
8996
- // If tx_buffered > 0 mark the tx_buffered_state if there are no
8997
- // flushable streams and there no inflight bytes.
8998
- //
8999
- // It is normal to have tx_buffered == 0 while there are inflight bytes
9000
- // since not QUIC frames are retransmittable; inflight tracks all bytes
9001
- // on the network which are subject to congestion control.
9002
- if self.tx_buffered > 0 &&
9003
- !self.streams.has_flushable() &&
9004
- !self
9005
- .paths
9006
- .iter()
9007
- .any(|(_, p)| p.recovery.bytes_in_flight() > 0)
9008
- {
9009
- self.tx_buffered_state = TxBufferTrackingState::Inconsistent;
9010
- }
9125
+ ((self.streams.tx_buffered() + self.dgram_send_queue_byte_size()) < cwin_available)
9126
+ && (self.tx_data.saturating_sub(self.last_tx_data)) < cwin_available as u64
9127
+ && cwin_available > 0
9011
9128
  }
9012
9129
 
9013
9130
  fn set_initial_dcid(
9014
- &mut self, cid: ConnectionId<'static>, reset_token: Option<u128>,
9131
+ &mut self,
9132
+ cid: ConnectionId<'static>,
9133
+ reset_token: Option<u128>,
9015
9134
  path_id: usize,
9016
9135
  ) -> Result<()> {
9017
9136
  self.ids.set_initial_dcid(cid, reset_token, Some(path_id));
@@ -9023,26 +9142,26 @@ impl<F: BufFactory> Connection<F> {
9023
9142
  /// Selects the path that the incoming packet belongs to, or creates a new
9024
9143
  /// one if no existing path matches.
9025
9144
  fn get_or_create_recv_path_id(
9026
- &mut self, recv_pid: Option<usize>, dcid: &ConnectionId, buf_len: usize,
9145
+ &mut self,
9146
+ recv_pid: Option<usize>,
9147
+ dcid: &ConnectionId,
9148
+ buf_len: usize,
9027
9149
  info: &RecvInfo,
9028
9150
  ) -> Result<usize> {
9029
9151
  let ids = &mut self.ids;
9030
9152
 
9031
- let (in_scid_seq, mut in_scid_pid) =
9032
- ids.find_scid_seq(dcid).ok_or(Error::InvalidState)?;
9153
+ let (in_scid_seq, mut in_scid_pid) = ids.find_scid_seq(dcid).ok_or(Error::InvalidState)?;
9033
9154
 
9034
9155
  if let Some(recv_pid) = recv_pid {
9035
9156
  // If the path observes a change of SCID used, note it.
9036
9157
  let recv_path = self.paths.get_mut(recv_pid)?;
9037
9158
 
9038
- let cid_entry =
9039
- recv_path.active_scid_seq.and_then(|v| ids.get_scid(v).ok());
9159
+ let cid_entry = recv_path.active_scid_seq.and_then(|v| ids.get_scid(v).ok());
9040
9160
 
9041
9161
  if cid_entry.map(|e| &e.cid) != Some(dcid) {
9042
9162
  let incoming_cid_entry = ids.get_scid(in_scid_seq)?;
9043
9163
 
9044
- let prev_recv_pid =
9045
- incoming_cid_entry.path_id.unwrap_or(recv_pid);
9164
+ let prev_recv_pid = incoming_cid_entry.path_id.unwrap_or(recv_pid);
9046
9165
 
9047
9166
  if prev_recv_pid != recv_pid {
9048
9167
  trace!(
@@ -9078,32 +9197,17 @@ impl<F: BufFactory> Connection<F> {
9078
9197
  in_scid_pid = None;
9079
9198
  }
9080
9199
 
9081
- if let Some(in_scid_pid) = in_scid_pid {
9082
- // This CID has been used by another path. If we have the
9083
- // room to do so, create a new `Path` structure holding this
9084
- // new 4-tuple. Otherwise, drop the packet.
9085
- let old_path = self.paths.get_mut(in_scid_pid)?;
9086
- let old_local_addr = old_path.local_addr();
9087
- let old_peer_addr = old_path.peer_addr();
9088
-
9089
- trace!(
9090
- "{} reused CID seq {} of ({},{}) (path {}) on ({},{})",
9091
- self.trace_id,
9092
- in_scid_seq,
9093
- old_local_addr,
9094
- old_peer_addr,
9095
- in_scid_pid,
9096
- info.to,
9097
- info.from
9098
- );
9200
+ // Capture old path info before insert_path() so we can emit the
9201
+ // ReusedSourceConnectionId event after successful insertion. This
9202
+ // ensures the event count is bounded by path Slab capacity.
9203
+ let reused_cid_info = match in_scid_pid {
9204
+ Some(pid) => {
9205
+ let old_path = self.paths.get(pid)?;
9206
+ Some((pid, old_path.local_addr(), old_path.peer_addr()))
9207
+ }
9099
9208
 
9100
- // Notify the application.
9101
- self.paths.notify_event(PathEvent::ReusedSourceConnectionId(
9102
- in_scid_seq,
9103
- (old_local_addr, old_peer_addr),
9104
- (info.to, info.from),
9105
- ));
9106
- }
9209
+ None => None,
9210
+ };
9107
9211
 
9108
9212
  // This is a new path using an unassigned CID; create it!
9109
9213
  let mut path = path::Path::new(
@@ -9123,21 +9227,44 @@ impl<F: BufFactory> Connection<F> {
9123
9227
 
9124
9228
  let pid = self.paths.insert_path(path, self.is_server)?;
9125
9229
 
9126
- // Do not record path reuse.
9127
- if in_scid_pid.is_none() {
9128
- ids.link_scid_to_path_id(in_scid_seq, pid)?;
9230
+ // Notify the application of CID reuse only after the path was
9231
+ // successfully admitted. This bounds event queue growth by path Slab
9232
+ // capacity, preventing an attacker from growing the queue unboundedly
9233
+ // by rotating source ports.
9234
+ match reused_cid_info {
9235
+ Some((old_pid, old_local_addr, old_peer_addr)) => {
9236
+ trace!(
9237
+ "{} reused CID seq {} of ({},{}) (path {}) on ({},{})",
9238
+ self.trace_id,
9239
+ in_scid_seq,
9240
+ old_local_addr,
9241
+ old_peer_addr,
9242
+ old_pid,
9243
+ info.to,
9244
+ info.from
9245
+ );
9246
+
9247
+ self.paths.notify_event(PathEvent::ReusedSourceConnectionId(
9248
+ in_scid_seq,
9249
+ (old_local_addr, old_peer_addr),
9250
+ (info.to, info.from),
9251
+ ));
9252
+ }
9253
+
9254
+ None => {
9255
+ ids.link_scid_to_path_id(in_scid_seq, pid)?;
9256
+ }
9129
9257
  }
9130
9258
 
9131
9259
  Ok(pid)
9132
9260
  }
9133
9261
 
9134
9262
  /// Selects the path on which the next packet must be sent.
9135
- fn get_send_path_id(
9136
- &self, from: Option<SocketAddr>, to: Option<SocketAddr>,
9137
- ) -> Result<usize> {
9263
+ fn get_send_path_id(&self, from: Option<SocketAddr>, to: Option<SocketAddr>) -> Result<usize> {
9138
9264
  // A probing packet must be sent, but only if the connection is fully
9139
- // established.
9140
- if self.is_established() {
9265
+ // established. A closing connection only sends the CONNECTION_CLOSE
9266
+ // frame, on the active path.
9267
+ if self.is_established() && self.local_error.is_none() {
9141
9268
  let mut probing = self
9142
9269
  .paths
9143
9270
  .iter()
@@ -9170,12 +9297,11 @@ impl<F: BufFactory> Connection<F> {
9170
9297
  /// Sets the path with identifier 'path_id' to be active.
9171
9298
  fn set_active_path(&mut self, path_id: usize, now: Instant) -> Result<()> {
9172
9299
  if let Ok(old_active_path) = self.paths.get_active_mut() {
9173
- for &e in packet::Epoch::epochs(
9174
- packet::Epoch::Initial..=packet::Epoch::Application,
9175
- ) {
9176
- let (lost_packets, lost_bytes) = old_active_path
9177
- .recovery
9178
- .on_path_change(e, now, &self.trace_id);
9300
+ for &e in packet::Epoch::epochs(packet::Epoch::Initial..=packet::Epoch::Application) {
9301
+ let (lost_packets, lost_bytes) =
9302
+ old_active_path
9303
+ .recovery
9304
+ .on_path_change(e, now, &self.trace_id);
9179
9305
 
9180
9306
  self.lost_count += lost_packets;
9181
9307
  self.lost_bytes += lost_bytes as u64;
@@ -9187,7 +9313,10 @@ impl<F: BufFactory> Connection<F> {
9187
9313
 
9188
9314
  /// Handles potential connection migration.
9189
9315
  fn on_peer_migrated(
9190
- &mut self, new_pid: usize, disable_dcid_reuse: bool, now: Instant,
9316
+ &mut self,
9317
+ new_pid: usize,
9318
+ disable_dcid_reuse: bool,
9319
+ now: Instant,
9191
9320
  ) -> Result<()> {
9192
9321
  let active_path_id = self.paths.get_active_path_id()?;
9193
9322
 
@@ -9197,8 +9326,7 @@ impl<F: BufFactory> Connection<F> {
9197
9326
 
9198
9327
  self.set_active_path(new_pid, now)?;
9199
9328
 
9200
- let no_spare_dcid =
9201
- self.paths.get_mut(new_pid)?.active_dcid_seq.is_none();
9329
+ let no_spare_dcid = self.paths.get_mut(new_pid)?.active_dcid_seq.is_none();
9202
9330
 
9203
9331
  if no_spare_dcid && !disable_dcid_reuse {
9204
9332
  self.paths.get_mut(new_pid)?.active_dcid_seq =
@@ -9210,7 +9338,9 @@ impl<F: BufFactory> Connection<F> {
9210
9338
 
9211
9339
  /// Creates a new client-side path.
9212
9340
  fn create_path_on_client(
9213
- &mut self, local_addr: SocketAddr, peer_addr: SocketAddr,
9341
+ &mut self,
9342
+ local_addr: SocketAddr,
9343
+ peer_addr: SocketAddr,
9214
9344
  ) -> Result<usize> {
9215
9345
  if self.is_server {
9216
9346
  return Err(Error::InvalidState);
@@ -9256,7 +9386,12 @@ impl<F: BufFactory> Connection<F> {
9256
9386
  fn mark_closed(&mut self) {
9257
9387
  #[cfg(feature = "qlog")]
9258
9388
  {
9259
- let cc = match (self.is_established(), self.timed_out, &self.peer_error, &self.local_error) {
9389
+ let cc = match (
9390
+ self.is_established(),
9391
+ self.timed_out,
9392
+ &self.peer_error,
9393
+ &self.local_error,
9394
+ ) {
9260
9395
  (false, _, _, _) => qlog::events::quic::ConnectionClosed {
9261
9396
  initiator: Some(TransportInitiator::Local),
9262
9397
  connection_error: None,
@@ -9264,7 +9399,7 @@ impl<F: BufFactory> Connection<F> {
9264
9399
  error_code: None,
9265
9400
  internal_code: None,
9266
9401
  reason: Some("Failed to establish connection".to_string()),
9267
- trigger: Some(qlog::events::quic::ConnectionClosedTrigger::HandshakeTimeout)
9402
+ trigger: Some(qlog::events::quic::ConnectionClosedTrigger::HandshakeTimeout),
9268
9403
  },
9269
9404
 
9270
9405
  (true, true, _, _) => qlog::events::quic::ConnectionClosed {
@@ -9274,7 +9409,7 @@ impl<F: BufFactory> Connection<F> {
9274
9409
  error_code: None,
9275
9410
  internal_code: None,
9276
9411
  reason: Some("Idle timeout".to_string()),
9277
- trigger: Some(qlog::events::quic::ConnectionClosedTrigger::IdleTimeout)
9412
+ trigger: Some(qlog::events::quic::ConnectionClosedTrigger::IdleTimeout),
9278
9413
  },
9279
9414
 
9280
9415
  (true, false, Some(peer_error), None) => {
@@ -9287,7 +9422,13 @@ impl<F: BufFactory> Connection<F> {
9287
9422
  Some(qlog::events::quic::ConnectionClosedTrigger::Error)
9288
9423
  };
9289
9424
 
9290
- (Some(qlog::events::ConnectionClosedEventError::TransportError(qlog::events::quic::TransportError::Unknown)), None, trigger)
9425
+ (
9426
+ Some(qlog::events::ConnectionClosedEventError::TransportError(
9427
+ qlog::events::quic::TransportError::Unknown,
9428
+ )),
9429
+ None,
9430
+ trigger,
9431
+ )
9291
9432
  };
9292
9433
 
9293
9434
  // TODO: select more appopriate connection_code and application_error than unknown.
@@ -9300,7 +9441,7 @@ impl<F: BufFactory> Connection<F> {
9300
9441
  reason: Some(String::from_utf8_lossy(&peer_error.reason).to_string()),
9301
9442
  trigger,
9302
9443
  }
9303
- },
9444
+ }
9304
9445
 
9305
9446
  (true, false, None, Some(local_error)) => {
9306
9447
  let (connection_code, application_error, trigger) = if local_error.is_app {
@@ -9312,7 +9453,13 @@ impl<F: BufFactory> Connection<F> {
9312
9453
  Some(qlog::events::quic::ConnectionClosedTrigger::Error)
9313
9454
  };
9314
9455
 
9315
- (Some(qlog::events::ConnectionClosedEventError::TransportError(qlog::events::quic::TransportError::Unknown)), None, trigger)
9456
+ (
9457
+ Some(qlog::events::ConnectionClosedEventError::TransportError(
9458
+ qlog::events::quic::TransportError::Unknown,
9459
+ )),
9460
+ None,
9461
+ trigger,
9462
+ )
9316
9463
  };
9317
9464
 
9318
9465
  // TODO: select more appopriate connection_code and application_error than unknown.
@@ -9325,7 +9472,7 @@ impl<F: BufFactory> Connection<F> {
9325
9472
  reason: Some(String::from_utf8_lossy(&local_error.reason).to_string()),
9326
9473
  trigger,
9327
9474
  }
9328
- },
9475
+ }
9329
9476
 
9330
9477
  _ => qlog::events::quic::ConnectionClosed {
9331
9478
  initiator: None,
@@ -9374,9 +9521,7 @@ impl<F: BufFactory> AsMut<boring::ssl::SslRef> for Connection<F> {
9374
9521
  /// This must only be used for errors preceding packet authentication. Failures
9375
9522
  /// happening after a packet has been authenticated should still cause the
9376
9523
  /// connection to be aborted.
9377
- fn drop_pkt_on_err(
9378
- e: Error, recv_count: usize, is_server: bool, trace_id: &str,
9379
- ) -> Error {
9524
+ fn drop_pkt_on_err(e: Error, recv_count: usize, is_server: bool, trace_id: &str) -> Error {
9380
9525
  // On the server, if no other packet has been successfully processed, abort
9381
9526
  // the connection to avoid keeping the connection open when only junk is
9382
9527
  // received.
@@ -9500,6 +9645,10 @@ pub struct Stats {
9500
9645
  pub bytes_in_flight_duration: Duration,
9501
9646
 
9502
9647
  /// Health state of the connection's tx_buffered.
9648
+ ///
9649
+ /// Indicates whether the streams.tx_buffered value is consistent with
9650
+ /// the actual sum of bytes buffered across all stream send buffers.
9651
+ /// Returns `Ok` if consistent, `Inconsistent` if there's a mismatch.
9503
9652
  pub tx_buffered_state: TxBufferTrackingState,
9504
9653
  }
9505
9654
 
@@ -9539,6 +9688,8 @@ pub use crate::path::SocketAddrIter;
9539
9688
 
9540
9689
  pub use crate::recovery::BbrBwLoReductionStrategy;
9541
9690
  pub use crate::recovery::BbrParams;
9691
+ #[cfg(feature = "internal")]
9692
+ pub use crate::recovery::BbrRttJumpDetector;
9542
9693
  pub use crate::recovery::CongestionControlAlgorithm;
9543
9694
  pub use crate::recovery::StartupExit;
9544
9695
  pub use crate::recovery::StartupExitReason;