rwebtransport 0.2.2 → 0.2.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/Cargo.lock +523 -140
  2. package/Cargo.toml +5 -5
  3. package/README.md +8 -4
  4. package/crates/echo-server/src/bin/gen-test-cert.rs +1 -1
  5. package/crates/native/Cargo.toml +2 -0
  6. package/crates/native/src/config.rs +69 -8
  7. package/crates/native/src/driver.rs +253 -90
  8. package/crates/native/src/lib.rs +80 -57
  9. package/crates/native/src/parser_memory.rs +203 -0
  10. package/crates/native/src/server.rs +215 -183
  11. package/crates/native/src/session.rs +1341 -243
  12. package/crates/native/src/test_support.rs +119 -0
  13. package/crates/wtcore/Cargo.toml +1 -0
  14. package/crates/wtcore/src/h3.rs +250 -46
  15. package/dist/index.js +218 -75
  16. package/dist/index.js.map +7 -1
  17. package/dist/index.mjs +214 -69
  18. package/dist/index.mjs.map +7 -1
  19. package/dist/native.d.ts +10 -4
  20. package/dist/server.d.ts +1 -0
  21. package/package.json +84 -78
  22. package/prebuilds/darwin-arm64/rwebtransport.node +0 -0
  23. package/prebuilds/linux-arm64/rwebtransport.node +0 -0
  24. package/prebuilds/linux-x64/rwebtransport.node +0 -0
  25. package/prebuilds/win32-x64/rwebtransport.node +0 -0
  26. package/scripts/build-linux-prebuilt.sh +46 -0
  27. package/scripts/build-ts.js +41 -0
  28. package/scripts/build.js +10 -3
  29. package/vendor/quiche/AGENTS.md +10 -0
  30. package/vendor/quiche/Cargo.lock +239 -248
  31. package/vendor/quiche/Cargo.toml +23 -10
  32. package/vendor/quiche/examples/Makefile +4 -4
  33. package/vendor/quiche/examples/client.rs +10 -28
  34. package/vendor/quiche/examples/http3-client.rs +20 -34
  35. package/vendor/quiche/examples/http3-server.rs +47 -78
  36. package/vendor/quiche/examples/server.rs +19 -49
  37. package/vendor/quiche/include/quiche.h +35 -4
  38. package/vendor/quiche/src/build.rs +17 -0
  39. package/vendor/quiche/src/cid.rs +129 -61
  40. package/vendor/quiche/src/crypto/boringssl.rs +98 -86
  41. package/vendor/quiche/src/crypto/mod.rs +64 -77
  42. package/vendor/quiche/src/dgram.rs +2 -3
  43. package/vendor/quiche/src/error.rs +22 -20
  44. package/vendor/quiche/src/ffi.rs +558 -280
  45. package/vendor/quiche/src/flowcontrol.rs +0 -7
  46. package/vendor/quiche/src/frame.rs +134 -135
  47. package/vendor/quiche/src/h3/ffi.rs +73 -60
  48. package/vendor/quiche/src/h3/frame.rs +70 -90
  49. package/vendor/quiche/src/h3/mod.rs +1024 -593
  50. package/vendor/quiche/src/h3/qpack/decoder.rs +248 -34
  51. package/vendor/quiche/src/h3/qpack/dynamic_table.rs +859 -0
  52. package/vendor/quiche/src/h3/qpack/encoder.rs +14 -17
  53. package/vendor/quiche/src/h3/qpack/mod.rs +48 -0
  54. package/vendor/quiche/src/h3/qpack/static_table.rs +86 -73
  55. package/vendor/quiche/src/h3/stream.rs +1009 -194
  56. package/vendor/quiche/src/lib.rs +1361 -1195
  57. package/vendor/quiche/src/minmax.rs +2 -6
  58. package/vendor/quiche/src/packet.rs +420 -552
  59. package/vendor/quiche/src/path.rs +202 -72
  60. package/vendor/quiche/src/pmtud.rs +18 -16
  61. package/vendor/quiche/src/ranges.rs +184 -82
  62. package/vendor/quiche/src/recovery/AGENTS.md +5 -2
  63. package/vendor/quiche/src/recovery/bandwidth.rs +59 -40
  64. package/vendor/quiche/src/recovery/bytes_in_flight.rs +2 -2
  65. package/vendor/quiche/src/recovery/congestion/cubic.rs +38 -47
  66. package/vendor/quiche/src/recovery/congestion/delivery_rate.rs +54 -60
  67. package/vendor/quiche/src/recovery/congestion/hystart.rs +6 -10
  68. package/vendor/quiche/src/recovery/congestion/mod.rs +110 -42
  69. package/vendor/quiche/src/recovery/congestion/prr.rs +4 -1
  70. package/vendor/quiche/src/recovery/congestion/recovery.rs +91 -95
  71. package/vendor/quiche/src/recovery/congestion/reno.rs +17 -12
  72. package/vendor/quiche/src/recovery/congestion/test_sender.rs +8 -27
  73. package/vendor/quiche/src/recovery/gcongestion/bbr/bandwidth_sampler.rs +159 -247
  74. package/vendor/quiche/src/recovery/gcongestion/bbr/windowed_filter.rs +9 -14
  75. package/vendor/quiche/src/recovery/gcongestion/bbr2/drain.rs +18 -15
  76. package/vendor/quiche/src/recovery/gcongestion/bbr2/mode.rs +43 -28
  77. package/vendor/quiche/src/recovery/gcongestion/bbr2/network_model.rs +349 -76
  78. package/vendor/quiche/src/recovery/gcongestion/bbr2/probe_bw.rs +147 -195
  79. package/vendor/quiche/src/recovery/gcongestion/bbr2/probe_rtt.rs +29 -27
  80. package/vendor/quiche/src/recovery/gcongestion/bbr2/rtt_jump_detector/global_min.rs +180 -0
  81. package/vendor/quiche/src/recovery/gcongestion/bbr2/rtt_jump_detector/global_min_tests.rs +221 -0
  82. package/vendor/quiche/src/recovery/gcongestion/bbr2/rtt_jump_detector/hmm.rs +395 -0
  83. package/vendor/quiche/src/recovery/gcongestion/bbr2/rtt_jump_detector/hmm_tests.rs +473 -0
  84. package/vendor/quiche/src/recovery/gcongestion/bbr2/rtt_jump_detector/mod.rs +175 -0
  85. package/vendor/quiche/src/recovery/gcongestion/bbr2/startup.rs +33 -26
  86. package/vendor/quiche/src/recovery/gcongestion/bbr2.rs +107 -60
  87. package/vendor/quiche/src/recovery/gcongestion/mod.rs +64 -11
  88. package/vendor/quiche/src/recovery/gcongestion/pacer.rs +34 -25
  89. package/vendor/quiche/src/recovery/gcongestion/recovery.rs +81 -86
  90. package/vendor/quiche/src/recovery/mod.rs +414 -173
  91. package/vendor/quiche/src/recovery/rtt.rs +8 -5
  92. package/vendor/quiche/src/stream/mod.rs +864 -133
  93. package/vendor/quiche/src/stream/recv_buf.rs +476 -172
  94. package/vendor/quiche/src/stream/send_buf.rs +343 -81
  95. package/vendor/quiche/src/test_utils.rs +78 -64
  96. package/vendor/quiche/src/tests.rs +2559 -1335
  97. package/vendor/quiche/src/tls/boringssl.rs +60 -63
  98. package/vendor/quiche/src/tls/mod.rs +146 -193
  99. package/vendor/quiche/src/transport_params.rs +87 -103
@@ -372,6 +372,9 @@
372
372
  //! initiating a connection. Dangerous if the DCID does not meet QUIC's
373
373
  //! unpredictability and length requirements.
374
374
  //!
375
+ //! * `congestion_window_unchecked_available`: Expose a congestion control
376
+ //! algorithm with a fixed, configurable window that defaults to `usize::MAX`.
377
+ //!
375
378
  //! [feature flags]: https://doc.rust-lang.org/cargo/reference/manifest.html#the-features-section
376
379
  //! [boring]: https://crates.io/crates/boring
377
380
  //! [qlog]: https://datatracker.ietf.org/doc/html/draft-ietf-quic-qlog-main-schema
@@ -392,6 +395,8 @@ use debug_panic::debug_panic;
392
395
 
393
396
  use std::net::SocketAddr;
394
397
 
398
+ use std::num::NonZeroUsize;
399
+
395
400
  use std::str::FromStr;
396
401
 
397
402
  use std::sync::Arc;
@@ -427,6 +432,7 @@ use crate::recovery::OnLossDetectionTimeoutOutcome;
427
432
  use crate::recovery::RecoveryOps;
428
433
  use crate::recovery::ReleaseDecision;
429
434
 
435
+ use crate::stream::ReceiveMemory;
430
436
  use crate::stream::RecvAction;
431
437
  use crate::stream::StreamPriorityKey;
432
438
 
@@ -481,9 +487,6 @@ const MAX_UNDECRYPTABLE_PACKETS: usize = 10;
481
487
 
482
488
  const RESERVED_VERSION_MASK: u32 = 0xfafafafa;
483
489
 
484
- // The default size of the receiver connection flow control window.
485
- const DEFAULT_CONNECTION_WINDOW: u64 = 48 * 1024;
486
-
487
490
  // The maximum size of the receiver connection flow control window.
488
491
  const MAX_CONNECTION_WINDOW: u64 = 24 * 1024 * 1024;
489
492
 
@@ -498,9 +501,18 @@ const MAX_PROBING_TIMEOUTS: usize = 3;
498
501
  // The default initial congestion window size in terms of packet count.
499
502
  const DEFAULT_INITIAL_CONGESTION_WINDOW_PACKETS: usize = 10;
500
503
 
504
+ /// The default number of sent packets that can be tracked for loss detection
505
+ /// before the connection is closed.
506
+ pub const DEFAULT_MAX_OUTSTANDING_SENT_PACKETS: NonZeroUsize = NonZeroUsize::new(22_000).unwrap();
507
+
501
508
  // The maximum data offset that can be stored in a crypto stream.
502
509
  const MAX_CRYPTO_STREAM_OFFSET: u64 = 1 << 16;
503
510
 
511
+ // TODO: Remove once https://github.com/cloudflare/quiche/pull/2453 is done.
512
+ // Preserve the previous app-limited threshold independently of STREAM frame
513
+ // header sizing.
514
+ const LEGACY_APP_LIMITED_BYTE_THRESHOLD: usize = 12;
515
+
504
516
  // The send capacity factor.
505
517
  const TX_CAP_FACTOR: f64 = 1.0;
506
518
 
@@ -540,7 +552,7 @@ pub struct SendInfo {
540
552
  #[derive(PartialEq, Eq)]
541
553
  pub enum Shutdown {
542
554
  /// Stop receiving stream data.
543
- Read = 0,
555
+ Read = 0,
544
556
 
545
557
  /// Stop sending stream data.
546
558
  Write = 1,
@@ -552,10 +564,10 @@ pub enum Shutdown {
552
564
  #[cfg_attr(docsrs, doc(cfg(feature = "qlog")))]
553
565
  pub enum QlogLevel {
554
566
  /// Logs any events of Core importance.
555
- Core = 0,
567
+ Core = 0,
556
568
 
557
569
  /// Logs any events of Core and Base importance.
558
- Base = 1,
570
+ Base = 1,
559
571
 
560
572
  /// Logs any events of Core, Base and Extra importance
561
573
  Extra = 2,
@@ -576,6 +588,7 @@ pub struct Config {
576
588
  cc_algorithm: CongestionControlAlgorithm,
577
589
  custom_bbr_params: Option<BbrParams>,
578
590
  initial_congestion_window_packets: usize,
591
+ unchecked_congestion_window: usize,
579
592
  enable_relaxed_loss_threshold: bool,
580
593
  enable_cubic_idle_restart_fix: bool,
581
594
  enable_send_streams_blocked: bool,
@@ -600,6 +613,7 @@ pub struct Config {
600
613
 
601
614
  max_connection_window: u64,
602
615
  max_stream_window: u64,
616
+ stream_recv_memory: Option<(usize, Arc<ReceiveMemory>)>,
603
617
 
604
618
  max_amplification_factor: usize,
605
619
 
@@ -609,9 +623,7 @@ pub struct Config {
609
623
 
610
624
  initial_rtt: Duration,
611
625
 
612
- /// When true, uses the initial max data (for connection
613
- /// and stream) as the initial flow control window.
614
- use_initial_max_data_as_flow_control_win: bool,
626
+ max_outstanding_sent_packets: Option<NonZeroUsize>,
615
627
  }
616
628
 
617
629
  // See https://quicwg.org/base-drafts/rfc9000.html#section-15
@@ -642,9 +654,10 @@ impl Config {
642
654
  #[cfg(feature = "boringssl-boring-crate")]
643
655
  #[cfg_attr(docsrs, doc(cfg(feature = "boringssl-boring-crate")))]
644
656
  pub fn with_boring_ssl_ctx_builder(
645
- version: u32, tls_ctx_builder: boring::ssl::SslContextBuilder,
657
+ version: u32,
658
+ tls_ctx_builder: boring::ssl::SslContextBuilder,
646
659
  ) -> Result<Config> {
647
- Self::with_tls_ctx(version, tls::Context::from_boring(tls_ctx_builder))
660
+ Self::with_tls_ctx(version, tls::Context::from_boring(tls_ctx_builder)?)
648
661
  }
649
662
 
650
663
  fn with_tls_ctx(version: u32, tls_ctx: tls::Context) -> Result<Config> {
@@ -660,8 +673,8 @@ impl Config {
660
673
  grease: true,
661
674
  cc_algorithm: CongestionControlAlgorithm::CUBIC,
662
675
  custom_bbr_params: None,
663
- initial_congestion_window_packets:
664
- DEFAULT_INITIAL_CONGESTION_WINDOW_PACKETS,
676
+ initial_congestion_window_packets: DEFAULT_INITIAL_CONGESTION_WINDOW_PACKETS,
677
+ unchecked_congestion_window: usize::MAX,
665
678
  enable_relaxed_loss_threshold: false,
666
679
  enable_cubic_idle_restart_fix: true,
667
680
  enable_send_streams_blocked: false,
@@ -676,13 +689,13 @@ impl Config {
676
689
  dgram_recv_max_queue_len: DEFAULT_MAX_DGRAM_QUEUE_LEN,
677
690
  dgram_send_max_queue_len: DEFAULT_MAX_DGRAM_QUEUE_LEN,
678
691
 
679
- path_challenge_recv_max_queue_len:
680
- DEFAULT_MAX_PATH_CHALLENGE_RX_QUEUE_LEN,
692
+ path_challenge_recv_max_queue_len: DEFAULT_MAX_PATH_CHALLENGE_RX_QUEUE_LEN,
681
693
 
682
694
  max_send_udp_payload_size: MAX_SEND_UDP_PAYLOAD_SIZE,
683
695
 
684
696
  max_connection_window: MAX_CONNECTION_WINDOW,
685
697
  max_stream_window: stream::MAX_STREAM_WINDOW,
698
+ stream_recv_memory: None,
686
699
 
687
700
  max_amplification_factor: MAX_AMPLIFICATION_FACTOR,
688
701
 
@@ -691,7 +704,7 @@ impl Config {
691
704
  track_unknown_transport_params: None,
692
705
  initial_rtt: DEFAULT_INITIAL_RTT,
693
706
 
694
- use_initial_max_data_as_flow_control_win: false,
707
+ max_outstanding_sent_packets: Some(DEFAULT_MAX_OUTSTANDING_SENT_PACKETS),
695
708
  })
696
709
  }
697
710
 
@@ -754,12 +767,20 @@ impl Config {
754
767
  /// config.load_verify_locations_from_directory("/path/to/certs")?;
755
768
  /// # Ok::<(), quiche::Error>(())
756
769
  /// ```
757
- pub fn load_verify_locations_from_directory(
758
- &mut self, dir: &str,
759
- ) -> Result<()> {
770
+ pub fn load_verify_locations_from_directory(&mut self, dir: &str) -> Result<()> {
760
771
  self.tls_ctx.load_verify_locations_from_directory(dir)
761
772
  }
762
773
 
774
+ /// Configures the TLS curve preference list.
775
+ ///
776
+ /// `curves` is a colon-separated list of curve (a.k.a. group) names, in
777
+ /// order of preference, e.g. `"X25519MLKEM768:X25519:P-256:P-384"`.
778
+ /// Corresponds to `SSL_CTX_set1_curves_list` (a.k.a.
779
+ /// `SSL_CTX_set1_groups_list`).
780
+ pub fn set_curves_list(&mut self, curves: &str) -> Result<()> {
781
+ self.tls_ctx.set_curves_list(curves)
782
+ }
783
+
763
784
  /// Configures whether to verify the peer's certificate.
764
785
  ///
765
786
  /// This should usually be `true` for client-side connections and `false`
@@ -781,6 +802,9 @@ impl Config {
781
802
 
782
803
  /// Configures whether to do path MTU discovery.
783
804
  ///
805
+ /// PMTUD-driven packet limit updates are reported to the application
806
+ /// through [`PathEvent::PmtuUpdated`].
807
+ ///
784
808
  /// The default value is `false`.
785
809
  pub fn discover_pmtu(&mut self, discover: bool) {
786
810
  self.pmtud = discover;
@@ -850,11 +874,8 @@ impl Config {
850
874
  /// config.set_application_protos(&[b"http/1.1", b"http/0.9"]);
851
875
  /// # Ok::<(), quiche::Error>(())
852
876
  /// ```
853
- pub fn set_application_protos(
854
- &mut self, protos_list: &[&[u8]],
855
- ) -> Result<()> {
856
- self.application_protos =
857
- protos_list.iter().map(|s| s.to_vec()).collect();
877
+ pub fn set_application_protos(&mut self, protos_list: &[&[u8]]) -> Result<()> {
878
+ self.application_protos = protos_list.iter().map(|s| s.to_vec()).collect();
858
879
 
859
880
  self.tls_ctx.set_alpn(protos_list)
860
881
  }
@@ -875,9 +896,7 @@ impl Config {
875
896
  /// config.set_application_protos_wire_format(b"\x08http/1.1\x08http/0.9")?;
876
897
  /// # Ok::<(), quiche::Error>(())
877
898
  /// ```
878
- pub fn set_application_protos_wire_format(
879
- &mut self, protos: &[u8],
880
- ) -> Result<()> {
899
+ pub fn set_application_protos_wire_format(&mut self, protos: &[u8]) -> Result<()> {
881
900
  let mut b = octets::Octets::with_slice(protos);
882
901
 
883
902
  let mut protos_list = Vec::new();
@@ -910,20 +929,33 @@ impl Config {
910
929
  self.initial_rtt = v;
911
930
  }
912
931
 
932
+ /// Sets the number of sent packets that can be tracked for loss detection
933
+ /// before the connection is closed.
934
+ ///
935
+ /// This limits the memory used to track sent packets. Packets that only
936
+ /// carry ACK frames are counted too. The limit should be larger than the
937
+ /// expected maximum congestion window, in packets, otherwise connections
938
+ /// that make progress can be closed.
939
+ ///
940
+ /// The connection is closed with a `PROTOCOL_VIOLATION` transport error.
941
+ ///
942
+ /// `None` disables this limit. The default value is `22,000`.
943
+ pub fn set_max_outstanding_sent_packets(&mut self, max_packets: Option<NonZeroUsize>) {
944
+ self.max_outstanding_sent_packets = max_packets;
945
+ }
946
+
913
947
  /// Sets the `max_idle_timeout` transport parameter, in milliseconds.
914
948
  ///
915
949
  /// The default value is infinite, that is, no timeout is used.
916
950
  pub fn set_max_idle_timeout(&mut self, v: u64) {
917
- self.local_transport_params.max_idle_timeout =
918
- cmp::min(v, octets::MAX_VAR_INT);
951
+ self.local_transport_params.max_idle_timeout = cmp::min(v, octets::MAX_VAR_INT);
919
952
  }
920
953
 
921
954
  /// Sets the `max_udp_payload_size transport` parameter.
922
955
  ///
923
956
  /// The default value is `65527`.
924
957
  pub fn set_max_recv_udp_payload_size(&mut self, v: usize) {
925
- self.local_transport_params.max_udp_payload_size =
926
- cmp::min(v as u64, octets::MAX_VAR_INT);
958
+ self.local_transport_params.max_udp_payload_size = cmp::min(v as u64, octets::MAX_VAR_INT);
927
959
  }
928
960
 
929
961
  /// Sets the maximum outgoing UDP payload size.
@@ -946,8 +978,7 @@ impl Config {
946
978
  ///
947
979
  /// The default value is `0`.
948
980
  pub fn set_initial_max_data(&mut self, v: u64) {
949
- self.local_transport_params.initial_max_data =
950
- cmp::min(v, octets::MAX_VAR_INT);
981
+ self.local_transport_params.initial_max_data = cmp::min(v, octets::MAX_VAR_INT);
951
982
  }
952
983
 
953
984
  /// Sets the `initial_max_stream_data_bidi_local` transport parameter.
@@ -965,8 +996,7 @@ impl Config {
965
996
  /// The default value is `0`.
966
997
  pub fn set_initial_max_stream_data_bidi_local(&mut self, v: u64) {
967
998
  self.local_transport_params
968
- .initial_max_stream_data_bidi_local =
969
- cmp::min(v, octets::MAX_VAR_INT);
999
+ .initial_max_stream_data_bidi_local = cmp::min(v, octets::MAX_VAR_INT);
970
1000
  }
971
1001
 
972
1002
  /// Sets the `initial_max_stream_data_bidi_remote` transport parameter.
@@ -984,8 +1014,7 @@ impl Config {
984
1014
  /// The default value is `0`.
985
1015
  pub fn set_initial_max_stream_data_bidi_remote(&mut self, v: u64) {
986
1016
  self.local_transport_params
987
- .initial_max_stream_data_bidi_remote =
988
- cmp::min(v, octets::MAX_VAR_INT);
1017
+ .initial_max_stream_data_bidi_remote = cmp::min(v, octets::MAX_VAR_INT);
989
1018
  }
990
1019
 
991
1020
  /// Sets the `initial_max_stream_data_uni` transport parameter.
@@ -1001,8 +1030,7 @@ impl Config {
1001
1030
  ///
1002
1031
  /// The default value is `0`.
1003
1032
  pub fn set_initial_max_stream_data_uni(&mut self, v: u64) {
1004
- self.local_transport_params.initial_max_stream_data_uni =
1005
- cmp::min(v, octets::MAX_VAR_INT);
1033
+ self.local_transport_params.initial_max_stream_data_uni = cmp::min(v, octets::MAX_VAR_INT);
1006
1034
  }
1007
1035
 
1008
1036
  /// Sets the `initial_max_streams_bidi` transport parameter.
@@ -1019,12 +1047,12 @@ impl Config {
1019
1047
  /// has been read by the application (up to the `fin` offset) or the
1020
1048
  /// stream's read direction has been shutdown, and all outgoing data has
1021
1049
  /// been acked by the peer (up to the `fin` offset) or the stream's write
1022
- /// direction has been shutdown.
1050
+ /// direction has been shutdown and the resulting `RESET_STREAM` frame has
1051
+ /// been acked by the peer.
1023
1052
  ///
1024
1053
  /// The default value is `0`.
1025
1054
  pub fn set_initial_max_streams_bidi(&mut self, v: u64) {
1026
- self.local_transport_params.initial_max_streams_bidi =
1027
- cmp::min(v, octets::MAX_VAR_INT);
1055
+ self.local_transport_params.initial_max_streams_bidi = cmp::min(v, octets::MAX_VAR_INT);
1028
1056
  }
1029
1057
 
1030
1058
  /// Sets the `initial_max_streams_uni` transport parameter.
@@ -1043,8 +1071,7 @@ impl Config {
1043
1071
  ///
1044
1072
  /// The default value is `0`.
1045
1073
  pub fn set_initial_max_streams_uni(&mut self, v: u64) {
1046
- self.local_transport_params.initial_max_streams_uni =
1047
- cmp::min(v, octets::MAX_VAR_INT);
1074
+ self.local_transport_params.initial_max_streams_uni = cmp::min(v, octets::MAX_VAR_INT);
1048
1075
  }
1049
1076
 
1050
1077
  /// Sets the `ack_delay_exponent` transport parameter.
@@ -1055,16 +1082,14 @@ impl Config {
1055
1082
  ///
1056
1083
  /// The default value is `3`.
1057
1084
  pub fn set_ack_delay_exponent(&mut self, v: u64) {
1058
- self.local_transport_params.ack_delay_exponent =
1059
- cmp::min(v, MAX_ACK_DELAY_EXPONENT);
1085
+ self.local_transport_params.ack_delay_exponent = cmp::min(v, MAX_ACK_DELAY_EXPONENT);
1060
1086
  }
1061
1087
 
1062
1088
  /// Sets the `max_ack_delay` transport parameter.
1063
1089
  ///
1064
1090
  /// The default value is `25`.
1065
1091
  pub fn set_max_ack_delay(&mut self, v: u64) {
1066
- self.local_transport_params.max_ack_delay =
1067
- cmp::min(v, octets::MAX_VAR_INT);
1092
+ self.local_transport_params.max_ack_delay = cmp::min(v, octets::MAX_VAR_INT);
1068
1093
  }
1069
1094
 
1070
1095
  /// Sets the `active_connection_id_limit` transport parameter.
@@ -1072,8 +1097,7 @@ impl Config {
1072
1097
  /// The default value is `2`. Lower values will be ignored.
1073
1098
  pub fn set_active_connection_id_limit(&mut self, v: u64) {
1074
1099
  if v >= 2 {
1075
- self.local_transport_params.active_conn_id_limit =
1076
- cmp::min(v, octets::MAX_VAR_INT);
1100
+ self.local_transport_params.active_conn_id_limit = cmp::min(v, octets::MAX_VAR_INT);
1077
1101
  }
1078
1102
  }
1079
1103
 
@@ -1130,6 +1154,23 @@ impl Config {
1130
1154
  self.initial_congestion_window_packets = packets;
1131
1155
  }
1132
1156
 
1157
+ /// Sets the fixed congestion window in bytes for the
1158
+ /// `congestion_window_unchecked` algorithm.
1159
+ ///
1160
+ /// The default is `usize::MAX`. Other congestion control algorithms ignore
1161
+ /// this setting. Recovery probes can temporarily bypass this window.
1162
+ ///
1163
+ /// Values are not clamped. Zero can prevent connection establishment, and
1164
+ /// windows too small for outgoing packets or PMTU probes can stall
1165
+ /// transmission. At least two maximum-sized datagrams are recommended.
1166
+ ///
1167
+ /// This API is available even when the
1168
+ /// `congestion_window_unchecked_available` feature is disabled, in which
1169
+ /// case the setting has no effect.
1170
+ pub fn set_unchecked_congestion_window(&mut self, window: usize) {
1171
+ self.unchecked_congestion_window = window;
1172
+ }
1173
+
1133
1174
  /// Configure whether to enable relaxed loss detection on spurious loss.
1134
1175
  ///
1135
1176
  /// The default value is false.
@@ -1186,9 +1227,7 @@ impl Config {
1186
1227
  /// to 65536 as recommended by draft-ietf-quic-datagram-01.
1187
1228
  ///
1188
1229
  /// The default is `false`.
1189
- pub fn enable_dgram(
1190
- &mut self, enabled: bool, recv_queue_len: usize, send_queue_len: usize,
1191
- ) {
1230
+ pub fn enable_dgram(&mut self, enabled: bool, recv_queue_len: usize, send_queue_len: usize) {
1192
1231
  self.local_transport_params.max_datagram_frame_size = if enabled {
1193
1232
  Some(MAX_DGRAM_FRAME_SIZE)
1194
1233
  } else {
@@ -1198,6 +1237,11 @@ impl Config {
1198
1237
  self.dgram_send_max_queue_len = send_queue_len;
1199
1238
  }
1200
1239
 
1240
+ /// Advertises support for QUIC RESET_STREAM_AT.
1241
+ pub fn enable_reliable_stream_reset(&mut self) {
1242
+ self.local_transport_params.reset_stream_at = true;
1243
+ }
1244
+
1201
1245
  /// Configures the max number of queued received PATH_CHALLENGE frames.
1202
1246
  ///
1203
1247
  /// When an endpoint receives a PATH_CHALLENGE frame and the queue is full,
@@ -1215,6 +1259,13 @@ impl Config {
1215
1259
  self.max_connection_window = v;
1216
1260
  }
1217
1261
 
1262
+ /// Sets budgets for retained stream and handshake receive allocations.
1263
+ /// The second limit is shared by all connections created from this config.
1264
+ /// Also bounds fragmented completed-stream history.
1265
+ pub fn set_stream_recv_memory_limits(&mut self, per_connection: usize, shared: usize) {
1266
+ self.stream_recv_memory = Some((per_connection, ReceiveMemory::new(shared, None)));
1267
+ }
1268
+
1218
1269
  /// Sets the maximum size of the stream window.
1219
1270
  ///
1220
1271
  /// The default value is MAX_STREAM_WINDOW (16MBytes).
@@ -1261,16 +1312,10 @@ impl Config {
1261
1312
  /// Sets whether the initial max data value should be used as the initial
1262
1313
  /// flow control window.
1263
1314
  ///
1264
- /// If set to true, the initial flow control window for streams and the
1265
- /// connection itself will be set to the initial max data value for streams
1266
- /// and the connection respectively. If false, the window is set to the
1267
- /// minimum of initial max data and `DEFAULT_STREAM_WINDOW` or
1268
- /// `DEFAULT_CONNECTION_WINDOW`
1269
- ///
1270
- /// The default is false.
1271
- pub fn set_use_initial_max_data_as_flow_control_win(&mut self, v: bool) {
1272
- self.use_initial_max_data_as_flow_control_win = v;
1273
- }
1315
+ /// This is now always enabled and this method is a no-op. It will be
1316
+ /// removed in a future release.
1317
+ #[deprecated(note = "This is now always enabled. This method is a no-op.")]
1318
+ pub fn set_use_initial_max_data_as_flow_control_win(&mut self, _v: bool) {}
1274
1319
  }
1275
1320
 
1276
1321
  /// Tracks the health of the tx_buffered value.
@@ -1425,12 +1470,6 @@ where
1425
1470
  /// The send capacity factor.
1426
1471
  tx_cap_factor: f64,
1427
1472
 
1428
- /// Number of bytes buffered in the send buffer.
1429
- tx_buffered: usize,
1430
-
1431
- /// Tracks the health of tx_buffered.
1432
- tx_buffered_state: TxBufferTrackingState,
1433
-
1434
1473
  /// Total number of bytes sent to the peer.
1435
1474
  tx_data: u64,
1436
1475
 
@@ -1457,7 +1496,7 @@ where
1457
1496
  lost_bytes: u64,
1458
1497
 
1459
1498
  /// Streams map, indexed by stream ID.
1460
- streams: stream::StreamMap<F>,
1499
+ pub(crate) streams: stream::StreamMap<F>,
1461
1500
 
1462
1501
  /// Peer's original destination connection ID. Used by the client to
1463
1502
  /// validate the server's transport parameter.
@@ -1610,6 +1649,10 @@ where
1610
1649
 
1611
1650
  /// The anti-amplification limit factor.
1612
1651
  max_amplification_factor: usize,
1652
+
1653
+ /// The number of tracked sent packets after which the connection is
1654
+ /// closed.
1655
+ max_outstanding_sent_packets: Option<NonZeroUsize>,
1613
1656
  }
1614
1657
 
1615
1658
  /// Creates a new server-side connection.
@@ -1634,24 +1677,29 @@ where
1634
1677
  /// ```
1635
1678
  #[inline(always)]
1636
1679
  pub fn accept(
1637
- scid: &ConnectionId, odcid: Option<&ConnectionId>, local: SocketAddr,
1638
- peer: SocketAddr, config: &mut Config,
1680
+ scid: &ConnectionId,
1681
+ odcid: Option<&ConnectionId>,
1682
+ local: SocketAddr,
1683
+ peer: SocketAddr,
1684
+ config: &mut Config,
1639
1685
  ) -> Result<Connection> {
1640
1686
  accept_with_buf_factory(scid, odcid, local, peer, config)
1641
1687
  }
1642
1688
 
1643
- /// Creates a new server-side connection, with a custom buffer generation
1644
- /// method.
1689
+ /// Creates a server-side connection with custom buffer generation.
1645
1690
  ///
1646
- /// The buffers generated can be anything that can be drereferenced as a byte
1647
- /// slice. See [`accept`] and [`BufFactory`] for more info.
1691
+ /// The buffers generated can be anything that can be dereferenced as a byte
1692
+ /// slice. See [`accept`] and [`BufFactory`] for more information.
1648
1693
  #[inline]
1649
1694
  pub fn accept_with_buf_factory<F: BufFactory>(
1650
- scid: &ConnectionId, odcid: Option<&ConnectionId>, local: SocketAddr,
1651
- peer: SocketAddr, config: &mut Config,
1695
+ scid: &ConnectionId,
1696
+ odcid: Option<&ConnectionId>,
1697
+ local: SocketAddr,
1698
+ peer: SocketAddr,
1699
+ config: &mut Config,
1652
1700
  ) -> Result<Connection<F>> {
1653
- // For connections with `odcid` set, we historically used `retry_source_cid =
1654
- // scid`. Keep this behavior to preserve backwards compatibility.
1701
+ // Connections with `odcid` historically used `scid` as the retry source
1702
+ // CID. Preserve this behavior for backwards compatibility.
1655
1703
  // `accept_with_retry` allows the SCIDs to be specified separately.
1656
1704
  let retry_cids = odcid.map(|odcid| RetryConnectionIds {
1657
1705
  original_destination_cid: odcid,
@@ -1688,8 +1736,11 @@ pub struct RetryConnectionIds<'a> {
1688
1736
  /// manner.
1689
1737
  #[inline]
1690
1738
  pub fn accept_with_retry<F: BufFactory>(
1691
- scid: &ConnectionId, retry_cids: RetryConnectionIds, local: SocketAddr,
1692
- peer: SocketAddr, config: &mut Config,
1739
+ scid: &ConnectionId,
1740
+ retry_cids: RetryConnectionIds,
1741
+ local: SocketAddr,
1742
+ peer: SocketAddr,
1743
+ config: &mut Config,
1693
1744
  ) -> Result<Connection<F>> {
1694
1745
  Connection::new(scid, Some(retry_cids), None, local, peer, config, true)
1695
1746
  }
@@ -1714,8 +1765,11 @@ pub fn accept_with_retry<F: BufFactory>(
1714
1765
  /// ```
1715
1766
  #[inline]
1716
1767
  pub fn connect(
1717
- server_name: Option<&str>, scid: &ConnectionId, local: SocketAddr,
1718
- peer: SocketAddr, config: &mut Config,
1768
+ server_name: Option<&str>,
1769
+ scid: &ConnectionId,
1770
+ local: SocketAddr,
1771
+ peer: SocketAddr,
1772
+ config: &mut Config,
1719
1773
  ) -> Result<Connection> {
1720
1774
  let mut conn = Connection::new(scid, None, None, local, peer, config, false)?;
1721
1775
 
@@ -1740,11 +1794,14 @@ pub fn connect(
1740
1794
  #[cfg(feature = "custom-client-dcid")]
1741
1795
  #[cfg_attr(docsrs, doc(cfg(feature = "custom-client-dcid")))]
1742
1796
  pub fn connect_with_dcid(
1743
- server_name: Option<&str>, scid: &ConnectionId, dcid: &ConnectionId,
1744
- local: SocketAddr, peer: SocketAddr, config: &mut Config,
1797
+ server_name: Option<&str>,
1798
+ scid: &ConnectionId,
1799
+ dcid: &ConnectionId,
1800
+ local: SocketAddr,
1801
+ peer: SocketAddr,
1802
+ config: &mut Config,
1745
1803
  ) -> Result<Connection> {
1746
- let mut conn =
1747
- Connection::new(scid, None, Some(dcid), local, peer, config, false)?;
1804
+ let mut conn = Connection::new(scid, None, Some(dcid), local, peer, config, false)?;
1748
1805
 
1749
1806
  if let Some(server_name) = server_name {
1750
1807
  conn.handshake.set_host_name(server_name)?;
@@ -1760,8 +1817,11 @@ pub fn connect_with_dcid(
1760
1817
  /// slice. See [`connect`] and [`BufFactory`] for more info.
1761
1818
  #[inline]
1762
1819
  pub fn connect_with_buffer_factory<F: BufFactory>(
1763
- server_name: Option<&str>, scid: &ConnectionId, local: SocketAddr,
1764
- peer: SocketAddr, config: &mut Config,
1820
+ server_name: Option<&str>,
1821
+ scid: &ConnectionId,
1822
+ local: SocketAddr,
1823
+ peer: SocketAddr,
1824
+ config: &mut Config,
1765
1825
  ) -> Result<Connection<F>> {
1766
1826
  let mut conn = Connection::new(scid, None, None, local, peer, config, false)?;
1767
1827
 
@@ -1783,11 +1843,14 @@ pub fn connect_with_buffer_factory<F: BufFactory>(
1783
1843
  #[cfg(feature = "custom-client-dcid")]
1784
1844
  #[cfg_attr(docsrs, doc(cfg(feature = "custom-client-dcid")))]
1785
1845
  pub fn connect_with_dcid_and_buffer_factory<F: BufFactory>(
1786
- server_name: Option<&str>, scid: &ConnectionId, dcid: &ConnectionId,
1787
- local: SocketAddr, peer: SocketAddr, config: &mut Config,
1846
+ server_name: Option<&str>,
1847
+ scid: &ConnectionId,
1848
+ dcid: &ConnectionId,
1849
+ local: SocketAddr,
1850
+ peer: SocketAddr,
1851
+ config: &mut Config,
1788
1852
  ) -> Result<Connection<F>> {
1789
- let mut conn =
1790
- Connection::new(scid, None, Some(dcid), local, peer, config, false)?;
1853
+ let mut conn = Connection::new(scid, None, Some(dcid), local, peer, config, false)?;
1791
1854
 
1792
1855
  if let Some(server_name) = server_name {
1793
1856
  conn.handshake.set_host_name(server_name)?;
@@ -1821,7 +1884,9 @@ pub fn connect_with_dcid_and_buffer_factory<F: BufFactory>(
1821
1884
  /// ```
1822
1885
  #[inline]
1823
1886
  pub fn negotiate_version(
1824
- scid: &ConnectionId, dcid: &ConnectionId, out: &mut [u8],
1887
+ scid: &ConnectionId,
1888
+ dcid: &ConnectionId,
1889
+ out: &mut [u8],
1825
1890
  ) -> Result<usize> {
1826
1891
  packet::negotiate_version(scid, dcid, out)
1827
1892
  }
@@ -1887,8 +1952,12 @@ pub fn negotiate_version(
1887
1952
  /// ```
1888
1953
  #[inline]
1889
1954
  pub fn retry(
1890
- scid: &ConnectionId, dcid: &ConnectionId, new_scid: &ConnectionId,
1891
- token: &[u8], version: u32, out: &mut [u8],
1955
+ scid: &ConnectionId,
1956
+ dcid: &ConnectionId,
1957
+ new_scid: &ConnectionId,
1958
+ token: &[u8],
1959
+ version: u32,
1960
+ out: &mut [u8],
1892
1961
  ) -> Result<usize> {
1893
1962
  packet::retry(scid, dcid, new_scid, token, version, out)
1894
1963
  }
@@ -1937,28 +2006,22 @@ macro_rules! qlog_with_type {
1937
2006
  }
1938
2007
 
1939
2008
  #[cfg(feature = "qlog")]
1940
- const QLOG_PARAMS_SET: EventType =
1941
- EventType::QuicEventType(QuicEventType::ParametersSet);
2009
+ const QLOG_PARAMS_SET: EventType = EventType::QuicEventType(QuicEventType::ParametersSet);
1942
2010
 
1943
2011
  #[cfg(feature = "qlog")]
1944
- const QLOG_PACKET_RX: EventType =
1945
- EventType::QuicEventType(QuicEventType::PacketReceived);
2012
+ const QLOG_PACKET_RX: EventType = EventType::QuicEventType(QuicEventType::PacketReceived);
1946
2013
 
1947
2014
  #[cfg(feature = "qlog")]
1948
- const QLOG_PACKET_TX: EventType =
1949
- EventType::QuicEventType(QuicEventType::PacketSent);
2015
+ const QLOG_PACKET_TX: EventType = EventType::QuicEventType(QuicEventType::PacketSent);
1950
2016
 
1951
2017
  #[cfg(feature = "qlog")]
1952
- const QLOG_DATA_MV: EventType =
1953
- EventType::QuicEventType(QuicEventType::StreamDataMoved);
2018
+ const QLOG_DATA_MV: EventType = EventType::QuicEventType(QuicEventType::StreamDataMoved);
1954
2019
 
1955
2020
  #[cfg(feature = "qlog")]
1956
- const QLOG_METRICS: EventType =
1957
- EventType::QuicEventType(QuicEventType::RecoveryMetricsUpdated);
2021
+ const QLOG_METRICS: EventType = EventType::QuicEventType(QuicEventType::RecoveryMetricsUpdated);
1958
2022
 
1959
2023
  #[cfg(feature = "qlog")]
1960
- const QLOG_CONNECTION_CLOSED: EventType =
1961
- EventType::QuicEventType(QuicEventType::ConnectionClosed);
2024
+ const QLOG_CONNECTION_CLOSED: EventType = EventType::QuicEventType(QuicEventType::ConnectionClosed);
1962
2025
 
1963
2026
  #[cfg(feature = "qlog")]
1964
2027
  struct QlogInfo {
@@ -1980,9 +2043,13 @@ impl Default for QlogInfo {
1980
2043
 
1981
2044
  impl<F: BufFactory> Connection<F> {
1982
2045
  fn new(
1983
- scid: &ConnectionId, retry_cids: Option<RetryConnectionIds>,
1984
- client_dcid: Option<&ConnectionId>, local: SocketAddr, peer: SocketAddr,
1985
- config: &mut Config, is_server: bool,
2046
+ scid: &ConnectionId,
2047
+ retry_cids: Option<RetryConnectionIds>,
2048
+ client_dcid: Option<&ConnectionId>,
2049
+ local: SocketAddr,
2050
+ peer: SocketAddr,
2051
+ config: &mut Config,
2052
+ is_server: bool,
1986
2053
  ) -> Result<Connection<F>> {
1987
2054
  let tls = config.tls_ctx.new_handshake()?;
1988
2055
  Connection::with_tls(
@@ -1999,9 +2066,14 @@ impl<F: BufFactory> Connection<F> {
1999
2066
 
2000
2067
  #[allow(clippy::too_many_arguments)]
2001
2068
  fn with_tls(
2002
- scid: &ConnectionId, retry_cids: Option<RetryConnectionIds>,
2003
- client_dcid: Option<&ConnectionId>, local: SocketAddr, peer: SocketAddr,
2004
- config: &Config, tls: tls::Handshake, is_server: bool,
2069
+ scid: &ConnectionId,
2070
+ retry_cids: Option<RetryConnectionIds>,
2071
+ client_dcid: Option<&ConnectionId>,
2072
+ local: SocketAddr,
2073
+ peer: SocketAddr,
2074
+ config: &Config,
2075
+ tls: tls::Handshake,
2076
+ is_server: bool,
2005
2077
  ) -> Result<Connection<F>> {
2006
2078
  if retry_cids.is_some() && client_dcid.is_some() {
2007
2079
  // These are exclusive, the caller should only specify one or the
@@ -2023,8 +2095,7 @@ impl<F: BufFactory> Connection<F> {
2023
2095
 
2024
2096
  let max_rx_data = config.local_transport_params.initial_max_data;
2025
2097
 
2026
- let scid_as_hex: Vec<String> =
2027
- scid.iter().map(|b| format!("{b:02x}")).collect();
2098
+ let scid_as_hex: Vec<String> = scid.iter().map(|b| format!("{b:02x}")).collect();
2028
2099
 
2029
2100
  let reset_token = if is_server {
2030
2101
  config.local_transport_params.stateless_reset_token
@@ -2064,12 +2135,15 @@ impl<F: BufFactory> Connection<F> {
2064
2135
  reset_token,
2065
2136
  );
2066
2137
 
2067
- let initial_flow_control_window =
2068
- if config.use_initial_max_data_as_flow_control_win {
2069
- max_rx_data
2070
- } else {
2071
- cmp::min(max_rx_data / 2 * 3, DEFAULT_CONNECTION_WINDOW)
2072
- };
2138
+ let initial_flow_control_window = max_rx_data;
2139
+ let mut streams = stream::StreamMap::new(
2140
+ config.local_transport_params.initial_max_streams_bidi,
2141
+ config.local_transport_params.initial_max_streams_uni,
2142
+ config.max_stream_window,
2143
+ );
2144
+ if let Some((limit, shared)) = &config.stream_recv_memory {
2145
+ streams.receive_memory = Some(ReceiveMemory::new(*limit, Some(shared.clone())));
2146
+ }
2073
2147
  let mut conn = Connection {
2074
2148
  version: config.version,
2075
2149
 
@@ -2095,8 +2169,7 @@ impl<F: BufFactory> Connection<F> {
2095
2169
 
2096
2170
  peer_transport_params: TransportParams::default(),
2097
2171
 
2098
- peer_transport_params_track_unknown: config
2099
- .track_unknown_transport_params,
2172
+ peer_transport_params_track_unknown: config.track_unknown_transport_params,
2100
2173
 
2101
2174
  local_transport_params: config.local_transport_params.clone(),
2102
2175
 
@@ -2107,8 +2180,7 @@ impl<F: BufFactory> Connection<F> {
2107
2180
  recovery_config,
2108
2181
 
2109
2182
  paths,
2110
- path_challenge_recv_max_queue_len: config
2111
- .path_challenge_recv_max_queue_len,
2183
+ path_challenge_recv_max_queue_len: config.path_challenge_recv_max_queue_len,
2112
2184
  path_challenge_rx_count: 0,
2113
2185
 
2114
2186
  application_protos: config.application_protos.clone(),
@@ -2138,20 +2210,13 @@ impl<F: BufFactory> Connection<F> {
2138
2210
  tx_cap: 0,
2139
2211
  tx_cap_factor: config.tx_cap_factor,
2140
2212
 
2141
- tx_buffered: 0,
2142
- tx_buffered_state: TxBufferTrackingState::Ok,
2143
-
2144
2213
  tx_data: 0,
2145
2214
  max_tx_data: 0,
2146
2215
  last_tx_data: 0,
2147
2216
 
2148
2217
  stream_retrans_bytes: 0,
2149
2218
 
2150
- streams: stream::StreamMap::new(
2151
- config.local_transport_params.initial_max_streams_bidi,
2152
- config.local_transport_params.initial_max_streams_uni,
2153
- config.max_stream_window,
2154
- ),
2219
+ streams,
2155
2220
 
2156
2221
  odcid: None,
2157
2222
 
@@ -2212,13 +2277,9 @@ impl<F: BufFactory> Connection<F> {
2212
2277
  #[cfg(feature = "qlog")]
2213
2278
  qlog: Default::default(),
2214
2279
 
2215
- dgram_recv_queue: dgram::DatagramQueue::new(
2216
- config.dgram_recv_max_queue_len,
2217
- ),
2280
+ dgram_recv_queue: dgram::DatagramQueue::new(config.dgram_recv_max_queue_len),
2218
2281
 
2219
- dgram_send_queue: dgram::DatagramQueue::new(
2220
- config.dgram_send_max_queue_len,
2221
- ),
2282
+ dgram_send_queue: dgram::DatagramQueue::new(config.dgram_send_max_queue_len),
2222
2283
 
2223
2284
  emit_dgram: true,
2224
2285
 
@@ -2243,11 +2304,9 @@ impl<F: BufFactory> Connection<F> {
2243
2304
  streams_blocked_uni_state: Default::default(),
2244
2305
 
2245
2306
  max_amplification_factor: config.max_amplification_factor,
2246
- };
2247
- conn.streams.set_use_initial_max_data_as_flow_control_win(
2248
- config.use_initial_max_data_as_flow_control_win,
2249
- );
2250
2307
 
2308
+ max_outstanding_sent_packets: config.max_outstanding_sent_packets,
2309
+ };
2251
2310
  if let Some(retry_cids) = retry_cids {
2252
2311
  conn.local_transport_params
2253
2312
  .original_destination_connection_id =
@@ -2282,19 +2341,11 @@ impl<F: BufFactory> Connection<F> {
2282
2341
  dcid.to_vec()
2283
2342
  };
2284
2343
 
2285
- let (aead_open, aead_seal) = crypto::derive_initial_key_material(
2286
- &dcid,
2287
- conn.version,
2288
- conn.is_server,
2289
- false,
2290
- )?;
2344
+ let (aead_open, aead_seal) =
2345
+ crypto::derive_initial_key_material(&dcid, conn.version, conn.is_server, false)?;
2291
2346
 
2292
2347
  let reset_token = conn.peer_transport_params.stateless_reset_token;
2293
- conn.set_initial_dcid(
2294
- dcid.to_vec().into(),
2295
- reset_token,
2296
- active_path_id,
2297
- )?;
2348
+ conn.set_initial_dcid(dcid.to_vec().into(), reset_token, active_path_id)?;
2298
2349
 
2299
2350
  conn.crypto_ctx[packet::Epoch::Initial].crypto_open = Some(aead_open);
2300
2351
  conn.crypto_ctx[packet::Epoch::Initial].crypto_seal = Some(aead_seal);
@@ -2328,7 +2379,9 @@ impl<F: BufFactory> Connection<F> {
2328
2379
  #[cfg(feature = "qlog")]
2329
2380
  #[cfg_attr(docsrs, doc(cfg(feature = "qlog")))]
2330
2381
  pub fn set_qlog(
2331
- &mut self, writer: Box<dyn std::io::Write + Send + Sync>, title: String,
2382
+ &mut self,
2383
+ writer: Box<dyn std::io::Write + Send + Sync>,
2384
+ title: String,
2332
2385
  description: String,
2333
2386
  ) {
2334
2387
  self.set_qlog_with_level(writer, title, description, QlogLevel::Base)
@@ -2346,8 +2399,11 @@ impl<F: BufFactory> Connection<F> {
2346
2399
  #[cfg(feature = "qlog")]
2347
2400
  #[cfg_attr(docsrs, doc(cfg(feature = "qlog")))]
2348
2401
  pub fn set_qlog_with_level(
2349
- &mut self, writer: Box<dyn std::io::Write + Send + Sync>, title: String,
2350
- description: String, qlog_level: QlogLevel,
2402
+ &mut self,
2403
+ writer: Box<dyn std::io::Write + Send + Sync>,
2404
+ title: String,
2405
+ description: String,
2406
+ qlog_level: QlogLevel,
2351
2407
  ) {
2352
2408
  use qlog::events::quic::TransportInitiator;
2353
2409
  use qlog::events::HTTP3_URI;
@@ -2460,8 +2516,7 @@ impl<F: BufFactory> Connection<F> {
2460
2516
  /// The default value is infinite, that is, no timeout is used unless
2461
2517
  /// already configured when creating the connection.
2462
2518
  pub fn set_max_idle_timeout(&mut self, v: u64) -> Result<()> {
2463
- self.local_transport_params.max_idle_timeout =
2464
- cmp::min(v, octets::MAX_VAR_INT);
2519
+ self.local_transport_params.max_idle_timeout = cmp::min(v, octets::MAX_VAR_INT);
2465
2520
 
2466
2521
  self.encode_transport_params()
2467
2522
  }
@@ -2478,7 +2533,8 @@ impl<F: BufFactory> Connection<F> {
2478
2533
  #[cfg(feature = "boringssl-boring-crate")]
2479
2534
  #[cfg_attr(docsrs, doc(cfg(feature = "boringssl-boring-crate")))]
2480
2535
  pub fn set_cc_algorithm_in_handshake(
2481
- ssl: &mut boring::ssl::SslRef, algo: CongestionControlAlgorithm,
2536
+ ssl: &mut boring::ssl::SslRef,
2537
+ algo: CongestionControlAlgorithm,
2482
2538
  ) -> Result<()> {
2483
2539
  let ex_data = tls::ExData::from_ssl_ref(ssl).ok_or(Error::TlsFail)?;
2484
2540
 
@@ -2505,7 +2561,8 @@ impl<F: BufFactory> Connection<F> {
2505
2561
  #[cfg_attr(docsrs, doc(cfg(feature = "boringssl-boring-crate")))]
2506
2562
  #[doc(hidden)]
2507
2563
  pub fn set_custom_bbr_settings_in_handshake(
2508
- ssl: &mut boring::ssl::SslRef, custom_bbr_params: BbrParams,
2564
+ ssl: &mut boring::ssl::SslRef,
2565
+ custom_bbr_params: BbrParams,
2509
2566
  ) -> Result<()> {
2510
2567
  let ex_data = tls::ExData::from_ssl_ref(ssl).ok_or(Error::TlsFail)?;
2511
2568
 
@@ -2526,7 +2583,8 @@ impl<F: BufFactory> Connection<F> {
2526
2583
  #[cfg(feature = "boringssl-boring-crate")]
2527
2584
  #[cfg_attr(docsrs, doc(cfg(feature = "boringssl-boring-crate")))]
2528
2585
  pub fn set_cc_algorithm_name_in_handshake(
2529
- ssl: &mut boring::ssl::SslRef, name: &str,
2586
+ ssl: &mut boring::ssl::SslRef,
2587
+ name: &str,
2530
2588
  ) -> Result<()> {
2531
2589
  let cc_algo = CongestionControlAlgorithm::from_str(name)?;
2532
2590
  Self::set_cc_algorithm_in_handshake(ssl, cc_algo)
@@ -2544,7 +2602,8 @@ impl<F: BufFactory> Connection<F> {
2544
2602
  #[cfg(feature = "boringssl-boring-crate")]
2545
2603
  #[cfg_attr(docsrs, doc(cfg(feature = "boringssl-boring-crate")))]
2546
2604
  pub fn set_initial_congestion_window_packets_in_handshake(
2547
- ssl: &mut boring::ssl::SslRef, packets: usize,
2605
+ ssl: &mut boring::ssl::SslRef,
2606
+ packets: usize,
2548
2607
  ) -> Result<()> {
2549
2608
  let ex_data = tls::ExData::from_ssl_ref(ssl).ok_or(Error::TlsFail)?;
2550
2609
 
@@ -2565,7 +2624,8 @@ impl<F: BufFactory> Connection<F> {
2565
2624
  #[cfg(feature = "boringssl-boring-crate")]
2566
2625
  #[cfg_attr(docsrs, doc(cfg(feature = "boringssl-boring-crate")))]
2567
2626
  pub fn set_enable_relaxed_loss_threshold_in_handshake(
2568
- ssl: &mut boring::ssl::SslRef, enable: bool,
2627
+ ssl: &mut boring::ssl::SslRef,
2628
+ enable: bool,
2569
2629
  ) -> Result<()> {
2570
2630
  let ex_data = tls::ExData::from_ssl_ref(ssl).ok_or(Error::TlsFail)?;
2571
2631
 
@@ -2586,7 +2646,8 @@ impl<F: BufFactory> Connection<F> {
2586
2646
  #[cfg(feature = "boringssl-boring-crate")]
2587
2647
  #[cfg_attr(docsrs, doc(cfg(feature = "boringssl-boring-crate")))]
2588
2648
  pub fn set_enable_cubic_idle_restart_fix_in_handshake(
2589
- ssl: &mut boring::ssl::SslRef, enable: bool,
2649
+ ssl: &mut boring::ssl::SslRef,
2650
+ enable: bool,
2590
2651
  ) -> Result<()> {
2591
2652
  let ex_data = tls::ExData::from_ssl_ref(ssl).ok_or(Error::TlsFail)?;
2592
2653
 
@@ -2606,9 +2667,7 @@ impl<F: BufFactory> Connection<F> {
2606
2667
  /// [`Config::enable_hystart()`]: struct.Config.html#method.enable_hystart
2607
2668
  #[cfg(feature = "boringssl-boring-crate")]
2608
2669
  #[cfg_attr(docsrs, doc(cfg(feature = "boringssl-boring-crate")))]
2609
- pub fn set_hystart_in_handshake(
2610
- ssl: &mut boring::ssl::SslRef, v: bool,
2611
- ) -> Result<()> {
2670
+ pub fn set_hystart_in_handshake(ssl: &mut boring::ssl::SslRef, v: bool) -> Result<()> {
2612
2671
  let ex_data = tls::ExData::from_ssl_ref(ssl).ok_or(Error::TlsFail)?;
2613
2672
 
2614
2673
  ex_data.recovery_config.hystart = v;
@@ -2627,9 +2686,7 @@ impl<F: BufFactory> Connection<F> {
2627
2686
  /// [`Config::enable_pacing()`]: struct.Config.html#method.enable_pacing
2628
2687
  #[cfg(feature = "boringssl-boring-crate")]
2629
2688
  #[cfg_attr(docsrs, doc(cfg(feature = "boringssl-boring-crate")))]
2630
- pub fn set_pacing_in_handshake(
2631
- ssl: &mut boring::ssl::SslRef, v: bool,
2632
- ) -> Result<()> {
2689
+ pub fn set_pacing_in_handshake(ssl: &mut boring::ssl::SslRef, v: bool) -> Result<()> {
2633
2690
  let ex_data = tls::ExData::from_ssl_ref(ssl).ok_or(Error::TlsFail)?;
2634
2691
 
2635
2692
  ex_data.recovery_config.pacing = v;
@@ -2649,7 +2706,8 @@ impl<F: BufFactory> Connection<F> {
2649
2706
  #[cfg(feature = "boringssl-boring-crate")]
2650
2707
  #[cfg_attr(docsrs, doc(cfg(feature = "boringssl-boring-crate")))]
2651
2708
  pub fn set_max_pacing_rate_in_handshake(
2652
- ssl: &mut boring::ssl::SslRef, v: Option<u64>,
2709
+ ssl: &mut boring::ssl::SslRef,
2710
+ v: Option<u64>,
2653
2711
  ) -> Result<()> {
2654
2712
  let ex_data = tls::ExData::from_ssl_ref(ssl).ok_or(Error::TlsFail)?;
2655
2713
 
@@ -2670,7 +2728,8 @@ impl<F: BufFactory> Connection<F> {
2670
2728
  #[cfg(feature = "boringssl-boring-crate")]
2671
2729
  #[cfg_attr(docsrs, doc(cfg(feature = "boringssl-boring-crate")))]
2672
2730
  pub fn set_max_send_udp_payload_size_in_handshake(
2673
- ssl: &mut boring::ssl::SslRef, v: usize,
2731
+ ssl: &mut boring::ssl::SslRef,
2732
+ v: usize,
2674
2733
  ) -> Result<()> {
2675
2734
  let ex_data = tls::ExData::from_ssl_ref(ssl).ok_or(Error::TlsFail)?;
2676
2735
 
@@ -2691,7 +2750,8 @@ impl<F: BufFactory> Connection<F> {
2691
2750
  #[cfg(feature = "boringssl-boring-crate")]
2692
2751
  #[cfg_attr(docsrs, doc(cfg(feature = "boringssl-boring-crate")))]
2693
2752
  pub fn set_send_capacity_factor_in_handshake(
2694
- ssl: &mut boring::ssl::SslRef, v: f64,
2753
+ ssl: &mut boring::ssl::SslRef,
2754
+ v: f64,
2695
2755
  ) -> Result<()> {
2696
2756
  let ex_data = tls::ExData::from_ssl_ref(ssl).ok_or(Error::TlsFail)?;
2697
2757
 
@@ -2712,7 +2772,9 @@ impl<F: BufFactory> Connection<F> {
2712
2772
  #[cfg(feature = "boringssl-boring-crate")]
2713
2773
  #[cfg_attr(docsrs, doc(cfg(feature = "boringssl-boring-crate")))]
2714
2774
  pub fn set_discover_pmtu_in_handshake(
2715
- ssl: &mut boring::ssl::SslRef, discover: bool, max_probes: u8,
2775
+ ssl: &mut boring::ssl::SslRef,
2776
+ discover: bool,
2777
+ max_probes: u8,
2716
2778
  ) -> Result<()> {
2717
2779
  let ex_data = tls::ExData::from_ssl_ref(ssl).ok_or(Error::TlsFail)?;
2718
2780
 
@@ -2732,9 +2794,7 @@ impl<F: BufFactory> Connection<F> {
2732
2794
  /// [`Config::set_max_idle_timeout()`]: struct.Config.html#method.set_max_idle_timeout
2733
2795
  #[cfg(feature = "boringssl-boring-crate")]
2734
2796
  #[cfg_attr(docsrs, doc(cfg(feature = "boringssl-boring-crate")))]
2735
- pub fn set_max_idle_timeout_in_handshake(
2736
- ssl: &mut boring::ssl::SslRef, v: u64,
2737
- ) -> Result<()> {
2797
+ pub fn set_max_idle_timeout_in_handshake(ssl: &mut boring::ssl::SslRef, v: u64) -> Result<()> {
2738
2798
  let ex_data = tls::ExData::from_ssl_ref(ssl).ok_or(Error::TlsFail)?;
2739
2799
 
2740
2800
  ex_data.local_transport_params.max_idle_timeout = v;
@@ -2746,6 +2806,27 @@ impl<F: BufFactory> Connection<F> {
2746
2806
  )
2747
2807
  }
2748
2808
 
2809
+ /// Sets the number of sent packets that can be tracked for loss detection
2810
+ /// before the connection is closed.
2811
+ ///
2812
+ /// This function can only be called inside one of BoringSSL's handshake
2813
+ /// callbacks, before any packet has been sent. Calling this function any
2814
+ /// other time will have no effect.
2815
+ ///
2816
+ /// See [`Config::set_max_outstanding_sent_packets()`].
2817
+ #[cfg(feature = "boringssl-boring-crate")]
2818
+ #[cfg_attr(docsrs, doc(cfg(feature = "boringssl-boring-crate")))]
2819
+ pub fn set_max_outstanding_sent_packets_in_handshake(
2820
+ ssl: &mut boring::ssl::SslRef,
2821
+ max_packets: Option<NonZeroUsize>,
2822
+ ) -> Result<()> {
2823
+ let ex_data = tls::ExData::from_ssl_ref(ssl).ok_or(Error::TlsFail)?;
2824
+
2825
+ ex_data.max_outstanding_sent_packets = max_packets;
2826
+
2827
+ Ok(())
2828
+ }
2829
+
2749
2830
  /// Sets the `initial_max_streams_bidi` transport parameter.
2750
2831
  ///
2751
2832
  /// This function can only be called inside one of BoringSSL's handshake
@@ -2758,7 +2839,8 @@ impl<F: BufFactory> Connection<F> {
2758
2839
  #[cfg(feature = "boringssl-boring-crate")]
2759
2840
  #[cfg_attr(docsrs, doc(cfg(feature = "boringssl-boring-crate")))]
2760
2841
  pub fn set_initial_max_streams_bidi_in_handshake(
2761
- ssl: &mut boring::ssl::SslRef, v: u64,
2842
+ ssl: &mut boring::ssl::SslRef,
2843
+ v: u64,
2762
2844
  ) -> Result<()> {
2763
2845
  let ex_data = tls::ExData::from_ssl_ref(ssl).ok_or(Error::TlsFail)?;
2764
2846
 
@@ -2773,7 +2855,9 @@ impl<F: BufFactory> Connection<F> {
2773
2855
 
2774
2856
  #[cfg(feature = "boringssl-boring-crate")]
2775
2857
  fn set_transport_parameters_in_hanshake(
2776
- params: TransportParams, is_server: bool, ssl: &mut boring::ssl::SslRef,
2858
+ params: TransportParams,
2859
+ is_server: bool,
2860
+ ssl: &mut boring::ssl::SslRef,
2777
2861
  ) -> Result<()> {
2778
2862
  use foreign_types_shared::ForeignTypeRef;
2779
2863
  use std::mem::ManuallyDrop;
@@ -2787,9 +2871,8 @@ impl<F: BufFactory> Connection<F> {
2787
2871
  // Wrap the temporary `Handshake` in `ManuallyDrop` because this is only
2788
2872
  // a borrowed view of `ssl`. The caller retains ownership of the
2789
2873
  // underlying BoringSSL object.
2790
- let mut handshake = ManuallyDrop::new(unsafe {
2791
- tls::Handshake::from_ptr(ssl.as_ptr() as _)
2792
- });
2874
+ let mut handshake =
2875
+ ManuallyDrop::new(unsafe { tls::Handshake::from_ptr(ssl.as_ptr() as _)? });
2793
2876
 
2794
2877
  handshake.set_quic_transport_params(&params, is_server)
2795
2878
  }
@@ -2797,19 +2880,14 @@ impl<F: BufFactory> Connection<F> {
2797
2880
  /// Sets the `use_initial_max_data_as_flow_control_win` flag during SSL
2798
2881
  /// handshake.
2799
2882
  ///
2800
- /// This function can only be called inside one of BoringSSL's handshake
2801
- /// callbacks, before any packet has been sent. Calling this function any
2802
- /// other time will have no effect.
2803
- ///
2804
- /// See [`Connection::enable_use_initial_max_data_as_flow_control_win()`].
2883
+ /// This is now always enabled and this method is a no-op. It will be
2884
+ /// removed in a future release.
2805
2885
  #[cfg(feature = "boringssl-boring-crate")]
2806
2886
  #[cfg_attr(docsrs, doc(cfg(feature = "boringssl-boring-crate")))]
2887
+ #[deprecated(note = "This is now always enabled. This method is a no-op.")]
2807
2888
  pub fn set_use_initial_max_data_as_flow_control_win_in_handshake(
2808
- ssl: &mut boring::ssl::SslRef,
2889
+ _ssl: &mut boring::ssl::SslRef,
2809
2890
  ) -> Result<()> {
2810
- let ex_data = tls::ExData::from_ssl_ref(ssl).ok_or(Error::TlsFail)?;
2811
-
2812
- ex_data.use_initial_max_data_as_flow_control_win = true;
2813
2891
  Ok(())
2814
2892
  }
2815
2893
 
@@ -2898,11 +2976,7 @@ impl<F: BufFactory> Connection<F> {
2898
2976
 
2899
2977
  // Process coalesced packets.
2900
2978
  while left > 0 {
2901
- let read = match self.recv_single(
2902
- &mut buf[len - left..len],
2903
- &info,
2904
- recv_pid,
2905
- ) {
2979
+ let read = match self.recv_single(&mut buf[len - left..len], &info, recv_pid) {
2906
2980
  Ok(v) => v,
2907
2981
 
2908
2982
  Err(Error::Done) => {
@@ -2915,14 +2989,19 @@ impl<F: BufFactory> Connection<F> {
2915
2989
  }
2916
2990
 
2917
2991
  left
2918
- },
2992
+ }
2919
2993
 
2920
2994
  Err(e) => {
2921
2995
  // In case of error processing the incoming packet, close
2922
2996
  // the connection.
2923
- self.close(false, e.to_wire(), b"").ok();
2997
+ let reason: &[u8] = if e == Error::StreamMemoryLimit {
2998
+ b"QUIC receive memory limit"
2999
+ } else {
3000
+ b""
3001
+ };
3002
+ self.close(false, e.to_wire(), reason).ok();
2924
3003
  return Err(e);
2925
- },
3004
+ }
2926
3005
  };
2927
3006
 
2928
3007
  done += read;
@@ -2944,8 +3023,7 @@ impl<F: BufFactory> Connection<F> {
2944
3023
  .crypto_0rtt_open
2945
3024
  .is_some()
2946
3025
  {
2947
- while let Some((mut pkt, info)) = self.undecryptable_pkts.pop_front()
2948
- {
3026
+ while let Some((mut pkt, info)) = self.undecryptable_pkts.pop_front() {
2949
3027
  if let Err(e) = self.recv(&mut pkt, info) {
2950
3028
  self.undecryptable_pkts.clear();
2951
3029
 
@@ -2975,7 +3053,7 @@ impl<F: BufFactory> Connection<F> {
2975
3053
  &buf[buf_len - token_len..buf_len],
2976
3054
  )
2977
3055
  .is_ok()
2978
- },
3056
+ }
2979
3057
 
2980
3058
  None => false,
2981
3059
  }
@@ -2996,7 +3074,10 @@ impl<F: BufFactory> Connection<F> {
2996
3074
  ///
2997
3075
  /// [`Done`]: enum.Error.html#variant.Done
2998
3076
  fn recv_single(
2999
- &mut self, buf: &mut [u8], info: &RecvInfo, recv_pid: Option<usize>,
3077
+ &mut self,
3078
+ buf: &mut [u8],
3079
+ info: &RecvInfo,
3080
+ recv_pid: Option<usize>,
3000
3081
  ) -> Result<usize> {
3001
3082
  let now = Instant::now();
3002
3083
 
@@ -3019,14 +3100,7 @@ impl<F: BufFactory> Connection<F> {
3019
3100
  let mut b = octets::OctetsMut::with_slice(buf);
3020
3101
 
3021
3102
  let mut hdr = Header::from_bytes(&mut b, self.source_id().len())
3022
- .map_err(|e| {
3023
- drop_pkt_on_err(
3024
- e,
3025
- self.recv_count,
3026
- self.is_server,
3027
- &self.trace_id,
3028
- )
3029
- })?;
3103
+ .map_err(|e| drop_pkt_on_err(e, self.recv_count, self.is_server, &self.trace_id))?;
3030
3104
 
3031
3105
  if hdr.ty == Type::VersionNegotiation {
3032
3106
  // Version negotiation packets can only be sent by the server.
@@ -3063,8 +3137,7 @@ impl<F: BufFactory> Connection<F> {
3063
3137
  return Err(Error::Done);
3064
3138
  }
3065
3139
 
3066
- let supported_versions =
3067
- versions.iter().filter(|&&v| version_is_supported(v));
3140
+ let supported_versions = versions.iter().filter(|&&v| version_is_supported(v));
3068
3141
 
3069
3142
  let mut found_version = false;
3070
3143
 
@@ -3131,13 +3204,7 @@ impl<F: BufFactory> Connection<F> {
3131
3204
  }
3132
3205
 
3133
3206
  // Check if Retry packet is valid.
3134
- if packet::verify_retry_integrity(
3135
- &b,
3136
- &self.destination_id(),
3137
- self.version,
3138
- )
3139
- .is_err()
3140
- {
3207
+ if packet::verify_retry_integrity(&b, &self.destination_id(), self.version).is_err() {
3141
3208
  return Err(Error::Done);
3142
3209
  }
3143
3210
 
@@ -3149,21 +3216,13 @@ impl<F: BufFactory> Connection<F> {
3149
3216
  // Remember peer's new connection ID.
3150
3217
  self.odcid = Some(self.destination_id().into_owned());
3151
3218
 
3152
- self.set_initial_dcid(
3153
- hdr.scid.clone(),
3154
- None,
3155
- self.paths.get_active_path_id()?,
3156
- )?;
3219
+ self.set_initial_dcid(hdr.scid.clone(), None, self.paths.get_active_path_id()?)?;
3157
3220
 
3158
3221
  self.rscid = Some(self.destination_id().into_owned());
3159
3222
 
3160
3223
  // Derive Initial secrets using the new connection ID.
3161
- let (aead_open, aead_seal) = crypto::derive_initial_key_material(
3162
- &hdr.scid,
3163
- self.version,
3164
- self.is_server,
3165
- true,
3166
- )?;
3224
+ let (aead_open, aead_seal) =
3225
+ crypto::derive_initial_key_material(&hdr.scid, self.version, self.is_server, true)?;
3167
3226
 
3168
3227
  // Reset connection state to force sending another Initial packet.
3169
3228
  self.drop_epoch_state(packet::Epoch::Initial, now);
@@ -3204,12 +3263,7 @@ impl<F: BufFactory> Connection<F> {
3204
3263
  b.cap()
3205
3264
  } else {
3206
3265
  b.get_varint().map_err(|e| {
3207
- drop_pkt_on_err(
3208
- e.into(),
3209
- self.recv_count,
3210
- self.is_server,
3211
- &self.trace_id,
3212
- )
3266
+ drop_pkt_on_err(e.into(), self.recv_count, self.is_server, &self.trace_id)
3213
3267
  })? as usize
3214
3268
  };
3215
3269
 
@@ -3256,9 +3310,9 @@ impl<F: BufFactory> Connection<F> {
3256
3310
  Some(v) => v,
3257
3311
 
3258
3312
  None => {
3259
- if hdr.ty == Type::ZeroRTT &&
3260
- self.undecryptable_pkts.len() < MAX_UNDECRYPTABLE_PACKETS &&
3261
- !self.is_established()
3313
+ if hdr.ty == Type::ZeroRTT
3314
+ && self.undecryptable_pkts.len() < MAX_UNDECRYPTABLE_PACKETS
3315
+ && !self.is_established()
3262
3316
  {
3263
3317
  // Buffer 0-RTT packets when the required read key is not
3264
3318
  // available yet, and process them later.
@@ -3280,14 +3334,13 @@ impl<F: BufFactory> Connection<F> {
3280
3334
  );
3281
3335
 
3282
3336
  return Err(e);
3283
- },
3337
+ }
3284
3338
  };
3285
3339
 
3286
3340
  let aead_tag_len = aead.alg().tag_len();
3287
3341
 
3288
- packet::decrypt_hdr(&mut b, &mut hdr, aead).map_err(|e| {
3289
- drop_pkt_on_err(e, self.recv_count, self.is_server, &self.trace_id)
3290
- })?;
3342
+ packet::decrypt_hdr(&mut b, &mut hdr, aead)
3343
+ .map_err(|e| drop_pkt_on_err(e, self.recv_count, self.is_server, &self.trace_id))?;
3291
3344
 
3292
3345
  let pn = packet::decode_pkt_num(
3293
3346
  self.pkt_num_spaces[epoch].largest_rx_pkt_num,
@@ -3312,17 +3365,12 @@ impl<F: BufFactory> Connection<F> {
3312
3365
  // Check for key update.
3313
3366
  let mut aead_next = None;
3314
3367
 
3315
- if self.handshake_confirmed &&
3316
- hdr.ty != Type::ZeroRTT &&
3317
- hdr.key_phase != self.key_phase
3318
- {
3368
+ if self.handshake_confirmed && hdr.ty != Type::ZeroRTT && hdr.key_phase != self.key_phase {
3319
3369
  // Check if this packet arrived before key update.
3320
3370
  if let Some(key_update) = self.crypto_ctx[epoch]
3321
3371
  .key_update
3322
3372
  .as_ref()
3323
- .and_then(|key_update| {
3324
- (pn < key_update.pn_on_update).then_some(key_update)
3325
- })
3373
+ .and_then(|key_update| (pn < key_update.pn_on_update).then_some(key_update))
3326
3374
  {
3327
3375
  aead = &key_update.crypto_open;
3328
3376
  } else {
@@ -3341,22 +3389,14 @@ impl<F: BufFactory> Connection<F> {
3341
3389
  .derive_next_packet_key()?,
3342
3390
  ));
3343
3391
 
3344
- // `aead_next` is always `Some()` at this point, so the `unwrap()`
3345
- // will never fail.
3392
+ // `aead_next` is always `Some` at this point, so the
3393
+ // `unwrap()` will never fail.
3346
3394
  aead = &aead_next.as_ref().unwrap().0;
3347
3395
  }
3348
3396
  }
3349
3397
 
3350
- let mut payload = packet::decrypt_pkt(
3351
- &mut b,
3352
- pn,
3353
- pn_len,
3354
- payload_len,
3355
- aead,
3356
- )
3357
- .map_err(|e| {
3358
- drop_pkt_on_err(e, self.recv_count, self.is_server, &self.trace_id)
3359
- })?;
3398
+ let mut payload = packet::decrypt_pkt(&mut b, pn, pn_len, payload_len, aead)
3399
+ .map_err(|e| drop_pkt_on_err(e, self.recv_count, self.is_server, &self.trace_id))?;
3360
3400
 
3361
3401
  if self.pkt_num_spaces[epoch].recv_pkt_num.contains(pn) {
3362
3402
  trace!("{} ignored duplicate packet {}", self.trace_id, pn);
@@ -3411,25 +3451,21 @@ impl<F: BufFactory> Connection<F> {
3411
3451
  self.key_phase = !self.key_phase;
3412
3452
 
3413
3453
  qlog_with_type!(QLOG_PACKET_RX, self.qlog, q, {
3414
- let trigger = Some(
3415
- qlog::events::quic::KeyUpdateOrRetiredTrigger::RemoteUpdate,
3416
- );
3454
+ let trigger = Some(qlog::events::quic::KeyUpdateOrRetiredTrigger::RemoteUpdate);
3417
3455
 
3418
- let ev_data_client =
3419
- EventData::QuicKeyUpdated(qlog::events::quic::KeyUpdated {
3420
- key_type: qlog::events::quic::KeyType::Client1RttSecret,
3421
- trigger: trigger.clone(),
3422
- ..Default::default()
3423
- });
3456
+ let ev_data_client = EventData::QuicKeyUpdated(qlog::events::quic::KeyUpdated {
3457
+ key_type: qlog::events::quic::KeyType::Client1RttSecret,
3458
+ trigger: trigger.clone(),
3459
+ ..Default::default()
3460
+ });
3424
3461
 
3425
3462
  q.add_event_data_with_instant(ev_data_client, now).ok();
3426
3463
 
3427
- let ev_data_server =
3428
- EventData::QuicKeyUpdated(qlog::events::quic::KeyUpdated {
3429
- key_type: qlog::events::quic::KeyType::Server1RttSecret,
3430
- trigger,
3431
- ..Default::default()
3432
- });
3464
+ let ev_data_server = EventData::QuicKeyUpdated(qlog::events::quic::KeyUpdated {
3465
+ key_type: qlog::events::quic::KeyType::Server1RttSecret,
3466
+ trigger,
3467
+ ..Default::default()
3468
+ });
3433
3469
 
3434
3470
  q.add_event_data_with_instant(ev_data_server, now).ok();
3435
3471
  });
@@ -3456,8 +3492,7 @@ impl<F: BufFactory> Connection<F> {
3456
3492
 
3457
3493
  if !self.did_retry {
3458
3494
  self.local_transport_params
3459
- .original_destination_connection_id =
3460
- Some(hdr.dcid.to_vec().into());
3495
+ .original_destination_connection_id = Some(hdr.dcid.to_vec().into());
3461
3496
 
3462
3497
  self.encode_transport_params()?;
3463
3498
  }
@@ -3494,8 +3529,7 @@ impl<F: BufFactory> Connection<F> {
3494
3529
  probing = false;
3495
3530
  }
3496
3531
 
3497
- if let Err(e) = self.process_frame(frame, &hdr, recv_pid, epoch, now)
3498
- {
3532
+ if let Err(e) = self.process_frame(frame, &hdr, recv_pid, epoch, now) {
3499
3533
  frame_processing_err = Some(e);
3500
3534
  break;
3501
3535
  }
@@ -3518,14 +3552,12 @@ impl<F: BufFactory> Connection<F> {
3518
3552
  data: None,
3519
3553
  };
3520
3554
 
3521
- let ev_data = EventData::QuicPacketReceived(
3522
- qlog::events::quic::PacketReceived {
3523
- header: qlog_pkt_hdr,
3524
- frames: Some(qlog_frames),
3525
- raw: Some(qlog_raw_info),
3526
- ..Default::default()
3527
- },
3528
- );
3555
+ let ev_data = EventData::QuicPacketReceived(qlog::events::quic::PacketReceived {
3556
+ header: qlog_pkt_hdr,
3557
+ frames: Some(qlog_frames),
3558
+ raw: Some(qlog_raw_info),
3559
+ ..Default::default()
3560
+ });
3529
3561
 
3530
3562
  q.add_event_data_with_instant(ev_data, now).ok();
3531
3563
  });
@@ -3546,10 +3578,9 @@ impl<F: BufFactory> Connection<F> {
3546
3578
  if self.is_established() {
3547
3579
  qlog_with_type!(QLOG_PARAMS_SET, self.qlog, q, {
3548
3580
  if !self.qlog.logged_peer_params {
3549
- let ev_data = self.peer_transport_params.to_qlog(
3550
- TransportInitiator::Remote,
3551
- self.handshake.cipher(),
3552
- );
3581
+ let ev_data = self
3582
+ .peer_transport_params
3583
+ .to_qlog(TransportInitiator::Remote, self.handshake.cipher());
3553
3584
 
3554
3585
  q.add_event_data_with_instant(ev_data, now).ok();
3555
3586
 
@@ -3560,54 +3591,54 @@ impl<F: BufFactory> Connection<F> {
3560
3591
 
3561
3592
  // Process acked frames. Note that several packets from several paths
3562
3593
  // might have been acked by the received packet.
3563
- for (_, p) in self.paths.iter_mut() {
3594
+ let (paths, path_events) = self.paths.iter_mut_and_events();
3595
+ for (_, p) in paths {
3564
3596
  while let Some(acked) = p.recovery.next_acked_frame(epoch) {
3565
3597
  match acked {
3566
3598
  frame::Frame::Ping {
3567
3599
  mtu_probe: Some(mtu_probe),
3568
3600
  } => {
3601
+ trace!(
3602
+ "{} pmtud probe acked; probe size {:?}",
3603
+ self.trace_id,
3604
+ mtu_probe
3605
+ );
3606
+
3607
+ let local = p.local_addr();
3608
+ let peer = p.peer_addr();
3569
3609
  if let Some(pmtud) = p.pmtud.as_mut() {
3570
- trace!(
3571
- "{} pmtud probe acked; probe size {:?}",
3572
- self.trace_id,
3573
- mtu_probe
3574
- );
3610
+ let old_pmtu = pmtud.get_current_mtu();
3611
+ let current_mtu = pmtud.successful_probe(mtu_probe);
3612
+ let new_pmtu = pmtud.get_current_mtu();
3575
3613
 
3576
- // Ensure the probe is within the supported MTU range
3577
- // before updating the max datagram size
3578
- if let Some(current_mtu) =
3579
- pmtud.successful_probe(mtu_probe)
3580
- {
3614
+ // Update the datagram size only after validating
3615
+ // the MTU.
3616
+ if let Some(current_mtu) = current_mtu {
3581
3617
  qlog_with_type!(
3582
- EventType::QuicEventType(
3583
- QuicEventType::MtuUpdated
3584
- ),
3618
+ EventType::QuicEventType(QuicEventType::MtuUpdated),
3585
3619
  self.qlog,
3586
3620
  q,
3587
3621
  {
3588
3622
  let pmtu_data = EventData::QuicMtuUpdated(
3589
3623
  qlog::events::quic::MtuUpdated {
3590
- old: Some(
3591
- p.recovery.max_datagram_size()
3592
- as u32,
3593
- ),
3624
+ old: Some(p.recovery.max_datagram_size() as u32),
3594
3625
  new: current_mtu as u32,
3595
3626
  done: Some(true),
3596
3627
  },
3597
3628
  );
3598
3629
 
3599
- q.add_event_data_with_instant(
3600
- pmtu_data, now,
3601
- )
3602
- .ok();
3630
+ q.add_event_data_with_instant(pmtu_data, now).ok();
3603
3631
  }
3604
3632
  );
3605
3633
 
3606
- p.recovery
3607
- .pmtud_update_max_datagram_size(current_mtu);
3634
+ p.recovery.pmtud_update_max_datagram_size(current_mtu);
3635
+ }
3636
+
3637
+ if let Some(event) = path::pmtu_event(local, peer, old_pmtu, new_pmtu) {
3638
+ path_events.push_back(event);
3608
3639
  }
3609
3640
  }
3610
- },
3641
+ }
3611
3642
 
3612
3643
  frame::Frame::ACK { ranges, .. } => {
3613
3644
  // Stop acknowledging packets less than or equal to the
@@ -3618,27 +3649,24 @@ impl<F: BufFactory> Connection<F> {
3618
3649
  .recv_pkt_need_ack
3619
3650
  .remove_until(largest_acked);
3620
3651
  }
3621
- },
3652
+ }
3622
3653
 
3623
3654
  frame::Frame::CryptoHeader { offset, length } => {
3624
3655
  self.crypto_ctx[epoch]
3625
3656
  .crypto_stream
3626
3657
  .send
3627
3658
  .ack_and_drop(offset, length);
3628
- },
3659
+ }
3629
3660
 
3630
3661
  frame::Frame::StreamHeader {
3631
3662
  stream_id,
3632
3663
  offset,
3633
3664
  length,
3634
- ..
3665
+ fin,
3635
3666
  } => {
3636
- // Update tx_buffered and emit qlog before checking if the
3637
- // stream still exists. The client does need to ACK
3638
- // frames that were received after the client sends a
3639
- // ResetStream.
3640
- self.tx_buffered =
3641
- self.tx_buffered.saturating_sub(length);
3667
+ // Emit qlog before checking if the stream still exists.
3668
+ // The client does need to ACK frames that were received
3669
+ // after the client sends a ResetStream.
3642
3670
 
3643
3671
  qlog_with_type!(QLOG_DATA_MV, self.qlog, q, {
3644
3672
  let ev_data = EventData::QuicStreamDataMoved(
@@ -3664,41 +3692,21 @@ impl<F: BufFactory> Connection<F> {
3664
3692
  None => continue,
3665
3693
  };
3666
3694
 
3667
- stream.send.ack_and_drop(offset, length);
3695
+ let dropped = stream.send.ack_and_drop(offset, length);
3668
3696
 
3669
- let priority_key = Arc::clone(&stream.priority_key);
3697
+ if fin {
3698
+ stream.send.ack_fin();
3699
+ }
3670
3700
 
3671
- // Only collect the stream if it is complete and not
3672
- // readable or writable.
3673
- //
3674
- // If it is readable, it will get collected when
3675
- // stream_recv() is next used.
3676
- //
3677
- // If it is writable, it might mean that the stream
3678
- // has been stopped by the peer (i.e. a STOP_SENDING
3679
- // frame is received), in which case before collecting
3680
- // the stream we will need to propagate the
3681
- // `StreamStopped` error to the application. It will
3682
- // instead get collected when one of stream_capacity(),
3683
- // stream_writable(), stream_send(), ... is next called.
3684
- //
3685
- // Note that we can't use `is_writable()` here because
3686
- // it returns false if the stream is stopped. Instead,
3687
- // since the stream is marked as writable when a
3688
- // STOP_SENDING frame is received, we check the writable
3689
- // queue directly instead.
3690
- let is_writable = priority_key.writable.is_linked() &&
3691
- // Ensure that the stream is actually stopped.
3692
- stream.send.is_stopped();
3693
-
3694
- let is_complete = stream.is_complete();
3695
- let is_readable = stream.is_readable();
3696
-
3697
- if is_complete && !is_readable && !is_writable {
3698
- let local = stream.local;
3699
- self.streams.collect(stream_id, local);
3701
+ self.streams.try_collect(stream_id);
3702
+
3703
+ // Update `tx_buffered` for data dropped from stream
3704
+ // buffers, such as retransmission data acknowledged
3705
+ // before it could be resent.
3706
+ if dropped > 0 {
3707
+ self.streams.sub_tx_buffered(dropped);
3700
3708
  }
3701
- },
3709
+ }
3702
3710
 
3703
3711
  frame::Frame::HandshakeDone => {
3704
3712
  // Explicitly set this to true, so that if the frame was
@@ -3706,48 +3714,15 @@ impl<F: BufFactory> Connection<F> {
3706
3714
  self.handshake_done_sent = true;
3707
3715
 
3708
3716
  self.handshake_done_acked = true;
3709
- },
3717
+ }
3710
3718
 
3711
3719
  frame::Frame::ResetStream { stream_id, .. } => {
3712
- let stream = match self.streams.get_mut(stream_id) {
3713
- Some(v) => v,
3714
-
3715
- None => continue,
3716
- };
3717
-
3718
- let priority_key = Arc::clone(&stream.priority_key);
3719
-
3720
- // Only collect the stream if it is complete and not
3721
- // readable or writable.
3722
- //
3723
- // If it is readable, it will get collected when
3724
- // stream_recv() is next used.
3725
- //
3726
- // If it is writable, it might mean that the stream
3727
- // has been stopped by the peer (i.e. a STOP_SENDING
3728
- // frame is received), in which case before collecting
3729
- // the stream we will need to propagate the
3730
- // `StreamStopped` error to the application. It will
3731
- // instead get collected when one of stream_capacity(),
3732
- // stream_writable(), stream_send(), ... is next called.
3733
- //
3734
- // Note that we can't use `is_writable()` here because
3735
- // it returns false if the stream is stopped. Instead,
3736
- // since the stream is marked as writable when a
3737
- // STOP_SENDING frame is received, we check the writable
3738
- // queue directly instead.
3739
- let is_writable = priority_key.writable.is_linked() &&
3740
- // Ensure that the stream is actually stopped.
3741
- stream.send.is_stopped();
3742
-
3743
- let is_complete = stream.is_complete();
3744
- let is_readable = stream.is_readable();
3745
-
3746
- if is_complete && !is_readable && !is_writable {
3747
- let local = stream.local;
3748
- self.streams.collect(stream_id, local);
3720
+ if let Some(stream) = self.streams.get_mut(stream_id) {
3721
+ stream.send.ack_reset_stream();
3749
3722
  }
3750
- },
3723
+ self.streams.remove_reset(stream_id);
3724
+ self.streams.try_collect(stream_id);
3725
+ }
3751
3726
 
3752
3727
  _ => (),
3753
3728
  }
@@ -3802,9 +3777,9 @@ impl<F: BufFactory> Connection<F> {
3802
3777
  // Did the peer migrated to another path?
3803
3778
  let active_path_id = self.paths.get_active_path_id()?;
3804
3779
 
3805
- if self.is_server &&
3806
- recv_pid != active_path_id &&
3807
- self.pkt_num_spaces[epoch].largest_rx_non_probing_pkt_num == pn
3780
+ if self.is_server
3781
+ && recv_pid != active_path_id
3782
+ && self.pkt_num_spaces[epoch].largest_rx_non_probing_pkt_num == pn
3808
3783
  {
3809
3784
  self.on_peer_migrated(recv_pid, self.disable_dcid_reuse, now)?;
3810
3785
  }
@@ -3988,7 +3963,9 @@ impl<F: BufFactory> Connection<F> {
3988
3963
  /// # Ok::<(), quiche::Error>(())
3989
3964
  /// ```
3990
3965
  pub fn send_on_path(
3991
- &mut self, out: &mut [u8], from: Option<SocketAddr>,
3966
+ &mut self,
3967
+ out: &mut [u8],
3968
+ from: Option<SocketAddr>,
3992
3969
  to: Option<SocketAddr>,
3993
3970
  ) -> Result<(usize, SendInfo)> {
3994
3971
  if out.is_empty() {
@@ -4001,6 +3978,14 @@ impl<F: BufFactory> Connection<F> {
4001
3978
 
4002
3979
  let now = Instant::now();
4003
3980
 
3981
+ if self.streams.collected_history_exhausted() {
3982
+ let _ = self.close(
3983
+ false,
3984
+ Error::StreamMemoryLimit.to_wire(),
3985
+ b"stream history limit",
3986
+ );
3987
+ }
3988
+
4004
3989
  if self.local_error.is_none() {
4005
3990
  self.do_handshake(now)?;
4006
3991
  }
@@ -4038,11 +4023,10 @@ impl<F: BufFactory> Connection<F> {
4038
4023
 
4039
4024
  let send_path = self.paths.get_mut(send_pid)?;
4040
4025
 
4041
- // Update max datagram size to allow path MTU discovery probe to be sent.
4026
+ // Increase the maximum datagram size for a PMTUD probe.
4042
4027
  if let Some(pmtud) = send_path.pmtud.as_mut() {
4043
4028
  if pmtud.should_probe() {
4044
- let size = if self.handshake_confirmed || self.handshake_completed
4045
- {
4029
+ let size = if self.handshake_confirmed || self.handshake_completed {
4046
4030
  pmtud.get_probe_size()
4047
4031
  } else {
4048
4032
  pmtud.get_current_mtu()
@@ -4050,8 +4034,7 @@ impl<F: BufFactory> Connection<F> {
4050
4034
 
4051
4035
  send_path.recovery.pmtud_update_max_datagram_size(size);
4052
4036
 
4053
- left =
4054
- cmp::min(out.len(), send_path.recovery.max_datagram_size());
4037
+ left = cmp::min(out.len(), send_path.recovery.max_datagram_size());
4055
4038
  }
4056
4039
  }
4057
4040
 
@@ -4063,18 +4046,14 @@ impl<F: BufFactory> Connection<F> {
4063
4046
 
4064
4047
  // Generate coalesced packets.
4065
4048
  while left > 0 {
4066
- let (ty, written) = match self.send_single(
4067
- &mut out[done..done + left],
4068
- send_pid,
4069
- has_initial,
4070
- now,
4071
- ) {
4072
- Ok(v) => v,
4049
+ let (ty, written) =
4050
+ match self.send_single(&mut out[done..done + left], send_pid, has_initial, now) {
4051
+ Ok(v) => v,
4073
4052
 
4074
- Err(Error::BufferTooShort) | Err(Error::Done) => break,
4053
+ Err(Error::BufferTooShort) | Err(Error::Done) => break,
4075
4054
 
4076
- Err(e) => return Err(e),
4077
- };
4055
+ Err(e) => return Err(e),
4056
+ };
4078
4057
 
4079
4058
  done += written;
4080
4059
  left -= written;
@@ -4097,9 +4076,7 @@ impl<F: BufFactory> Connection<F> {
4097
4076
  }
4098
4077
 
4099
4078
  // Don't coalesce packets that must go on different paths.
4100
- if !(from.is_some() && to.is_some()) &&
4101
- self.get_send_path_id(from, to)? != send_pid
4102
- {
4079
+ if !(from.is_some() && to.is_some()) && self.get_send_path_id(from, to)? != send_pid {
4103
4080
  break;
4104
4081
  }
4105
4082
  }
@@ -4133,7 +4110,10 @@ impl<F: BufFactory> Connection<F> {
4133
4110
  }
4134
4111
 
4135
4112
  fn send_single(
4136
- &mut self, out: &mut [u8], send_pid: usize, has_initial: bool,
4113
+ &mut self,
4114
+ out: &mut [u8],
4115
+ send_pid: usize,
4116
+ has_initial: bool,
4137
4117
  now: Instant,
4138
4118
  ) -> Result<(Type, usize)> {
4139
4119
  if out.is_empty() {
@@ -4144,14 +4124,61 @@ impl<F: BufFactory> Connection<F> {
4144
4124
  return Err(Error::Done);
4145
4125
  }
4146
4126
 
4127
+ if self.local_error.is_none() {
4128
+ for &epoch in packet::Epoch::epochs(packet::Epoch::Initial..=packet::Epoch::Application)
4129
+ {
4130
+ let has_probe = self.crypto_ctx[epoch].crypto_seal.is_some()
4131
+ && self.paths.get(send_pid)?.recovery.loss_probes(epoch) > 0;
4132
+
4133
+ if !has_probe {
4134
+ continue;
4135
+ }
4136
+
4137
+ match self.send_single_for_type(
4138
+ out,
4139
+ send_pid,
4140
+ has_initial,
4141
+ now,
4142
+ Type::from_epoch(epoch),
4143
+ true,
4144
+ ) {
4145
+ Err(Error::Done) => (),
4146
+
4147
+ result => return result,
4148
+ }
4149
+ }
4150
+ }
4151
+
4152
+ let pkt_type = self.write_pkt_type(send_pid)?;
4153
+ let epoch = pkt_type.to_epoch()?;
4154
+ let send_path = self.paths.get(send_pid)?;
4155
+
4156
+ // A closing connection sends a single packet with a CONNECTION_CLOSE
4157
+ // frame before draining, which must not be delayed when the congestion
4158
+ // window is full of unacknowledged packets.
4159
+ let sends_close =
4160
+ self.local_error.is_some() && (send_path.active() || self.paths.len() == 1);
4161
+
4162
+ let bypass_cwnd = send_path.recovery.loss_probes(epoch) > 0 || sends_close;
4163
+
4164
+ self.send_single_for_type(out, send_pid, has_initial, now, pkt_type, bypass_cwnd)
4165
+ }
4166
+
4167
+ fn send_single_for_type(
4168
+ &mut self,
4169
+ out: &mut [u8],
4170
+ send_pid: usize,
4171
+ has_initial: bool,
4172
+ now: Instant,
4173
+ pkt_type: Type,
4174
+ bypass_cwnd: bool,
4175
+ ) -> Result<(Type, usize)> {
4147
4176
  let is_closing = self.local_error.is_some();
4148
4177
 
4149
4178
  let out_len = out.len();
4150
4179
 
4151
4180
  let mut b = octets::OctetsMut::with_slice(out);
4152
4181
 
4153
- let pkt_type = self.write_pkt_type(send_pid)?;
4154
-
4155
4182
  let max_dgram_len = if !self.dgram_send_queue.is_empty() {
4156
4183
  self.dgram_max_writable_len()
4157
4184
  } else {
@@ -4163,7 +4190,8 @@ impl<F: BufFactory> Connection<F> {
4163
4190
  let crypto_ctx = &mut self.crypto_ctx[epoch];
4164
4191
 
4165
4192
  // Process lost frames. There might be several paths having lost frames.
4166
- for (_, p) in self.paths.iter_mut() {
4193
+ let (paths, path_events) = self.paths.iter_mut_and_events();
4194
+ for (_, p) in paths {
4167
4195
  while let Some(lost) = p.recovery.next_lost_frame(epoch) {
4168
4196
  match lost {
4169
4197
  frame::Frame::CryptoHeader { offset, length } => {
@@ -4174,7 +4202,7 @@ impl<F: BufFactory> Connection<F> {
4174
4202
 
4175
4203
  self.retrans_count += 1;
4176
4204
  p.retrans_count += 1;
4177
- },
4205
+ }
4178
4206
 
4179
4207
  frame::Frame::StreamHeader {
4180
4208
  stream_id,
@@ -4184,16 +4212,13 @@ impl<F: BufFactory> Connection<F> {
4184
4212
  } => {
4185
4213
  let stream = match self.streams.get_mut(stream_id) {
4186
4214
  // Only retransmit data if the stream is not closed
4187
- // or stopped.
4188
- Some(v) if !v.send.is_stopped() => v,
4215
+ // or reset.
4216
+ Some(v) if v.send.can_send_data() => v,
4189
4217
 
4190
4218
  // Data on a closed stream will not be retransmitted
4191
- // or acked after it is declared lost, so update
4192
- // tx_buffered and qlog.
4219
+ // or acked after it is declared lost, so just drop
4220
+ // it.
4193
4221
  _ => {
4194
- self.tx_buffered =
4195
- self.tx_buffered.saturating_sub(length);
4196
-
4197
4222
  qlog_with_type!(QLOG_DATA_MV, self.qlog, q, {
4198
4223
  let ev_data = EventData::QuicStreamDataMoved(
4199
4224
  qlog::events::quic::StreamDataMoved {
@@ -4209,19 +4234,18 @@ impl<F: BufFactory> Connection<F> {
4209
4234
  },
4210
4235
  );
4211
4236
 
4212
- q.add_event_data_with_instant(ev_data, now)
4213
- .ok();
4237
+ q.add_event_data_with_instant(ev_data, now).ok();
4214
4238
  });
4215
4239
 
4216
4240
  continue;
4217
- },
4241
+ }
4218
4242
  };
4219
4243
 
4220
4244
  let was_flushable = stream.is_flushable();
4221
4245
 
4222
4246
  let empty_fin = length == 0 && fin;
4223
4247
 
4224
- stream.send.retransmit(offset, length);
4248
+ let retransmitted = stream.send.retransmit(offset, length);
4225
4249
 
4226
4250
  // If the stream is now flushable push it to the
4227
4251
  // flushable queue, but only if it wasn't already
@@ -4230,31 +4254,37 @@ impl<F: BufFactory> Connection<F> {
4230
4254
  // Consider the stream flushable also when we are
4231
4255
  // sending a zero-length frame that has the fin flag
4232
4256
  // set.
4233
- if (stream.is_flushable() || empty_fin) && !was_flushable
4234
- {
4257
+ if (stream.is_flushable() || empty_fin) && !was_flushable {
4235
4258
  let priority_key = Arc::clone(&stream.priority_key);
4236
4259
  self.streams.insert_flushable(&priority_key);
4237
4260
  }
4238
4261
 
4262
+ // Update tx_buffered when data is marked for
4263
+ // retransmission (it was decremented when emitted).
4264
+ // Only increment by the actual amount retransmitted,
4265
+ // which may be less than `length` if some data was
4266
+ // already acked.
4267
+ self.streams.add_tx_buffered(retransmitted);
4268
+
4239
4269
  self.stream_retrans_bytes += length as u64;
4240
4270
  p.stream_retrans_bytes += length as u64;
4241
4271
 
4242
4272
  self.retrans_count += 1;
4243
4273
  p.retrans_count += 1;
4244
- },
4274
+ }
4245
4275
 
4246
4276
  frame::Frame::ACK { .. } => {
4247
4277
  pkt_space.ack_elicited = true;
4248
- },
4278
+ }
4249
4279
 
4250
4280
  frame::Frame::ResetStream {
4251
4281
  stream_id,
4252
4282
  error_code,
4253
4283
  final_size,
4284
+ ..
4254
4285
  } => {
4255
- self.streams
4256
- .insert_reset(stream_id, error_code, final_size);
4257
- },
4286
+ self.streams.insert_reset(stream_id, error_code, final_size);
4287
+ }
4258
4288
 
4259
4289
  frame::Frame::StopSending {
4260
4290
  stream_id,
@@ -4264,37 +4294,39 @@ impl<F: BufFactory> Connection<F> {
4264
4294
  // the stream is still active and not FIN'd. Even if the
4265
4295
  // packet was lost, if the application has the final
4266
4296
  // size at this point there is no need to retransmit.
4297
+ {
4267
4298
  if let Some(stream) = self.streams.get(stream_id) {
4268
4299
  if !stream.recv.is_fin() {
4269
- self.streams
4270
- .insert_stopped(stream_id, error_code);
4300
+ self.streams.insert_stopped(stream_id, error_code);
4271
4301
  }
4272
- },
4302
+ }
4303
+ }
4273
4304
 
4274
4305
  // Retransmit HANDSHAKE_DONE only if it hasn't been acked at
4275
4306
  // least once already.
4276
- frame::Frame::HandshakeDone =>
4307
+ frame::Frame::HandshakeDone => {
4277
4308
  if !self.handshake_done_acked {
4278
4309
  self.handshake_done_sent = false;
4279
- },
4310
+ }
4311
+ }
4280
4312
 
4281
4313
  frame::Frame::MaxStreamData { stream_id, .. } => {
4282
4314
  if self.streams.get(stream_id).is_some() {
4283
4315
  self.streams.insert_almost_full(stream_id);
4284
4316
  }
4285
- },
4317
+ }
4286
4318
 
4287
4319
  frame::Frame::MaxData { .. } => {
4288
4320
  self.should_send_max_data = true;
4289
- },
4321
+ }
4290
4322
 
4291
4323
  frame::Frame::MaxStreamsUni { .. } => {
4292
4324
  self.should_send_max_streams_uni = true;
4293
- },
4325
+ }
4294
4326
 
4295
4327
  frame::Frame::MaxStreamsBidi { .. } => {
4296
4328
  self.should_send_max_streams_bidi = true;
4297
- },
4329
+ }
4298
4330
 
4299
4331
  // Retransmit STREAMS_BLOCKED frames if the frame with the
4300
4332
  // most recent limit is lost. These are informational
@@ -4304,30 +4336,42 @@ impl<F: BufFactory> Connection<F> {
4304
4336
  frame::Frame::StreamsBlockedBidi { limit } => {
4305
4337
  self.streams_blocked_bidi_state
4306
4338
  .force_retransmit_sent_limit_eq(limit);
4307
- },
4339
+ }
4308
4340
 
4309
4341
  frame::Frame::StreamsBlockedUni { limit } => {
4310
4342
  self.streams_blocked_uni_state
4311
4343
  .force_retransmit_sent_limit_eq(limit);
4312
- },
4344
+ }
4313
4345
 
4314
4346
  frame::Frame::NewConnectionId { seq_num, .. } => {
4315
4347
  self.ids.mark_advertise_new_scid_seq(seq_num, true);
4316
- },
4348
+ }
4317
4349
 
4318
4350
  frame::Frame::RetireConnectionId { seq_num } => {
4319
4351
  self.ids.mark_retire_dcid_seq(seq_num, true)?;
4320
- },
4352
+ }
4321
4353
 
4322
4354
  frame::Frame::Ping { mtu_probe } => {
4323
4355
  // Ping frames are not retransmitted.
4324
4356
  if let Some(failed_probe) = mtu_probe {
4357
+ trace!("pmtud probe dropped: {failed_probe}");
4358
+
4359
+ let local = p.local_addr();
4360
+ let peer = p.peer_addr();
4325
4361
  if let Some(pmtud) = p.pmtud.as_mut() {
4326
- trace!("pmtud probe dropped: {failed_probe}");
4362
+ let old_pmtu = pmtud.get_current_mtu();
4327
4363
  pmtud.failed_probe(failed_probe);
4364
+ let new_pmtu = pmtud.get_current_mtu();
4365
+
4366
+ if let Some(event) =
4367
+ path::pmtu_event(local, peer, old_pmtu, new_pmtu)
4368
+ {
4369
+ p.recovery.pmtud_update_max_datagram_size(new_pmtu);
4370
+ path_events.push_back(event);
4371
+ }
4328
4372
  }
4329
4373
  }
4330
- },
4374
+ }
4331
4375
 
4332
4376
  // Sent as StreamHeader frames. Stream frames are never
4333
4377
  // generated by quiche.
@@ -4337,7 +4381,7 @@ impl<F: BufFactory> Connection<F> {
4337
4381
  have tracked retransmittable stream data as \
4338
4382
  StreamHeader frames."
4339
4383
  );
4340
- },
4384
+ }
4341
4385
 
4342
4386
  // Sent as CryptoHeader frames. Crypto frames are never
4343
4387
  // generated by quiche.
@@ -4347,7 +4391,7 @@ impl<F: BufFactory> Connection<F> {
4347
4391
  have tracked retransmittable crypto data as \
4348
4392
  CryptoHeader frames."
4349
4393
  );
4350
- },
4394
+ }
4351
4395
 
4352
4396
  // NewToken frames are never sent by quiche; they are not
4353
4397
  // implemented.
@@ -4357,13 +4401,12 @@ impl<F: BufFactory> Connection<F> {
4357
4401
  not implement NewToken frames, retransmission of \
4358
4402
  these frames is not implemented."
4359
4403
  );
4360
- },
4404
+ }
4361
4405
 
4362
4406
  // Data blocked frames are an optional advisory
4363
4407
  // signal. We choose to not retransmit them to
4364
4408
  // avoid unnecessary network usage.
4365
- frame::Frame::DataBlocked { .. } |
4366
- frame::Frame::StreamDataBlocked { .. } => (),
4409
+ frame::Frame::DataBlocked { .. } | frame::Frame::StreamDataBlocked { .. } => (),
4367
4410
 
4368
4411
  // Path challenge and response have their own
4369
4412
  // retry logic. They should not be retransmitted
@@ -4371,32 +4414,31 @@ impl<F: BufFactory> Connection<F> {
4371
4414
  // 8.2.2: "An endpoint MUST NOT send more than one
4372
4415
  // PATH_RESPONSE frame in response to one
4373
4416
  // PATH_CHALLENGE frame".
4374
- frame::Frame::PathChallenge { .. } |
4375
- frame::Frame::PathResponse { .. } => (),
4417
+ frame::Frame::PathChallenge { .. } | frame::Frame::PathResponse { .. } => (),
4376
4418
 
4377
4419
  // From RFC 9000 Section 13.3: CONNECTION_CLOSE
4378
4420
  // frames, are not sent again when packet loss is
4379
4421
  // detected. Resending these signals is described
4380
4422
  // in Section 10.
4381
- frame::Frame::ConnectionClose { .. } |
4382
- frame::Frame::ApplicationClose { .. } => (),
4423
+ frame::Frame::ConnectionClose { .. }
4424
+ | frame::Frame::ApplicationClose { .. } => (),
4383
4425
 
4384
4426
  // Padding doesn't require retransmission.
4385
4427
  frame::Frame::Padding { .. } => (),
4386
4428
 
4387
- frame::Frame::DatagramHeader { .. } |
4388
- frame::Frame::Datagram { .. } => {
4429
+ frame::Frame::DatagramHeader { .. } | frame::Frame::Datagram { .. } => {
4389
4430
  // Datagrams do not require retransmission. Just update
4390
4431
  // stats.
4391
4432
  p.dgram_lost_count = p.dgram_lost_count.saturating_add(1);
4392
- },
4393
- // IMPORTANT: Do not add an exhaustive catch
4394
- // all. We want to add explicit handling for frame
4395
- // types that can be safely ignored when lost.
4433
+ } // IMPORTANT: Do not add an exhaustive catch
4434
+ // all. We want to add explicit handling for frame
4435
+ // types that can be safely ignored when lost.
4396
4436
  }
4397
4437
  }
4398
4438
  }
4399
- self.check_tx_buffered_invariant();
4439
+
4440
+ #[cfg(debug_assertions)]
4441
+ self.streams.debug_check_tx_buffered_consistency();
4400
4442
 
4401
4443
  let is_app_limited = self.delivery_rate_check_if_app_limited();
4402
4444
  let n_paths = self.paths.len();
@@ -4419,8 +4461,7 @@ impl<F: BufFactory> Connection<F> {
4419
4461
  };
4420
4462
  let pn = self.next_pkt_num;
4421
4463
 
4422
- let largest_acked_pkt =
4423
- path.recovery.get_largest_acked_on_epoch(epoch).unwrap_or(0);
4464
+ let largest_acked_pkt = path.recovery.get_largest_acked_on_epoch(epoch).unwrap_or(0);
4424
4465
  let pn_len = packet::pkt_num_len(pn, largest_acked_pkt);
4425
4466
 
4426
4467
  // The AEAD overhead at the current encryption level.
@@ -4428,8 +4469,7 @@ impl<F: BufFactory> Connection<F> {
4428
4469
 
4429
4470
  let dcid_seq = path.active_dcid_seq.ok_or(Error::OutOfIdentifiers)?;
4430
4471
 
4431
- let dcid =
4432
- ConnectionId::from_ref(self.ids.get_dcid(dcid_seq)?.cid.as_ref());
4472
+ let dcid = ConnectionId::from_ref(self.ids.get_dcid(dcid_seq)?.cid.as_ref());
4433
4473
 
4434
4474
  let scid = if let Some(scid_seq) = path.active_scid_seq {
4435
4475
  ConnectionId::from_ref(self.ids.get_scid(scid_seq)?.cid.as_ref())
@@ -4507,7 +4547,7 @@ impl<F: BufFactory> Connection<F> {
4507
4547
  // is set to false here to make cwnd grow when ACK is received.
4508
4548
  path.recovery.update_app_limited(false);
4509
4549
  return Err(Error::Done);
4510
- },
4550
+ }
4511
4551
  }
4512
4552
 
4513
4553
  // Make sure there is enough space for the minimum payload length.
@@ -4522,10 +4562,11 @@ impl<F: BufFactory> Connection<F> {
4522
4562
  let mut in_flight = false;
4523
4563
  let mut is_pmtud_probe = false;
4524
4564
  let mut has_data = false;
4565
+ let mut stream_data_skipped = false;
4525
4566
 
4526
- // Whether or not we should explicitly elicit an ACK via PING frame if we
4527
- // implicitly elicit one otherwise.
4528
- let ack_elicit_required = path.recovery.should_elicit_ack(epoch);
4567
+ // Whether a PING frame must explicitly elicit an ACK when no other
4568
+ // frame does so implicitly.
4569
+ let ack_elicit_required = !is_closing && path.recovery.should_elicit_ack(epoch);
4529
4570
 
4530
4571
  let header_offset = b.off();
4531
4572
 
@@ -4541,8 +4582,15 @@ impl<F: BufFactory> Connection<F> {
4541
4582
 
4542
4583
  let payload_offset = b.off();
4543
4584
 
4544
- let cwnd_available =
4545
- path.recovery.cwnd_available().saturating_sub(overhead);
4585
+ // `usize::MAX` bypasses cwnd when authorized by packet selection;
4586
+ // `left` still bounds the packet. Keep this local so other cwnd users
4587
+ // see the actual window.
4588
+ let send_limit_from_cwnd = if bypass_cwnd {
4589
+ usize::MAX
4590
+ } else {
4591
+ path.recovery.cwnd_available()
4592
+ }
4593
+ .saturating_sub(overhead);
4546
4594
 
4547
4595
  let left_before_packing_ack_frame = left;
4548
4596
 
@@ -4552,22 +4600,22 @@ impl<F: BufFactory> Connection<F> {
4552
4600
  // generate an ACK (if there's anything to ACK) since we're going to
4553
4601
  // send a packet with PING anyways, even if we haven't received anything
4554
4602
  // ACK eliciting.
4555
- if pkt_space.recv_pkt_need_ack.len() > 0 &&
4556
- (pkt_space.ack_elicited || ack_elicit_required) &&
4557
- (!is_closing ||
4558
- (pkt_type == Type::Handshake &&
4559
- self.local_error
4560
- .as_ref()
4561
- .is_some_and(|le| le.is_app))) &&
4562
- path.active()
4603
+ //
4604
+ // While closing, PING frames are suppressed, so ACK elicitation must
4605
+ // not generate ACK-only packets.
4606
+ if pkt_space.recv_pkt_need_ack.len() > 0
4607
+ && (pkt_space.ack_elicited || ack_elicit_required)
4608
+ && (!is_closing
4609
+ || (pkt_type == Type::Handshake
4610
+ && self.local_error.as_ref().is_some_and(|le| le.is_app)))
4611
+ && path.active()
4563
4612
  {
4564
4613
  #[cfg(not(feature = "fuzzing"))]
4565
4614
  let ack_delay = pkt_space.largest_rx_pkt_time.elapsed();
4566
4615
 
4567
4616
  #[cfg(not(feature = "fuzzing"))]
4568
- let ack_delay = ack_delay.as_micros() as u64 /
4569
- 2_u64
4570
- .pow(self.local_transport_params.ack_delay_exponent as u32);
4617
+ let ack_delay = ack_delay.as_micros() as u64
4618
+ / 2_u64.pow(self.local_transport_params.ack_delay_exponent as u32);
4571
4619
 
4572
4620
  // pseudo-random reproducible ack delays when fuzzing
4573
4621
  #[cfg(feature = "fuzzing")]
@@ -4583,7 +4631,7 @@ impl<F: BufFactory> Connection<F> {
4583
4631
  // there is not enough cwnd available for both (note that PING
4584
4632
  // frames are always 1 byte, so we just need to check that the
4585
4633
  // ACK's length is lower than cwnd).
4586
- if pkt_space.ack_elicited || frame.wire_len() < cwnd_available {
4634
+ if pkt_space.ack_elicited || frame.wire_len() < send_limit_from_cwnd {
4587
4635
  // ACK-only packets are not congestion controlled so ACKs must
4588
4636
  // be bundled considering the buffer capacity only, and not the
4589
4637
  // available cwnd.
@@ -4597,7 +4645,7 @@ impl<F: BufFactory> Connection<F> {
4597
4645
  left = cmp::min(
4598
4646
  left,
4599
4647
  // Bytes consumed by ACK frames.
4600
- cwnd_available.saturating_sub(left_before_packing_ack_frame - left),
4648
+ send_limit_from_cwnd.saturating_sub(left_before_packing_ack_frame - left),
4601
4649
  );
4602
4650
 
4603
4651
  let mut challenge_data = None;
@@ -4605,15 +4653,12 @@ impl<F: BufFactory> Connection<F> {
4605
4653
  if pkt_type == Type::Short {
4606
4654
  // Create PMTUD probe.
4607
4655
  //
4608
- // In order to send a PMTUD probe the current `left` value, which was
4609
- // already limited by the current PMTU measure, needs to be ignored,
4610
- // but the outgoing packet still needs to be limited by
4611
- // the output buffer size, as well as the congestion
4612
- // window.
4656
+ // A PMTUD probe must ignore `left`, which is already limited by the
4657
+ // current PMTU. The probe remains limited by the output buffer and
4658
+ // congestion window.
4613
4659
  //
4614
- // In addition, the PMTUD probe is only generated when the handshake
4615
- // is confirmed, to avoid interfering with the handshake
4616
- // (e.g. due to the anti-amplification limits).
4660
+ // Generate PMTUD probes only after handshake confirmation to avoid
4661
+ // interference from anti-amplification limits.
4617
4662
  if let Ok(active_path) = self.paths.get_active_mut() {
4618
4663
  let should_probe_pmtu = active_path.should_send_pmtu_probe(
4619
4664
  self.handshake_confirmed,
@@ -4627,11 +4672,11 @@ impl<F: BufFactory> Connection<F> {
4627
4672
  if let Some(pmtud) = active_path.pmtud.as_mut() {
4628
4673
  let probe_size = pmtud.get_probe_size();
4629
4674
  trace!(
4630
- "{} sending pmtud probe pmtu_probe={} estimated_pmtu={}",
4631
- self.trace_id,
4632
- probe_size,
4633
- pmtud.get_current_mtu(),
4634
- );
4675
+ "{} sending pmtud probe pmtu_probe={} estimated_pmtu={}",
4676
+ self.trace_id,
4677
+ probe_size,
4678
+ pmtud.get_current_mtu(),
4679
+ );
4635
4680
 
4636
4681
  left = probe_size;
4637
4682
 
@@ -4642,14 +4687,14 @@ impl<F: BufFactory> Connection<F> {
4642
4687
  // We can't send more because there isn't enough
4643
4688
  // space available in the output buffer.
4644
4689
  //
4645
- // This usually happens when we try to send a new
4646
- // packet but failed because cwnd is almost full.
4690
+ // The congestion window is nearly full. A new
4691
+ // packet does not fit.
4647
4692
  //
4648
- // In such case app_limited is set to false here
4649
- // to make cwnd grow when ACK is received.
4693
+ // Clear the app-limited state so ACKs can grow
4694
+ // the congestion window.
4650
4695
  active_path.recovery.update_app_limited(false);
4651
4696
  return Err(Error::Done);
4652
- },
4697
+ }
4653
4698
  }
4654
4699
 
4655
4700
  let frame = frame::Frame::Padding {
@@ -4733,10 +4778,7 @@ impl<F: BufFactory> Connection<F> {
4733
4778
  if pkt_type == Type::Short && !is_closing && path.active() {
4734
4779
  // Create HANDSHAKE_DONE frame.
4735
4780
  // self.should_send_handshake_done() but without the need to borrow
4736
- if self.handshake_completed &&
4737
- !self.handshake_done_sent &&
4738
- self.is_server
4739
- {
4781
+ if self.handshake_completed && !self.handshake_done_sent && self.is_server {
4740
4782
  let frame = frame::Frame::HandshakeDone;
4741
4783
 
4742
4784
  if push_frame_to_pkt!(b, frames, frame, left) {
@@ -4748,9 +4790,7 @@ impl<F: BufFactory> Connection<F> {
4748
4790
  }
4749
4791
 
4750
4792
  // Create MAX_STREAMS_BIDI frame.
4751
- if self.streams.should_update_max_streams_bidi() ||
4752
- self.should_send_max_streams_bidi
4753
- {
4793
+ if self.streams.should_update_max_streams_bidi() || self.should_send_max_streams_bidi {
4754
4794
  let frame = frame::Frame::MaxStreamsBidi {
4755
4795
  max: self.streams.max_streams_bidi_next(),
4756
4796
  };
@@ -4765,9 +4805,7 @@ impl<F: BufFactory> Connection<F> {
4765
4805
  }
4766
4806
 
4767
4807
  // Create MAX_STREAMS_UNI frame.
4768
- if self.streams.should_update_max_streams_uni() ||
4769
- self.should_send_max_streams_uni
4770
- {
4808
+ if self.streams.should_update_max_streams_uni() || self.should_send_max_streams_uni {
4771
4809
  let frame = frame::Frame::MaxStreamsUni {
4772
4810
  max: self.streams.max_streams_uni_next(),
4773
4811
  };
@@ -4787,8 +4825,7 @@ impl<F: BufFactory> Connection<F> {
4787
4825
 
4788
4826
  if push_frame_to_pkt!(b, frames, frame, left) {
4789
4827
  self.blocked_limit = None;
4790
- self.data_blocked_sent_count =
4791
- self.data_blocked_sent_count.saturating_add(1);
4828
+ self.data_blocked_sent_count = self.data_blocked_sent_count.saturating_add(1);
4792
4829
 
4793
4830
  ack_eliciting = true;
4794
4831
  in_flight = true;
@@ -4808,8 +4845,7 @@ impl<F: BufFactory> Connection<F> {
4808
4845
  // Record the limit we just notified the peer about so
4809
4846
  // that redundant frames for the same limit are
4810
4847
  // suppressed.
4811
- self.streams_blocked_bidi_state.blocked_sent =
4812
- Some(limit);
4848
+ self.streams_blocked_bidi_state.blocked_sent = Some(limit);
4813
4849
 
4814
4850
  ack_eliciting = true;
4815
4851
  in_flight = true;
@@ -4848,7 +4884,7 @@ impl<F: BufFactory> Connection<F> {
4848
4884
  // the almost full set.
4849
4885
  self.streams.remove_almost_full(stream_id);
4850
4886
  continue;
4851
- },
4887
+ }
4852
4888
  };
4853
4889
 
4854
4890
  // Autotune the stream window size, but only if this is not a
@@ -4883,8 +4919,8 @@ impl<F: BufFactory> Connection<F> {
4883
4919
  }
4884
4920
 
4885
4921
  // Create MAX_DATA frame as needed.
4886
- if flow_control.should_update_max_data() &&
4887
- flow_control.max_data() < flow_control.max_data_next()
4922
+ if flow_control.should_update_max_data()
4923
+ && flow_control.max_data() < flow_control.max_data_next()
4888
4924
  {
4889
4925
  // Autotune the connection window size. We only tune the window
4890
4926
  // if we are sending an "organic" update, not on retransmits.
@@ -4929,16 +4965,13 @@ impl<F: BufFactory> Connection<F> {
4929
4965
  }
4930
4966
 
4931
4967
  // Create RESET_STREAM frames as needed.
4932
- for (stream_id, (error_code, final_size)) in self
4933
- .streams
4934
- .reset()
4935
- .map(|(&k, &v)| (k, v))
4936
- .collect::<Vec<(u64, (u64, u64))>>()
4937
- {
4968
+ let resets: Vec<_> = self.streams.reset().map(|(&k, &v)| (k, v)).collect();
4969
+ for (stream_id, (error_code, final_size, reliable_size)) in resets {
4938
4970
  let frame = frame::Frame::ResetStream {
4939
4971
  stream_id,
4940
4972
  error_code,
4941
4973
  final_size,
4974
+ reliable_size,
4942
4975
  };
4943
4976
 
4944
4977
  if push_frame_to_pkt!(b, frames, frame, left) {
@@ -5034,10 +5067,10 @@ impl<F: BufFactory> Connection<F> {
5034
5067
  }
5035
5068
 
5036
5069
  // Create CRYPTO frame.
5037
- if crypto_ctx.crypto_stream.is_flushable() &&
5038
- left > frame::MAX_CRYPTO_OVERHEAD &&
5039
- !is_closing &&
5040
- path.active()
5070
+ if crypto_ctx.crypto_stream.is_flushable()
5071
+ && left > frame::MAX_CRYPTO_OVERHEAD
5072
+ && !is_closing
5073
+ && path.active()
5041
5074
  {
5042
5075
  let crypto_off = crypto_ctx.crypto_stream.send.off_front();
5043
5076
 
@@ -5060,8 +5093,7 @@ impl<F: BufFactory> Connection<F> {
5060
5093
  2; // length, always encode as 2-byte varint
5061
5094
 
5062
5095
  if let Some(max_len) = left.checked_sub(hdr_len) {
5063
- let (mut crypto_hdr, mut crypto_payload) =
5064
- b.split_at(hdr_off + hdr_len)?;
5096
+ let (mut crypto_hdr, mut crypto_payload) = b.split_at(hdr_off + hdr_len)?;
5065
5097
 
5066
5098
  // Write stream data into the packet buffer.
5067
5099
  let (len, _) = crypto_ctx
@@ -5077,11 +5109,7 @@ impl<F: BufFactory> Connection<F> {
5077
5109
  // initial frame offset.
5078
5110
  crypto_hdr.skip(hdr_off)?;
5079
5111
 
5080
- frame::encode_crypto_header(
5081
- crypto_off,
5082
- len as u64,
5083
- &mut crypto_hdr,
5084
- )?;
5112
+ frame::encode_crypto_header(crypto_off, len as u64, &mut crypto_hdr)?;
5085
5113
 
5086
5114
  // Advance the packet buffer's offset.
5087
5115
  b.skip(hdr_len + len)?;
@@ -5116,11 +5144,11 @@ impl<F: BufFactory> Connection<F> {
5116
5144
  }
5117
5145
 
5118
5146
  // Create DATAGRAM frame.
5119
- if (pkt_type == Type::Short || pkt_type == Type::ZeroRTT) &&
5120
- left > frame::MAX_DGRAM_OVERHEAD &&
5121
- !is_closing &&
5122
- path.active() &&
5123
- do_dgram
5147
+ if (pkt_type == Type::Short || pkt_type == Type::ZeroRTT)
5148
+ && left > frame::MAX_DGRAM_OVERHEAD
5149
+ && !is_closing
5150
+ && path.active()
5151
+ && do_dgram
5124
5152
  {
5125
5153
  if let Some(max_dgram_payload) = max_dgram_len {
5126
5154
  while let Some(len) = self.dgram_send_queue.peek_front_len() {
@@ -5151,8 +5179,7 @@ impl<F: BufFactory> Connection<F> {
5151
5179
  let (mut dgram_hdr, mut dgram_payload) =
5152
5180
  b.split_at(hdr_off + hdr_len)?;
5153
5181
 
5154
- dgram_payload.as_mut()[..len]
5155
- .copy_from_slice(data.as_ref());
5182
+ dgram_payload.as_mut()[..len].copy_from_slice(data.as_ref());
5156
5183
 
5157
5184
  // Encode the frame's header.
5158
5185
  //
@@ -5163,27 +5190,21 @@ impl<F: BufFactory> Connection<F> {
5163
5190
  // offset.
5164
5191
  dgram_hdr.skip(hdr_off)?;
5165
5192
 
5166
- frame::encode_dgram_header(
5167
- len as u64,
5168
- &mut dgram_hdr,
5169
- )?;
5193
+ frame::encode_dgram_header(len as u64, &mut dgram_hdr)?;
5170
5194
 
5171
5195
  // Advance the packet buffer's offset.
5172
5196
  b.skip(hdr_len + len)?;
5173
5197
 
5174
- let frame =
5175
- frame::Frame::DatagramHeader { length: len };
5198
+ let frame = frame::Frame::DatagramHeader { length: len };
5176
5199
 
5177
5200
  if push_frame_to_pkt!(b, frames, frame, left) {
5178
5201
  ack_eliciting = true;
5179
5202
  in_flight = true;
5180
5203
  dgram_emitted = true;
5181
- self.dgram_sent_count =
5182
- self.dgram_sent_count.saturating_add(1);
5183
- path.dgram_sent_count =
5184
- path.dgram_sent_count.saturating_add(1);
5204
+ self.dgram_sent_count = self.dgram_sent_count.saturating_add(1);
5205
+ path.dgram_sent_count = path.dgram_sent_count.saturating_add(1);
5185
5206
  }
5186
- },
5207
+ }
5187
5208
 
5188
5209
  None => continue,
5189
5210
  };
@@ -5198,24 +5219,21 @@ impl<F: BufFactory> Connection<F> {
5198
5219
  }
5199
5220
 
5200
5221
  // Create a single STREAM frame for the first stream that is flushable.
5201
- if (pkt_type == Type::Short || pkt_type == Type::ZeroRTT) &&
5202
- left > frame::MAX_STREAM_OVERHEAD &&
5203
- !is_closing &&
5204
- path.active() &&
5205
- !dgram_emitted
5222
+ if (pkt_type == Type::Short || pkt_type == Type::ZeroRTT)
5223
+ && !is_closing
5224
+ && path.active()
5225
+ && !dgram_emitted
5206
5226
  {
5207
5227
  while let Some(priority_key) = self.streams.peek_flushable() {
5208
5228
  let stream_id = priority_key.id;
5209
5229
  let stream = match self.streams.get_mut(stream_id) {
5210
- // Avoid sending frames for streams that were already stopped.
5211
- //
5212
- // This might happen if stream data was buffered but not yet
5213
- // flushed on the wire when a STOP_SENDING frame is received.
5214
- Some(v) if !v.send.is_stopped() => v,
5230
+ // A reset can abandon queued data or FINs while the stream
5231
+ // remains alive waiting for the reset to be acknowledged.
5232
+ Some(stream) if stream.send.can_send_data() => stream,
5215
5233
  _ => {
5216
5234
  self.streams.remove_flushable(&priority_key);
5217
5235
  continue;
5218
- },
5236
+ }
5219
5237
  };
5220
5238
 
5221
5239
  let stream_off = stream.send.off_front();
@@ -5239,22 +5257,38 @@ impl<F: BufFactory> Connection<F> {
5239
5257
  octets::varint_len(stream_off) + // offset
5240
5258
  2; // length, always encode as 2-byte varint
5241
5259
 
5260
+ // A non-FIN STREAM frame must carry at least one byte. If the
5261
+ // header and payload don't fit, leave the stream flushable for
5262
+ // a later packet. Empty FIN frames only need the header.
5242
5263
  let max_len = match left.checked_sub(hdr_len) {
5243
- Some(v) => v,
5244
- None => {
5245
- let priority_key = Arc::clone(&stream.priority_key);
5246
- self.streams.remove_flushable(&priority_key);
5264
+ Some(v) if v > 0 || stream.send.empty_fin_next() => v,
5265
+ _ => {
5266
+ stream_data_skipped = true;
5247
5267
 
5248
- continue;
5249
- },
5268
+ if stream.incremental {
5269
+ let priority_key = Arc::clone(&stream.priority_key);
5270
+ self.streams.remove_flushable(&priority_key);
5271
+ self.streams.insert_flushable(&priority_key);
5272
+ }
5273
+
5274
+ break;
5275
+ }
5250
5276
  };
5251
5277
 
5252
- let (mut stream_hdr, mut stream_payload) =
5253
- b.split_at(hdr_off + hdr_len)?;
5278
+ let (mut stream_hdr, mut stream_payload) = b.split_at(hdr_off + hdr_len)?;
5254
5279
 
5255
5280
  // Write stream data into the packet buffer.
5256
- let (len, fin) =
5257
- stream.send.emit(&mut stream_payload.as_mut()[..max_len])?;
5281
+ let (len, fin) = stream.send.emit(&mut stream_payload.as_mut()[..max_len])?;
5282
+
5283
+ // ACK processing can leave a stream in the flushable queue
5284
+ // after removing all of its retransmission data. Unlink the
5285
+ // stale entry and continue with the next stream;
5286
+ // future retransmits or writedata will queue the stream again.
5287
+ if len == 0 && !fin {
5288
+ let priority_key = Arc::clone(&stream.priority_key);
5289
+ self.streams.remove_flushable(&priority_key);
5290
+ continue;
5291
+ }
5258
5292
 
5259
5293
  // Encode the frame's header.
5260
5294
  //
@@ -5289,7 +5323,7 @@ impl<F: BufFactory> Connection<F> {
5289
5323
  }
5290
5324
 
5291
5325
  let priority_key = Arc::clone(&stream.priority_key);
5292
- // If the stream is no longer flushable, remove it from the queue
5326
+ // Remove the stream when it is no longer flushable.
5293
5327
  if !stream.is_flushable() {
5294
5328
  self.streams.remove_flushable(&priority_key);
5295
5329
  } else if stream.incremental {
@@ -5299,12 +5333,14 @@ impl<F: BufFactory> Connection<F> {
5299
5333
  self.streams.insert_flushable(&priority_key);
5300
5334
  }
5301
5335
 
5336
+ // Update tx_buffered when data is emitted.
5337
+ self.streams.sub_tx_buffered(len);
5338
+
5302
5339
  #[cfg(feature = "fuzzing")]
5303
5340
  // Coalesce STREAM frames when fuzzing.
5304
- if left > frame::MAX_STREAM_OVERHEAD {
5305
- continue;
5306
- }
5341
+ continue;
5307
5342
 
5343
+ #[cfg(not(feature = "fuzzing"))]
5308
5344
  break;
5309
5345
  }
5310
5346
  }
@@ -5317,10 +5353,10 @@ impl<F: BufFactory> Connection<F> {
5317
5353
  // - if we've sent too many non ack-eliciting packets without having
5318
5354
  // sent an ACK eliciting one; OR
5319
5355
  // - the application requested an ack-eliciting frame be sent.
5320
- if (ack_elicit_required || path.needs_ack_eliciting) &&
5321
- !ack_eliciting &&
5322
- left >= 1 &&
5323
- !is_closing
5356
+ if (ack_elicit_required || path.needs_ack_eliciting)
5357
+ && !ack_eliciting
5358
+ && left >= 1
5359
+ && !is_closing
5324
5360
  {
5325
5361
  let frame = frame::Frame::Ping { mtu_probe: None };
5326
5362
 
@@ -5335,9 +5371,11 @@ impl<F: BufFactory> Connection<F> {
5335
5371
  path.recovery.ping_sent(epoch);
5336
5372
  }
5337
5373
 
5338
- if !has_data &&
5339
- !dgram_emitted &&
5340
- cwnd_available > frame::MAX_STREAM_OVERHEAD
5374
+ // Pending stream data means the sender is size-, not app-limited.
5375
+ if !has_data
5376
+ && !stream_data_skipped
5377
+ && !dgram_emitted
5378
+ && send_limit_from_cwnd > LEGACY_APP_LIMITED_BYTE_THRESHOLD
5341
5379
  {
5342
5380
  path.recovery.on_app_limited();
5343
5381
  }
@@ -5357,10 +5395,7 @@ impl<F: BufFactory> Connection<F> {
5357
5395
  // as Initial always requires padding.
5358
5396
  //
5359
5397
  // 2) this is a probing packet towards an unvalidated peer address.
5360
- if (has_initial || !path.validated()) &&
5361
- pkt_type == Type::Short &&
5362
- left >= 1
5363
- {
5398
+ if (has_initial || !path.validated()) && pkt_type == Type::Short && left >= 1 {
5364
5399
  let frame = frame::Frame::Padding { len: left };
5365
5400
 
5366
5401
  if push_frame_to_pkt!(b, frames, frame, left) {
@@ -5389,8 +5424,7 @@ impl<F: BufFactory> Connection<F> {
5389
5424
  let len = pn_len + payload_len + crypto_overhead;
5390
5425
 
5391
5426
  let (_, mut payload_with_len) = b.split_at(header_offset)?;
5392
- payload_with_len
5393
- .put_varint_with_len(len as u64, PAYLOAD_LENGTH_LEN)?;
5427
+ payload_with_len.put_varint_with_len(len as u64, PAYLOAD_LENGTH_LEN)?;
5394
5428
  }
5395
5429
 
5396
5430
  trace!(
@@ -5403,8 +5437,7 @@ impl<F: BufFactory> Connection<F> {
5403
5437
  );
5404
5438
 
5405
5439
  #[cfg(feature = "qlog")]
5406
- let mut qlog_frames: Vec<qlog::events::quic::QuicFrame> =
5407
- Vec::with_capacity(frames.len());
5440
+ let mut qlog_frames: Vec<qlog::events::quic::QuicFrame> = Vec::with_capacity(frames.len());
5408
5441
 
5409
5442
  for frame in &mut frames {
5410
5443
  trace!("{} tx frm {:?}", self.trace_id, frame);
@@ -5427,17 +5460,15 @@ impl<F: BufFactory> Connection<F> {
5427
5460
  data: None,
5428
5461
  };
5429
5462
 
5430
- let send_at_time =
5431
- now.duration_since(q.start_time()).as_secs_f64() * 1000.0;
5463
+ let send_at_time = now.duration_since(q.start_time()).as_secs_f64() * 1000.0;
5432
5464
 
5433
- let ev_data =
5434
- EventData::QuicPacketSent(qlog::events::quic::PacketSent {
5435
- header,
5436
- frames: Some(qlog_frames),
5437
- raw: Some(qlog_raw_info),
5438
- send_at_time: Some(send_at_time),
5439
- ..Default::default()
5440
- });
5465
+ let ev_data = EventData::QuicPacketSent(qlog::events::quic::PacketSent {
5466
+ header,
5467
+ frames: Some(qlog_frames),
5468
+ raw: Some(qlog_raw_info),
5469
+ send_at_time: Some(send_at_time),
5470
+ ..Default::default()
5471
+ });
5441
5472
 
5442
5473
  q.add_event_data_with_instant(ev_data, now).ok();
5443
5474
  }
@@ -5448,15 +5479,8 @@ impl<F: BufFactory> Connection<F> {
5448
5479
  None => return Err(Error::InvalidState),
5449
5480
  };
5450
5481
 
5451
- let written = packet::encrypt_pkt(
5452
- &mut b,
5453
- pn,
5454
- pn_len,
5455
- payload_len,
5456
- payload_offset,
5457
- None,
5458
- aead,
5459
- )?;
5482
+ let written =
5483
+ packet::encrypt_pkt(&mut b, pn, pn_len, payload_len, payload_offset, None, aead)?;
5460
5484
 
5461
5485
  let sent_pkt_has_data = if path.recovery.gcongestion_enabled() {
5462
5486
  has_data || dgram_emitted
@@ -5490,8 +5514,7 @@ impl<F: BufFactory> Connection<F> {
5490
5514
  self.next_pkt_num += 1;
5491
5515
 
5492
5516
  let handshake_status = recovery::HandshakeStatus {
5493
- has_handshake_keys: self.crypto_ctx[packet::Epoch::Handshake]
5494
- .has_keys(),
5517
+ has_handshake_keys: self.crypto_ctx[packet::Epoch::Handshake].has_keys(),
5495
5518
  peer_verified_address: self.peer_verified_initial_address,
5496
5519
  completed: self.handshake_completed,
5497
5520
  };
@@ -5519,13 +5542,12 @@ impl<F: BufFactory> Connection<F> {
5519
5542
 
5520
5543
  let had_send_budget = path.max_send_bytes > 0;
5521
5544
  path.max_send_bytes = path.max_send_bytes.saturating_sub(written);
5522
- if self.is_server &&
5523
- !path.verified_peer_address &&
5524
- had_send_budget &&
5525
- path.max_send_bytes == 0
5545
+ if self.is_server
5546
+ && !path.verified_peer_address
5547
+ && had_send_budget
5548
+ && path.max_send_bytes == 0
5526
5549
  {
5527
- self.amplification_limited_count =
5528
- self.amplification_limited_count.saturating_add(1);
5550
+ self.amplification_limited_count = self.amplification_limited_count.saturating_add(1);
5529
5551
  }
5530
5552
 
5531
5553
  // On the client, drop initial state after sending an Handshake packet.
@@ -5549,29 +5571,54 @@ impl<F: BufFactory> Connection<F> {
5549
5571
  }
5550
5572
 
5551
5573
  fn on_packet_sent(
5552
- &mut self, send_pid: usize, sent_pkt: recovery::Sent,
5553
- epoch: packet::Epoch, handshake_status: recovery::HandshakeStatus,
5574
+ &mut self,
5575
+ send_pid: usize,
5576
+ sent_pkt: recovery::Sent,
5577
+ epoch: packet::Epoch,
5578
+ handshake_status: recovery::HandshakeStatus,
5554
5579
  now: Instant,
5555
5580
  ) -> Result<()> {
5556
5581
  let path = self.paths.get_mut(send_pid)?;
5557
5582
 
5558
- // It's fine to set the skip counter based on a non-active path's values.
5583
+ // The skip counter may use values from an inactive path.
5559
5584
  let cwnd = path.recovery.cwnd();
5560
5585
  let max_datagram_size = path.recovery.max_datagram_size();
5561
5586
  self.pkt_num_spaces[epoch].on_packet_sent(&sent_pkt);
5562
- self.pkt_num_manager.on_packet_sent(
5563
- cwnd,
5564
- max_datagram_size,
5565
- self.handshake_completed,
5566
- );
5587
+ self.pkt_num_manager
5588
+ .on_packet_sent(cwnd, max_datagram_size, self.handshake_completed);
5567
5589
 
5568
- path.recovery.on_packet_sent(
5569
- sent_pkt,
5570
- epoch,
5571
- handshake_status,
5572
- now,
5573
- &self.trace_id,
5574
- );
5590
+ path.recovery
5591
+ .on_packet_sent(sent_pkt, epoch, handshake_status, now, &self.trace_id);
5592
+
5593
+ // Bound recovery metadata when sent packet records accumulate.
5594
+ if self.local_error.is_none() {
5595
+ if let Some(max_outstanding_sent_packets) = self.max_outstanding_sent_packets {
5596
+ let tracked_sent_packets: usize = self
5597
+ .paths
5598
+ .iter()
5599
+ .flat_map(|(_, p)| {
5600
+ packet::Epoch::epochs(packet::Epoch::Initial..=packet::Epoch::Application)
5601
+ .iter()
5602
+ .map(|&e| p.recovery.sent_packets_len(e))
5603
+ })
5604
+ .sum();
5605
+
5606
+ if tracked_sent_packets > max_outstanding_sent_packets.get() {
5607
+ trace!(
5608
+ "{} too many outstanding sent packets: {}",
5609
+ self.trace_id,
5610
+ tracked_sent_packets
5611
+ );
5612
+
5613
+ self.close(
5614
+ false,
5615
+ WireErrorCode::ProtocolViolation as u64,
5616
+ b"too many outstanding sent packets",
5617
+ )
5618
+ .ok();
5619
+ }
5620
+ }
5621
+ }
5575
5622
 
5576
5623
  Ok(())
5577
5624
  }
@@ -5640,9 +5687,7 @@ impl<F: BufFactory> Connection<F> {
5640
5687
  ///
5641
5688
  /// If the (`local_addr`, peer_addr`) 4-tuple relates to a non-existing
5642
5689
  /// path, this method returns 0.
5643
- pub fn send_quantum_on_path(
5644
- &self, local_addr: SocketAddr, peer_addr: SocketAddr,
5645
- ) -> usize {
5690
+ pub fn send_quantum_on_path(&self, local_addr: SocketAddr, peer_addr: SocketAddr) -> usize {
5646
5691
  self.paths
5647
5692
  .path_id_from_addrs(&(local_addr, peer_addr))
5648
5693
  .and_then(|pid| self.paths.get(pid).ok())
@@ -5681,12 +5726,25 @@ impl<F: BufFactory> Connection<F> {
5681
5726
  /// # Ok::<(), quiche::Error>(())
5682
5727
  /// ```
5683
5728
  #[inline]
5684
- pub fn stream_recv(
5685
- &mut self, stream_id: u64, out: &mut [u8],
5686
- ) -> Result<(usize, bool)> {
5729
+ pub fn stream_recv(&mut self, stream_id: u64, out: &mut [u8]) -> Result<(usize, bool)> {
5687
5730
  self.stream_recv_buf(stream_id, out)
5688
5731
  }
5689
5732
 
5733
+ /// Returns the receive-side final size before stream state is collected.
5734
+ /// This is needed for WebTransport session flow control after RESET_STREAM.
5735
+ pub fn stream_recv_final_size(&self, stream_id: u64) -> Option<u64> {
5736
+ self.streams
5737
+ .get(stream_id)
5738
+ .and_then(|stream| stream.recv.final_size())
5739
+ }
5740
+
5741
+ /// Returns the send-side final size, including locally or remotely reset streams.
5742
+ pub fn stream_send_final_size(&self, stream_id: u64) -> Option<u64> {
5743
+ self.streams
5744
+ .get(stream_id)
5745
+ .and_then(|stream| stream.send.final_size())
5746
+ }
5747
+
5690
5748
  /// Reads contiguous data from a stream into the provided [`bytes::BufMut`].
5691
5749
  ///
5692
5750
  /// **NOTE**:
@@ -5729,7 +5787,9 @@ impl<F: BufFactory> Connection<F> {
5729
5787
  /// # Ok::<(), quiche::Error>(())
5730
5788
  /// ```
5731
5789
  pub fn stream_recv_buf<B: bytes::BufMut>(
5732
- &mut self, stream_id: u64, out: B,
5790
+ &mut self,
5791
+ stream_id: u64,
5792
+ out: B,
5733
5793
  ) -> Result<(usize, bool)> {
5734
5794
  self.do_stream_recv(stream_id, RecvAction::Emit { out })
5735
5795
  }
@@ -5761,9 +5821,7 @@ impl<F: BufFactory> Connection<F> {
5761
5821
  /// }
5762
5822
  /// # Ok::<(), quiche::Error>(())
5763
5823
  /// ```
5764
- pub fn stream_discard(
5765
- &mut self, stream_id: u64, len: usize,
5766
- ) -> Result<(usize, bool)> {
5824
+ pub fn stream_discard(&mut self, stream_id: u64, len: usize) -> Result<(usize, bool)> {
5767
5825
  // `do_stream_recv()` is generic on the kind of `BufMut` in RecvAction.
5768
5826
  // Since we are discarding, it doesn't matter, but the compiler still
5769
5827
  // wants to know, so we say `&mut [u8]`.
@@ -5789,12 +5847,12 @@ impl<F: BufFactory> Connection<F> {
5789
5847
  // [`Done`]: enum.Error.html#variant.Done
5790
5848
  // [`send()`]: struct.Connection.html#method.send
5791
5849
  fn do_stream_recv<B: bytes::BufMut>(
5792
- &mut self, stream_id: u64, action: RecvAction<B>,
5850
+ &mut self,
5851
+ stream_id: u64,
5852
+ action: RecvAction<B>,
5793
5853
  ) -> Result<(usize, bool)> {
5794
5854
  // We can't read on our own unidirectional streams.
5795
- if !stream::is_bidi(stream_id) &&
5796
- stream::is_local(stream_id, self.is_server)
5797
- {
5855
+ if !stream::is_bidi(stream_id) && stream::is_local(stream_id, self.is_server) {
5798
5856
  return Err(Error::InvalidStreamState(stream_id));
5799
5857
  }
5800
5858
 
@@ -5824,24 +5882,22 @@ impl<F: BufFactory> Connection<F> {
5824
5882
  Ok(v) => v,
5825
5883
 
5826
5884
  Err(e) => {
5827
- // Collect the stream if it is now complete. This can happen if
5828
- // we got a `StreamReset` error which will now be propagated to
5829
- // the application, so we don't need to keep the stream's state
5830
- // anymore.
5831
- if stream.is_complete() {
5885
+ // A StreamReset read completes the receive side, but an
5886
+ // unreported STOP error still needs the send-side state.
5887
+ if stream.state() == stream::StreamState::Collectable {
5832
5888
  self.streams.collect(stream_id, local);
5833
5889
  }
5834
5890
 
5835
5891
  self.streams.remove_readable(&priority_key);
5836
5892
  return Err(e);
5837
- },
5893
+ }
5838
5894
  };
5839
5895
 
5840
5896
  self.flow_control.add_consumed(read as u64);
5841
5897
 
5842
5898
  let readable = stream.is_readable();
5843
5899
 
5844
- let complete = stream.is_complete();
5900
+ let state = stream.state();
5845
5901
 
5846
5902
  if stream.recv.almost_full() {
5847
5903
  self.streams.insert_almost_full(stream_id);
@@ -5851,25 +5907,22 @@ impl<F: BufFactory> Connection<F> {
5851
5907
  self.streams.remove_readable(&priority_key);
5852
5908
  }
5853
5909
 
5854
- if complete {
5910
+ if state == stream::StreamState::Collectable {
5855
5911
  self.streams.collect(stream_id, local);
5856
5912
  }
5857
5913
 
5858
5914
  qlog_with_type!(QLOG_DATA_MV, self.qlog, q, {
5859
- let ev_data = EventData::QuicStreamDataMoved(
5860
- qlog::events::quic::StreamDataMoved {
5861
- stream_id: Some(stream_id),
5862
- offset: Some(offset),
5863
- raw: Some(RawInfo {
5864
- length: Some(read as u64),
5865
- ..Default::default()
5866
- }),
5867
- from: Some(DataRecipient::Transport),
5868
- to,
5869
- additional_info: fin
5870
- .then_some(DataMovedAdditionalInfo::FinSet),
5871
- },
5872
- );
5915
+ let ev_data = EventData::QuicStreamDataMoved(qlog::events::quic::StreamDataMoved {
5916
+ stream_id: Some(stream_id),
5917
+ offset: Some(offset),
5918
+ raw: Some(RawInfo {
5919
+ length: Some(read as u64),
5920
+ ..Default::default()
5921
+ }),
5922
+ from: Some(DataRecipient::Transport),
5923
+ to,
5924
+ additional_info: fin.then_some(DataMovedAdditionalInfo::FinSet),
5925
+ });
5873
5926
 
5874
5927
  let now = Instant::now();
5875
5928
  q.add_event_data_with_instant(ev_data, now).ok();
@@ -5931,17 +5984,12 @@ impl<F: BufFactory> Connection<F> {
5931
5984
  /// conn.stream_send(stream_id, b"hello", true)?;
5932
5985
  /// # Ok::<(), quiche::Error>(())
5933
5986
  /// ```
5934
- pub fn stream_send(
5935
- &mut self, stream_id: u64, buf: &[u8], fin: bool,
5936
- ) -> Result<usize> {
5987
+ pub fn stream_send(&mut self, stream_id: u64, buf: &[u8], fin: bool) -> Result<usize> {
5937
5988
  self.stream_do_send(
5938
5989
  stream_id,
5939
5990
  buf,
5940
5991
  fin,
5941
- |stream: &mut stream::Stream<F>,
5942
- buf: &[u8],
5943
- cap: usize,
5944
- fin: bool| {
5992
+ |stream: &mut stream::Stream<F>, buf: &[u8], cap: usize, fin: bool| {
5945
5993
  stream.send.write(&buf[..cap], fin).map(|v| (v, v))
5946
5994
  },
5947
5995
  )
@@ -5956,7 +6004,10 @@ impl<F: BufFactory> Connection<F> {
5956
6004
  /// The application should retry the operation once the stream is
5957
6005
  /// reported as writable again.
5958
6006
  pub fn stream_send_zc(
5959
- &mut self, stream_id: u64, buf: F::Buf, fin: bool,
6007
+ &mut self,
6008
+ stream_id: u64,
6009
+ buf: F::Buf,
6010
+ fin: bool,
5960
6011
  ) -> Result<(usize, Option<F::Buf>)>
5961
6012
  where
5962
6013
  F::Buf: BufSplit,
@@ -5965,10 +6016,7 @@ impl<F: BufFactory> Connection<F> {
5965
6016
  stream_id,
5966
6017
  buf,
5967
6018
  fin,
5968
- |stream: &mut stream::Stream<F>,
5969
- buf: F::Buf,
5970
- cap: usize,
5971
- fin: bool| {
6019
+ |stream: &mut stream::Stream<F>, buf: F::Buf, cap: usize, fin: bool| {
5972
6020
  let (sent, remaining) = stream.send.append_buf(buf, cap, fin)?;
5973
6021
  Ok((sent, (sent, remaining)))
5974
6022
  },
@@ -5976,16 +6024,18 @@ impl<F: BufFactory> Connection<F> {
5976
6024
  }
5977
6025
 
5978
6026
  fn stream_do_send<B, R, SND>(
5979
- &mut self, stream_id: u64, buf: B, fin: bool, write_fn: SND,
6027
+ &mut self,
6028
+ stream_id: u64,
6029
+ buf: B,
6030
+ fin: bool,
6031
+ write_fn: SND,
5980
6032
  ) -> Result<R>
5981
6033
  where
5982
6034
  B: AsRef<[u8]>,
5983
6035
  SND: FnOnce(&mut stream::Stream<F>, B, usize, bool) -> Result<(usize, R)>,
5984
6036
  {
5985
6037
  // We can't write on the peer's unidirectional streams.
5986
- if !stream::is_bidi(stream_id) &&
5987
- !stream::is_local(stream_id, self.is_server)
5988
- {
6038
+ if !stream::is_bidi(stream_id) && !stream::is_local(stream_id, self.is_server) {
5989
6039
  return Err(Error::InvalidStreamState(stream_id));
5990
6040
  }
5991
6041
 
@@ -6010,9 +6060,7 @@ impl<F: BufFactory> Connection<F> {
6010
6060
  // If the local endpoint has exhausted the peer's stream count
6011
6061
  // limit, record the current limit so that a STREAMS_BLOCKED
6012
6062
  // frame can be sent.
6013
- if self.enable_send_streams_blocked &&
6014
- stream::is_local(stream_id, self.is_server)
6015
- {
6063
+ if self.enable_send_streams_blocked && stream::is_local(stream_id, self.is_server) {
6016
6064
  if stream::is_bidi(stream_id) {
6017
6065
  let limit = self.streams.peer_max_streams_bidi();
6018
6066
  self.streams_blocked_bidi_state.update_at(limit);
@@ -6023,7 +6071,7 @@ impl<F: BufFactory> Connection<F> {
6023
6071
  }
6024
6072
 
6025
6073
  return Err(Error::StreamLimit);
6026
- },
6074
+ }
6027
6075
 
6028
6076
  Err(e) => return Err(e),
6029
6077
  };
@@ -6035,27 +6083,14 @@ impl<F: BufFactory> Connection<F> {
6035
6083
 
6036
6084
  let was_flushable = stream.is_flushable();
6037
6085
 
6038
- let is_complete = stream.is_complete();
6039
- let is_readable = stream.is_readable();
6040
-
6041
- let priority_key = Arc::clone(&stream.priority_key);
6042
-
6043
- // Return early if the stream has been stopped, and collect its state
6044
- // if complete.
6045
- if let Err(Error::StreamStopped(e)) = stream.send.cap() {
6046
- // Only collect the stream if it is complete and not readable.
6047
- // If it is readable, it will get collected when stream_recv()
6048
- // is used.
6049
- //
6050
- // The stream can't be writable if it has been stopped.
6051
- if is_complete && !is_readable {
6052
- let local = stream.local;
6053
- self.streams.collect(stream_id, local);
6054
- }
6055
-
6086
+ // Return the stop before collecting a completed stream.
6087
+ if let Err(Error::StreamStopped(e)) = stream.send.cap_and_report_stop_sending() {
6088
+ self.streams.on_stop_sending_reported(stream_id);
6056
6089
  return Err(Error::StreamStopped(e));
6057
6090
  };
6058
6091
 
6092
+ let priority_key = Arc::clone(&stream.priority_key);
6093
+
6059
6094
  // Truncate the input buffer based on the connection's send capacity if
6060
6095
  // necessary.
6061
6096
  //
@@ -6087,7 +6122,7 @@ impl<F: BufFactory> Connection<F> {
6087
6122
  Err(e) => {
6088
6123
  self.streams.remove_writable(&priority_key);
6089
6124
  return Err(e);
6090
- },
6125
+ }
6091
6126
  };
6092
6127
 
6093
6128
  let incremental = stream.incremental;
@@ -6136,24 +6171,20 @@ impl<F: BufFactory> Connection<F> {
6136
6171
 
6137
6172
  self.tx_data += sent as u64;
6138
6173
 
6139
- self.tx_buffered += sent;
6140
- self.check_tx_buffered_invariant();
6174
+ self.streams.add_tx_buffered(sent);
6141
6175
 
6142
6176
  qlog_with_type!(QLOG_DATA_MV, self.qlog, q, {
6143
- let ev_data = EventData::QuicStreamDataMoved(
6144
- qlog::events::quic::StreamDataMoved {
6145
- stream_id: Some(stream_id),
6146
- offset: Some(offset),
6147
- raw: Some(RawInfo {
6148
- length: Some(sent as u64),
6149
- ..Default::default()
6150
- }),
6151
- from: Some(DataRecipient::Application),
6152
- to: Some(DataRecipient::Transport),
6153
- additional_info: fin
6154
- .then_some(DataMovedAdditionalInfo::FinSet),
6155
- },
6156
- );
6177
+ let ev_data = EventData::QuicStreamDataMoved(qlog::events::quic::StreamDataMoved {
6178
+ stream_id: Some(stream_id),
6179
+ offset: Some(offset),
6180
+ raw: Some(RawInfo {
6181
+ length: Some(sent as u64),
6182
+ ..Default::default()
6183
+ }),
6184
+ from: Some(DataRecipient::Application),
6185
+ to: Some(DataRecipient::Transport),
6186
+ additional_info: fin.then_some(DataMovedAdditionalInfo::FinSet),
6187
+ });
6157
6188
 
6158
6189
  let now = Instant::now();
6159
6190
  q.add_event_data_with_instant(ev_data, now).ok();
@@ -6181,7 +6212,10 @@ impl<F: BufFactory> Connection<F> {
6181
6212
  /// The target stream is created if it did not exist before calling this
6182
6213
  /// method.
6183
6214
  pub fn stream_priority(
6184
- &mut self, stream_id: u64, urgency: u8, incremental: bool,
6215
+ &mut self,
6216
+ stream_id: u64,
6217
+ urgency: u8,
6218
+ incremental: bool,
6185
6219
  ) -> Result<()> {
6186
6220
  // Get existing stream or create a new one, but if the stream
6187
6221
  // has already been closed and collected, ignore the prioritization.
@@ -6229,7 +6263,8 @@ impl<F: BufFactory> Connection<F> {
6229
6263
  /// data in the stream's send buffer is dropped, and no additional data is
6230
6264
  /// added to it. Data passed to [`stream_send()`] after calling this method
6231
6265
  /// will be ignored. In addition, a `RESET_STREAM` frame will be sent to the
6232
- /// peer to signal the reset.
6266
+ /// peer to signal the reset, unless all outgoing data has already been
6267
+ /// acked by the peer (up to the `fin` offset).
6233
6268
  ///
6234
6269
  /// Locally-initiated unidirectional streams can only be closed in the
6235
6270
  /// [`Shutdown::Write`] direction. Remotely-initiated unidirectional streams
@@ -6241,21 +6276,19 @@ impl<F: BufFactory> Connection<F> {
6241
6276
  /// [`stream_recv()`]: struct.Connection.html#method.stream_recv
6242
6277
  /// [`stream_send()`]: struct.Connection.html#method.stream_send
6243
6278
  /// [`InvalidStreamState`]: enum.Error.html#variant.InvalidStreamState
6244
- pub fn stream_shutdown(
6245
- &mut self, stream_id: u64, direction: Shutdown, err: u64,
6246
- ) -> Result<()> {
6279
+ pub fn stream_shutdown(&mut self, stream_id: u64, direction: Shutdown, err: u64) -> Result<()> {
6247
6280
  // Don't try to stop a local unidirectional stream.
6248
- if direction == Shutdown::Read &&
6249
- stream::is_local(stream_id, self.is_server) &&
6250
- !stream::is_bidi(stream_id)
6281
+ if direction == Shutdown::Read
6282
+ && stream::is_local(stream_id, self.is_server)
6283
+ && !stream::is_bidi(stream_id)
6251
6284
  {
6252
6285
  return Err(Error::InvalidStreamState(stream_id));
6253
6286
  }
6254
6287
 
6255
6288
  // Don't try to reset a remote unidirectional stream.
6256
- if direction == Shutdown::Write &&
6257
- !stream::is_local(stream_id, self.is_server) &&
6258
- !stream::is_bidi(stream_id)
6289
+ if direction == Shutdown::Write
6290
+ && !stream::is_local(stream_id, self.is_server)
6291
+ && !stream::is_bidi(stream_id)
6259
6292
  {
6260
6293
  return Err(Error::InvalidStreamState(stream_id));
6261
6294
  }
@@ -6270,6 +6303,11 @@ impl<F: BufFactory> Connection<F> {
6270
6303
  let consumed = stream.recv.shutdown()?;
6271
6304
  self.flow_control.add_consumed(consumed);
6272
6305
 
6306
+ // Discarding unread data can complete the receive side, so no
6307
+ // later read would collect the stream.
6308
+ let state = stream.state();
6309
+ let local = stream.local;
6310
+
6273
6311
  if !stream.recv.is_fin() {
6274
6312
  self.streams.insert_stopped(stream_id, err);
6275
6313
  }
@@ -6277,29 +6315,36 @@ impl<F: BufFactory> Connection<F> {
6277
6315
  // Once shutdown, the stream is guaranteed to be non-readable.
6278
6316
  self.streams.remove_readable(&priority_key);
6279
6317
 
6280
- self.stopped_stream_local_count =
6281
- self.stopped_stream_local_count.saturating_add(1);
6282
- },
6318
+ self.stopped_stream_local_count = self.stopped_stream_local_count.saturating_add(1);
6319
+
6320
+ if state == stream::StreamState::Collectable {
6321
+ self.streams.collect(stream_id, local);
6322
+ }
6323
+ }
6283
6324
 
6284
6325
  Shutdown::Write => {
6326
+ // Save the buffered length before shutdown (shutdown clears the
6327
+ // buffer).
6328
+ let buffered_len = stream.send.buffered_bytes() as usize;
6329
+
6285
6330
  let (final_size, unsent) = stream.send.shutdown()?;
6331
+ let dropped = buffered_len.saturating_sub(stream.send.buffered_bytes() as usize);
6332
+ let flushable = stream.is_flushable();
6286
6333
 
6287
6334
  // Claw back some flow control allowance from data that was
6288
6335
  // buffered but not actually sent before the stream was reset.
6289
6336
  self.tx_data = self.tx_data.saturating_sub(unsent);
6290
6337
 
6291
- self.tx_buffered =
6292
- self.tx_buffered.saturating_sub(unsent as usize);
6338
+ // Update tx_buffered: subtract only the buffered data, not
6339
+ // inflight data.
6340
+ self.streams.sub_tx_buffered(dropped);
6293
6341
 
6294
- // These drops in qlog are a bit weird, but the only way to ensure
6295
- // that all bytes that are moved from App to Transport in
6296
- // stream_do_send are eventually moved from Transport to Dropped.
6297
- // Ideally we would add a Transport to Network transition also as
6298
- // a way to indicate when bytes were transmitted vs dropped
6299
- // without ever being sent.
6342
+ // Match App-to-Transport moves with Transport-to-Dropped moves.
6343
+ // A Network transition would distinguish sent bytes from drops
6344
+ // before transmission.
6300
6345
  qlog_with_type!(QLOG_DATA_MV, self.qlog, q, {
6301
- let ev_data = EventData::QuicStreamDataMoved(
6302
- qlog::events::quic::StreamDataMoved {
6346
+ let ev_data =
6347
+ EventData::QuicStreamDataMoved(qlog::events::quic::StreamDataMoved {
6303
6348
  stream_id: Some(stream_id),
6304
6349
  offset: Some(final_size),
6305
6350
  raw: Some(RawInfo {
@@ -6309,8 +6354,7 @@ impl<F: BufFactory> Connection<F> {
6309
6354
  from: Some(DataRecipient::Transport),
6310
6355
  to: Some(DataRecipient::Dropped),
6311
6356
  ..Default::default()
6312
- },
6313
- );
6357
+ });
6314
6358
 
6315
6359
  q.add_event_data_with_instant(ev_data, Instant::now()).ok();
6316
6360
  });
@@ -6320,17 +6364,40 @@ impl<F: BufFactory> Connection<F> {
6320
6364
 
6321
6365
  self.streams.insert_reset(stream_id, err, final_size);
6322
6366
 
6323
- // Once shutdown, the stream is guaranteed to be non-writable.
6324
6367
  self.streams.remove_writable(&priority_key);
6368
+ if !flushable {
6369
+ self.streams.remove_flushable(&priority_key);
6370
+ }
6325
6371
 
6326
- self.reset_stream_local_count =
6327
- self.reset_stream_local_count.saturating_add(1);
6328
- },
6372
+ self.reset_stream_local_count = self.reset_stream_local_count.saturating_add(1);
6373
+
6374
+ self.streams.try_collect(stream_id);
6375
+ }
6329
6376
  }
6330
6377
 
6331
6378
  Ok(())
6332
6379
  }
6333
6380
 
6381
+ /// Returns whether both endpoints advertised QUIC RESET_STREAM_AT.
6382
+ pub fn reliable_stream_reset_negotiated(&self) -> bool {
6383
+ self.local_transport_params.reset_stream_at && self.peer_transport_params.reset_stream_at
6384
+ }
6385
+
6386
+ /// Use RESET_STREAM_AT for this stream and retain the given prefix on a
6387
+ /// local write shutdown or a peer STOP_SENDING. The prefix must already be
6388
+ /// queued with stream_send.
6389
+ pub fn stream_set_reliable_reset(&mut self, id: u64, size: u64) -> Result<()> {
6390
+ if !self.reliable_stream_reset_negotiated() {
6391
+ return Err(Error::InvalidState);
6392
+ }
6393
+ let stream = self.streams.get_mut(id).ok_or(Error::Done)?;
6394
+ if stream.send.is_reset() || size > stream.send.off_back() {
6395
+ return Err(Error::InvalidStreamState(id));
6396
+ }
6397
+ stream.send.reliable_reset_size = Some(size);
6398
+ Ok(())
6399
+ }
6400
+
6334
6401
  /// Returns the stream's send capacity in bytes.
6335
6402
  ///
6336
6403
  /// The returned capacity takes into account the stream's flow control limit
@@ -6348,21 +6415,14 @@ impl<F: BufFactory> Connection<F> {
6348
6415
  /// [`StreamStopped`]: enum.Error.html#variant.StreamStopped
6349
6416
  #[inline]
6350
6417
  pub fn stream_capacity(&mut self, stream_id: u64) -> Result<usize> {
6351
- if let Some(stream) = self.streams.get(stream_id) {
6352
- let stream_cap = match stream.send.cap() {
6418
+ if let Some(stream) = self.streams.get_mut(stream_id) {
6419
+ let stream_cap = match stream.send.cap_and_report_stop_sending() {
6353
6420
  Ok(v) => v,
6354
6421
 
6355
6422
  Err(Error::StreamStopped(e)) => {
6356
- // Only collect the stream if it is complete and not
6357
- // readable. If it is readable, it will get collected when
6358
- // stream_recv() is used.
6359
- if stream.is_complete() && !stream.is_readable() {
6360
- let local = stream.local;
6361
- self.streams.collect(stream_id, local);
6362
- }
6363
-
6423
+ self.streams.on_stop_sending_reported(stream_id);
6364
6424
  return Err(Error::StreamStopped(e));
6365
- },
6425
+ }
6366
6426
 
6367
6427
  Err(e) => return Err(e),
6368
6428
  };
@@ -6405,6 +6465,33 @@ impl<F: BufFactory> Connection<F> {
6405
6465
  stream.is_readable()
6406
6466
  }
6407
6467
 
6468
+ /// Returns the number of contiguous bytes buffered for a stream, up to
6469
+ /// `max_len`.
6470
+ ///
6471
+ /// This is the length of the contiguous, in-order data buffered at the
6472
+ /// stream's current read offset, i.e. the bytes a call to [`stream_recv`]
6473
+ /// would return right now. Data received out of order that sits behind a
6474
+ /// gap is not counted, so this never reports bytes that are not yet
6475
+ /// readable.
6476
+ ///
6477
+ /// This is a companion to [`stream_readable`], which only reports *whether*
6478
+ /// data is available; this reports *how much*. It is intended for sizing a
6479
+ /// receive buffer. The cost is proportional to the number of contiguous
6480
+ /// buffered chunks at the front of the stream, up to `max_len`, and no data
6481
+ /// is copied.
6482
+ ///
6483
+ /// Returns 0 if the stream does not exist.
6484
+ ///
6485
+ /// [`stream_recv`]: struct.Connection.html#method.stream_recv
6486
+ /// [`stream_readable`]: struct.Connection.html#method.stream_readable
6487
+ pub fn stream_readable_len(&self, stream_id: u64, max_len: usize) -> usize {
6488
+ match self.streams.get(stream_id) {
6489
+ Some(s) => s.recv.readable_len(max_len),
6490
+
6491
+ None => 0,
6492
+ }
6493
+ }
6494
+
6408
6495
  /// Returns the next stream that can be written to.
6409
6496
  ///
6410
6497
  /// Note that once returned by this method, a stream ID will not be returned
@@ -6418,13 +6505,15 @@ impl<F: BufFactory> Connection<F> {
6418
6505
  /// The [`stream_writable()`] method can also be used to fine-tune when a
6419
6506
  /// stream is reported as writable again.
6420
6507
  ///
6508
+ /// A stopped stream is returned even without send capacity. Querying its
6509
+ /// capacity or writing to it returns [`StreamStopped`].
6510
+ ///
6421
6511
  /// [`stream_writable()`]: struct.Connection.html#method.stream_writable
6422
6512
  /// [`writable()`]: struct.Connection.html#method.writable
6513
+ /// [`StreamStopped`]: enum.Error.html#variant.StreamStopped
6423
6514
  pub fn stream_writable_next(&mut self) -> Option<u64> {
6424
- // If there is not enough connection-level send capacity, none of the
6425
- // streams are writable.
6426
6515
  if self.tx_cap == 0 {
6427
- return None;
6516
+ return self.streams.pop_stopped_writable();
6428
6517
  }
6429
6518
 
6430
6519
  let mut cursor = self.streams.writable.front();
@@ -6436,12 +6525,13 @@ impl<F: BufFactory> Connection<F> {
6436
6525
 
6437
6526
  // Return the stream to the application immediately if it's
6438
6527
  // stopped.
6439
- Err(_) =>
6528
+ Err(_) => {
6440
6529
  return {
6441
6530
  self.streams.remove_writable(&priority_key);
6442
6531
 
6443
6532
  Some(priority_key.id)
6444
- },
6533
+ }
6534
+ }
6445
6535
  };
6446
6536
 
6447
6537
  if cmp::min(self.tx_cap, cap) >= stream.send_lowat {
@@ -6479,9 +6569,7 @@ impl<F: BufFactory> Connection<F> {
6479
6569
  /// [`InvalidStreamState`]: enum.Error.html#variant.InvalidStreamState
6480
6570
  /// [`StreamStopped`]: enum.Error.html#variant.StreamStopped
6481
6571
  #[inline]
6482
- pub fn stream_writable(
6483
- &mut self, stream_id: u64, len: usize,
6484
- ) -> Result<bool> {
6572
+ pub fn stream_writable(&mut self, stream_id: u64, len: usize) -> Result<bool> {
6485
6573
  if self.stream_capacity(stream_id)? >= len {
6486
6574
  return Ok(true);
6487
6575
  }
@@ -6634,6 +6722,9 @@ impl<F: BufFactory> Connection<F> {
6634
6722
  /// streams are only allowed to buffer outgoing data up to the amount that
6635
6723
  /// the peer allows to send.
6636
6724
  ///
6725
+ /// Stopped streams are also included so that a write or capacity query can
6726
+ /// return [`StreamStopped`], even without connection send capacity.
6727
+ ///
6637
6728
  /// Note that the iterator will only include streams that were writable at
6638
6729
  /// the time the iterator itself was created (i.e. when `writable()` was
6639
6730
  /// called). To account for newly writable streams, the iterator needs to be
@@ -6659,12 +6750,12 @@ impl<F: BufFactory> Connection<F> {
6659
6750
  /// # Ok::<(), quiche::Error>(())
6660
6751
  /// ```
6661
6752
  /// [`stream_priority()`]: struct.Connection.html#method.stream_priority
6753
+ /// [`StreamStopped`]: enum.Error.html#variant.StreamStopped
6662
6754
  #[inline]
6663
6755
  pub fn writable(&self) -> StreamIter {
6664
- // If there is not enough connection-level send capacity, none of the
6665
- // streams are writable, so return an empty iterator.
6756
+ // Stopped streams must be reported even without send capacity.
6666
6757
  if self.tx_cap == 0 {
6667
- return StreamIter::default();
6758
+ return self.streams.stopped_writable();
6668
6759
  }
6669
6760
 
6670
6761
  self.streams.writable()
@@ -6692,8 +6783,8 @@ impl<F: BufFactory> Connection<F> {
6692
6783
 
6693
6784
  if let Some(max_datagram_size) = max_datagram_size {
6694
6785
  if self.is_established() {
6695
- // We cap the maximum packet size to 16KB or so, so that it can be
6696
- // always encoded with a 2-byte varint.
6786
+ // Cap the packet size at 16,383 bytes so a two-byte varint can
6787
+ // always encode it.
6697
6788
  return cmp::min(16383, max_datagram_size);
6698
6789
  }
6699
6790
  }
@@ -6730,7 +6821,9 @@ impl<F: BufFactory> Connection<F> {
6730
6821
  /// [`send_ack_eliciting()`]: struct.Connection.html#method.send_ack_eliciting
6731
6822
  /// [`InvalidState`]: enum.Error.html#variant.InvalidState
6732
6823
  pub fn send_ack_eliciting_on_path(
6733
- &mut self, local: SocketAddr, peer: SocketAddr,
6824
+ &mut self,
6825
+ local: SocketAddr,
6826
+ peer: SocketAddr,
6734
6827
  ) -> Result<()> {
6735
6828
  if self.is_closed() || self.is_draining() {
6736
6829
  return Ok(());
@@ -6782,7 +6875,7 @@ impl<F: BufFactory> Connection<F> {
6782
6875
 
6783
6876
  buf[..len].copy_from_slice(d.as_ref());
6784
6877
  Ok(len)
6785
- },
6878
+ }
6786
6879
 
6787
6880
  None => Err(Error::Done),
6788
6881
  }
@@ -6916,9 +7009,7 @@ impl<F: BufFactory> Connection<F> {
6916
7009
 
6917
7010
  let active_path = self.paths.get_active_mut()?;
6918
7011
 
6919
- if self.dgram_send_queue.byte_size() >
6920
- active_path.recovery.cwnd_available()
6921
- {
7012
+ if self.dgram_send_queue.byte_size() > active_path.recovery.cwnd_available() {
6922
7013
  active_path.recovery.update_app_limited(false);
6923
7014
  }
6924
7015
 
@@ -6982,16 +7073,14 @@ impl<F: BufFactory> Connection<F> {
6982
7073
  // ...subtract the packet number (max len)...
6983
7074
  max_len = max_len.saturating_sub(packet::MAX_PKT_NUM_LEN);
6984
7075
  // ...subtract the crypto overhead...
6985
- max_len = max_len.saturating_sub(
6986
- self.crypto_ctx[packet::Epoch::Application]
6987
- .crypto_overhead()?,
6988
- );
7076
+ max_len = max_len
7077
+ .saturating_sub(self.crypto_ctx[packet::Epoch::Application].crypto_overhead()?);
6989
7078
  // ...clamp to what peer can support...
6990
7079
  max_len = cmp::min(peer_frame_len as usize, max_len);
6991
7080
  // ...subtract frame overhead, checked for underflow.
6992
7081
  // (1 byte of frame type + len of length )
6993
7082
  max_len.checked_sub(1 + frame::MAX_DGRAM_OVERHEAD)
6994
- },
7083
+ }
6995
7084
  }
6996
7085
  }
6997
7086
 
@@ -7138,12 +7227,12 @@ impl<F: BufFactory> Connection<F> {
7138
7227
  // The connection cannot continue.
7139
7228
  self.mark_closed();
7140
7229
  }
7141
- },
7230
+ }
7142
7231
 
7143
7232
  // The connection cannot continue.
7144
7233
  None => {
7145
7234
  self.mark_closed();
7146
- },
7235
+ }
7147
7236
  }
7148
7237
  }
7149
7238
  }
@@ -7178,9 +7267,7 @@ impl<F: BufFactory> Connection<F> {
7178
7267
  /// [`InvalidState`]: enum.Error.html#InvalidState
7179
7268
  /// [`send()`]: struct.Connection.html#method.send
7180
7269
  /// [`send_on_path()`]: struct.Connection.html#method.send_on_path
7181
- pub fn probe_path(
7182
- &mut self, local_addr: SocketAddr, peer_addr: SocketAddr,
7183
- ) -> Result<u64> {
7270
+ pub fn probe_path(&mut self, local_addr: SocketAddr, peer_addr: SocketAddr) -> Result<u64> {
7184
7271
  // We may want to probe an existing path.
7185
7272
  let pid = match self.paths.path_id_from_addrs(&(local_addr, peer_addr)) {
7186
7273
  Some(pid) => pid,
@@ -7220,61 +7307,58 @@ impl<F: BufFactory> Connection<F> {
7220
7307
  ///
7221
7308
  /// [`OutOfIdentifiers`]: enum.Error.html#OutOfIdentifiers
7222
7309
  /// [`InvalidState`]: enum.Error.html#InvalidState
7223
- pub fn migrate(
7224
- &mut self, local_addr: SocketAddr, peer_addr: SocketAddr,
7225
- ) -> Result<u64> {
7310
+ pub fn migrate(&mut self, local_addr: SocketAddr, peer_addr: SocketAddr) -> Result<u64> {
7226
7311
  if self.is_server {
7227
7312
  return Err(Error::InvalidState);
7228
7313
  }
7229
7314
 
7230
7315
  // If the path already exists, mark it as the active one.
7231
- let (pid, dcid_seq) = if let Some(pid) =
7232
- self.paths.path_id_from_addrs(&(local_addr, peer_addr))
7233
- {
7234
- let path = self.paths.get_mut(pid)?;
7316
+ let (pid, dcid_seq) =
7317
+ if let Some(pid) = self.paths.path_id_from_addrs(&(local_addr, peer_addr)) {
7318
+ let path = self.paths.get_mut(pid)?;
7235
7319
 
7236
- // If it is already active, do nothing.
7237
- if path.active() {
7238
- return path.active_dcid_seq.ok_or(Error::OutOfIdentifiers);
7239
- }
7320
+ // If it is already active, do nothing.
7321
+ if path.active() {
7322
+ return path.active_dcid_seq.ok_or(Error::OutOfIdentifiers);
7323
+ }
7240
7324
 
7241
- // Ensures that a Source Connection ID has been dedicated to this
7242
- // path, or a free one is available. This is only required if the
7243
- // host uses non-zero length Source Connection IDs.
7244
- if !self.ids.zero_length_scid() &&
7245
- path.active_scid_seq.is_none() &&
7246
- self.ids.available_scids() == 0
7247
- {
7248
- return Err(Error::OutOfIdentifiers);
7249
- }
7325
+ // Ensures that a Source Connection ID has been dedicated to this
7326
+ // path, or a free one is available. This is only required if the
7327
+ // host uses non-zero length Source Connection IDs.
7328
+ if !self.ids.zero_length_scid()
7329
+ && path.active_scid_seq.is_none()
7330
+ && self.ids.available_scids() == 0
7331
+ {
7332
+ return Err(Error::OutOfIdentifiers);
7333
+ }
7250
7334
 
7251
- // Ensures that the migrated path has a Destination Connection ID.
7252
- let dcid_seq = if let Some(dcid_seq) = path.active_dcid_seq {
7253
- dcid_seq
7254
- } else {
7255
- let dcid_seq = self
7256
- .ids
7257
- .lowest_available_dcid_seq()
7258
- .ok_or(Error::OutOfIdentifiers)?;
7335
+ // Ensures that the migrated path has a Destination Connection ID.
7336
+ let dcid_seq = if let Some(dcid_seq) = path.active_dcid_seq {
7337
+ dcid_seq
7338
+ } else {
7339
+ let dcid_seq = self
7340
+ .ids
7341
+ .lowest_available_dcid_seq()
7342
+ .ok_or(Error::OutOfIdentifiers)?;
7259
7343
 
7260
- self.ids.link_dcid_to_path_id(dcid_seq, pid)?;
7261
- path.active_dcid_seq = Some(dcid_seq);
7344
+ self.ids.link_dcid_to_path_id(dcid_seq, pid)?;
7345
+ path.active_dcid_seq = Some(dcid_seq);
7262
7346
 
7263
- dcid_seq
7264
- };
7347
+ dcid_seq
7348
+ };
7265
7349
 
7266
- (pid, dcid_seq)
7267
- } else {
7268
- let pid = self.create_path_on_client(local_addr, peer_addr)?;
7350
+ (pid, dcid_seq)
7351
+ } else {
7352
+ let pid = self.create_path_on_client(local_addr, peer_addr)?;
7269
7353
 
7270
- let dcid_seq = self
7271
- .paths
7272
- .get(pid)?
7273
- .active_dcid_seq
7274
- .ok_or(Error::InvalidState)?;
7354
+ let dcid_seq = self
7355
+ .paths
7356
+ .get(pid)?
7357
+ .active_dcid_seq
7358
+ .ok_or(Error::InvalidState)?;
7275
7359
 
7276
- (pid, dcid_seq)
7277
- };
7360
+ (pid, dcid_seq)
7361
+ };
7278
7362
 
7279
7363
  // Change the active path.
7280
7364
  self.set_active_path(pid, Instant::now())?;
@@ -7315,7 +7399,10 @@ impl<F: BufFactory> Connection<F> {
7315
7399
  /// [`IdLimit`]: enum.Error.html#IdLimit
7316
7400
  /// [`InvalidState`]: enum.Error.html#InvalidState
7317
7401
  pub fn new_scid(
7318
- &mut self, scid: &ConnectionId, reset_token: u128, retire_if_needed: bool,
7402
+ &mut self,
7403
+ scid: &ConnectionId,
7404
+ reset_token: u128,
7405
+ retire_if_needed: bool,
7319
7406
  ) -> Result<u64> {
7320
7407
  self.ids.new_scid(
7321
7408
  scid.to_vec().into(),
@@ -7389,9 +7476,9 @@ impl<F: BufFactory> Connection<F> {
7389
7476
 
7390
7477
  let active_path_id = self.paths.get_active_path_id()?;
7391
7478
 
7392
- if active_path_dcid_seq == dcid_seq &&
7393
- self.ids.lowest_available_dcid_seq().is_none() &&
7394
- !self
7479
+ if active_path_dcid_seq == dcid_seq
7480
+ && self.ids.lowest_available_dcid_seq().is_none()
7481
+ && !self
7395
7482
  .paths
7396
7483
  .iter()
7397
7484
  .any(|(pid, p)| pid != active_path_id && p.usable())
@@ -7552,8 +7639,7 @@ impl<F: BufFactory> Connection<F> {
7552
7639
  return Err(Error::Done);
7553
7640
  }
7554
7641
 
7555
- let is_safe_to_send_app_data =
7556
- self.is_established() || self.is_in_early_data();
7642
+ let is_safe_to_send_app_data = self.is_established() || self.is_in_early_data();
7557
7643
 
7558
7644
  if app && !is_safe_to_send_app_data {
7559
7645
  // Clear error information.
@@ -7570,7 +7656,7 @@ impl<F: BufFactory> Connection<F> {
7570
7656
  });
7571
7657
  }
7572
7658
 
7573
- // When no packet was successfully processed close connection immediately.
7659
+ // Close immediately if no packet was processed successfully.
7574
7660
  if self.recv_count == 0 {
7575
7661
  self.mark_closed();
7576
7662
  }
@@ -7601,7 +7687,10 @@ impl<F: BufFactory> Connection<F> {
7601
7687
  /// optional `context`, into `out`. Only available once the handshake has
7602
7688
  /// completed; calling it earlier returns [`Error::TlsFail`].
7603
7689
  pub fn export_keying_material(
7604
- &mut self, out: &mut [u8], label: &[u8], context: Option<&[u8]>,
7690
+ &mut self,
7691
+ out: &mut [u8],
7692
+ label: &[u8],
7693
+ context: Option<&[u8]>,
7605
7694
  ) -> Result<()> {
7606
7695
  self.handshake.export_keying_material(out, label, context)
7607
7696
  }
@@ -7704,13 +7793,30 @@ impl<F: BufFactory> Connection<F> {
7704
7793
  /// Revalidates the PMTU for the active path by sending a new probe packet
7705
7794
  /// of PMTU size. If the probe is dropped PMTUD will restart and find a new
7706
7795
  /// valid PMTU.
7796
+ ///
7797
+ /// If revalidation invalidates a previously discovered larger size, a
7798
+ /// [`PathEvent::PmtuUpdated`] event is queued with QUIC's minimum packet
7799
+ /// size. Further events report larger sizes as probes validate them.
7707
7800
  #[inline]
7708
7801
  pub fn revalidate_pmtu(&mut self) {
7709
- if let Ok(active_path) = self.paths.get_active_mut() {
7710
- if let Some(pmtud) = active_path.pmtud.as_mut() {
7711
- pmtud.revalidate_pmtu();
7712
- }
7713
- }
7802
+ let Ok(active_path) = self.paths.get_active_mut() else {
7803
+ return;
7804
+ };
7805
+
7806
+ let local = active_path.local_addr();
7807
+ let peer = active_path.peer_addr();
7808
+ let Some(pmtud) = active_path.pmtud.as_mut() else {
7809
+ return;
7810
+ };
7811
+
7812
+ let old_pmtu = pmtud.get_current_mtu();
7813
+ pmtud.revalidate_pmtu();
7814
+
7815
+ let Some(event) = path::pmtu_event(local, peer, old_pmtu, pmtud.get_current_mtu()) else {
7816
+ return;
7817
+ };
7818
+
7819
+ self.paths.notify_event(event);
7714
7820
  }
7715
7821
 
7716
7822
  /// Returns true if the connection handshake is complete.
@@ -7756,9 +7862,7 @@ impl<F: BufFactory> Connection<F> {
7756
7862
  /// [`InvalidState`].
7757
7863
  ///
7758
7864
  /// [`InvalidState`]: enum.Error.html#variant.InvalidState
7759
- pub fn is_path_validated(
7760
- &self, from: SocketAddr, to: SocketAddr,
7761
- ) -> Result<bool> {
7865
+ pub fn is_path_validated(&self, from: SocketAddr, to: SocketAddr) -> Result<bool> {
7762
7866
  let pid = self
7763
7867
  .paths
7764
7868
  .path_id_from_addrs(&(from, to))
@@ -7857,7 +7961,11 @@ impl<F: BufFactory> Connection<F> {
7857
7961
  path_challenge_rx_count: self.path_challenge_rx_count,
7858
7962
  amplification_limited_count: self.amplification_limited_count,
7859
7963
  bytes_in_flight_duration: self.bytes_in_flight_duration(),
7860
- tx_buffered_state: self.tx_buffered_state,
7964
+ tx_buffered_state: if self.streams.tx_buffered_is_consistent() {
7965
+ TxBufferTrackingState::Ok
7966
+ } else {
7967
+ TxBufferTrackingState::Inconsistent
7968
+ },
7861
7969
  }
7862
7970
  }
7863
7971
 
@@ -7897,19 +8005,14 @@ impl<F: BufFactory> Connection<F> {
7897
8005
  }
7898
8006
 
7899
8007
  fn encode_transport_params(&mut self) -> Result<()> {
7900
- self.handshake.set_quic_transport_params(
7901
- &self.local_transport_params,
7902
- self.is_server,
7903
- )
8008
+ self.handshake
8009
+ .set_quic_transport_params(&self.local_transport_params, self.is_server)
7904
8010
  }
7905
8011
 
7906
- fn parse_peer_transport_params(
7907
- &mut self, peer_params: TransportParams,
7908
- ) -> Result<()> {
8012
+ fn parse_peer_transport_params(&mut self, peer_params: TransportParams) -> Result<()> {
7909
8013
  // Validate initial_source_connection_id.
7910
8014
  match &peer_params.initial_source_connection_id {
7911
- Some(v) if v != &self.destination_id() =>
7912
- return Err(Error::InvalidTransportParam),
8015
+ Some(v) if v != &self.destination_id() => return Err(Error::InvalidTransportParam),
7913
8016
 
7914
8017
  Some(_) => (),
7915
8018
 
@@ -7921,15 +8024,13 @@ impl<F: BufFactory> Connection<F> {
7921
8024
  // Validate original_destination_connection_id.
7922
8025
  if let Some(odcid) = &self.odcid {
7923
8026
  match &peer_params.original_destination_connection_id {
7924
- Some(v) if v != odcid =>
7925
- return Err(Error::InvalidTransportParam),
8027
+ Some(v) if v != odcid => return Err(Error::InvalidTransportParam),
7926
8028
 
7927
8029
  Some(_) => (),
7928
8030
 
7929
8031
  // original_destination_connection_id must be
7930
8032
  // sent by the server.
7931
- None if !self.is_server =>
7932
- return Err(Error::InvalidTransportParam),
8033
+ None if !self.is_server => return Err(Error::InvalidTransportParam),
7933
8034
 
7934
8035
  None => (),
7935
8036
  }
@@ -7938,8 +8039,7 @@ impl<F: BufFactory> Connection<F> {
7938
8039
  // Validate retry_source_connection_id.
7939
8040
  if let Some(rscid) = &self.rscid {
7940
8041
  match &peer_params.retry_source_connection_id {
7941
- Some(v) if v != rscid =>
7942
- return Err(Error::InvalidTransportParam),
8042
+ Some(v) if v != rscid => return Err(Error::InvalidTransportParam),
7943
8043
 
7944
8044
  Some(_) => (),
7945
8045
 
@@ -7956,9 +8056,7 @@ impl<F: BufFactory> Connection<F> {
7956
8056
  Ok(())
7957
8057
  }
7958
8058
 
7959
- fn process_peer_transport_params(
7960
- &mut self, peer_params: TransportParams,
7961
- ) -> Result<()> {
8059
+ fn process_peer_transport_params(&mut self, peer_params: TransportParams) -> Result<()> {
7962
8060
  self.max_tx_data = peer_params.initial_max_data;
7963
8061
 
7964
8062
  // Update send capacity.
@@ -7970,8 +8068,12 @@ impl<F: BufFactory> Connection<F> {
7970
8068
  .update_peer_max_streams_uni(peer_params.initial_max_streams_uni);
7971
8069
 
7972
8070
  let max_ack_delay = Duration::from_millis(peer_params.max_ack_delay);
8071
+ let max_udp_payload_size =
8072
+ usize::try_from(peer_params.max_udp_payload_size).unwrap_or(usize::MAX);
7973
8073
 
7974
8074
  self.recovery_config.max_ack_delay = max_ack_delay;
8075
+ self.recovery_config
8076
+ .update_max_send_udp_payload_size(max_udp_payload_size);
7975
8077
 
7976
8078
  let active_path = self.paths.get_active_mut()?;
7977
8079
 
@@ -7989,12 +8091,12 @@ impl<F: BufFactory> Connection<F> {
7989
8091
  .as_mut()
7990
8092
  .expect("PMTUD existence verified above")
7991
8093
  .get_probe_size()
7992
- .min(peer_params.max_udp_payload_size as usize),
8094
+ .min(max_udp_payload_size),
7993
8095
  );
7994
8096
  } else {
7995
- active_path.recovery.update_max_datagram_size(
7996
- peer_params.max_udp_payload_size as usize,
7997
- );
8097
+ active_path
8098
+ .recovery
8099
+ .update_max_datagram_size(max_udp_payload_size);
7998
8100
  }
7999
8101
 
8000
8102
  // Record the max_active_conn_id parameter advertised by the peer.
@@ -8029,89 +8131,93 @@ impl<F: BufFactory> Connection<F> {
8029
8131
 
8030
8132
  tx_cap_factor: self.tx_cap_factor,
8031
8133
 
8134
+ max_outstanding_sent_packets: self.max_outstanding_sent_packets,
8135
+
8032
8136
  pmtud: None,
8033
8137
 
8034
8138
  is_server: self.is_server,
8035
-
8036
- use_initial_max_data_as_flow_control_win: false,
8037
8139
  };
8038
8140
 
8039
8141
  if self.handshake_completed {
8040
8142
  return self.handshake.process_post_handshake(&mut ex_data);
8041
8143
  }
8042
8144
 
8043
- match self.handshake.do_handshake(&mut ex_data) {
8044
- Ok(_) => (),
8145
+ let handshake_needs_retry = match self.handshake.do_handshake(&mut ex_data) {
8146
+ Ok(_) => false,
8147
+ Err(Error::Done) => true,
8148
+ Err(e) => return Err(e),
8149
+ };
8045
8150
 
8046
- Err(Error::Done) => {
8047
- // Apply in-handshake configuration from callbacks if the path's
8048
- // Recovery module can still be reinitilized.
8049
- if self
8050
- .paths
8051
- .get_active()
8052
- .map(|p| p.can_reinit_recovery())
8053
- .unwrap_or(false)
8054
- {
8055
- if ex_data.recovery_config != self.recovery_config {
8056
- if let Ok(path) = self.paths.get_active_mut() {
8057
- self.recovery_config = ex_data.recovery_config;
8058
- path.reinit_recovery(&self.recovery_config);
8059
- }
8060
- }
8151
+ // BoringSSL reports success when entering early data before the
8152
+ // handshake completes. Apply callback configuration after either
8153
+ // non-fatal outcome so it is not lost on that path.
8154
+ if self
8155
+ .paths
8156
+ .get_active()
8157
+ .map(|p| p.can_reinit_recovery())
8158
+ .unwrap_or(false)
8159
+ {
8160
+ if ex_data.recovery_config != self.recovery_config {
8161
+ // If we received peer transport parameters already, respect the
8162
+ // peer's `max_udp_payload_size`.
8163
+ if self.parsed_peer_transport_params {
8164
+ let max_udp_payload_size =
8165
+ usize::try_from(self.peer_transport_params.max_udp_payload_size)
8166
+ .unwrap_or(usize::MAX);
8167
+ ex_data
8168
+ .recovery_config
8169
+ .update_max_send_udp_payload_size(max_udp_payload_size);
8170
+ }
8061
8171
 
8062
- if ex_data.tx_cap_factor != self.tx_cap_factor {
8063
- self.tx_cap_factor = ex_data.tx_cap_factor;
8064
- }
8172
+ if let Ok(path) = self.paths.get_active_mut() {
8173
+ self.recovery_config = ex_data.recovery_config;
8174
+ path.reinit_recovery(&self.recovery_config);
8175
+ }
8176
+ }
8065
8177
 
8066
- if let Some((discover, max_probes)) = ex_data.pmtud {
8067
- self.paths.set_discover_pmtu_on_existing_paths(
8068
- discover,
8069
- self.recovery_config.max_send_udp_payload_size,
8070
- max_probes,
8071
- );
8072
- }
8178
+ if ex_data.tx_cap_factor != self.tx_cap_factor {
8179
+ self.tx_cap_factor = ex_data.tx_cap_factor;
8180
+ }
8073
8181
 
8074
- if ex_data.local_transport_params !=
8075
- self.local_transport_params
8076
- {
8077
- self.streams.set_max_streams_bidi(
8078
- ex_data
8079
- .local_transport_params
8080
- .initial_max_streams_bidi,
8081
- );
8182
+ self.max_outstanding_sent_packets = ex_data.max_outstanding_sent_packets;
8082
8183
 
8083
- self.local_transport_params =
8084
- ex_data.local_transport_params;
8085
- }
8086
- }
8184
+ if let Some((discover, max_probes)) = ex_data.pmtud {
8185
+ self.paths.set_discover_pmtu_on_existing_paths(
8186
+ discover,
8187
+ self.recovery_config.max_send_udp_payload_size,
8188
+ max_probes,
8189
+ );
8190
+ }
8087
8191
 
8088
- if ex_data.use_initial_max_data_as_flow_control_win {
8089
- self.enable_use_initial_max_data_as_flow_control_win();
8090
- }
8192
+ if ex_data.local_transport_params != self.local_transport_params {
8193
+ self.streams
8194
+ .set_max_streams_bidi(ex_data.local_transport_params.initial_max_streams_bidi);
8091
8195
 
8092
- // Try to parse transport parameters as soon as the first flight
8093
- // of handshake data is processed.
8094
- //
8095
- // This is potentially dangerous as the handshake hasn't been
8096
- // completed yet, though it's required to be able to send data
8097
- // in 0.5 RTT.
8098
- let raw_params = self.handshake.quic_transport_params();
8099
-
8100
- if !self.parsed_peer_transport_params && !raw_params.is_empty() {
8101
- let peer_params = TransportParams::decode(
8102
- raw_params,
8103
- self.is_server,
8104
- self.peer_transport_params_track_unknown,
8105
- )?;
8196
+ self.local_transport_params = ex_data.local_transport_params;
8197
+ }
8198
+ }
8106
8199
 
8107
- self.parse_peer_transport_params(peer_params)?;
8108
- }
8200
+ if handshake_needs_retry {
8201
+ // Try to parse transport parameters as soon as the first flight of
8202
+ // handshake data is processed.
8203
+ //
8204
+ // This is potentially dangerous as the handshake hasn't been
8205
+ // completed yet, though it's required to be able to send data in
8206
+ // 0.5 RTT.
8207
+ let raw_params = self.handshake.quic_transport_params();
8208
+
8209
+ if !self.parsed_peer_transport_params && !raw_params.is_empty() {
8210
+ let peer_params = TransportParams::decode(
8211
+ raw_params,
8212
+ self.is_server,
8213
+ self.peer_transport_params_track_unknown,
8214
+ )?;
8109
8215
 
8110
- return Ok(());
8111
- },
8216
+ self.parse_peer_transport_params(peer_params)?;
8217
+ }
8112
8218
 
8113
- Err(e) => return Err(e),
8114
- };
8219
+ return Ok(());
8220
+ }
8115
8221
 
8116
8222
  self.handshake_completed = self.handshake.is_completed();
8117
8223
 
@@ -8156,19 +8262,6 @@ impl<F: BufFactory> Connection<F> {
8156
8262
  Ok(())
8157
8263
  }
8158
8264
 
8159
- /// Use the value of the intial max_data / initial stream max_data setting
8160
- /// as the initial flow control window for the connection and streams.
8161
- /// The connection-level flow control window will only be changed if it
8162
- /// hasn't been auto tuned yet. For streams: only newly created streams
8163
- /// receive the new setting.
8164
- fn enable_use_initial_max_data_as_flow_control_win(&mut self) {
8165
- self.flow_control.set_window_if_not_tuned_yet(
8166
- self.local_transport_params.initial_max_data,
8167
- );
8168
- self.streams
8169
- .set_use_initial_max_data_as_flow_control_win(true);
8170
- }
8171
-
8172
8265
  /// Selects the packet type for the next outgoing packet.
8173
8266
  fn write_pkt_type(&self, send_pid: usize) -> Result<Type> {
8174
8267
  // On error send packet in the latest epoch available, but only send
@@ -8195,7 +8288,9 @@ impl<F: BufFactory> Connection<F> {
8195
8288
  // not be able to decrypt handshake packets yet.
8196
8289
  packet::Epoch::Handshake
8197
8290
  if self.crypto_ctx[packet::Epoch::Initial].has_keys() =>
8198
- return Ok(Type::Initial),
8291
+ {
8292
+ return Ok(Type::Initial)
8293
+ }
8199
8294
 
8200
8295
  _ => (),
8201
8296
  };
@@ -8204,9 +8299,19 @@ impl<F: BufFactory> Connection<F> {
8204
8299
  return Ok(Type::from_epoch(epoch));
8205
8300
  }
8206
8301
 
8207
- for &epoch in packet::Epoch::epochs(
8208
- packet::Epoch::Initial..=packet::Epoch::Application,
8209
- ) {
8302
+ // APPLICATION_CLOSE can only be sent in a 1-RTT packet. Prioritize it
8303
+ // over obsolete lower-epoch PTO probes, which cannot be consumed while
8304
+ // closing because PING frames are suppressed.
8305
+ if self.is_established()
8306
+ && self
8307
+ .local_error
8308
+ .as_ref()
8309
+ .is_some_and(|conn_err| conn_err.is_app)
8310
+ {
8311
+ return Ok(Type::Short);
8312
+ }
8313
+
8314
+ for &epoch in packet::Epoch::epochs(packet::Epoch::Initial..=packet::Epoch::Application) {
8210
8315
  let crypto_ctx = &self.crypto_ctx[epoch];
8211
8316
  let pkt_space = &self.pkt_num_spaces[epoch];
8212
8317
 
@@ -8236,36 +8341,39 @@ impl<F: BufFactory> Connection<F> {
8236
8341
  // If there are flushable, almost full or blocked streams, use the
8237
8342
  // Application epoch.
8238
8343
  let send_path = self.paths.get(send_pid)?;
8239
- if (self.is_established() || self.is_in_early_data()) &&
8240
- (self.should_send_handshake_done() ||
8241
- self.flow_control.should_update_max_data() ||
8242
- self.should_send_max_data ||
8243
- self.blocked_limit.is_some() ||
8244
- self.streams_blocked_bidi_state
8245
- .has_pending_stream_blocked_frame() ||
8246
- self.streams_blocked_uni_state
8247
- .has_pending_stream_blocked_frame() ||
8248
- self.dgram_send_queue.has_pending() ||
8249
- self.local_error
8344
+ if (self.is_established() || self.is_in_early_data())
8345
+ && (self.should_send_handshake_done()
8346
+ || self.flow_control.should_update_max_data()
8347
+ || self.should_send_max_data
8348
+ || self.blocked_limit.is_some()
8349
+ || self
8350
+ .streams_blocked_bidi_state
8351
+ .has_pending_stream_blocked_frame()
8352
+ || self
8353
+ .streams_blocked_uni_state
8354
+ .has_pending_stream_blocked_frame()
8355
+ || self.dgram_send_queue.has_pending()
8356
+ || self
8357
+ .local_error
8250
8358
  .as_ref()
8251
- .is_some_and(|conn_err| conn_err.is_app) ||
8252
- self.should_send_max_streams_bidi ||
8253
- self.streams.should_update_max_streams_bidi() ||
8254
- self.should_send_max_streams_uni ||
8255
- self.streams.should_update_max_streams_uni() ||
8256
- self.streams.has_flushable() ||
8257
- self.streams.has_almost_full() ||
8258
- self.streams.has_blocked() ||
8259
- self.streams.has_reset() ||
8260
- self.streams.has_stopped() ||
8261
- self.ids.has_new_scids() ||
8262
- self.ids.has_retire_dcids() ||
8263
- send_path
8359
+ .is_some_and(|conn_err| conn_err.is_app)
8360
+ || self.should_send_max_streams_bidi
8361
+ || self.streams.should_update_max_streams_bidi()
8362
+ || self.should_send_max_streams_uni
8363
+ || self.streams.should_update_max_streams_uni()
8364
+ || self.streams.has_flushable()
8365
+ || self.streams.has_almost_full()
8366
+ || self.streams.has_blocked()
8367
+ || self.streams.has_reset()
8368
+ || self.streams.has_stopped()
8369
+ || self.ids.has_new_scids()
8370
+ || self.ids.has_retire_dcids()
8371
+ || send_path
8264
8372
  .pmtud
8265
8373
  .as_ref()
8266
- .is_some_and(|pmtud| pmtud.should_probe()) ||
8267
- send_path.needs_ack_eliciting ||
8268
- send_path.probing_required())
8374
+ .is_some_and(|pmtud| pmtud.should_probe())
8375
+ || send_path.needs_ack_eliciting
8376
+ || send_path.probing_required())
8269
8377
  {
8270
8378
  // Only clients can send 0-RTT packets.
8271
8379
  if !self.is_server && self.is_in_early_data() {
@@ -8280,9 +8388,7 @@ impl<F: BufFactory> Connection<F> {
8280
8388
 
8281
8389
  /// Returns the mutable stream with the given ID if it exists, or creates
8282
8390
  /// a new one otherwise.
8283
- fn get_or_create_stream(
8284
- &mut self, id: u64, local: bool,
8285
- ) -> Result<&mut stream::Stream<F>> {
8391
+ fn get_or_create_stream(&mut self, id: u64, local: bool) -> Result<&mut stream::Stream<F>> {
8286
8392
  self.streams.get_or_create(
8287
8393
  id,
8288
8394
  &self.local_transport_params,
@@ -8292,10 +8398,30 @@ impl<F: BufFactory> Connection<F> {
8292
8398
  )
8293
8399
  }
8294
8400
 
8401
+ /// Gets or creates a stream referenced by a received frame.
8402
+ fn get_or_create_stream_for_received_frame(
8403
+ &mut self,
8404
+ id: u64,
8405
+ ) -> Result<&mut stream::Stream<F>> {
8406
+ let local = stream::is_local(id, self.is_server);
8407
+
8408
+ // Opening a higher-numbered stream also opens lower streams of the
8409
+ // same type, even if they do not have Stream objects yet.
8410
+ if local && !self.streams.local_stream_opened(id) {
8411
+ return Err(Error::InvalidStreamState(id));
8412
+ }
8413
+
8414
+ self.get_or_create_stream(id, local)
8415
+ }
8416
+
8295
8417
  /// Processes an incoming frame.
8296
8418
  fn process_frame(
8297
- &mut self, frame: frame::Frame, hdr: &Header, recv_path_id: usize,
8298
- epoch: packet::Epoch, now: Instant,
8419
+ &mut self,
8420
+ frame: frame::Frame,
8421
+ hdr: &Header,
8422
+ recv_path_id: usize,
8423
+ epoch: packet::Epoch,
8424
+ now: Instant,
8299
8425
  ) -> Result<()> {
8300
8426
  trace!("{} rx frm {:?}", self.trace_id, frame);
8301
8427
 
@@ -8308,14 +8434,11 @@ impl<F: BufFactory> Connection<F> {
8308
8434
  ranges, ack_delay, ..
8309
8435
  } => {
8310
8436
  let ack_delay = ack_delay
8311
- .checked_mul(2_u64.pow(
8312
- self.peer_transport_params.ack_delay_exponent as u32,
8313
- ))
8437
+ .checked_mul(2_u64.pow(self.peer_transport_params.ack_delay_exponent as u32))
8314
8438
  .ok_or(Error::InvalidFrame)?;
8315
8439
 
8316
- if epoch == packet::Epoch::Handshake ||
8317
- (epoch == packet::Epoch::Application &&
8318
- self.is_established())
8440
+ if epoch == packet::Epoch::Handshake
8441
+ || (epoch == packet::Epoch::Application && self.is_established())
8319
8442
  {
8320
8443
  self.peer_verified_initial_address = true;
8321
8444
  }
@@ -8324,9 +8447,9 @@ impl<F: BufFactory> Connection<F> {
8324
8447
 
8325
8448
  let is_app_limited = self.delivery_rate_check_if_app_limited();
8326
8449
 
8327
- let largest_acked = ranges.last().expect(
8328
- "ACK frames should always have at least one ack range",
8329
- );
8450
+ let largest_acked = ranges
8451
+ .last()
8452
+ .expect("ACK frames should always have at least one ack range");
8330
8453
 
8331
8454
  for (_, p) in self.paths.iter_mut() {
8332
8455
  if self.pkt_num_spaces[epoch]
@@ -8360,14 +8483,10 @@ impl<F: BufFactory> Connection<F> {
8360
8483
  )?;
8361
8484
 
8362
8485
  let skip_pn = self.pkt_num_manager.skip_pn();
8363
- let largest_acked =
8364
- p.recovery.get_largest_acked_on_epoch(epoch);
8486
+ let largest_acked = p.recovery.get_largest_acked_on_epoch(epoch);
8365
8487
 
8366
- // Consider the skip_pn validated if the peer has sent an ack
8367
- // for a larger pkt number.
8368
- if let Some((largest_acked, skip_pn)) =
8369
- largest_acked.zip(skip_pn)
8370
- {
8488
+ // A higher ACK validates `skip_pn`.
8489
+ if let Some((largest_acked, skip_pn)) = largest_acked.zip(skip_pn) {
8371
8490
  if largest_acked > skip_pn {
8372
8491
  self.pkt_num_manager.set_skip_pn(None);
8373
8492
  }
@@ -8378,17 +8497,23 @@ impl<F: BufFactory> Connection<F> {
8378
8497
  self.acked_bytes += acked_bytes as u64;
8379
8498
  self.spurious_lost_count += spurious_losses;
8380
8499
  }
8381
- },
8500
+ }
8382
8501
 
8383
8502
  frame::Frame::ResetStream {
8384
8503
  stream_id,
8385
8504
  error_code,
8386
8505
  final_size,
8506
+ reliable_size,
8387
8507
  } => {
8508
+ if reliable_size.is_some() && !self.local_transport_params.reset_stream_at {
8509
+ return Err(Error::InvalidState);
8510
+ }
8511
+ if reliable_size.is_some_and(|size| size > final_size) {
8512
+ return Err(Error::InvalidFrame);
8513
+ }
8514
+ let check_error_code = self.local_transport_params.reset_stream_at;
8388
8515
  // Peer can't send on our unidirectional streams.
8389
- if !stream::is_bidi(stream_id) &&
8390
- stream::is_local(stream_id, self.is_server)
8391
- {
8516
+ if !stream::is_bidi(stream_id) && stream::is_local(stream_id, self.is_server) {
8392
8517
  return Err(Error::InvalidStreamState(stream_id));
8393
8518
  }
8394
8519
 
@@ -8404,7 +8529,7 @@ impl<F: BufFactory> Connection<F> {
8404
8529
  // Note that it makes it impossible to check if the frame is
8405
8530
  // illegal, since we have no state, but since we ignore the
8406
8531
  // frame, it should be fine.
8407
- let stream = match self.get_or_create_stream(stream_id, false) {
8532
+ let stream = match self.get_or_create_stream_for_received_frame(stream_id) {
8408
8533
  Ok(v) => v,
8409
8534
 
8410
8535
  Err(Error::Done) => return Ok(()),
@@ -8415,15 +8540,31 @@ impl<F: BufFactory> Connection<F> {
8415
8540
  let was_readable = stream.is_readable();
8416
8541
  let priority_key = Arc::clone(&stream.priority_key);
8417
8542
 
8543
+ if check_error_code
8544
+ && stream
8545
+ .recv
8546
+ .reset_error()
8547
+ .is_some_and(|old| old != error_code)
8548
+ {
8549
+ return Err(Error::InvalidStreamState(stream_id));
8550
+ }
8418
8551
  let stream::RecvBufResetReturn {
8419
8552
  max_data_delta,
8420
8553
  consumed_flowcontrol,
8421
- } = stream.recv.reset(error_code, final_size)?;
8554
+ } = match reliable_size {
8555
+ Some(size) => stream.recv.reset_at(error_code, final_size, size)?,
8556
+ None => stream.recv.reset(error_code, final_size)?,
8557
+ };
8422
8558
 
8423
8559
  if max_data_delta > max_rx_data_left {
8424
8560
  return Err(Error::FlowControl);
8425
8561
  }
8426
8562
 
8563
+ // The receive side can complete without the stream becoming
8564
+ // readable, so no later read would collect it.
8565
+ let state = stream.state();
8566
+ let local = stream.local;
8567
+
8427
8568
  if !was_readable && stream.is_readable() {
8428
8569
  self.streams.insert_readable(&priority_key);
8429
8570
  }
@@ -8433,18 +8574,19 @@ impl<F: BufFactory> Connection<F> {
8433
8574
  // flow-control
8434
8575
  self.flow_control.add_consumed(consumed_flowcontrol);
8435
8576
 
8436
- self.reset_stream_remote_count =
8437
- self.reset_stream_remote_count.saturating_add(1);
8438
- },
8577
+ self.reset_stream_remote_count = self.reset_stream_remote_count.saturating_add(1);
8578
+
8579
+ if state == stream::StreamState::Collectable {
8580
+ self.streams.collect(stream_id, local);
8581
+ }
8582
+ }
8439
8583
 
8440
8584
  frame::Frame::StopSending {
8441
8585
  stream_id,
8442
8586
  error_code,
8443
8587
  } => {
8444
8588
  // STOP_SENDING on a receive-only stream is a fatal error.
8445
- if !stream::is_local(stream_id, self.is_server) &&
8446
- !stream::is_bidi(stream_id)
8447
- {
8589
+ if !stream::is_local(stream_id, self.is_server) && !stream::is_bidi(stream_id) {
8448
8590
  return Err(Error::InvalidStreamState(stream_id));
8449
8591
  }
8450
8592
 
@@ -8458,7 +8600,7 @@ impl<F: BufFactory> Connection<F> {
8458
8600
  // Note that it makes it impossible to check if the frame is
8459
8601
  // illegal, since we have no state, but since we ignore the
8460
8602
  // frame, it should be fine.
8461
- let stream = match self.get_or_create_stream(stream_id, false) {
8603
+ let stream = match self.get_or_create_stream_for_received_frame(stream_id) {
8462
8604
  Ok(v) => v,
8463
8605
 
8464
8606
  Err(Error::Done) => return Ok(()),
@@ -8466,12 +8608,17 @@ impl<F: BufFactory> Connection<F> {
8466
8608
  Err(e) => return Err(e),
8467
8609
  };
8468
8610
 
8469
- let was_writable = stream.is_writable();
8470
-
8471
8611
  let priority_key = Arc::clone(&stream.priority_key);
8472
8612
 
8613
+ // Save the buffered length before stopping (stop clears the
8614
+ // buffer).
8615
+ let buffered_len = stream.send.buffered_bytes() as usize;
8616
+
8473
8617
  // Try stopping the stream.
8474
8618
  if let Ok((final_size, unsent)) = stream.send.stop(error_code) {
8619
+ let dropped =
8620
+ buffered_len.saturating_sub(stream.send.buffered_bytes() as usize);
8621
+ let flushable = stream.is_flushable();
8475
8622
  // Claw back some flow control allowance from data that was
8476
8623
  // buffered but not actually sent before the stream was
8477
8624
  // reset.
@@ -8480,18 +8627,17 @@ impl<F: BufFactory> Connection<F> {
8480
8627
  // to touch it here.
8481
8628
  self.tx_data = self.tx_data.saturating_sub(unsent);
8482
8629
 
8483
- self.tx_buffered =
8484
- self.tx_buffered.saturating_sub(unsent as usize);
8630
+ // Update tx_buffered: subtract only the buffered data, not
8631
+ // inflight data.
8632
+ self.streams.sub_tx_buffered(dropped);
8485
8633
 
8486
- // These drops in qlog are a bit weird, but the only way to
8487
- // ensure that all bytes that are moved from App to Transport
8488
- // in stream_do_send are eventually moved from Transport to
8489
- // Dropped. Ideally we would add a Transport to Network
8490
- // transition also as a way to indicate when bytes were
8491
- // transmitted vs dropped without ever being sent.
8634
+ // Match moves from App to Transport with moves from
8635
+ // Transport to Dropped.
8636
+ // A Network transition would distinguish sent bytes from
8637
+ // drops before transmission.
8492
8638
  qlog_with_type!(QLOG_DATA_MV, self.qlog, q, {
8493
- let ev_data = EventData::QuicStreamDataMoved(
8494
- qlog::events::quic::StreamDataMoved {
8639
+ let ev_data =
8640
+ EventData::QuicStreamDataMoved(qlog::events::quic::StreamDataMoved {
8495
8641
  stream_id: Some(stream_id),
8496
8642
  offset: Some(final_size),
8497
8643
  raw: Some(RawInfo {
@@ -8501,24 +8647,24 @@ impl<F: BufFactory> Connection<F> {
8501
8647
  from: Some(DataRecipient::Transport),
8502
8648
  to: Some(DataRecipient::Dropped),
8503
8649
  ..Default::default()
8504
- },
8505
- );
8650
+ });
8506
8651
 
8507
8652
  q.add_event_data_with_instant(ev_data, now).ok();
8508
8653
  });
8509
8654
 
8510
8655
  self.streams.insert_reset(stream_id, error_code, final_size);
8511
8656
 
8512
- if !was_writable {
8513
- self.streams.insert_writable(&priority_key);
8657
+ self.streams.insert_stopped_writable(&priority_key);
8658
+
8659
+ if !flushable {
8660
+ self.streams.remove_flushable(&priority_key);
8514
8661
  }
8515
8662
 
8516
8663
  self.stopped_stream_remote_count =
8517
8664
  self.stopped_stream_remote_count.saturating_add(1);
8518
- self.reset_stream_local_count =
8519
- self.reset_stream_local_count.saturating_add(1);
8665
+ self.reset_stream_local_count = self.reset_stream_local_count.saturating_add(1);
8520
8666
  }
8521
- },
8667
+ }
8522
8668
 
8523
8669
  frame::Frame::Crypto { data } => {
8524
8670
  if data.max_off() >= MAX_CRYPTO_STREAM_OFFSET {
@@ -8526,7 +8672,11 @@ impl<F: BufFactory> Connection<F> {
8526
8672
  }
8527
8673
 
8528
8674
  // Push the data to the stream so it can be re-ordered.
8529
- self.crypto_ctx[epoch].crypto_stream.recv.write(data)?;
8675
+ let stream = &mut self.crypto_ctx[epoch].crypto_stream;
8676
+ stream
8677
+ .recv
8678
+ .set_memory_budget(self.streams.receive_memory.clone());
8679
+ stream.recv.write(data)?;
8530
8680
 
8531
8681
  // Feed crypto data to the TLS state, if there's data
8532
8682
  // available at the expected offset.
@@ -8534,29 +8684,26 @@ impl<F: BufFactory> Connection<F> {
8534
8684
 
8535
8685
  let level = crypto::Level::from_epoch(epoch);
8536
8686
 
8537
- let stream = &mut self.crypto_ctx[epoch].crypto_stream;
8538
-
8539
8687
  while let Ok((read, _)) = stream.recv.emit(&mut crypto_buf) {
8540
8688
  let recv_buf = &crypto_buf[..read];
8541
8689
  self.handshake.provide_data(level, recv_buf)?;
8542
8690
  }
8543
8691
 
8544
8692
  self.do_handshake(now)?;
8545
- },
8693
+ }
8546
8694
 
8547
8695
  frame::Frame::CryptoHeader { .. } => unreachable!(),
8548
8696
 
8549
8697
  // TODO: implement stateless retry
8550
- frame::Frame::NewToken { .. } =>
8698
+ frame::Frame::NewToken { .. } => {
8551
8699
  if self.is_server {
8552
8700
  return Err(Error::InvalidPacket);
8553
- },
8701
+ }
8702
+ }
8554
8703
 
8555
8704
  frame::Frame::Stream { stream_id, data } => {
8556
8705
  // Peer can't send on our unidirectional streams.
8557
- if !stream::is_bidi(stream_id) &&
8558
- stream::is_local(stream_id, self.is_server)
8559
- {
8706
+ if !stream::is_bidi(stream_id) && stream::is_local(stream_id, self.is_server) {
8560
8707
  return Err(Error::InvalidStreamState(stream_id));
8561
8708
  }
8562
8709
 
@@ -8572,7 +8719,7 @@ impl<F: BufFactory> Connection<F> {
8572
8719
  // Note that it makes it impossible to check if the frame is
8573
8720
  // illegal, since we have no state, but since we ignore the
8574
8721
  // frame, it should be fine.
8575
- let stream = match self.get_or_create_stream(stream_id, false) {
8722
+ let stream = match self.get_or_create_stream_for_received_frame(stream_id) {
8576
8723
  Ok(v) => v,
8577
8724
 
8578
8725
  Err(Error::Done) => return Ok(()),
@@ -8581,8 +8728,7 @@ impl<F: BufFactory> Connection<F> {
8581
8728
  };
8582
8729
 
8583
8730
  // Check for the connection-level flow control limit.
8584
- let max_off_delta =
8585
- data.max_off().saturating_sub(stream.recv.max_off());
8731
+ let max_off_delta = data.max_off().saturating_sub(stream.recv.max_off());
8586
8732
 
8587
8733
  if max_off_delta > max_rx_data_left {
8588
8734
  return Err(Error::FlowControl);
@@ -8595,6 +8741,11 @@ impl<F: BufFactory> Connection<F> {
8595
8741
 
8596
8742
  stream.recv.write(data)?;
8597
8743
 
8744
+ // The receive side can complete without the stream becoming
8745
+ // readable, so no later read would collect it.
8746
+ let state = stream.state();
8747
+ let local = stream.local;
8748
+
8598
8749
  if !was_readable && stream.is_readable() {
8599
8750
  self.streams.insert_readable(&priority_key);
8600
8751
  }
@@ -8608,19 +8759,21 @@ impl<F: BufFactory> Connection<F> {
8608
8759
  // control update.
8609
8760
  self.flow_control.add_consumed(max_off_delta);
8610
8761
  }
8611
- },
8762
+
8763
+ if state == stream::StreamState::Collectable {
8764
+ self.streams.collect(stream_id, local);
8765
+ }
8766
+ }
8612
8767
 
8613
8768
  frame::Frame::StreamHeader { .. } => unreachable!(),
8614
8769
 
8615
8770
  frame::Frame::MaxData { max } => {
8616
8771
  self.max_tx_data = cmp::max(self.max_tx_data, max);
8617
- },
8772
+ }
8618
8773
 
8619
8774
  frame::Frame::MaxStreamData { stream_id, max } => {
8620
8775
  // Peer can't receive on its own unidirectional streams.
8621
- if !stream::is_bidi(stream_id) &&
8622
- !stream::is_local(stream_id, self.is_server)
8623
- {
8776
+ if !stream::is_bidi(stream_id) && !stream::is_local(stream_id, self.is_server) {
8624
8777
  return Err(Error::InvalidStreamState(stream_id));
8625
8778
  }
8626
8779
 
@@ -8634,7 +8787,7 @@ impl<F: BufFactory> Connection<F> {
8634
8787
  // Note that it makes it impossible to check if the frame is
8635
8788
  // illegal, since we have no state, but since we ignore the
8636
8789
  // frame, it should be fine.
8637
- let stream = match self.get_or_create_stream(stream_id, false) {
8790
+ let stream = match self.get_or_create_stream_for_received_frame(stream_id) {
8638
8791
  Ok(v) => v,
8639
8792
 
8640
8793
  Err(Error::Done) => return Ok(()),
@@ -8650,8 +8803,8 @@ impl<F: BufFactory> Connection<F> {
8650
8803
 
8651
8804
  let priority_key = Arc::clone(&stream.priority_key);
8652
8805
 
8653
- // If the stream is now flushable push it to the flushable queue,
8654
- // but only if it wasn't already queued.
8806
+ // If the stream became flushable, add it to the queue unless it
8807
+ // is already present.
8655
8808
  if stream.is_flushable() && !was_flushable {
8656
8809
  let priority_key = Arc::clone(&stream.priority_key);
8657
8810
  self.streams.insert_flushable(&priority_key);
@@ -8660,7 +8813,7 @@ impl<F: BufFactory> Connection<F> {
8660
8813
  if writable {
8661
8814
  self.streams.insert_writable(&priority_key);
8662
8815
  }
8663
- },
8816
+ }
8664
8817
 
8665
8818
  frame::Frame::MaxStreamsBidi { max } => {
8666
8819
  if max > MAX_STREAM_ID {
@@ -8668,7 +8821,7 @@ impl<F: BufFactory> Connection<F> {
8668
8821
  }
8669
8822
 
8670
8823
  self.streams.update_peer_max_streams_bidi(max);
8671
- },
8824
+ }
8672
8825
 
8673
8826
  frame::Frame::MaxStreamsUni { max } => {
8674
8827
  if max > MAX_STREAM_ID {
@@ -8676,17 +8829,16 @@ impl<F: BufFactory> Connection<F> {
8676
8829
  }
8677
8830
 
8678
8831
  self.streams.update_peer_max_streams_uni(max);
8679
- },
8832
+ }
8680
8833
 
8681
8834
  frame::Frame::DataBlocked { .. } => {
8682
- self.data_blocked_recv_count =
8683
- self.data_blocked_recv_count.saturating_add(1);
8684
- },
8835
+ self.data_blocked_recv_count = self.data_blocked_recv_count.saturating_add(1);
8836
+ }
8685
8837
 
8686
8838
  frame::Frame::StreamDataBlocked { .. } => {
8687
8839
  self.stream_data_blocked_recv_count =
8688
8840
  self.stream_data_blocked_recv_count.saturating_add(1);
8689
- },
8841
+ }
8690
8842
 
8691
8843
  frame::Frame::StreamsBlockedBidi { limit } => {
8692
8844
  if limit > MAX_STREAM_ID {
@@ -8695,7 +8847,7 @@ impl<F: BufFactory> Connection<F> {
8695
8847
 
8696
8848
  self.streams_blocked_bidi_recv_count =
8697
8849
  self.streams_blocked_bidi_recv_count.saturating_add(1);
8698
- },
8850
+ }
8699
8851
 
8700
8852
  frame::Frame::StreamsBlockedUni { limit } => {
8701
8853
  if limit > MAX_STREAM_ID {
@@ -8704,7 +8856,7 @@ impl<F: BufFactory> Connection<F> {
8704
8856
 
8705
8857
  self.streams_blocked_uni_recv_count =
8706
8858
  self.streams_blocked_uni_recv_count.saturating_add(1);
8707
- },
8859
+ }
8708
8860
 
8709
8861
  frame::Frame::NewConnectionId {
8710
8862
  seq_num,
@@ -8736,16 +8888,17 @@ impl<F: BufFactory> Connection<F> {
8736
8888
  continue;
8737
8889
  }
8738
8890
 
8739
- if let Some(new_dcid_seq) =
8740
- self.ids.lowest_available_dcid_seq()
8741
- {
8891
+ if let Some(new_dcid_seq) = self.ids.lowest_available_dcid_seq() {
8742
8892
  path.active_dcid_seq = Some(new_dcid_seq);
8743
8893
 
8744
8894
  self.ids.link_dcid_to_path_id(new_dcid_seq, pid)?;
8745
8895
 
8746
8896
  trace!(
8747
8897
  "{} path ID {} changed DCID: old seq num {} new seq num {}",
8748
- self.trace_id, pid, dcid_seq, new_dcid_seq,
8898
+ self.trace_id,
8899
+ pid,
8900
+ dcid_seq,
8901
+ new_dcid_seq,
8749
8902
  );
8750
8903
  } else {
8751
8904
  // We cannot use this path anymore for now.
@@ -8753,14 +8906,16 @@ impl<F: BufFactory> Connection<F> {
8753
8906
 
8754
8907
  trace!(
8755
8908
  "{} path ID {} cannot be used; DCID seq num {} has been retired",
8756
- self.trace_id, pid, dcid_seq,
8909
+ self.trace_id,
8910
+ pid,
8911
+ dcid_seq,
8757
8912
  );
8758
8913
  }
8759
8914
  }
8760
8915
 
8761
8916
  // Propagate error (if any) now...
8762
8917
  new_dcid_res?;
8763
- },
8918
+ }
8764
8919
 
8765
8920
  frame::Frame::RetireConnectionId { seq_num } => {
8766
8921
  if self.ids.zero_length_scid() {
@@ -8778,7 +8933,7 @@ impl<F: BufFactory> Connection<F> {
8778
8933
  path.active_scid_seq = None;
8779
8934
  }
8780
8935
  }
8781
- },
8936
+ }
8782
8937
 
8783
8938
  frame::Frame::PathChallenge { data } => {
8784
8939
  self.path_challenge_rx_count += 1;
@@ -8786,11 +8941,11 @@ impl<F: BufFactory> Connection<F> {
8786
8941
  self.paths
8787
8942
  .get_mut(recv_path_id)?
8788
8943
  .on_challenge_received(data);
8789
- },
8944
+ }
8790
8945
 
8791
8946
  frame::Frame::PathResponse { data } => {
8792
8947
  self.paths.on_response_received(data)?;
8793
- },
8948
+ }
8794
8949
 
8795
8950
  frame::Frame::ConnectionClose {
8796
8951
  error_code, reason, ..
@@ -8803,7 +8958,7 @@ impl<F: BufFactory> Connection<F> {
8803
8958
 
8804
8959
  let path = self.paths.get_active()?;
8805
8960
  self.draining_timer = Some(now + (path.recovery.pto() * 3));
8806
- },
8961
+ }
8807
8962
 
8808
8963
  frame::Frame::ApplicationClose { error_code, reason } => {
8809
8964
  self.peer_error = Some(ConnectionError {
@@ -8814,7 +8969,7 @@ impl<F: BufFactory> Connection<F> {
8814
8969
 
8815
8970
  let path = self.paths.get_active()?;
8816
8971
  self.draining_timer = Some(now + (path.recovery.pto() * 3));
8817
- },
8972
+ }
8818
8973
 
8819
8974
  frame::Frame::HandshakeDone => {
8820
8975
  if self.is_server {
@@ -8827,7 +8982,7 @@ impl<F: BufFactory> Connection<F> {
8827
8982
 
8828
8983
  // Once the handshake is confirmed, we can drop Handshake keys.
8829
8984
  self.drop_epoch_state(packet::Epoch::Handshake, now);
8830
- },
8985
+ }
8831
8986
 
8832
8987
  frame::Frame::Datagram { data } => {
8833
8988
  // Close the connection if DATAGRAMs are not enabled.
@@ -8849,7 +9004,7 @@ impl<F: BufFactory> Connection<F> {
8849
9004
 
8850
9005
  let path = self.paths.get_mut(recv_path_id)?;
8851
9006
  path.dgram_recv_count = path.dgram_recv_count.saturating_add(1);
8852
- },
9007
+ }
8853
9008
 
8854
9009
  frame::Frame::DatagramHeader { .. } => unreachable!(),
8855
9010
  }
@@ -8892,8 +9047,8 @@ impl<F: BufFactory> Connection<F> {
8892
9047
  // If the transport parameter is set to 0, then the respective endpoint
8893
9048
  // decided to disable the idle timeout. If both are disabled we should
8894
9049
  // not set any timeout.
8895
- if self.local_transport_params.max_idle_timeout == 0 &&
8896
- self.peer_transport_params.max_idle_timeout == 0
9050
+ if self.local_transport_params.max_idle_timeout == 0
9051
+ && self.peer_transport_params.max_idle_timeout == 0
8897
9052
  {
8898
9053
  return None;
8899
9054
  }
@@ -8925,8 +9080,7 @@ impl<F: BufFactory> Connection<F> {
8925
9080
  /// Returns the connection's handshake status for use in loss recovery.
8926
9081
  fn handshake_status(&self) -> recovery::HandshakeStatus {
8927
9082
  recovery::HandshakeStatus {
8928
- has_handshake_keys: self.crypto_ctx[packet::Epoch::Handshake]
8929
- .has_keys(),
9083
+ has_handshake_keys: self.crypto_ctx[packet::Epoch::Handshake].has_keys(),
8930
9084
 
8931
9085
  peer_verified_address: self.peer_verified_initial_address,
8932
9086
 
@@ -8941,8 +9095,7 @@ impl<F: BufFactory> Connection<F> {
8941
9095
  Err(_) => 0,
8942
9096
  };
8943
9097
 
8944
- let cap =
8945
- cmp::min(cwin_available, self.max_tx_data - self.tx_data) as usize;
9098
+ let cap = cmp::min(cwin_available, self.max_tx_data - self.tx_data) as usize;
8946
9099
  self.tx_cap = (cap as f64 * self.tx_cap_factor).ceil() as usize;
8947
9100
  }
8948
9101
 
@@ -8950,18 +9103,18 @@ impl<F: BufFactory> Connection<F> {
8950
9103
  // Enter the app-limited phase of delivery rate when these conditions
8951
9104
  // are met:
8952
9105
  //
8953
- // - The remaining capacity is higher than available bytes in cwnd (there
9106
+ // - The remaining capacity exceeds the available bytes in CWND (there
8954
9107
  // is more room to send).
8955
- // - New data since the last send() is smaller than available bytes in
8956
- // cwnd (we queued less than what we can send).
8957
- // - There is room to send more data in cwnd.
9108
+ // - New data since the last `send()` is smaller than available bytes in
9109
+ // CWND (we queued less than what we can send).
9110
+ // - CWND has room for more data.
8958
9111
  //
8959
9112
  // In application-limited phases the transmission rate is limited by the
8960
9113
  // application rather than the congestion control algorithm.
8961
9114
  //
8962
- // Note that this is equivalent to CheckIfApplicationLimited() from the
8963
- // delivery rate draft. This is also separate from `recovery.app_limited`
8964
- // and only applies to delivery rate calculation.
9115
+ // This mirrors `CheckIfApplicationLimited()` from the delivery-rate
9116
+ // draft but affects only delivery-rate calculation, not
9117
+ // `recovery.app_limited`.
8965
9118
  let cwin_available = self
8966
9119
  .paths
8967
9120
  .iter()
@@ -8969,34 +9122,15 @@ impl<F: BufFactory> Connection<F> {
8969
9122
  .map(|(_, p)| p.recovery.cwnd_available())
8970
9123
  .sum();
8971
9124
 
8972
- ((self.tx_buffered + self.dgram_send_queue_byte_size()) < cwin_available) &&
8973
- (self.tx_data.saturating_sub(self.last_tx_data)) <
8974
- cwin_available as u64 &&
8975
- cwin_available > 0
8976
- }
8977
-
8978
- fn check_tx_buffered_invariant(&mut self) {
8979
- // tx_buffered should track bytes queued in the stream buffers
8980
- // and unacked retransmitable bytes in the network.
8981
- // If tx_buffered > 0 mark the tx_buffered_state if there are no
8982
- // flushable streams and there no inflight bytes.
8983
- //
8984
- // It is normal to have tx_buffered == 0 while there are inflight bytes
8985
- // since not QUIC frames are retransmittable; inflight tracks all bytes
8986
- // on the network which are subject to congestion control.
8987
- if self.tx_buffered > 0 &&
8988
- !self.streams.has_flushable() &&
8989
- !self
8990
- .paths
8991
- .iter()
8992
- .any(|(_, p)| p.recovery.bytes_in_flight() > 0)
8993
- {
8994
- self.tx_buffered_state = TxBufferTrackingState::Inconsistent;
8995
- }
9125
+ ((self.streams.tx_buffered() + self.dgram_send_queue_byte_size()) < cwin_available)
9126
+ && (self.tx_data.saturating_sub(self.last_tx_data)) < cwin_available as u64
9127
+ && cwin_available > 0
8996
9128
  }
8997
9129
 
8998
9130
  fn set_initial_dcid(
8999
- &mut self, cid: ConnectionId<'static>, reset_token: Option<u128>,
9131
+ &mut self,
9132
+ cid: ConnectionId<'static>,
9133
+ reset_token: Option<u128>,
9000
9134
  path_id: usize,
9001
9135
  ) -> Result<()> {
9002
9136
  self.ids.set_initial_dcid(cid, reset_token, Some(path_id));
@@ -9008,26 +9142,26 @@ impl<F: BufFactory> Connection<F> {
9008
9142
  /// Selects the path that the incoming packet belongs to, or creates a new
9009
9143
  /// one if no existing path matches.
9010
9144
  fn get_or_create_recv_path_id(
9011
- &mut self, recv_pid: Option<usize>, dcid: &ConnectionId, buf_len: usize,
9145
+ &mut self,
9146
+ recv_pid: Option<usize>,
9147
+ dcid: &ConnectionId,
9148
+ buf_len: usize,
9012
9149
  info: &RecvInfo,
9013
9150
  ) -> Result<usize> {
9014
9151
  let ids = &mut self.ids;
9015
9152
 
9016
- let (in_scid_seq, mut in_scid_pid) =
9017
- ids.find_scid_seq(dcid).ok_or(Error::InvalidState)?;
9153
+ let (in_scid_seq, mut in_scid_pid) = ids.find_scid_seq(dcid).ok_or(Error::InvalidState)?;
9018
9154
 
9019
9155
  if let Some(recv_pid) = recv_pid {
9020
9156
  // If the path observes a change of SCID used, note it.
9021
9157
  let recv_path = self.paths.get_mut(recv_pid)?;
9022
9158
 
9023
- let cid_entry =
9024
- recv_path.active_scid_seq.and_then(|v| ids.get_scid(v).ok());
9159
+ let cid_entry = recv_path.active_scid_seq.and_then(|v| ids.get_scid(v).ok());
9025
9160
 
9026
9161
  if cid_entry.map(|e| &e.cid) != Some(dcid) {
9027
9162
  let incoming_cid_entry = ids.get_scid(in_scid_seq)?;
9028
9163
 
9029
- let prev_recv_pid =
9030
- incoming_cid_entry.path_id.unwrap_or(recv_pid);
9164
+ let prev_recv_pid = incoming_cid_entry.path_id.unwrap_or(recv_pid);
9031
9165
 
9032
9166
  if prev_recv_pid != recv_pid {
9033
9167
  trace!(
@@ -9063,32 +9197,17 @@ impl<F: BufFactory> Connection<F> {
9063
9197
  in_scid_pid = None;
9064
9198
  }
9065
9199
 
9066
- if let Some(in_scid_pid) = in_scid_pid {
9067
- // This CID has been used by another path. If we have the
9068
- // room to do so, create a new `Path` structure holding this
9069
- // new 4-tuple. Otherwise, drop the packet.
9070
- let old_path = self.paths.get_mut(in_scid_pid)?;
9071
- let old_local_addr = old_path.local_addr();
9072
- let old_peer_addr = old_path.peer_addr();
9073
-
9074
- trace!(
9075
- "{} reused CID seq {} of ({},{}) (path {}) on ({},{})",
9076
- self.trace_id,
9077
- in_scid_seq,
9078
- old_local_addr,
9079
- old_peer_addr,
9080
- in_scid_pid,
9081
- info.to,
9082
- info.from
9083
- );
9200
+ // Capture old path info before insert_path() so we can emit the
9201
+ // ReusedSourceConnectionId event after successful insertion. This
9202
+ // ensures the event count is bounded by path Slab capacity.
9203
+ let reused_cid_info = match in_scid_pid {
9204
+ Some(pid) => {
9205
+ let old_path = self.paths.get(pid)?;
9206
+ Some((pid, old_path.local_addr(), old_path.peer_addr()))
9207
+ }
9084
9208
 
9085
- // Notify the application.
9086
- self.paths.notify_event(PathEvent::ReusedSourceConnectionId(
9087
- in_scid_seq,
9088
- (old_local_addr, old_peer_addr),
9089
- (info.to, info.from),
9090
- ));
9091
- }
9209
+ None => None,
9210
+ };
9092
9211
 
9093
9212
  // This is a new path using an unassigned CID; create it!
9094
9213
  let mut path = path::Path::new(
@@ -9108,21 +9227,44 @@ impl<F: BufFactory> Connection<F> {
9108
9227
 
9109
9228
  let pid = self.paths.insert_path(path, self.is_server)?;
9110
9229
 
9111
- // Do not record path reuse.
9112
- if in_scid_pid.is_none() {
9113
- ids.link_scid_to_path_id(in_scid_seq, pid)?;
9230
+ // Notify the application of CID reuse only after the path was
9231
+ // successfully admitted. This bounds event queue growth by path Slab
9232
+ // capacity, preventing an attacker from growing the queue unboundedly
9233
+ // by rotating source ports.
9234
+ match reused_cid_info {
9235
+ Some((old_pid, old_local_addr, old_peer_addr)) => {
9236
+ trace!(
9237
+ "{} reused CID seq {} of ({},{}) (path {}) on ({},{})",
9238
+ self.trace_id,
9239
+ in_scid_seq,
9240
+ old_local_addr,
9241
+ old_peer_addr,
9242
+ old_pid,
9243
+ info.to,
9244
+ info.from
9245
+ );
9246
+
9247
+ self.paths.notify_event(PathEvent::ReusedSourceConnectionId(
9248
+ in_scid_seq,
9249
+ (old_local_addr, old_peer_addr),
9250
+ (info.to, info.from),
9251
+ ));
9252
+ }
9253
+
9254
+ None => {
9255
+ ids.link_scid_to_path_id(in_scid_seq, pid)?;
9256
+ }
9114
9257
  }
9115
9258
 
9116
9259
  Ok(pid)
9117
9260
  }
9118
9261
 
9119
9262
  /// Selects the path on which the next packet must be sent.
9120
- fn get_send_path_id(
9121
- &self, from: Option<SocketAddr>, to: Option<SocketAddr>,
9122
- ) -> Result<usize> {
9263
+ fn get_send_path_id(&self, from: Option<SocketAddr>, to: Option<SocketAddr>) -> Result<usize> {
9123
9264
  // A probing packet must be sent, but only if the connection is fully
9124
- // established.
9125
- if self.is_established() {
9265
+ // established. A closing connection only sends the CONNECTION_CLOSE
9266
+ // frame, on the active path.
9267
+ if self.is_established() && self.local_error.is_none() {
9126
9268
  let mut probing = self
9127
9269
  .paths
9128
9270
  .iter()
@@ -9155,12 +9297,11 @@ impl<F: BufFactory> Connection<F> {
9155
9297
  /// Sets the path with identifier 'path_id' to be active.
9156
9298
  fn set_active_path(&mut self, path_id: usize, now: Instant) -> Result<()> {
9157
9299
  if let Ok(old_active_path) = self.paths.get_active_mut() {
9158
- for &e in packet::Epoch::epochs(
9159
- packet::Epoch::Initial..=packet::Epoch::Application,
9160
- ) {
9161
- let (lost_packets, lost_bytes) = old_active_path
9162
- .recovery
9163
- .on_path_change(e, now, &self.trace_id);
9300
+ for &e in packet::Epoch::epochs(packet::Epoch::Initial..=packet::Epoch::Application) {
9301
+ let (lost_packets, lost_bytes) =
9302
+ old_active_path
9303
+ .recovery
9304
+ .on_path_change(e, now, &self.trace_id);
9164
9305
 
9165
9306
  self.lost_count += lost_packets;
9166
9307
  self.lost_bytes += lost_bytes as u64;
@@ -9172,7 +9313,10 @@ impl<F: BufFactory> Connection<F> {
9172
9313
 
9173
9314
  /// Handles potential connection migration.
9174
9315
  fn on_peer_migrated(
9175
- &mut self, new_pid: usize, disable_dcid_reuse: bool, now: Instant,
9316
+ &mut self,
9317
+ new_pid: usize,
9318
+ disable_dcid_reuse: bool,
9319
+ now: Instant,
9176
9320
  ) -> Result<()> {
9177
9321
  let active_path_id = self.paths.get_active_path_id()?;
9178
9322
 
@@ -9182,8 +9326,7 @@ impl<F: BufFactory> Connection<F> {
9182
9326
 
9183
9327
  self.set_active_path(new_pid, now)?;
9184
9328
 
9185
- let no_spare_dcid =
9186
- self.paths.get_mut(new_pid)?.active_dcid_seq.is_none();
9329
+ let no_spare_dcid = self.paths.get_mut(new_pid)?.active_dcid_seq.is_none();
9187
9330
 
9188
9331
  if no_spare_dcid && !disable_dcid_reuse {
9189
9332
  self.paths.get_mut(new_pid)?.active_dcid_seq =
@@ -9195,7 +9338,9 @@ impl<F: BufFactory> Connection<F> {
9195
9338
 
9196
9339
  /// Creates a new client-side path.
9197
9340
  fn create_path_on_client(
9198
- &mut self, local_addr: SocketAddr, peer_addr: SocketAddr,
9341
+ &mut self,
9342
+ local_addr: SocketAddr,
9343
+ peer_addr: SocketAddr,
9199
9344
  ) -> Result<usize> {
9200
9345
  if self.is_server {
9201
9346
  return Err(Error::InvalidState);
@@ -9241,7 +9386,12 @@ impl<F: BufFactory> Connection<F> {
9241
9386
  fn mark_closed(&mut self) {
9242
9387
  #[cfg(feature = "qlog")]
9243
9388
  {
9244
- let cc = match (self.is_established(), self.timed_out, &self.peer_error, &self.local_error) {
9389
+ let cc = match (
9390
+ self.is_established(),
9391
+ self.timed_out,
9392
+ &self.peer_error,
9393
+ &self.local_error,
9394
+ ) {
9245
9395
  (false, _, _, _) => qlog::events::quic::ConnectionClosed {
9246
9396
  initiator: Some(TransportInitiator::Local),
9247
9397
  connection_error: None,
@@ -9249,7 +9399,7 @@ impl<F: BufFactory> Connection<F> {
9249
9399
  error_code: None,
9250
9400
  internal_code: None,
9251
9401
  reason: Some("Failed to establish connection".to_string()),
9252
- trigger: Some(qlog::events::quic::ConnectionClosedTrigger::HandshakeTimeout)
9402
+ trigger: Some(qlog::events::quic::ConnectionClosedTrigger::HandshakeTimeout),
9253
9403
  },
9254
9404
 
9255
9405
  (true, true, _, _) => qlog::events::quic::ConnectionClosed {
@@ -9259,7 +9409,7 @@ impl<F: BufFactory> Connection<F> {
9259
9409
  error_code: None,
9260
9410
  internal_code: None,
9261
9411
  reason: Some("Idle timeout".to_string()),
9262
- trigger: Some(qlog::events::quic::ConnectionClosedTrigger::IdleTimeout)
9412
+ trigger: Some(qlog::events::quic::ConnectionClosedTrigger::IdleTimeout),
9263
9413
  },
9264
9414
 
9265
9415
  (true, false, Some(peer_error), None) => {
@@ -9272,7 +9422,13 @@ impl<F: BufFactory> Connection<F> {
9272
9422
  Some(qlog::events::quic::ConnectionClosedTrigger::Error)
9273
9423
  };
9274
9424
 
9275
- (Some(qlog::events::ConnectionClosedEventError::TransportError(qlog::events::quic::TransportError::Unknown)), None, trigger)
9425
+ (
9426
+ Some(qlog::events::ConnectionClosedEventError::TransportError(
9427
+ qlog::events::quic::TransportError::Unknown,
9428
+ )),
9429
+ None,
9430
+ trigger,
9431
+ )
9276
9432
  };
9277
9433
 
9278
9434
  // TODO: select more appopriate connection_code and application_error than unknown.
@@ -9285,7 +9441,7 @@ impl<F: BufFactory> Connection<F> {
9285
9441
  reason: Some(String::from_utf8_lossy(&peer_error.reason).to_string()),
9286
9442
  trigger,
9287
9443
  }
9288
- },
9444
+ }
9289
9445
 
9290
9446
  (true, false, None, Some(local_error)) => {
9291
9447
  let (connection_code, application_error, trigger) = if local_error.is_app {
@@ -9297,7 +9453,13 @@ impl<F: BufFactory> Connection<F> {
9297
9453
  Some(qlog::events::quic::ConnectionClosedTrigger::Error)
9298
9454
  };
9299
9455
 
9300
- (Some(qlog::events::ConnectionClosedEventError::TransportError(qlog::events::quic::TransportError::Unknown)), None, trigger)
9456
+ (
9457
+ Some(qlog::events::ConnectionClosedEventError::TransportError(
9458
+ qlog::events::quic::TransportError::Unknown,
9459
+ )),
9460
+ None,
9461
+ trigger,
9462
+ )
9301
9463
  };
9302
9464
 
9303
9465
  // TODO: select more appopriate connection_code and application_error than unknown.
@@ -9310,7 +9472,7 @@ impl<F: BufFactory> Connection<F> {
9310
9472
  reason: Some(String::from_utf8_lossy(&local_error.reason).to_string()),
9311
9473
  trigger,
9312
9474
  }
9313
- },
9475
+ }
9314
9476
 
9315
9477
  _ => qlog::events::quic::ConnectionClosed {
9316
9478
  initiator: None,
@@ -9359,9 +9521,7 @@ impl<F: BufFactory> AsMut<boring::ssl::SslRef> for Connection<F> {
9359
9521
  /// This must only be used for errors preceding packet authentication. Failures
9360
9522
  /// happening after a packet has been authenticated should still cause the
9361
9523
  /// connection to be aborted.
9362
- fn drop_pkt_on_err(
9363
- e: Error, recv_count: usize, is_server: bool, trace_id: &str,
9364
- ) -> Error {
9524
+ fn drop_pkt_on_err(e: Error, recv_count: usize, is_server: bool, trace_id: &str) -> Error {
9365
9525
  // On the server, if no other packet has been successfully processed, abort
9366
9526
  // the connection to avoid keeping the connection open when only junk is
9367
9527
  // received.
@@ -9485,6 +9645,10 @@ pub struct Stats {
9485
9645
  pub bytes_in_flight_duration: Duration,
9486
9646
 
9487
9647
  /// Health state of the connection's tx_buffered.
9648
+ ///
9649
+ /// Indicates whether the streams.tx_buffered value is consistent with
9650
+ /// the actual sum of bytes buffered across all stream send buffers.
9651
+ /// Returns `Ok` if consistent, `Inconsistent` if there's a mismatch.
9488
9652
  pub tx_buffered_state: TxBufferTrackingState,
9489
9653
  }
9490
9654
 
@@ -9524,6 +9688,8 @@ pub use crate::path::SocketAddrIter;
9524
9688
 
9525
9689
  pub use crate::recovery::BbrBwLoReductionStrategy;
9526
9690
  pub use crate::recovery::BbrParams;
9691
+ #[cfg(feature = "internal")]
9692
+ pub use crate::recovery::BbrRttJumpDetector;
9527
9693
  pub use crate::recovery::CongestionControlAlgorithm;
9528
9694
  pub use crate::recovery::StartupExit;
9529
9695
  pub use crate::recovery::StartupExitReason;