refix-engine 0.0.1__tar.gz → 0.1.0__tar.gz

{refix_engine-0.0.1 → refix_engine-0.1.0}/Cargo.lock

@@ -109,14 +109,14 @@ dependencies = [
 
 [[package]]
 name = "refix"
-version = "0.0.2"
+version = "0.1.0"
 dependencies = [
  "refix-message",
 ]
 
 [[package]]
 name = "refix-engine"
-version = "0.0.1"
+version = "0.1.0"
 dependencies = [
  "bytes",
  "pyo3",
@@ -125,7 +125,7 @@ dependencies = [
 
 [[package]]
 name = "refix-message"
-version = "0.1.0"
+version = "0.2.0"
 dependencies = [
  "bytes",
 ]

{refix_engine-0.0.1 → refix_engine-0.1.0}/Cargo.toml

@@ -15,5 +15,5 @@ categories = ["encoding", "parsing", "finance"]
 [workspace.dependencies]
 bytes = "1.12.1"
 pyo3 = "0.29.0"
-refix-message = { version = "0.1.0", path = "crates/refix-message" }
+refix-message = { version = "0.2.0", path = "crates/refix-message" }
 serde = "1.0.229"

{refix_engine-0.0.1 → refix_engine-0.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: refix-engine
-Version: 0.0.1
+Version: 0.1.0
 Classifier: Development Status :: 2 - Pre-Alpha
 Classifier: Intended Audience :: Financial and Insurance Industry
 Classifier: Programming Language :: Python :: 3

{refix_engine-0.0.1 → refix_engine-0.1.0}/crates/refix-message/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "refix-message"
 description = "Framing, tokenisation and raw message representation for the ReFIX engine"
-version = "0.1.0"
+version = "0.2.0"
 authors.workspace = true
 edition.workspace = true
 license.workspace = true

refix_engine-0.1.0/crates/refix-message/src/length_tags.rs

@@ -0,0 +1,25 @@
+/// Length tags whose value gives the byte count of the immediately following
+/// data field.
+///
+/// Derived from the QuickFIX FIX 4.0 - 5.0 SP2 dictionaries:
+/// every LENGTH field paired with a DATA field.
+///
+/// *Note*: these values must remain ordered.
+pub(crate) const STANDARD: &[u32] = &[
+    90, 93, 95, 212, 348, 350, 352, 354, 356, 358, 360, 362, 364, 445, 618, 621, 1184, 1277, 1280,
+    1282, 1397, 1401, 1403, 1468, 1525, 1578, 1620, 1664, 1678, 1733, 1871, 1874, 2072, 2074, 2111,
+    2179, 2287, 2351, 2372, 2481, 2494, 2522, 2637, 2651, 2665, 2715, 2718, 2721, 2797, 2802, 2809,
+    2815, 40004, 40008, 40978, 40980, 40982, 40984, 40986, 40988, 41083, 41101, 41107, 41256,
+    41320, 41324, 41458, 41476, 41482, 41653, 41710, 41806, 41811, 41873, 41969, 42025, 42171,
+    42451, 42652, 42947, 43109, 43110, 43111,
+];
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn standard_is_sorted_and_unique() {
+        assert!(STANDARD.windows(2).all(|pair| pair[0] < pair[1]));
+    }
+}

{refix_engine-0.0.1 → refix_engine-0.1.0}/crates/refix-message/src/lib.rs

@@ -1,8 +1,11 @@
 pub mod framing;
+mod length_tags;
 mod message;
+pub mod stream;
 #[cfg(test)]
 mod test_utils;
 mod tokenizer;
 
 pub use self::message::{MALFORMED_TAG, RawMessage};
+pub use self::stream::MessageStream;
 pub use self::tokenizer::{TokenizeError, Tokenizer};

refix_engine-0.1.0/crates/refix-message/src/stream.rs

@@ -0,0 +1,229 @@
+use crate::framing::{GarbledReason, Outcome as FramingOutcome, Scanner};
+use crate::{RawMessage, Tokenizer};
+use bytes::{Bytes, BytesMut};
+
+/// Reads a stream of FIX messages from incrementally fed bytes.
+#[derive(Default)]
+pub struct MessageStream {
+    tokenizer: Tokenizer,
+    scanner: Scanner,
+    buf: BytesMut,
+}
+
+impl MessageStream {
+    /// Creates a stream that tokenises with `tokenizer`.
+    pub fn new(tokenizer: Tokenizer) -> Self {
+        Self {
+            tokenizer,
+            scanner: Scanner::default(),
+            buf: BytesMut::new(),
+        }
+    }
+
+    /// Appends bytes to the stream's buffer.
+    pub fn feed(&mut self, bytes: &[u8]) {
+        self.buf.extend_from_slice(bytes);
+    }
+
+    /// Consumes and returns the next outcome at the front of the buffer.
+    pub fn next_message(&mut self) -> Outcome {
+        match self.scanner.scan(&self.buf) {
+            FramingOutcome::Frame(frame) => {
+                let len = frame.bytes.len();
+                let bytes = self.buf.split_to(len).freeze();
+                let fields = self.tokenizer.tokenize_fields(&bytes);
+                Outcome::Message(RawMessage::new(bytes, fields))
+            }
+            FramingOutcome::Incomplete => Outcome::Incomplete,
+            FramingOutcome::Garbled { reason, skipped } => Outcome::Garbled {
+                reason,
+                bytes: self.buf.split_to(skipped).freeze(),
+            },
+        }
+    }
+}
+
+/// One step of reading the stream.
+pub enum Outcome {
+    /// A complete, checksum-verified message, consumed from the buffer.
+    Message(RawMessage),
+    /// Bytes that cannot begin a message, consumed from the buffer.
+    Garbled { reason: GarbledReason, bytes: Bytes },
+    /// The buffer holds no complete message, feed more bytes.
+    Incomplete,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::Tokenizer;
+    use crate::test_utils::construct_valid_frame;
+
+    /// Collects outcomes until the stream reports `Incomplete`.
+    fn drain(stream: &mut MessageStream) -> Vec<Outcome> {
+        let mut outcomes = Vec::new();
+        loop {
+            match stream.next_message() {
+                Outcome::Incomplete => return outcomes,
+                outcome => outcomes.push(outcome),
+            }
+        }
+    }
+
+    #[track_caller]
+    fn expect_message(outcome: &Outcome) -> &RawMessage {
+        match outcome {
+            Outcome::Message(message) => message,
+            Outcome::Garbled { .. } => panic!("expected a message, got a garble"),
+            Outcome::Incomplete => panic!("expected a message, got Incomplete"),
+        }
+    }
+
+    #[track_caller]
+    fn expect_garble(outcome: &Outcome) -> (GarbledReason, &Bytes) {
+        match outcome {
+            Outcome::Garbled { reason, bytes } => (*reason, bytes),
+            Outcome::Message(_) => panic!("expected a garble, got a message"),
+            Outcome::Incomplete => panic!("expected a garble, got Incomplete"),
+        }
+    }
+
+    #[test]
+    fn empty_stream_is_incomplete() {
+        let mut stream = MessageStream::default();
+        assert!(matches!(stream.next_message(), Outcome::Incomplete));
+    }
+
+    #[test]
+    fn single_message() {
+        let frame = construct_valid_frame("FIX.4.4", "35=0|58=hi|");
+        let mut stream = MessageStream::default();
+        stream.feed(&frame);
+
+        let outcomes = drain(&mut stream);
+
+        assert_eq!(outcomes.len(), 1);
+        assert_eq!(expect_message(&outcomes[0]).bytes(), &frame);
+    }
+
+    #[test]
+    fn several_messages_in_one_feed() {
+        let first = construct_valid_frame("FIX.4.4", "35=0|58=first|");
+        let second = construct_valid_frame("FIX.4.4", "35=0|58=second|");
+        let mut stream = MessageStream::default();
+        stream.feed(&first);
+        stream.feed(&second);
+
+        let outcomes = drain(&mut stream);
+
+        assert_eq!(outcomes.len(), 2);
+        assert_eq!(expect_message(&outcomes[0]).bytes(), &first);
+        assert_eq!(expect_message(&outcomes[1]).bytes(), &second);
+    }
+
+    /// Splitting a frame at every byte boundary covers every boundary class:
+    /// mid-preamble, mid-tag, mid-body and mid-data-field. The data field's
+    /// extent must survive the split.
+    #[test]
+    fn message_completes_across_feeds_at_every_split() {
+        let frame = construct_valid_frame("FIX.4.4", "35=0|95=3|96=a|b|58=ok|");
+        for split in 1..frame.len() {
+            let mut stream = MessageStream::default();
+            stream.feed(&frame[..split]);
+            assert!(
+                matches!(stream.next_message(), Outcome::Incomplete),
+                "prefix of {split} bytes must be incomplete",
+            );
+
+            stream.feed(&frame[split..]);
+            let outcomes = drain(&mut stream);
+            assert_eq!(outcomes.len(), 1, "split at {split}");
+            let message = expect_message(&outcomes[0]);
+            assert_eq!(message.bytes(), &frame, "split at {split}");
+            assert_eq!(message.get(96), Some(b"a\x01b".as_slice()));
+        }
+    }
+
+    #[test]
+    fn dialect_extras_flow_through() {
+        let frame = construct_valid_frame("FIX.4.4", "35=0|5001=3|5002=a|b|");
+        let mut stream = MessageStream::new(Tokenizer::with_extra_length_tags([5001]));
+        stream.feed(&frame);
+
+        let outcomes = drain(&mut stream);
+
+        assert_eq!(
+            expect_message(&outcomes[0]).get(5002),
+            Some(b"a\x01b".as_slice())
+        );
+    }
+
+    #[test]
+    fn garbage_between_messages() {
+        let first = construct_valid_frame("FIX.4.4", "35=0|58=first|");
+        let second = construct_valid_frame("FIX.4.4", "35=0|58=second|");
+        let mut stream = MessageStream::default();
+        stream.feed(&first);
+        stream.feed(b"YY8=Z");
+        stream.feed(&second);
+
+        let outcomes = drain(&mut stream);
+
+        assert_eq!(outcomes.len(), 3);
+        assert_eq!(expect_message(&outcomes[0]).bytes(), &first);
+        let (reason, bytes) = expect_garble(&outcomes[1]);
+        assert_eq!(reason, GarbledReason::MissingBeginString);
+        assert_eq!(bytes.as_ref(), b"YY8=Z");
+        assert_eq!(expect_message(&outcomes[2]).bytes(), &second);
+    }
+
+    #[test]
+    fn leading_garbage_is_skipped() {
+        let frame = construct_valid_frame("FIX.4.4", "35=0|");
+        let mut stream = MessageStream::default();
+        stream.feed(b"junk");
+        stream.feed(&frame);
+
+        let outcomes = drain(&mut stream);
+
+        assert_eq!(outcomes.len(), 2);
+        let (reason, bytes) = expect_garble(&outcomes[0]);
+        assert_eq!(reason, GarbledReason::MissingBeginString);
+        assert_eq!(bytes.as_ref(), b"junk");
+        assert_eq!(expect_message(&outcomes[1]).bytes(), &frame);
+    }
+
+    #[test]
+    fn garbled_frame_does_not_stall_the_stream() {
+        let mut bad = construct_valid_frame("FIX.4.4", "35=0|58=bad|");
+        let last_digit = bad.len() - 2;
+        bad[last_digit] = if bad[last_digit] == b'0' { b'1' } else { b'0' };
+        let good = construct_valid_frame("FIX.4.4", "35=0|58=good|");
+        let mut stream = MessageStream::default();
+        stream.feed(&bad);
+        stream.feed(&good);
+
+        let outcomes = drain(&mut stream);
+
+        let (reason, _) = expect_garble(&outcomes[0]);
+        assert_eq!(reason, GarbledReason::ChecksumMismatch);
+        let last = outcomes.last().unwrap();
+        assert_eq!(expect_message(last).bytes(), &good);
+    }
+
+    /// An oversized frame is a garble like any other, not a terminal state.
+    #[test]
+    fn oversized_frame_is_skipped_not_fatal() {
+        let frame = construct_valid_frame("FIX.4.4", "35=0|");
+        let mut stream = MessageStream::default();
+        stream.feed(b"8=FIX.4.4\x019=1048577\x01");
+        stream.feed(&frame);
+
+        let outcomes = drain(&mut stream);
+
+        let (reason, _) = expect_garble(&outcomes[0]);
+        assert_eq!(reason, GarbledReason::FrameTooLarge);
+        let last = outcomes.last().unwrap();
+        assert_eq!(expect_message(last).bytes(), &frame);
+    }
+}

{refix_engine-0.0.1 → refix_engine-0.1.0}/crates/refix-message/src/tokenizer.rs

@@ -1,58 +1,106 @@
 use crate::framing::{GarbledReason, Outcome, SOH, Scanner};
 use crate::message::RawField;
-use crate::{MALFORMED_TAG, RawMessage};
+use crate::{MALFORMED_TAG, RawMessage, length_tags};
 use bytes::Bytes;
 
-#[derive(Default)]
-pub struct Tokenizer;
+/// Splits a framed FIX message into its fields.
+#[derive(Clone, Debug)]
+pub struct Tokenizer {
+    length_tags: Vec<u32>,
+}
 
-/// Issues that can arise when the bytes handed to the tokeniser are not a valid FIX message.
-#[derive(Clone, Copy, Debug, PartialEq)]
-pub enum TokenizeError {
-    /// The bytes fail the frame-level checks.
-    Garbled(GarbledReason),
-    /// The bytes end mid-message.
-    Incomplete,
-    /// One message was found, but the input continues past it.
-    TrailingBytes { frame_len: usize },
-    /// The frame is too long for the index's `u32` offsets to address.
-    TooLargeToIndex,
+impl Default for Tokenizer {
+    fn default() -> Self {
+        Self {
+            length_tags: length_tags::STANDARD.to_vec(),
+        }
+    }
 }
 
 impl Tokenizer {
+    /// Constructs a [`Tokenizer`] with additional (non-standard) length tags.
+    ///
+    /// The supplied values are additive, they never replace the standard set.
+    /// Tag 0 is reserved as the malformed-field sentinel and is ignored.
+    pub fn with_extra_length_tags(extras: impl IntoIterator<Item = u32>) -> Self {
+        let mut length_tags: Vec<u32> = length_tags::STANDARD
+            .iter()
+            .copied()
+            .chain(extras.into_iter().filter(|&tag| tag != MALFORMED_TAG))
+            .collect();
+        length_tags.sort_unstable();
+        length_tags.dedup();
+        Self { length_tags }
+    }
+
+    /// Tokenises exactly one complete FIX message into a [`RawMessage`].
+    ///
+    /// The bytes must contain a single well-framed message.
+    /// Frame-level issues surface as [`TokenizeError`], while malformed
+    /// fields inside a valid frame are indexed under [`MALFORMED_TAG`].
     pub fn tokenize(&self, bytes: Bytes) -> Result<RawMessage, TokenizeError> {
         check_frame(&bytes)?;
-        let fields = tokenize_fields(&bytes);
+        let fields = self.tokenize_fields(&bytes);
 
         Ok(RawMessage::new(bytes, fields))
     }
-}
 
-fn tokenize_fields(bytes: &[u8]) -> Vec<RawField> {
-    // TODO: decide what capacity to start with
-    let mut fields = Vec::new();
-    let mut pos = 0;
+    pub(crate) fn tokenize_fields(&self, bytes: &[u8]) -> Vec<RawField> {
+        let mut fields = Vec::with_capacity(bytes.len() / 8);
+        let mut pos = 0;
+        let mut data_len: Option<usize> = None;
+
+        while let Some((field, next)) = next_field(bytes, pos, data_len.take()) {
+            data_len = self.pending_data_len(bytes, field);
+            fields.push(field);
+            pos = next;
+        }
+
+        fields
+    }
 
-    while let Some((field, next)) = next_field(bytes, pos) {
-        fields.push(field);
-        pos = next;
+    /// The value of `field` when it is a well-formed length tag, so the next
+    /// field's value can be read by extent instead of scanned for SOH.
+    fn pending_data_len(&self, bytes: &[u8], field: RawField) -> Option<usize> {
+        if !self.is_length_tag(field.tag) {
+            return None;
+        }
+        let value = &bytes[field.value_start as usize..field.value_end as usize];
+        parse_u32(value).map(|len| len as usize)
     }
 
-    fields
+    fn is_length_tag(&self, tag: u32) -> bool {
+        self.length_tags.first().is_some_and(|&min| tag >= min)
+            && self.length_tags.binary_search(&tag).is_ok()
+    }
+}
+
+/// Issues that can arise when the bytes handed to the tokeniser are not a valid FIX message.
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub enum TokenizeError {
+    /// The bytes fail the frame-level checks.
+    Garbled(GarbledReason),
+    /// The bytes end mid-message.
+    Incomplete,
+    /// One message was found, but the input continues past it.
+    TrailingBytes { frame_len: usize },
+    /// The frame is too long for the index's `u32` offsets to address.
+    TooLargeToIndex,
 }
 
-fn next_field(bytes: &[u8], pos: usize) -> Option<(RawField, usize)> {
+fn next_field(bytes: &[u8], pos: usize, data_len: Option<usize>) -> Option<(RawField, usize)> {
     match find_tag_end(bytes, pos) {
         None => None,
         Some(TagEnd::Equals(delimiter_pos)) => {
-            let value_end = match find_soh(bytes, delimiter_pos) {
-                None => bytes.len(),
-                Some(end) => end,
-            };
             let tag = parse_u32(&bytes[pos..delimiter_pos]).unwrap_or(MALFORMED_TAG);
+            let value_start = delimiter_pos + 1;
+            let value_end = data_len
+                .and_then(|len| data_value_end(bytes, value_start, len))
+                .or_else(|| find_soh(bytes, value_start))
+                .unwrap_or(bytes.len());
             let field = RawField {
                 tag,
-                value_start: (delimiter_pos + 1) as u32,
+                value_start: value_start as u32,
                 value_end: value_end as u32,
             };
 
@@ -113,6 +161,16 @@ fn find_soh(bytes: &[u8], from: usize) -> Option<usize> {
         .map(|rel| from + rel)
 }
 
+/// End of a length-delimited value.
+///
+/// Returns `None` unless the byte immediately after the claimed extent is SOH.
+/// A length that overruns the frame or lands mid-value is distrusted and the
+/// caller falls back to SOH scanning.
+fn data_value_end(bytes: &[u8], value_start: usize, len: usize) -> Option<usize> {
+    let end = value_start.checked_add(len)?;
+    (bytes.get(end) == Some(&SOH)).then_some(end)
+}
+
 fn check_frame(bytes: &Bytes) -> Result<(), TokenizeError> {
     let scanner = Scanner::new(u32::MAX as usize);
     match scanner.scan(bytes) {
@@ -145,7 +203,7 @@ mod tests {
     #[track_caller]
     fn tokenize_body(body: &str) -> RawMessage {
         let frame = construct_valid_frame("FIX.4.4", body);
-        let message = Tokenizer.tokenize(Bytes::from(frame)).unwrap();
+        let message = Tokenizer::default().tokenize(Bytes::from(frame)).unwrap();
         assert_tiles(&message);
         message
     }
@@ -216,7 +274,7 @@ mod tests {
         #[test]
         fn valid_message() {
             let frame = to_wire("8=FIX.4.4|9=41|35=0|49=A|56=B|34=1|52=20260730-10:00:00|10=123|");
-            let message = Tokenizer.tokenize(Bytes::from(frame)).unwrap();
+            let message = Tokenizer::default().tokenize(Bytes::from(frame)).unwrap();
             assert_tiles(&message);
 
             let expected_fields = [
@@ -269,7 +327,9 @@ mod tests {
             let mut bytes = construct_valid_frame("FIX.4.4", "35=0|");
             bytes.pop();
 
-            let error = Tokenizer.tokenize(Bytes::from(bytes)).unwrap_err();
+            let error = Tokenizer::default()
+                .tokenize(Bytes::from(bytes))
+                .unwrap_err();
             assert_eq!(error, TokenizeError::Incomplete);
         }
 
@@ -277,7 +337,9 @@ mod tests {
         fn absurd_body_length_is_too_large_to_index() {
             let bytes = to_wire("8=FIX.4.4|9=4294967295|35=0|");
 
-            let error = Tokenizer.tokenize(Bytes::from(bytes)).unwrap_err();
+            let error = Tokenizer::default()
+                .tokenize(Bytes::from(bytes))
+                .unwrap_err();
             assert_eq!(error, TokenizeError::TooLargeToIndex);
         }
 
@@ -287,7 +349,9 @@ mod tests {
             let frame_len = bytes.len();
             bytes.extend_from_slice(&construct_valid_frame("FIX.4.4", "35=0|"));
 
-            let error = Tokenizer.tokenize(Bytes::from(bytes)).unwrap_err();
+            let error = Tokenizer::default()
+                .tokenize(Bytes::from(bytes))
+                .unwrap_err();
             assert_eq!(error, TokenizeError::TrailingBytes { frame_len });
         }
     }
@@ -347,4 +411,121 @@ mod tests {
             );
         }
     }
+
+    /// Tests for length-delimited data fields, whose values may contain SOH.
+    mod data_fields {
+        use super::*;
+        use crate::message::MALFORMED_TAG;
+
+        #[test]
+        fn value_with_embedded_soh() {
+            let message = tokenize_body("35=0|95=3|96=a|b|58=ok|");
+            assert_body_entries(
+                &message,
+                &[(35, "0"), (95, "3"), (96, "a\x01b"), (58, "ok")],
+            );
+        }
+
+        #[test]
+        fn value_with_several_sohs() {
+            let message = tokenize_body("35=0|95=5|96=a|b|c|58=ok|");
+            assert_body_entries(
+                &message,
+                &[(35, "0"), (95, "5"), (96, "a\x01b\x01c"), (58, "ok")],
+            );
+        }
+
+        #[test]
+        fn zero_length_value() {
+            let message = tokenize_body("35=0|95=0|96=|58=ok|");
+            assert_body_entries(&message, &[(35, "0"), (95, "0"), (96, ""), (58, "ok")]);
+        }
+
+        #[test]
+        fn data_value_last_in_body() {
+            let message = tokenize_body("35=0|95=3|96=a|b|");
+            assert_body_entries(&message, &[(35, "0"), (95, "3"), (96, "a\x01b")]);
+        }
+
+        #[test]
+        fn overrunning_length_is_distrusted() {
+            let message = tokenize_body("35=0|95=4294967295|96=ab|58=ok|");
+            assert_body_entries(
+                &message,
+                &[(35, "0"), (95, "4294967295"), (96, "ab"), (58, "ok")],
+            );
+        }
+
+        #[test]
+        fn non_numeric_length_falls_back_to_scanning() {
+            let message = tokenize_body("35=0|95=abc|96=x|y|58=ok|");
+            assert_body_entries(
+                &message,
+                &[
+                    (35, "0"),
+                    (95, "abc"),
+                    (96, "x"),
+                    (MALFORMED_TAG, "y"),
+                    (58, "ok"),
+                ],
+            );
+        }
+
+        #[test]
+        fn short_length_surfaces_junk_after_the_data() {
+            let message = tokenize_body("35=0|95=1|96=a|b|58=ok|");
+            assert_body_entries(
+                &message,
+                &[
+                    (35, "0"),
+                    (95, "1"),
+                    (96, "a"),
+                    (MALFORMED_TAG, "b"),
+                    (58, "ok"),
+                ],
+            );
+        }
+
+        #[test]
+        fn dialect_extra_delimits_data() {
+            let frame = construct_valid_frame("FIX.4.4", "35=0|5001=3|5002=a|b|58=ok|");
+            let message = Tokenizer::with_extra_length_tags([5001])
+                .tokenize(Bytes::from(frame))
+                .unwrap();
+            assert_tiles(&message);
+            assert_body_entries(
+                &message,
+                &[(35, "0"), (5001, "3"), (5002, "a\x01b"), (58, "ok")],
+            );
+        }
+
+        #[test]
+        fn standard_set_survives_extras() {
+            let frame = construct_valid_frame("FIX.4.4", "35=0|95=3|96=a|b|");
+            let message = Tokenizer::with_extra_length_tags([5001])
+                .tokenize(Bytes::from(frame))
+                .unwrap();
+            assert_tiles(&message);
+            assert_body_entries(&message, &[(35, "0"), (95, "3"), (96, "a\x01b")]);
+        }
+
+        #[test]
+        fn extra_below_the_standard_minimum() {
+            let frame = construct_valid_frame("FIX.4.4", "35=0|42=3|58=a|b|11=ok|");
+            let message = Tokenizer::with_extra_length_tags([42])
+                .tokenize(Bytes::from(frame))
+                .unwrap();
+            assert_tiles(&message);
+            assert_body_entries(
+                &message,
+                &[(35, "0"), (42, "3"), (58, "a\x01b"), (11, "ok")],
+            );
+        }
+
+        #[test]
+        fn tag_zero_extra_is_ignored() {
+            let tokenizer = Tokenizer::with_extra_length_tags([0]);
+            assert!(!tokenizer.is_length_tag(MALFORMED_TAG));
+        }
+    }
 }

{refix_engine-0.0.1 → refix_engine-0.1.0}/pyproject.toml

@@ -37,6 +37,7 @@ dev = [
     "maturin>=1.9.4,<2",
     "pyright>=1.1",
     "pytest>=8",
+    "pytest-benchmark>=5",
     "pytest-cov>=6",
 ]
 

refix_engine-0.1.0/python/refix/__init__.py

@@ -0,0 +1,6 @@
+from refix._core import MALFORMED_TAG, Garble, MessageStream, RawMessage, Tokenizer, version
+from refix.log import read_log
+
+__version__ = version()
+
+__all__ = ["MALFORMED_TAG", "Garble", "MessageStream", "RawMessage", "read_log", "Tokenizer", "__version__"]

refix_engine-0.1.0/python/refix/_core.pyi

@@ -0,0 +1,41 @@
+import builtins
+from collections.abc import Sequence
+
+from refix.errors import GarbledReason
+
+
+class RawMessage:
+    @property
+    def bytes(self) -> builtins.bytes: ...
+
+    def get(self, tag: int) -> builtins.bytes | None: ...
+
+    def entries(self) -> list[tuple[int, builtins.bytes]]: ...
+
+
+class Tokenizer:
+    def __init__(self, *, extra_length_tags: Sequence[int] = ...) -> None: ...
+
+    def tokenize(self, data: bytes) -> RawMessage: ...
+
+
+class Garble:
+    @property
+    def bytes(self) -> builtins.bytes: ...
+
+    @property
+    def reason(self) -> GarbledReason: ...
+
+
+class MessageStream:
+    def __init__(self, *, extra_length_tags: Sequence[int] = ...) -> None: ...
+
+    def feed(self, data: bytes) -> None: ...
+
+    def next_message(self) -> RawMessage | Garble | None: ...
+
+
+MALFORMED_TAG: int
+
+
+def version() -> str: ...

refix_engine-0.1.0/python/refix/log.py

@@ -0,0 +1,30 @@
+import os
+from collections.abc import Iterator, Sequence
+from typing import BinaryIO
+
+from refix._core import Garble, MessageStream, RawMessage
+
+_CHUNK_SIZE = 65536
+
+
+def read_log(source: str | os.PathLike[str] | BinaryIO,
+             *,
+             extra_length_tags: Sequence[int] = (), ) -> Iterator[RawMessage | Garble]:
+    """Yields each message in a FIX message log, in order.
+
+    Bytes between frames that are not FIX (timestamps, newlines, junk) are yielded as `Garble`.
+    A truncated frame at the end of the log is ignored.
+    """
+    if isinstance(source, (str, os.PathLike)):
+        with open(source, "rb") as f:
+            yield from _drive(f, extra_length_tags)
+    else:
+        yield from _drive(source, extra_length_tags)
+
+
+def _drive(file: BinaryIO, extra_length_tags: Sequence[int]) -> Iterator[RawMessage | Garble]:
+    stream = MessageStream(extra_length_tags=extra_length_tags)
+    while chunk := file.read(_CHUNK_SIZE):
+        stream.feed(chunk)
+        while (outcome := stream.next_message()) is not None:
+            yield outcome

{refix_engine-0.0.1 → refix_engine-0.1.0}/python/refix-engine/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "refix-engine"
-version = "0.0.1"
+version = "0.1.0"
 publish = false
 authors.workspace = true
 edition.workspace = true

{refix_engine-0.0.1 → refix_engine-0.1.0}/python/refix-engine/src/lib.rs

@@ -1,4 +1,5 @@
 mod message;
+mod stream;
 mod tokenizer;
 
 use pyo3::prelude::*;
@@ -11,6 +12,8 @@ fn version() -> &'static str {
 #[pymodule]
 fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_function(wrap_pyfunction!(version, m)?)?;
+    m.add_class::<stream::Garble>()?;
+    m.add_class::<stream::MessageStream>()?;
     m.add_class::<message::RawMessage>()?;
     m.add_class::<tokenizer::Tokenizer>()?;
     m.add("MALFORMED_TAG", refix_message::MALFORMED_TAG)?;

refix_engine-0.1.0/python/refix-engine/src/stream.rs

@@ -0,0 +1,58 @@
+use crate::tokenizer::reason_str;
+use bytes::Bytes;
+use pyo3::types::{PyAnyMethods, PyBytes, PyModule};
+use pyo3::{Bound, Py, PyAny, PyResult, Python, pyclass, pymethods};
+use refix_message::framing::GarbledReason;
+use refix_message::stream::Outcome;
+use refix_message::{MessageStream as CoreMessageStream, Tokenizer as CoreTokenizer};
+
+#[pyclass(module = "refix._core")]
+pub(crate) struct MessageStream(CoreMessageStream);
+
+#[pymethods]
+impl MessageStream {
+    #[new]
+    #[pyo3(signature = (*, extra_length_tags = Vec::new()))]
+    fn new(extra_length_tags: Vec<u32>) -> Self {
+        let tokenizer = CoreTokenizer::with_extra_length_tags(extra_length_tags);
+        Self(CoreMessageStream::new(tokenizer))
+    }
+
+    fn feed(&mut self, data: &[u8]) {
+        self.0.feed(data);
+    }
+
+    fn next_message(&mut self, py: Python<'_>) -> PyResult<Option<Py<PyAny>>> {
+        match self.0.next_message() {
+            Outcome::Message(message) => {
+                let message = crate::message::RawMessage::new(message);
+                Ok(Some(Py::new(py, message)?.into_any()))
+            }
+            Outcome::Garbled { reason, bytes } => {
+                Ok(Some(Py::new(py, Garble { reason, bytes })?.into_any()))
+            }
+            Outcome::Incomplete => Ok(None),
+        }
+    }
+}
+
+/// Bytes between messages that cannot begin one.
+#[pyclass(frozen, module = "refix._core")]
+pub(crate) struct Garble {
+    reason: GarbledReason,
+    bytes: Bytes,
+}
+
+#[pymethods]
+impl Garble {
+    #[getter]
+    fn bytes<'py>(&self, py: Python<'py>) -> Bound<'py, PyBytes> {
+        PyBytes::new(py, &self.bytes)
+    }
+
+    #[getter]
+    fn reason<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
+        let reasons = PyModule::import(py, "refix.errors")?.getattr("GarbledReason")?;
+        reasons.call1((reason_str(self.reason),))
+    }
+}

{refix_engine-0.0.1 → refix_engine-0.1.0}/python/refix-engine/src/tokenizer.rs

@@ -14,8 +14,9 @@ pub(crate) struct Tokenizer(CoreTokenizer);
 #[pymethods]
 impl Tokenizer {
     #[new]
-    fn new() -> Self {
-        Self(CoreTokenizer)
+    #[pyo3(signature = (*, extra_length_tags = Vec::new()))]
+    fn new(extra_length_tags: Vec<u32>) -> Self {
+        Self(CoreTokenizer::with_extra_length_tags(extra_length_tags))
     }
 
     fn tokenize(&self, data: &[u8]) -> PyResult<crate::message::RawMessage> {
@@ -35,7 +36,7 @@ fn to_py_err(error: TokenizeError) -> PyErr {
     }
 }
 
-fn reason_str(reason: GarbledReason) -> &'static str {
+pub(crate) fn reason_str(reason: GarbledReason) -> &'static str {
     match reason {
         GarbledReason::MissingBeginString => "missing_begin_string",
         GarbledReason::MalformedBeginString => "malformed_begin_string",

refix_engine-0.1.0/python/refix-engine/tests/test_benchmarks.py

@@ -0,0 +1,57 @@
+from typing import Any, Callable, Protocol, TypeVar
+
+import refix
+from test_tokenizer import construct_valid_frame
+
+T = TypeVar("T")
+
+
+class Benchmark(Protocol):
+    """The callable interface of pytest-benchmark's `benchmark` fixture."""
+
+    def __call__(self, target: Callable[..., T], *args: Any) -> T: ...
+
+
+EXECUTION_REPORT_BODY = (
+    "35=8|34=123|49=BROKER|56=CLIENT|52=20260731-10:00:00.123|"
+    "37=ORD0001|11=CLT0001|17=EXEC0001|150=F|39=2|55=EURUSD|54=1|"
+    "38=1000000|44=1.0925|32=1000000|31=1.0925|14=1000000|151=0|"
+    "6=1.0925|60=20260731-10:00:00.120|"
+)
+
+
+def execution_report() -> bytes:
+    return construct_valid_frame("FIX.4.4", EXECUTION_REPORT_BODY)
+
+
+def test_bench_tokenize(benchmark: Benchmark) -> None:
+    frame = execution_report()
+    tokenizer = refix.Tokenizer()
+
+    message = benchmark(tokenizer.tokenize, frame)
+
+    assert message.get(35) == b"8"
+
+
+def test_bench_get_hit(benchmark: Benchmark) -> None:
+    message = refix.Tokenizer().tokenize(execution_report())
+
+    value = benchmark(message.get, 44)
+
+    assert value == b"1.0925"
+
+
+def test_bench_get_miss(benchmark: Benchmark) -> None:
+    message = refix.Tokenizer().tokenize(execution_report())
+
+    value = benchmark(message.get, 9999)
+
+    assert value is None
+
+
+def test_bench_entries(benchmark: Benchmark) -> None:
+    message = refix.Tokenizer().tokenize(execution_report())
+
+    entries = benchmark(message.entries)
+
+    assert len(entries) == 23

refix_engine-0.1.0/python/refix-engine/tests/test_log.py

@@ -0,0 +1,74 @@
+from io import BytesIO
+from pathlib import Path
+
+import refix
+from test_tokenizer import construct_valid_frame
+
+
+def messages_of(outcomes: list[refix.RawMessage | refix.Garble]) -> list[refix.RawMessage]:
+    return [outcome for outcome in outcomes if isinstance(outcome, refix.RawMessage)]
+
+
+class TestReadLog:
+    def test_reads_messages_from_a_path(self, tmp_path: Path):
+        first = construct_valid_frame("FIX.4.4", "35=0|58=first|")
+        second = construct_valid_frame("FIX.4.4", "35=0|58=second|")
+        path = tmp_path / "messages.log"
+        path.write_bytes(first + second)
+
+        outcomes = list(refix.read_log(path))
+
+        assert [message.bytes for message in messages_of(outcomes)] == [first, second]
+
+    def test_reads_from_a_file_object(self):
+        frame = construct_valid_frame("FIX.4.4", "35=0|")
+
+        outcomes = list(refix.read_log(BytesIO(frame)))
+
+        assert len(outcomes) == 1
+        assert messages_of(outcomes)[0].bytes == frame
+
+    def test_timestamped_log_yields_messages_and_garbles(self):
+        frames = [
+            construct_valid_frame("FIX.4.4", "35=0|58=first|"),
+            construct_valid_frame("FIX.4.4", "35=0|58=second|"),
+        ]
+        log = b"".join(
+            b"20260803-10:00:00.%03d : " % i + frame + b"\n"
+            for i, frame in enumerate(frames)
+        )
+
+        outcomes = list(refix.read_log(BytesIO(log)))
+
+        assert [message.bytes for message in messages_of(outcomes)] == frames
+        assert b"".join(outcome.bytes for outcome in outcomes) == log
+
+    def test_truncated_tail_is_ignored(self):
+        frame = construct_valid_frame("FIX.4.4", "35=0|58=hello|")
+        log = frame + frame[: len(frame) // 2]
+
+        outcomes = list(refix.read_log(BytesIO(log)))
+
+        assert len(outcomes) == 1
+        assert messages_of(outcomes)[0].bytes == frame
+
+    def test_extra_length_tags_flow_through(self):
+        frame = construct_valid_frame("FIX.4.4", "35=0|5001=3|5002=a|b|")
+
+        outcomes = list(refix.read_log(BytesIO(frame), extra_length_tags=[5001]))
+
+        assert messages_of(outcomes)[0].get(5002) == b"a\x01b"
+
+    def test_log_larger_than_one_chunk(self):
+        frame = construct_valid_frame("FIX.4.4", "35=0|58=hello|")
+        count = 5000
+        log = frame * count
+        assert len(log) > 65536, "log must span several read chunks"
+
+        outcomes = list(refix.read_log(BytesIO(log)))
+
+        assert len(outcomes) == count
+        assert all(
+            message.bytes == frame for message in messages_of(outcomes)
+        )
+        assert len(messages_of(outcomes)) == count

refix_engine-0.1.0/python/refix-engine/tests/test_stream.py

@@ -0,0 +1,129 @@
+import refix
+from refix.errors import GarbledReason
+from test_tokenizer import construct_valid_frame, to_wire
+
+
+def drain(stream: refix.MessageStream) -> list[refix.RawMessage | refix.Garble]:
+    """Collects outcomes until the stream reports no complete message."""
+    outcomes: list[refix.RawMessage | refix.Garble] = []
+    while (outcome := stream.next_message()) is not None:
+        outcomes.append(outcome)
+    return outcomes
+
+
+def expect_message(outcome: object) -> refix.RawMessage:
+    assert isinstance(outcome, refix.RawMessage)
+    return outcome
+
+
+def expect_garble(outcome: object) -> refix.Garble:
+    assert isinstance(outcome, refix.Garble)
+    return outcome
+
+
+class TestDelivery:
+    def test_empty_stream_has_no_message(self):
+        assert refix.MessageStream().next_message() is None
+
+    def test_single_message(self):
+        frame = construct_valid_frame("FIX.4.4", "35=0|58=hi|")
+        stream = refix.MessageStream()
+        stream.feed(frame)
+
+        outcomes = drain(stream)
+
+        assert len(outcomes) == 1
+        assert expect_message(outcomes[0]).bytes == frame
+
+    def test_several_messages_in_one_feed(self):
+        first = construct_valid_frame("FIX.4.4", "35=0|58=first|")
+        second = construct_valid_frame("FIX.4.4", "35=0|58=second|")
+        stream = refix.MessageStream()
+        stream.feed(first)
+        stream.feed(second)
+
+        outcomes = drain(stream)
+
+        assert len(outcomes) == 2
+        assert expect_message(outcomes[0]).bytes == first
+        assert expect_message(outcomes[1]).bytes == second
+
+    def test_message_completes_across_feeds_at_every_split(self):
+        frame = construct_valid_frame("FIX.4.4", "35=0|95=3|96=a|b|58=ok|")
+        for split in range(1, len(frame)):
+            stream = refix.MessageStream()
+            stream.feed(frame[:split])
+            assert stream.next_message() is None, f"prefix of {split} bytes"
+
+            stream.feed(frame[split:])
+            outcomes = drain(stream)
+            assert len(outcomes) == 1, f"split at {split}"
+            message = expect_message(outcomes[0])
+            assert message.bytes == frame, f"split at {split}"
+            assert message.get(96) == b"a\x01b"
+
+    def test_dialect_extras_flow_through(self):
+        frame = construct_valid_frame("FIX.4.4", "35=0|5001=3|5002=a|b|")
+        stream = refix.MessageStream(extra_length_tags=[5001])
+        stream.feed(frame)
+
+        outcomes = drain(stream)
+
+        assert expect_message(outcomes[0]).get(5002) == b"a\x01b"
+
+
+class TestFaults:
+    def test_garbage_between_messages(self):
+        first = construct_valid_frame("FIX.4.4", "35=0|58=first|")
+        second = construct_valid_frame("FIX.4.4", "35=0|58=second|")
+        stream = refix.MessageStream()
+        stream.feed(first)
+        stream.feed(b"YY8=Z")
+        stream.feed(second)
+
+        outcomes = drain(stream)
+
+        assert len(outcomes) == 3
+        assert expect_message(outcomes[0]).bytes == first
+        garble = expect_garble(outcomes[1])
+        assert garble.reason is GarbledReason.MISSING_BEGIN_STRING
+        assert garble.bytes == b"YY8=Z"
+        assert expect_message(outcomes[2]).bytes == second
+
+    def test_leading_garbage_is_skipped(self):
+        frame = construct_valid_frame("FIX.4.4", "35=0|")
+        stream = refix.MessageStream()
+        stream.feed(b"junk")
+        stream.feed(frame)
+
+        outcomes = drain(stream)
+
+        assert len(outcomes) == 2
+        garble = expect_garble(outcomes[0])
+        assert garble.reason is GarbledReason.MISSING_BEGIN_STRING
+        assert garble.bytes == b"junk"
+        assert expect_message(outcomes[1]).bytes == frame
+
+    def test_garbled_frame_does_not_stall_the_stream(self):
+        bad = bytearray(construct_valid_frame("FIX.4.4", "35=0|58=bad|"))
+        bad[-2] = ord("1") if bad[-2] == ord("0") else ord("0")
+        good = construct_valid_frame("FIX.4.4", "35=0|58=good|")
+        stream = refix.MessageStream()
+        stream.feed(bytes(bad))
+        stream.feed(good)
+
+        outcomes = drain(stream)
+
+        assert expect_garble(outcomes[0]).reason is GarbledReason.CHECKSUM_MISMATCH
+        assert expect_message(outcomes[-1]).bytes == good
+
+    def test_oversized_frame_is_skipped_not_fatal(self):
+        frame = construct_valid_frame("FIX.4.4", "35=0|")
+        stream = refix.MessageStream()
+        stream.feed(to_wire("8=FIX.4.4|9=1048577|"))
+        stream.feed(frame)
+
+        outcomes = drain(stream)
+
+        assert expect_garble(outcomes[0]).reason is GarbledReason.FRAME_TOO_LARGE
+        assert expect_message(outcomes[-1]).bytes == frame

{refix_engine-0.0.1 → refix_engine-0.1.0}/python/refix-engine/tests/test_tokenizer.py

@@ -111,6 +111,7 @@ class TestErrors:
             refix.Tokenizer().tokenize(frame + frame)
 
         assert excinfo.value.frame_len == len(frame)
+        assert str(excinfo.value) == f"one message of {len(frame)} bytes, then trailing input"
 
 
 class TestGarbledFrames:
@@ -136,6 +137,7 @@ class TestGarbledFrames:
             refix.Tokenizer().tokenize(to_wire(frame))
 
         assert excinfo.value.reason is reason
+        assert str(excinfo.value) == f"garbled frame: {reason.value}"
 
     def test_garbled_error_pickles(self):
         with pytest.raises(GarbledError) as excinfo:
@@ -184,3 +186,35 @@ class TestSentinelRuns:
             (refix.MALFORMED_TAG, b"more"),
             (58, b"ok"),
         ]
+
+
+class TestLengthTags:
+    """Length-delimited data fields, whose values may contain SOH."""
+
+    def test_standard_length_tag_delimits_data(self):
+        message = tokenize_body("35=0|95=3|96=a|b|58=ok|")
+        assert body_entries(message) == [
+            (35, b"0"),
+            (95, b"3"),
+            (96, b"a\x01b"),
+            (58, b"ok"),
+        ]
+
+    def test_dialect_extra_delimits_data(self):
+        frame = construct_valid_frame("FIX.4.4", "35=0|5001=3|5002=a|b|58=ok|")
+
+        message = refix.Tokenizer(extra_length_tags=[5001]).tokenize(frame)
+
+        assert body_entries(message) == [
+            (35, b"0"),
+            (5001, b"3"),
+            (5002, b"a\x01b"),
+            (58, b"ok"),
+        ]
+
+    def test_standard_set_survives_extras(self):
+        frame = construct_valid_frame("FIX.4.4", "35=0|95=3|96=a|b|")
+
+        message = refix.Tokenizer(extra_length_tags=[5001]).tokenize(frame)
+
+        assert body_entries(message) == [(35, b"0"), (95, b"3"), (96, b"a\x01b")]

{refix_engine-0.0.1 → refix_engine-0.1.0}/python/refix-engine/uv.lock

@@ -137,6 +137,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" },
 ]
 
+[[package]]
+name = "py-cpuinfo"
+version = "9.0.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716, upload-time = "2022-10-25T20:38:06.303Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335, upload-time = "2022-10-25T20:38:27.636Z" },
+]
+
 [[package]]
 name = "pygments"
 version = "2.20.0"
@@ -175,6 +184,19 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" },
 ]
 
+[[package]]
+name = "pytest-benchmark"
+version = "5.2.3"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "py-cpuinfo" },
+    { name = "pytest" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/24/34/9f732b76456d64faffbef6232f1f9dbec7a7c4999ff46282fa418bd1af66/pytest_benchmark-5.2.3.tar.gz", hash = "sha256:deb7317998a23c650fd4ff76e1230066a76cb45dcece0aca5607143c619e7779", size = 341340, upload-time = "2025-11-09T18:48:43.215Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/33/29/e756e715a48959f1c0045342088d7ca9762a2f509b945f362a316e9412b7/pytest_benchmark-5.2.3-py3-none-any.whl", hash = "sha256:bc839726ad20e99aaa0d11a127445457b4219bdb9e80a1afc4b51da7f96b0803", size = 45255, upload-time = "2025-11-09T18:48:39.765Z" },
+]
+
 [[package]]
 name = "pytest-cov"
 version = "7.1.0"
@@ -198,6 +220,7 @@ dev = [
     { name = "maturin" },
     { name = "pyright" },
     { name = "pytest" },
+    { name = "pytest-benchmark" },
     { name = "pytest-cov" },
 ]
 
@@ -208,6 +231,7 @@ dev = [
     { name = "maturin", specifier = ">=1.9.4,<2" },
     { name = "pyright", specifier = ">=1.1" },
     { name = "pytest", specifier = ">=8" },
+    { name = "pytest-benchmark", specifier = ">=5" },
     { name = "pytest-cov", specifier = ">=6" },
 ]
 

refix_engine-0.0.1/python/refix/__init__.py

@@ -1,5 +0,0 @@
-from refix._core import MALFORMED_TAG, RawMessage, Tokenizer, version
-
-__version__ = version()
-
-__all__ = ["MALFORMED_TAG", "RawMessage", "Tokenizer", "__version__"]

refix_engine-0.0.1/python/refix/_core.pyi

@@ -1,20 +0,0 @@
-import builtins
-
-
-class RawMessage:
-    @property
-    def bytes(self) -> builtins.bytes: ...
-
-    def get(self, tag: int) -> builtins.bytes | None: ...
-
-    def entries(self) -> list[tuple[int, builtins.bytes]]: ...
-
-
-class Tokenizer:
-    def tokenize(self, data: bytes) -> RawMessage: ...
-
-
-MALFORMED_TAG: int
-
-
-def version() -> str: ...