llguidance 0.7.27__tar.gz → 0.7.29__tar.gz

{llguidance-0.7.27 → llguidance-0.7.29}/CHANGELOG.md

@@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file. Dates are d
 
 If a release doesn't introduce any interesting changes (build fixes etc.), it's skipped.
 
+#### [0.7.29](https://github.com/guidance-ai/llguidance/compare/v0.7.28...0.7.29) 2025-06-06
+
+- cargo fmt
+
+#### [0.7.28](https://github.com/guidance-ai/llguidance/compare/v0.7.27...0.7.28) 2025-06-06
+
+- fix lexer_stack=... panic with numeric tokens [`4e91b0f`](https://github.com/guidance-ai/llguidance/commit/4e91b0fa0c03572a5fc221ac0e0b05035af9dcfa)
+
 #### [0.7.27](https://github.com/guidance-ai/llguidance/compare/v0.7.26...0.7.27) 2025-06-04
 
 - add toktrie_tiktoken and llguidance.tiktoken.lltokenizer_from_encoding [`#154`](https://github.com/guidance-ai/llguidance/issues/154)

{llguidance-0.7.27 → llguidance-0.7.29}/Cargo.lock

@@ -1211,7 +1211,7 @@ checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856"
 
 [[package]]
 name = "llguidance"
-version = "0.7.27"
+version = "0.7.29"
 dependencies = [
  "anyhow",
  "derivre",
@@ -1230,7 +1230,7 @@ dependencies = [
 
 [[package]]
 name = "llguidance_py"
-version = "0.7.27"
+version = "0.7.29"
 dependencies = [
  "anyhow",
  "bytemuck",
@@ -2395,7 +2395,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie"
-version = "0.7.27"
+version = "0.7.29"
 dependencies = [
  "anyhow",
  "bytemuck",
@@ -2406,7 +2406,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_hf_downloader"
-version = "0.7.27"
+version = "0.7.29"
 dependencies = [
  "anyhow",
  "hf-hub",
@@ -2417,7 +2417,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_hf_tokenizers"
-version = "0.7.27"
+version = "0.7.29"
 dependencies = [
  "anyhow",
  "log",
@@ -2429,7 +2429,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_tiktoken"
-version = "0.7.27"
+version = "0.7.29"
 dependencies = [
  "anyhow",
  "log",

{llguidance-0.7.27 → llguidance-0.7.29}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llguidance
-Version: 0.7.27
+Version: 0.7.29
 License-File: LICENSE
 Summary: Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance
 Author: Michal Moskal

{llguidance-0.7.27 → llguidance-0.7.29}/parser/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "llguidance"
-version = "0.7.27"
+version = "0.7.29"
 edition = "2021"
 license = "MIT"
 description = "Super-fast Structured Outputs"

{llguidance-0.7.27 → llguidance-0.7.29}/parser/src/earley/parser.rs

@@ -435,6 +435,7 @@ struct ParserState {
     // history - items are not popped in definitive mode.
     lexer_stack: Vec<LexerState>,
     lexer_stack_top_eos: bool,
+    lexer_stack_flush_position: usize,
     rows: Vec<Row>,
     rows_valid_end: usize,
 
@@ -665,6 +666,7 @@ impl ParserState {
             limits,
             backtrack_byte_count: 0,
             lexer_stack_top_eos: false,
+            lexer_stack_flush_position: 0,
             lexer_stack: vec![LexerState {
                 row_idx: 0,
                 lexer_state,
@@ -793,12 +795,6 @@ impl ParserState {
             });
         }
 
-        if set.is_zero() {
-            // nothing allowed
-            // we're going to be stopped outside - we better flush the lexer
-            let _ = self.flush_lexer();
-        }
-
         let eos = computer.trie().eos_token();
         if eos != INVALID_TOKEN && start.is_empty() && self.lexer_allows_eos() {
             set.allow_token(eos);
@@ -935,7 +931,7 @@ impl ParserState {
         self.stats = ParserStats::default();
     }
 
-    fn assert_definitive(&self) {
+    fn assert_definitive_inner(&self) {
         assert!(self.scratch.definitive);
         assert!(self.backtrack_byte_count == 0);
         if self.num_rows() != self.row_infos.len() {
@@ -947,6 +943,14 @@ impl ParserState {
         }
     }
 
+    fn assert_definitive(&self) {
+        self.assert_definitive_inner();
+
+        if self.lexer_spec().can_rollback() {
+            self.check_lexer_bytes_invariant();
+        }
+    }
+
     pub fn get_bytes(&self) -> &[u8] {
         &self.bytes
     }
@@ -995,7 +999,6 @@ impl ParserState {
             n_bytes,
             self.byte_to_token_idx.len()
         );
-        self.check_lexer_bytes_invariant();
 
         let new_len = self.byte_to_token_idx.len() - n_bytes;
 
@@ -1010,7 +1013,6 @@ impl ParserState {
         self.rows_valid_end = self.num_rows();
 
         self.assert_definitive();
-        self.check_lexer_bytes_invariant();
 
         Ok(())
     }
@@ -1097,6 +1099,12 @@ impl ParserState {
                     .push(self.token_idx.try_into().unwrap());
             }
         }
+        debug_def!(
+            self,
+            "add_numeric_token: idx={:?} bytes={:?}",
+            idx,
+            tok_bytes
+        );
         let ok = self.advance_parser(PreLexeme::just_idx(MatchingLexemesIdx::Single(idx)));
         ensure!(
             ok,
@@ -1156,9 +1164,20 @@ impl ParserState {
                 let row_idx = self.num_rows() - 1;
                 self.row_infos[row_idx].apply_token_idx(self.token_idx);
 
+                self.lexer_stack_flush_position = 0;
                 let idx = self.flush_and_check_numeric(tok_id).unwrap();
                 self.add_numeric_token(idx, tok_bytes)?;
 
+                // if flush_lexer() added a stack entry
+                if self.lexer_stack_flush_position > 0 {
+                    // we make sure it's not on the top
+                    assert!(self.lexer_stack_flush_position + 1 < self.lexer_stack.len());
+                    // and remove it
+                    self.lexer_stack.remove(self.lexer_stack_flush_position);
+                }
+
+                self.assert_definitive();
+
                 return Ok(0);
             }
         }
@@ -1314,6 +1333,8 @@ impl ParserState {
             self.print_row(self.num_rows() - 1);
         }
 
+        self.assert_definitive();
+
         Ok(0)
     }
 
@@ -1474,10 +1495,6 @@ impl ParserState {
         // debug!("trie_started: rows={} lexer={}", self.num_rows(), self.lexer_stack.len());
         self.assert_definitive();
 
-        if self.lexer_spec().can_rollback() {
-            self.check_lexer_bytes_invariant();
-        }
-
         self.trie_lexer_stack = self.lexer_stack.len();
         self.trie_grammar_stack = self.scratch.grammar_stack.len();
         self.scratch.definitive = false;
@@ -1514,6 +1531,7 @@ impl ParserState {
         self.assert_definitive();
         self.rows_valid_end = self.num_rows();
         self.scratch.log_override = false; // reset
+        self.lexer_stack_flush_position = 0;
     }
 
     fn run_speculative<T>(&mut self, lbl: &str, f: impl FnOnce(&mut Self) -> T) -> T {
@@ -1672,16 +1690,19 @@ impl ParserState {
         }
         let curr = self.lexer_state();
         let lex_result = self.lexer_mut().try_lexeme_end(curr.lexer_state);
+        let prev_len = self.lexer_stack.len();
         let r = self.advance_lexer_or_parser(lex_result, curr);
+        if self.lexer_stack.len() != prev_len {
+            assert!(self.lexer_stack.len() == prev_len + 1);
+            assert!(prev_len > 0);
+            self.lexer_stack_flush_position = prev_len;
+        }
         assert!(self.backtrack_byte_count == 0);
         r
     }
 
     pub fn scan_eos(&mut self) -> bool {
         self.assert_definitive(); // ???
-        if self.lexer_spec().can_rollback() {
-            self.check_lexer_bytes_invariant();
-        }
 
         let lexer_eos = self.lexer_allows_eos();
 
@@ -1710,9 +1731,7 @@ impl ParserState {
             self.lexer_stack_top_eos = true;
         }
 
-        if self.lexer_spec().can_rollback() {
-            self.check_lexer_bytes_invariant();
-        }
+        self.assert_definitive(); // ???
 
         false
     }
@@ -2361,7 +2380,7 @@ impl ParserState {
                 });
             }
             if self.scratch.definitive {
-                self.assert_definitive();
+                self.assert_definitive_inner();
             }
         } else {
             if trace_here {
@@ -2374,7 +2393,7 @@ impl ParserState {
                     byte: None,
                     ..no_hidden
                 });
-                self.assert_definitive();
+                self.assert_definitive_inner();
                 self.backtrack_byte_count = hidden_bytes.len();
             } else {
                 // prevent any further matches in this branch
@@ -2499,10 +2518,11 @@ impl ParserState {
                         }
                     }
                 }
+                debug_def!(self, "  push normal: {no_hidden:?}");
                 self.lexer_stack.push(no_hidden);
             }
             if self.scratch.definitive {
-                self.assert_definitive();
+                self.assert_definitive_inner();
             }
             true
         } else {

{llguidance-0.7.27 → llguidance-0.7.29}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "llguidance"
-version = "0.7.27"
+version = "0.7.29"
 description = "Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance"
 requires-python = ">=3.9"
 license = "MIT"

{llguidance-0.7.27 → llguidance-0.7.29}/python_ext/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "llguidance_py"
-version = "0.7.27"
+version = "0.7.29"
 edition = "2021"
 license = "MIT"
 description = "Super-fast Structured Outputs"

{llguidance-0.7.27 → llguidance-0.7.29}/sample_parser/tests/test_raw_parser.rs

@@ -268,3 +268,19 @@ fn test_trigger_lexer_error() {
         unreachable!();
     }
 }
+
+#[test]
+fn test_lexer_inv_crash() {
+    let tokenv = get_tok_env();
+    let t1 = tokenv.tokenize("#");
+    let t2 = tokenv.tokenize("?");
+    let tokens = tokenv.tokenize("a#");
+    assert!(t1.len() == 1);
+    assert!(t2.len() == 1);
+    let grm = format!("start: /[a-z]+/ ( <[{}]> | <[{}]> )", t1[0], t2[0]);
+    let parser = make_parser(&grm);
+    let mut matcher = Matcher::new(Ok(parser));
+    for t in tokens {
+        matcher.consume_token(t).unwrap();
+    }
+}

{llguidance-0.7.27 → llguidance-0.7.29}/toktrie/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "toktrie"
-version = "0.7.27"
+version = "0.7.29"
 edition = "2021"
 license = "MIT"
 description = "LLM Token Trie library"

{llguidance-0.7.27 → llguidance-0.7.29}/toktrie_hf_downloader/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "toktrie_hf_downloader"
-version = "0.7.27"
+version = "0.7.29"
 edition = "2021"
 license = "MIT"
 description = "HuggingFace Hub download library support for toktrie and llguidance"

{llguidance-0.7.27 → llguidance-0.7.29}/toktrie_hf_tokenizers/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "toktrie_hf_tokenizers"
-version = "0.7.27"
+version = "0.7.29"
 edition = "2021"
 license = "MIT"
 description = "HuggingFace tokenizers library support for toktrie and llguidance"

{llguidance-0.7.27 → llguidance-0.7.29}/toktrie_tiktoken/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "toktrie_tiktoken"
-version = "0.7.27"
+version = "0.7.29"
 edition = "2021"
 license = "MIT"
 description = "HuggingFace tokenizers library support for toktrie and llguidance"