llguidance 0.7.26__tar.gz → 0.7.29__tar.gz

{llguidance-0.7.26 → llguidance-0.7.29}/CHANGELOG.md

@@ -4,6 +4,19 @@ All notable changes to this project will be documented in this file. Dates are d
 
 If a release doesn't introduce any interesting changes (build fixes etc.), it's skipped.
 
+#### [0.7.29](https://github.com/guidance-ai/llguidance/compare/v0.7.28...0.7.29) 2025-06-06
+
+- cargo fmt
+
+#### [0.7.28](https://github.com/guidance-ai/llguidance/compare/v0.7.27...0.7.28) 2025-06-06
+
+- fix lexer_stack=... panic with numeric tokens [`4e91b0f`](https://github.com/guidance-ai/llguidance/commit/4e91b0fa0c03572a5fc221ac0e0b05035af9dcfa)
+
+#### [0.7.27](https://github.com/guidance-ai/llguidance/compare/v0.7.26...0.7.27) 2025-06-04
+
+- add toktrie_tiktoken and llguidance.tiktoken.lltokenizer_from_encoding [`#154`](https://github.com/guidance-ai/llguidance/issues/154)
+- implement clone on StopController [`#185`](https://github.com/guidance-ai/llguidance/issues/185)
+
 #### [0.7.26](https://github.com/guidance-ai/llguidance/compare/v0.7.25...0.7.26) 2025-05-30
 
 - add support for & and ~ in lark regexes [`96fcee3`](https://github.com/guidance-ai/llguidance/commit/96fcee373697b57bead94d1bc06c17cf1c6134e4)

{llguidance-0.7.26 → llguidance-0.7.29}/Cargo.lock

@@ -135,15 +135,30 @@ version = "0.22.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
 
+[[package]]
+name = "bit-set"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1"
+dependencies = [
+ "bit-vec 0.6.3",
+]
+
 [[package]]
 name = "bit-set"
 version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3"
 dependencies = [
- "bit-vec",
+ "bit-vec 0.8.0",
 ]
 
+[[package]]
+name = "bit-vec"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb"
+
 [[package]]
 name = "bit-vec"
 version = "0.8.0"
@@ -162,6 +177,17 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3eeab4423108c5d7c744f4d234de88d18d636100093ae04caf4825134b9c3a32"
 
+[[package]]
+name = "bstr"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4"
+dependencies = [
+ "memchr",
+ "regex-automata",
+ "serde",
+]
+
 [[package]]
 name = "bumpalo"
 version = "3.17.0"
@@ -492,13 +518,24 @@ version = "0.1.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6"
 
+[[package]]
+name = "fancy-regex"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2"
+dependencies = [
+ "bit-set 0.5.3",
+ "regex-automata",
+ "regex-syntax",
+]
+
 [[package]]
 name = "fancy-regex"
 version = "0.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298"
 dependencies = [
- "bit-set",
+ "bit-set 0.8.0",
  "regex-automata",
  "regex-syntax",
 ]
@@ -1123,7 +1160,7 @@ dependencies = [
  "base64 0.22.1",
  "bytecount",
  "email_address",
- "fancy-regex",
+ "fancy-regex 0.14.0",
  "fraction",
  "idna",
  "itoa",
@@ -1174,7 +1211,7 @@ checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856"
 
 [[package]]
 name = "llguidance"
-version = "0.7.26"
+version = "0.7.29"
 dependencies = [
  "anyhow",
  "derivre",
@@ -1193,7 +1230,7 @@ dependencies = [
 
 [[package]]
 name = "llguidance_py"
-version = "0.7.26"
+version = "0.7.29"
 dependencies = [
  "anyhow",
  "bytemuck",
@@ -1203,6 +1240,7 @@ dependencies = [
  "serde",
  "serde_json",
  "toktrie_hf_tokenizers",
+ "toktrie_tiktoken",
 ]
 
 [[package]]
@@ -1865,6 +1903,12 @@ version = "0.1.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
 
+[[package]]
+name = "rustc-hash"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
+
 [[package]]
 name = "rustix"
 version = "1.0.5"
@@ -2233,6 +2277,21 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "tiktoken-rs"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "25563eeba904d770acf527e8b370fe9a5547bacd20ff84a0b6c3bc41288e5625"
+dependencies = [
+ "anyhow",
+ "base64 0.22.1",
+ "bstr",
+ "fancy-regex 0.13.0",
+ "lazy_static",
+ "regex",
+ "rustc-hash",
+]
+
 [[package]]
 name = "tinystr"
 version = "0.7.6"
@@ -2252,7 +2311,7 @@ dependencies = [
  "aho-corasick",
  "derive_builder",
  "esaxx-rs",
- "fancy-regex",
+ "fancy-regex 0.14.0",
  "getrandom 0.2.15",
  "itertools 0.13.0",
  "lazy_static",
@@ -2336,7 +2395,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie"
-version = "0.7.26"
+version = "0.7.29"
 dependencies = [
  "anyhow",
  "bytemuck",
@@ -2347,7 +2406,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_hf_downloader"
-version = "0.7.26"
+version = "0.7.29"
 dependencies = [
  "anyhow",
  "hf-hub",
@@ -2358,7 +2417,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_hf_tokenizers"
-version = "0.7.26"
+version = "0.7.29"
 dependencies = [
  "anyhow",
  "log",
@@ -2368,6 +2427,18 @@ dependencies = [
  "toktrie",
 ]
 
+[[package]]
+name = "toktrie_tiktoken"
+version = "0.7.29"
+dependencies = [
+ "anyhow",
+ "log",
+ "serde",
+ "serde_json",
+ "tiktoken-rs",
+ "toktrie",
+]
+
 [[package]]
 name = "tower"
 version = "0.5.2"

{llguidance-0.7.26 → llguidance-0.7.29}/Cargo.toml

@@ -7,6 +7,7 @@ members = [
     "toktrie",
     "toktrie_hf_tokenizers",
     "toktrie_hf_downloader",
+    "toktrie_tiktoken",
 ]
 # just exclude python_ext since it doesn't build without maturin
 default-members = [
@@ -16,6 +17,7 @@ default-members = [
     "toktrie",
     "toktrie_hf_tokenizers",
     "toktrie_hf_downloader",
+    "toktrie_tiktoken",
 ]
 resolver = "2"
 
@@ -36,4 +38,5 @@ opt-level = 3
 toktrie = { path = "toktrie" }
 llguidance = { path = "parser" }
 toktrie_hf_tokenizers = { path = "toktrie_hf_tokenizers" }
-toktrie_hf_downloader = { path = "toktrie_hf_downloader" }
+toktrie_hf_downloader = { path = "toktrie_hf_downloader" }
+toktrie_tiktoken = { path = "toktrie_tiktoken" }

{llguidance-0.7.26 → llguidance-0.7.29}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llguidance
-Version: 0.7.26
+Version: 0.7.29
 License-File: LICENSE
 Summary: Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance
 Author: Michal Moskal

{llguidance-0.7.26 → llguidance-0.7.29}/parser/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "llguidance"
-version = "0.7.26"
+version = "0.7.29"
 edition = "2021"
 license = "MIT"
 description = "Super-fast Structured Outputs"

{llguidance-0.7.26 → llguidance-0.7.29}/parser/llguidance.h

@@ -433,6 +433,13 @@ const char *llg_stop_commit_token(struct LlgStopController *stop_ctrl,
                                   size_t *output_len_p,
                                   bool *is_stopped_p);
 
+/**
+ * Clone the stop-sequence controller.
+ * The cloned controller shares (under mutex) regex caches if any, so that
+ * cloning is cheap.
+ */
+struct LlgStopController *llg_clone_stop_controller(const struct LlgStopController *stop_ctrl);
+
 /**
  * Free the stop-sequence controller
  */

{llguidance-0.7.26 → llguidance-0.7.29}/parser/src/earley/parser.rs

@@ -53,6 +53,14 @@ macro_rules! debug {
     }
 }
 
+macro_rules! debug_def {
+    ($s:expr, $($arg:tt)*) => {
+        if cfg!(feature = "logging") && DEBUG && $s.scratch.log_enabled() {
+            eprintln!($($arg)*);
+        }
+    }
+}
+
 macro_rules! item_trace {
     ($($arg:tt)*) => {
         if ITEM_TRACE {
@@ -322,6 +330,8 @@ struct Scratch {
     // mode, which is used for computing the token mask on the
     // pre-lexemes.
     definitive: bool,
+
+    log_override: bool,
 }
 
 #[derive(Clone)]
@@ -425,6 +435,7 @@ struct ParserState {
     // history - items are not popped in definitive mode.
     lexer_stack: Vec<LexerState>,
     lexer_stack_top_eos: bool,
+    lexer_stack_flush_position: usize,
     rows: Vec<Row>,
     rows_valid_end: usize,
 
@@ -488,9 +499,14 @@ impl Scratch {
             items: vec![],
             grammar_stack: vec![],
             definitive: true,
+            log_override: false,
         }
     }
 
+    fn log_enabled(&self) -> bool {
+        self.definitive || self.log_override
+    }
+
     // Set current working Earley to empty set
     // The set backing data is at `pos`
     fn new_row(&mut self, pos: usize) {
@@ -523,7 +539,7 @@ impl Scratch {
     }
 
     fn push_grammar_stack(&mut self, node: GrammarStackNode) {
-        if self.definitive {
+        if self.log_enabled() {
             debug!("push_grammar_stack: {:?}", node);
         }
         let ptr = GrammarStackPtr::new(self.grammar_stack.len());
@@ -543,7 +559,7 @@ impl Scratch {
         } else {
             self.items[self.row_end] = item;
         }
-        if self.definitive {
+        if self.log_enabled() {
             debug!(
                 "      addu: {} ({})",
                 self.item_to_string(self.row_end),
@@ -650,6 +666,7 @@ impl ParserState {
             limits,
             backtrack_byte_count: 0,
             lexer_stack_top_eos: false,
+            lexer_stack_flush_position: 0,
             lexer_stack: vec![LexerState {
                 row_idx: 0,
                 lexer_state,
@@ -778,12 +795,6 @@ impl ParserState {
             });
         }
 
-        if set.is_zero() {
-            // nothing allowed
-            // we're going to be stopped outside - we better flush the lexer
-            let _ = self.flush_lexer();
-        }
-
         let eos = computer.trie().eos_token();
         if eos != INVALID_TOKEN && start.is_empty() && self.lexer_allows_eos() {
             set.allow_token(eos);
@@ -920,7 +931,7 @@ impl ParserState {
         self.stats = ParserStats::default();
     }
 
-    fn assert_definitive(&self) {
+    fn assert_definitive_inner(&self) {
         assert!(self.scratch.definitive);
         assert!(self.backtrack_byte_count == 0);
         if self.num_rows() != self.row_infos.len() {
@@ -932,6 +943,14 @@ impl ParserState {
         }
     }
 
+    fn assert_definitive(&self) {
+        self.assert_definitive_inner();
+
+        if self.lexer_spec().can_rollback() {
+            self.check_lexer_bytes_invariant();
+        }
+    }
+
     pub fn get_bytes(&self) -> &[u8] {
         &self.bytes
     }
@@ -980,7 +999,6 @@ impl ParserState {
             n_bytes,
             self.byte_to_token_idx.len()
         );
-        self.check_lexer_bytes_invariant();
 
         let new_len = self.byte_to_token_idx.len() - n_bytes;
 
@@ -995,7 +1013,6 @@ impl ParserState {
         self.rows_valid_end = self.num_rows();
 
         self.assert_definitive();
-        self.check_lexer_bytes_invariant();
 
         Ok(())
     }
@@ -1003,6 +1020,7 @@ impl ParserState {
     pub fn validate_tokens(&mut self, tokens: &[TokenId]) -> usize {
         self.assert_definitive();
         self.run_speculative("validate_tokens", |state| {
+            state.scratch.log_override = true;
             let mut applied_idx = state.byte_to_token_idx.len();
             let tok_env = state.tok_env.clone();
             let trie = tok_env.tok_trie();
@@ -1081,6 +1099,12 @@ impl ParserState {
                     .push(self.token_idx.try_into().unwrap());
             }
         }
+        debug_def!(
+            self,
+            "add_numeric_token: idx={:?} bytes={:?}",
+            idx,
+            tok_bytes
+        );
         let ok = self.advance_parser(PreLexeme::just_idx(MatchingLexemesIdx::Single(idx)));
         ensure!(
             ok,
@@ -1140,9 +1164,20 @@ impl ParserState {
                 let row_idx = self.num_rows() - 1;
                 self.row_infos[row_idx].apply_token_idx(self.token_idx);
 
+                self.lexer_stack_flush_position = 0;
                 let idx = self.flush_and_check_numeric(tok_id).unwrap();
                 self.add_numeric_token(idx, tok_bytes)?;
 
+                // if flush_lexer() added a stack entry
+                if self.lexer_stack_flush_position > 0 {
+                    // we make sure it's not on the top
+                    assert!(self.lexer_stack_flush_position + 1 < self.lexer_stack.len());
+                    // and remove it
+                    self.lexer_stack.remove(self.lexer_stack_flush_position);
+                }
+
+                self.assert_definitive();
+
                 return Ok(0);
             }
         }
@@ -1298,6 +1333,8 @@ impl ParserState {
             self.print_row(self.num_rows() - 1);
         }
 
+        self.assert_definitive();
+
         Ok(0)
     }
 
@@ -1458,10 +1495,6 @@ impl ParserState {
         // debug!("trie_started: rows={} lexer={}", self.num_rows(), self.lexer_stack.len());
         self.assert_definitive();
 
-        if self.lexer_spec().can_rollback() {
-            self.check_lexer_bytes_invariant();
-        }
-
         self.trie_lexer_stack = self.lexer_stack.len();
         self.trie_grammar_stack = self.scratch.grammar_stack.len();
         self.scratch.definitive = false;
@@ -1497,6 +1530,8 @@ impl ParserState {
         self.scratch.definitive = true;
         self.assert_definitive();
         self.rows_valid_end = self.num_rows();
+        self.scratch.log_override = false; // reset
+        self.lexer_stack_flush_position = 0;
     }
 
     fn run_speculative<T>(&mut self, lbl: &str, f: impl FnOnce(&mut Self) -> T) -> T {
@@ -1655,16 +1690,19 @@ impl ParserState {
         }
         let curr = self.lexer_state();
         let lex_result = self.lexer_mut().try_lexeme_end(curr.lexer_state);
+        let prev_len = self.lexer_stack.len();
         let r = self.advance_lexer_or_parser(lex_result, curr);
+        if self.lexer_stack.len() != prev_len {
+            assert!(self.lexer_stack.len() == prev_len + 1);
+            assert!(prev_len > 0);
+            self.lexer_stack_flush_position = prev_len;
+        }
         assert!(self.backtrack_byte_count == 0);
         r
     }
 
     pub fn scan_eos(&mut self) -> bool {
         self.assert_definitive(); // ???
-        if self.lexer_spec().can_rollback() {
-            self.check_lexer_bytes_invariant();
-        }
 
         let lexer_eos = self.lexer_allows_eos();
 
@@ -1693,9 +1731,7 @@ impl ParserState {
             self.lexer_stack_top_eos = true;
         }
 
-        if self.lexer_spec().can_rollback() {
-            self.check_lexer_bytes_invariant();
-        }
+        self.assert_definitive(); // ???
 
         false
     }
@@ -1762,14 +1798,13 @@ impl ParserState {
         self.scratch.new_row(items.end);
         self.scratch.push_lexeme_idx = lexeme.idx;
 
-        if self.scratch.definitive {
-            debug!(
-                "  scan: {} at row={} token={}",
-                self.lexer().dbg_lexeme(lexeme),
-                row_idx,
-                self.token_idx,
-            );
-        }
+        debug_def!(
+            self,
+            "  scan: {} at row={} token={}",
+            self.lexer().dbg_lexeme(lexeme),
+            row_idx,
+            self.token_idx,
+        );
 
         // This loop performs the scan inference rule
         // (slide 21 of Kallmeyer 2018).  It is an
@@ -1890,9 +1925,7 @@ impl ParserState {
             let item_idx = agenda_ptr;
             let item = self.scratch.items[agenda_ptr];
             agenda_ptr += 1;
-            if self.scratch.definitive {
-                debug!("    agenda: {}", self.item_to_string(item_idx));
-            }
+            debug_def!(self, "    agenda: {}", self.item_to_string(item_idx));
 
             let rule = item.rhs_ptr();
             let after_dot = self.grammar.sym_idx_dot(rule);
@@ -1988,13 +2021,12 @@ impl ParserState {
                     .start_state(&self.scratch.push_allowed_lexemes)
             };
 
-            if self.scratch.definitive {
-                debug!(
-                    "  push row: {} {:?}",
-                    self.allowed_lexemes_dbg(lex_start),
-                    grammar_id
-                );
-            }
+            debug_def!(
+                self,
+                "  push row: {} {:?}",
+                self.allowed_lexemes_dbg(lex_start),
+                grammar_id
+            );
 
             // Add the working row to the parser state
             let idx = self.num_rows();
@@ -2042,9 +2074,7 @@ impl ParserState {
     }
 
     fn process_max_tokens(&mut self, ptr: GrammarStackPtr, lexeme: &Lexeme) {
-        if self.scratch.definitive {
-            debug!("  process_max_tokens");
-        }
+        debug_def!(self, "  process_max_tokens");
         let curr_idx = self.num_rows();
         let top = &self.scratch.grammar_stack[ptr.as_usize()];
         self.scratch.push_grm_top = top.back_ptr;
@@ -2118,12 +2148,13 @@ impl ParserState {
 
         while grm_stack_top.as_usize() > 0 {
             let grm_top = &self.scratch.grammar_stack[grm_stack_top.as_usize()];
-            if self.scratch.definitive {
-                debug!(
-                    "  pop grammar_stack: top={:?}, curr={:?}, #{}",
-                    grm_top.grammar_id, grammar_ids, self.token_idx
-                );
-            }
+            debug_def!(
+                self,
+                "  pop grammar_stack: top={:?}, curr={:?}, #{}",
+                grm_top.grammar_id,
+                grammar_ids,
+                self.token_idx
+            );
             if grammar_ids.contains(&grm_top.grammar_id) {
                 // token_idx is one behind
                 if grm_top.token_horizon <= self.token_idx as u32 {
@@ -2132,12 +2163,12 @@ impl ParserState {
                     // We only pop one grammar off the stack.
                     // If more grammars have the same token horizon, they will get popped
                     // in the next step - we might overrun a bit.
-                    if self.scratch.definitive {
-                        debug!(
-                            "  hit token limit horizon={} token_idx={}",
-                            grm_top.token_horizon, self.token_idx
-                        );
-                    }
+                    debug_def!(
+                        self,
+                        "  hit token limit horizon={} token_idx={}",
+                        grm_top.token_horizon,
+                        self.token_idx
+                    );
                     max_token_ptr = Some(grm_stack_top);
                 }
                 break;
@@ -2252,13 +2283,14 @@ impl ParserState {
                     .saturating_sub(1);
                 self.row_infos[added_row].start_byte_idx -= new_start;
             }
-            debug!(
-                "lex: re-start {:?} (via {:?}); allowed: {}",
-                no_hidden.lexer_state,
-                transition_byte.map(|b| b as char),
-                self.allowed_lexemes_dbg(added_row_start_state)
-            );
         }
+        debug_def!(
+            self,
+            "lex: re-start {:?} (via {:?}); allowed: {}",
+            no_hidden.lexer_state,
+            transition_byte.map(|b| b as char),
+            self.allowed_lexemes_dbg(added_row_start_state)
+        );
 
         no_hidden
     }
@@ -2277,7 +2309,7 @@ impl ParserState {
 
         let hidden_bytes = lexeme.hidden_bytes();
 
-        let trace_here = self.scratch.definitive;
+        let trace_here = self.scratch.log_enabled();
 
         if trace_here {
             trace!(
@@ -2348,7 +2380,7 @@ impl ParserState {
                 });
             }
             if self.scratch.definitive {
-                self.assert_definitive();
+                self.assert_definitive_inner();
             }
         } else {
             if trace_here {
@@ -2361,7 +2393,7 @@ impl ParserState {
                     byte: None,
                     ..no_hidden
                 });
-                self.assert_definitive();
+                self.assert_definitive_inner();
                 self.backtrack_byte_count = hidden_bytes.len();
             } else {
                 // prevent any further matches in this branch
@@ -2467,9 +2499,7 @@ impl ParserState {
                         .lexer_mut()
                         .check_for_single_byte_lexeme(no_hidden.lexer_state, b);
                     if let Some(second_lexeme) = single {
-                        if self.scratch.definitive {
-                            debug!("single byte lexeme: {:?}", second_lexeme);
-                        }
+                        debug_def!(self, "single byte lexeme: {:?}", second_lexeme);
                         no_hidden.byte = None;
                         self.lexer_stack.push(no_hidden);
 
@@ -2488,16 +2518,15 @@ impl ParserState {
                         }
                     }
                 }
+                debug_def!(self, "  push normal: {no_hidden:?}");
                 self.lexer_stack.push(no_hidden);
             }
             if self.scratch.definitive {
-                self.assert_definitive();
+                self.assert_definitive_inner();
             }
             true
         } else {
-            if self.scratch.definitive {
-                debug!("  scan failed");
-            }
+            debug_def!(self, "  scan failed");
             false
         }
     }