llguidance 0.7.12__tar.gz → 0.7.13__tar.gz

{llguidance-0.7.12 → llguidance-0.7.13}/CHANGELOG.md

@@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. Dates are d
 
 If a release doesn't introduce any interesting changes (build fixes etc.), it's skipped.
 
+#### [0.7.13](https://github.com/guidance-ai/llguidance/compare/v0.7.12...0.7.13) 2025-04-05
+
+- expose LLParserLimits in Python API [`598dc8f`](https://github.com/guidance-ai/llguidance/commit/598dc8f37f69f51244e54d9885445abf02a515a7)
+- pre-compute lexer states for particularly large regexes (can be disabled in ParserLimits)
+
 #### [0.7.12](https://github.com/guidance-ai/llguidance/compare/v0.7.11...0.7.12) 2025-04-04
 
 - performance optimizations

{llguidance-0.7.12 → llguidance-0.7.13}/Cargo.lock

@@ -401,9 +401,9 @@ dependencies = [
 
 [[package]]
 name = "derivre"
-version = "0.3.4"
+version = "0.3.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "310c9990c5a531352e274c8c929ca667a84b6bbaceb1e095c177e6a979807f57"
+checksum = "2c15da97393af596fc344dc746f804c6742f1ddfd2e88fd7b75232a989c85c4e"
 dependencies = [
  "ahash",
  "anyhow",
@@ -1177,7 +1177,7 @@ checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104"
 
 [[package]]
 name = "llguidance"
-version = "0.7.12"
+version = "0.7.13"
 dependencies = [
  "anyhow",
  "derivre",
@@ -1196,7 +1196,7 @@ dependencies = [
 
 [[package]]
 name = "llguidance_py"
-version = "0.7.12"
+version = "0.7.13"
 dependencies = [
  "anyhow",
  "bytemuck",
@@ -2356,7 +2356,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie"
-version = "0.7.12"
+version = "0.7.13"
 dependencies = [
  "anyhow",
  "bytemuck",
@@ -2367,7 +2367,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_hf_downloader"
-version = "0.7.12"
+version = "0.7.13"
 dependencies = [
  "anyhow",
  "hf-hub",
@@ -2378,7 +2378,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_hf_tokenizers"
-version = "0.7.12"
+version = "0.7.13"
 dependencies = [
  "anyhow",
  "log",

{llguidance-0.7.12 → llguidance-0.7.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llguidance
-Version: 0.7.12
+Version: 0.7.13
 License-File: LICENSE
 Summary: Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance
 Author: Michal Moskal

{llguidance-0.7.12 → llguidance-0.7.13}/json_stats/src/json_stats.rs

@@ -5,11 +5,7 @@ use json_stats::SchemaStats;
 use jsonschema::Validator;
 use llguidance::{
     api::{GrammarInit, StopReason, TopLevelGrammar},
-    earley::{
-        perf::{num_with_commas, ParserPerfCounters},
-        regexvec::LexerStats,
-        XorShift,
-    },
+    earley::{perf::num_with_commas, regexvec::LexerStats, XorShift},
     toktrie::{InferenceCapabilities, SimpleVob, TokEnv},
     Constraint, HashMap, JsonCompileOptions, ParserFactory, TokenParser,
 };
@@ -290,7 +286,6 @@ struct TestEnv {
     factory: Arc<ParserFactory>,
     ref_factory: Arc<ParserFactory>,
     file_name: String,
-    perf_counters: Arc<ParserPerfCounters>,
     hash_rnd: Arc<ahash::RandomState>,
 }
 
@@ -739,8 +734,7 @@ impl TestEnv {
 
         let t2 = std::time::Instant::now();
         let parser = match parser {
-            Ok(mut parser) => {
-                parser.parser.set_perf_counters(self.perf_counters.clone());
+            Ok(parser) => {
                 let mut constraint = Constraint::new(parser.clone());
                 constraint.compute_mask().unwrap();
                 res.first_mask_us = t2.elapsed().as_micros() as usize;
@@ -996,7 +990,6 @@ fn main() {
 
     let t0 = std::time::Instant::now();
     let par = num_threads > 1;
-    let perf_counters = Arc::new(ParserPerfCounters::new());
     let hash_rnd = Arc::new(ahash::RandomState::new());
     let do_file = |file: &String| {
         let env = TestEnv {
@@ -1005,7 +998,6 @@ fn main() {
             ref_factory: ref_factory.clone(),
             file_name: file.to_string(),
             cli: options.clone(),
-            perf_counters: perf_counters.clone(),
             hash_rnd: hash_rnd.clone(),
         };
         env.run_test()
@@ -1155,7 +1147,7 @@ fn main() {
         eprintln!(
             "{}\n{}",
             serde_json::to_string_pretty(&total).unwrap(),
-            perf_counters
+            &factory.perf_counters(),
         );
     }
     eprintln!(

{llguidance-0.7.12 → llguidance-0.7.13}/parser/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "llguidance"
-version = "0.7.12"
+version = "0.7.13"
 edition = "2021"
 license = "MIT"
 description = "Super-fast Structured Outputs"
@@ -8,7 +8,7 @@ repository = "https://github.com/guidance-ai/llguidance"
 
 [dependencies]
 toktrie = { workspace = true }
-derivre = { version = "=0.3.4", default-features = false, features = ["compress"] }
+derivre = { version = "=0.3.5", default-features = false, features = ["compress"] }
 serde = { version = "1.0.217", features = ["derive"] }
 serde_json = { version = "1.0.138", features = ["preserve_order"] }
 anyhow = "1.0.95"

{llguidance-0.7.12 → llguidance-0.7.13}/parser/llguidance.h

@@ -70,6 +70,14 @@ typedef struct LlgParserLimits {
    * Default: 500_000 (a few megabytes of JSON)
    */
   size_t max_grammar_size;
+  /**
+   * If true, we'll run any extremely large regexes against the whole
+   * trie of the tokenizer while constructing the lexer.
+   * This reduces future mask computation time, but increases
+   * the time it takes to construct the lexer.
+   * Default: true
+   */
+  bool precompute_large_lexemes;
 } LlgParserLimits;
 
 typedef struct LlgConstraintInit {

{llguidance-0.7.12 → llguidance-0.7.13}/parser/src/api.rs

@@ -249,6 +249,13 @@ pub struct ParserLimits {
     /// Maximum size of the grammar (symbols in productions)
     /// Default: 500_000 (a few megabytes of JSON)
     pub max_grammar_size: usize,
+
+    /// If true, we'll run any extremely large regexes against the whole
+    /// trie of the tokenizer while constructing the lexer.
+    /// This reduces future mask computation time, but increases
+    /// the time it takes to construct the lexer.
+    /// Default: true
+    pub precompute_large_lexemes: bool,
 }
 
 impl Default for ParserLimits {
@@ -260,6 +267,7 @@ impl Default for ParserLimits {
             max_lexer_states: 250_000,     //
             max_grammar_size: 500_000,     // fhir schema => 200k
             step_max_items: 50_000,        //
+            precompute_large_lexemes: true,
         }
     }
 }

{llguidance-0.7.12 → llguidance-0.7.13}/parser/src/earley/lexer.rs

@@ -1,6 +1,6 @@
 use anyhow::Result;
 use std::fmt::Debug;
-use toktrie::SimpleVob;
+use toktrie::{Recognizer, SimpleVob, TokTrie};
 
 use crate::api::ParserLimits;
 
@@ -59,6 +59,29 @@ pub enum LexerResult {
     Error,
 }
 
+struct LexerPrecomputer<'a> {
+    states: Vec<StateID>,
+    lex: &'a mut Lexer,
+}
+
+impl Recognizer for LexerPrecomputer<'_> {
+    fn collapse(&mut self) {}
+    fn trie_finished(&mut self) {}
+    fn pop_bytes(&mut self, num: usize) {
+        self.states.truncate(self.states.len() - num);
+    }
+    fn try_push_byte(&mut self, byte: u8) -> bool {
+        let state = *self.states.last().unwrap();
+        match self.lex.advance(state, byte, false) {
+            LexerResult::State(next_state, _) => {
+                self.states.push(next_state);
+                true
+            }
+            _ => false,
+        }
+    }
+}
+
 impl Lexer {
     pub fn from(spec: &LexerSpec, limits: &mut ParserLimits, dbg: bool) -> Result<Self> {
         let mut dfa = spec.to_regex_vec(limits)?;
@@ -92,6 +115,15 @@ impl Lexer {
         self.dfa.initial_state(allowed_lexemes)
     }
 
+    pub fn precompute_for(&mut self, trie: &TokTrie, allowed_lexemes: &LexemeSet) {
+        let state = self.start_state(allowed_lexemes);
+        let mut states = Vec::with_capacity(300);
+        states.push(state);
+        let mut pre = LexerPrecomputer { states, lex: self };
+        let mut toks = trie.alloc_token_set();
+        trie.add_bias(&mut pre, &mut toks, &[]);
+    }
+
     pub fn transition_start_state(&mut self, s: StateID, first_byte: Option<u8>) -> StateID {
         first_byte.map(|b| self.dfa.transition(s, b)).unwrap_or(s)
     }

{llguidance-0.7.12 → llguidance-0.7.13}/parser/src/earley/lexerspec.rs

@@ -48,7 +48,7 @@ pub struct LexemeSpec {
     pub(crate) name: String,
     pub(crate) rx: RegexAst,
     class: LexemeClass,
-    compiled_rx: ExprRef,
+    pub(crate) compiled_rx: ExprRef,
     ends_at_eos: bool,
     lazy: bool,
     contextual: bool,
@@ -299,6 +299,7 @@ impl LexerSpec {
         } else {
             compiled
         };
+
         if let Some(idx) = self.lexemes.iter().position(|lex| {
             lex.compiled_rx == compiled
                 && lex.class == spec.class

{llguidance-0.7.12 → llguidance-0.7.13}/parser/src/earley/parser.rs

@@ -5,7 +5,7 @@
 // (Retrieved 18 Sep 2024).
 
 use std::{
-    fmt::Debug,
+    fmt::{Debug, Display},
     hash::Hash,
     ops::Range,
     sync::{Arc, Mutex},
@@ -207,6 +207,12 @@ impl ParserStats {
     }
 }
 
+impl Display for ParserStats {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", serde_json::to_string_pretty(self).unwrap())
+    }
+}
+
 id32_type!(GrammarStackPtr);
 
 #[derive(Clone, Debug)]
@@ -580,9 +586,26 @@ impl ParserState {
         tok_env: TokEnv,
         grammar: Arc<CGrammar>,
         mut limits: ParserLimits,
+        perf_counters: Arc<ParserPerfCounters>,
     ) -> Result<(Self, Lexer)> {
         let start = grammar.start();
-        let lexer = Lexer::from(grammar.lexer_spec(), &mut limits, true)?;
+        let mut lexer = Lexer::from(grammar.lexer_spec(), &mut limits, true)?;
+        if limits.precompute_large_lexemes {
+            let t0 = crate::Instant::now();
+            for spec in &grammar.lexer_spec().lexemes {
+                let w = lexer.dfa.lexeme_weight(spec.idx);
+                if w > 1000 {
+                    // println!(
+                    //     "precomputing lexeme {} (w={w})",
+                    //     lexer.lexer_spec().lexeme_def_to_string(spec.idx)
+                    // );
+                    let mut allowed = grammar.lexer_spec().alloc_lexeme_set();
+                    allowed.add(spec.idx);
+                    lexer.precompute_for(tok_env.tok_trie(), &allowed);
+                }
+            }
+            perf_counters.precompute.record(t0.elapsed());
+        }
         let scratch = Scratch::new(Arc::clone(&grammar));
         let lexer_state = lexer.a_dead_state(); // placeholder
         let spec_tok = tok_env
@@ -626,7 +649,7 @@ impl ParserState {
             shared_box: Box::new(SharedState {
                 lexer_opt: Some(lexer),
             }),
-            perf_counters: Arc::new(ParserPerfCounters::new()),
+            perf_counters,
         };
 
         r.scratch.grammar_stack.push(GrammarStackNode {
@@ -2576,8 +2599,13 @@ impl ParserError {
 }
 
 impl Parser {
-    pub fn new(tok_env: TokEnv, grammar: Arc<CGrammar>, limits: ParserLimits) -> Result<Self> {
-        let (state, lexer) = ParserState::new(tok_env, grammar, limits)?;
+    pub fn new(
+        tok_env: TokEnv,
+        grammar: Arc<CGrammar>,
+        limits: ParserLimits,
+        perf_counters: Arc<ParserPerfCounters>,
+    ) -> Result<Self> {
+        let (state, lexer) = ParserState::new(tok_env, grammar, limits, perf_counters)?;
         let shared = Arc::new(Mutex::new(Box::new(SharedState {
             lexer_opt: Some(lexer),
         })));
@@ -2603,10 +2631,6 @@ impl Parser {
         &self.state.stats
     }
 
-    pub fn set_perf_counters(&mut self, counters: Arc<ParserPerfCounters>) {
-        self.state.perf_counters = counters;
-    }
-
     #[inline(always)]
     pub fn perf_counters(&self) -> &ParserPerfCounters {
         &self.state.perf_counters

{llguidance-0.7.12 → llguidance-0.7.13}/parser/src/earley/perf.rs

@@ -87,6 +87,7 @@ pub struct ParserPerfCounters {
     pub tokenize_ff: PerfTimer,
     pub compute_bias: PerfTimer,
     pub compute_mask: PerfTimer,
+    pub precompute: PerfTimer,
 }
 
 impl Default for ParserPerfCounters {
@@ -104,6 +105,7 @@ impl ParserPerfCounters {
             tokenize_ff: PerfTimer::new("tokenize_ff"),
             compute_bias: PerfTimer::new("compute_bias"),
             compute_mask: PerfTimer::new("compute_mask"),
+            precompute: PerfTimer::new("precompute"),
         }
     }
 
@@ -115,6 +117,7 @@ impl ParserPerfCounters {
             &self.compute_bias,
             &self.compute_mask,
             &self.tmp_counter,
+            &self.precompute,
         ]
     }
 }

{llguidance-0.7.12 → llguidance-0.7.13}/parser/src/earley/regexvec.rs

@@ -490,6 +490,11 @@ impl RegexVec {
         self.exprs.cost()
     }
 
+    pub fn lexeme_weight(&mut self, lexeme_idx: LexemeIdx) -> u32 {
+        let e = self.rx_list[lexeme_idx.as_usize()];
+        self.exprs.get_weight(e)
+    }
+
     pub fn set_max_states(&mut self, max_states: usize) {
         if !self.has_error() {
             self.max_states = max_states;

{llguidance-0.7.12 → llguidance-0.7.13}/parser/src/factory.rs

@@ -5,7 +5,7 @@ use toktrie::{InferenceCapabilities, TokEnv};
 
 use crate::{
     api::{GrammarInit, ParserLimits, TopLevelGrammar},
-    earley::{SlicedBiasComputer, XorShift},
+    earley::{perf::ParserPerfCounters, SlicedBiasComputer, XorShift},
     Logger, TokenParser,
 };
 
@@ -17,6 +17,7 @@ pub struct ParserFactory {
     buffer_log_level: u32,
     limits: ParserLimits,
     seed: Mutex<XorShift>,
+    perf_counters: Arc<ParserPerfCounters>,
 }
 
 impl ParserFactory {
@@ -34,9 +35,14 @@ impl ParserFactory {
             buffer_log_level: 0,
             seed: Mutex::new(XorShift::default()),
             limits: ParserLimits::default(),
+            perf_counters: Arc::new(ParserPerfCounters::default()),
         })
     }
 
+    pub fn perf_counters(&self) -> Arc<ParserPerfCounters> {
+        self.perf_counters.clone()
+    }
+
     pub fn new_simple(tok_env: &TokEnv) -> Result<Self> {
         Self::new(
             tok_env,
@@ -55,6 +61,7 @@ impl ParserFactory {
             buffer_log_level: self.buffer_log_level,
             seed: Mutex::new(XorShift::default()),
             limits: self.limits.clone(),
+            perf_counters: self.perf_counters.clone(),
         })
     }
 

{llguidance-0.7.12 → llguidance-0.7.13}/parser/src/tokenparser.rs

@@ -80,7 +80,12 @@ impl TokenParser {
             limits.clone(),
             factory.extra_lexemes(),
         )?;
-        let mut parser = Parser::new(token_env.clone(), compiled_grammar, limits.clone())?;
+        let mut parser = Parser::new(
+            token_env.clone(),
+            compiled_grammar,
+            limits.clone(),
+            factory.perf_counters(),
+        )?;
         parser.metrics_mut().rand = factory.next_rng();
         let eos_token = token_env.tok_trie().eos_token();
 

{llguidance-0.7.12 → llguidance-0.7.13}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "llguidance"
-version = "0.7.12"
+version = "0.7.13"
 description = "Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance"
 requires-python = ">=3.9"
 license = "MIT"

{llguidance-0.7.12 → llguidance-0.7.13}/python/llguidance/__init__.py

@@ -6,6 +6,7 @@ from ._lib import (
     RegexCompiler,
     LLExecutor,
     LLMatcher,
+    LLParserLimits,
 )
 from ._tokenizer import TokenizerWrapper
 from ._grammar_from import GrammarFormat, grammar_from
@@ -16,6 +17,7 @@ __all__ = [
     "LLMatcher",
     "LLInterpreter",
     "LLExecutor",
+    "LLParserLimits",
     "JsonCompiler",
     "LarkCompiler",
     "RegexCompiler",

{llguidance-0.7.12 → llguidance-0.7.13}/python/llguidance/_lib.pyi

@@ -121,9 +121,11 @@ class LLInterpreter:
         cls,
         tokenizer: LLTokenizer,
         grammar: str,
+        /,
         enable_backtrack: bool = True,
         enable_ff_tokens: bool = True,
         log_level: int = 1,
+        limits: Optional[LLParserLimits] = None,
     ) -> "LLInterpreter":
         """
         Create a new interpreter.
@@ -218,10 +220,14 @@ class LLInterpreter:
 
 class LLMatcher:
 
-    def __new__(cls,
-                tokenizer: LLTokenizer,
-                grammar: str,
-                log_level: int = 1) -> "LLMatcher":
+    def __new__(
+        cls,
+        tokenizer: LLTokenizer,
+        grammar: str,
+        /,
+        log_level: int = 1,
+        limits: Optional[LLParserLimits] = None,
+    ) -> "LLMatcher":
         """
         Create a new LLMatcher.
         Args:
@@ -246,7 +252,8 @@ class LLMatcher:
         """
 
     @staticmethod
-    def validate_grammar(grammar: str, tokenizer: Optional[LLTokenizer] = None) -> str:
+    def validate_grammar(grammar: str,
+                         tokenizer: Optional[LLTokenizer] = None) -> str:
         """
         Validate the grammar, for example one returned by LLMatcher.grammar_from_*().
         Returns empty string if the grammar is valid, otherwise an error message.
@@ -504,3 +511,77 @@ class JsonCompileOptions(TypedDict, total=False):
     whitespace_flexible: Optional[bool]
     # defaults to false
     coerce_one_of: Optional[bool]
+
+
+class LLParserLimits:
+
+    def __init__(
+        self,
+        max_items_in_row: Optional[int] = None,
+        initial_lexer_fuel: Optional[int] = None,
+        step_lexer_fuel: Optional[int] = None,
+        step_max_items: Optional[int] = None,
+        max_lexer_states: Optional[int] = None,
+        max_grammar_size: Optional[int] = None,
+        precompute_large_lexemes: Optional[bool] = None,
+    ) -> None:
+        """
+    ParserLimits configuration for controlling parser and lexer resource usage.
+
+    Args:
+        max_items_in_row (Optional[int]):
+            Maximum branching factor for a single production row in the grammar.
+            Affects ambiguity and parsing explosion risk. Default: 2000.
+
+        initial_lexer_fuel (Optional[int]):
+            Fuel for building the initial regex ASTs in the lexer.
+            Limits complexity of regex analysis. Speed: ~50k/ms. Default: 1_000_000.
+
+        step_lexer_fuel (Optional[int]):
+            Maximum fuel used during a single lexer mask computation step.
+            Controls performance per token analysis phase. Speed: ~14k/ms. Default: 200_000.
+
+        step_max_items (Optional[int]):
+            Cap on the number of Earley items generated per mask step.
+            Controls parsing granularity and performance. Speed: ~20k/ms. Default: 50_000.
+
+        max_lexer_states (Optional[int]):
+            Maximum number of distinct states the lexer can construct.
+            Affects memory use (approx. 1–2kB per state). Default: 250_000.
+
+        max_grammar_size (Optional[int]):
+            Maximum number of symbols in grammar productions.
+            Acts as a limit on total grammar complexity and size. Default: 500_000.
+
+        precompute_large_lexemes (Optional[bool]):
+            Whether to run large regexes eagerly on the entire token trie during lexer build.
+            Increases lexer construction time, but speeds up mask computation. Default: True.
+    """
+
+    @property
+    def max_items_in_row(self) -> int:
+        """Maximum branching factor for a grammar row. Default: 2000"""
+
+    @property
+    def initial_lexer_fuel(self) -> int:
+        """Fuel used to build initial lexer regex ASTs. Default: 1_000_000"""
+
+    @property
+    def step_lexer_fuel(self) -> int:
+        """Lexer fuel for mask computation steps. Default: 200_000"""
+
+    @property
+    def step_max_items(self) -> int:
+        """Maximum Earley items per step. Default: 50_000"""
+
+    @property
+    def max_lexer_states(self) -> int:
+        """Maximum lexer states (affects memory). Default: 250_000"""
+
+    @property
+    def max_grammar_size(self) -> int:
+        """Maximum grammar size (symbols in productions). Default: 500_000"""
+
+    @property
+    def precompute_large_lexemes(self) -> bool:
+        """Precompute large regexes during lexer construction. Default: True"""

{llguidance-0.7.12 → llguidance-0.7.13}/python/torch_tests/test_matcher.py

@@ -1,7 +1,7 @@
 from typing import Any, Dict, List, Tuple, Union
 import llguidance
 from llguidance.numpy import fill_next_token_bitmask_par, allocate_token_bitmask
-from llguidance import LLMatcher, LLTokenizer, StructTag
+from llguidance import LLMatcher, LLTokenizer, StructTag, LLParserLimits
 import pytest
 from numpy.typing import NDArray
 import numpy as np
@@ -42,7 +42,7 @@ def check_one_grammar(grm: str, s: str, passing: bool) -> None:
         del tokens[-1]
     else:
         tokens = tokenizer().tokenize_str(s)
-    
+
     print("Check: " + tokenizer().dbg_tokens(tokens))
 
     for i, t in enumerate(tokens):
@@ -74,8 +74,7 @@ def check_one_grammar(grm: str, s: str, passing: bool) -> None:
     assert interp.is_accepting()
 
 
-def check_grammar(grm: str, passing: List[str],
-                  failing: List[str]) -> None:
+def check_grammar(grm: str, passing: List[str], failing: List[str]) -> None:
     for s in passing:
         check_one_grammar(grm, s, True)
     for s in failing:
@@ -391,6 +390,31 @@ def test_struct_tag_0() -> None:
                     # this should pass since it doesn't actually use special tokens:
                     '<|tool|>qux(1)',
                 ],
-                [
-                    "⁂<|tool|>qux(11)"
-                ])
+                ["⁂<|tool|>qux(11)"])
+
+
+def test_parser_limits() -> None:
+    # this one needs more than initial_lexer_fuel=20
+    m = LLMatcher(tokenizer(),
+                  "start: /(foo[12]23|bar)/",
+                  limits=LLParserLimits(initial_lexer_fuel=5))
+    assert "initial lexer configuration (grammar) too big" in m.get_error()
+
+    # m = LLMatcher(tokenizer(),
+    #               "start: /(foo[12]23|bar|qux|mux)/",
+    #               limits=LLParserLimits(
+    #                   step_lexer_fuel=1,
+    #                   precompute_large_lexemes=False,
+    #               ),
+    #               log_level=2)
+    # toks = tokenizer().tokenize_str("foo223")
+    # assert not m.get_error()
+    # m.consume_token(toks[0])
+    # mask = m.compute_logit_bias()
+    # assert not m.get_error()
+    # m.consume_token(toks[1])
+    # mask = m.compute_logit_bias()
+    # assert not m.get_error()
+    # m.consume_token(toks[2])
+    # mask = m.compute_logit_bias()
+    # assert not m.get_error()

{llguidance-0.7.12 → llguidance-0.7.13}/python_ext/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "llguidance_py"
-version = "0.7.12"
+version = "0.7.13"
 edition = "2021"
 license = "MIT"
 description = "Super-fast Structured Outputs"

{llguidance-0.7.12 → llguidance-0.7.13}/python_ext/src/lib.rs

@@ -2,6 +2,7 @@ use pyo3::prelude::*;
 
 mod llinterpreter;
 mod llmatcher;
+mod parserlimits;
 mod py;
 mod pyjson;
 
@@ -9,6 +10,7 @@ mod pyjson;
 #[pymodule]
 fn _lib(_py: Python<'_>, m: &Bound<PyModule>) -> PyResult<()> {
     py::init(m)?;
+    parserlimits::init(m)?;
     llinterpreter::init(m)?;
     llmatcher::init(m)?;
     Ok(())