llguidance 1.0.1__tar.gz → 1.1.1__tar.gz

{llguidance-1.0.1 → llguidance-1.1.1}/CHANGELOG.md

@@ -4,6 +4,18 @@ All notable changes to this project will be documented in this file. Dates are d
 
 If a release doesn't introduce any interesting changes (build fixes etc.), it's skipped.
 
+#### [1.1.1](https://github.com/guidance-ai/llguidance/compare/v1.1.0...1.1.1) 2025-07-23
+
+- prevent error state when calling `try_consume_tokens` after parser is stopped [`#213`](https://github.com/guidance-ai/llguidance/pull/213); fixes [`#211`](https://github.com/guidance-ai/llguidance/issues/211)
+- set parser stop condition in `try_consume_tokens` even when some tokens are rejected [`#212`](https://github.com/guidance-ai/llguidance/pull/212)
+
+#### [1.1.0](https://github.com/guidance-ai/llguidance/compare/v1.0.1...1.1.0) 2025-07-18
+
+- disable hf tokenizer truncation and padding [`#205`](https://github.com/guidance-ai/llguidance/pull/205); fixes [`#1322`](https://github.com/guidance-ai/guidance/issues/1322)
+- llama_cpp tokenizers: infer added tokens starting/ending with < and > to be special tokens [`#202`](https://github.com/guidance-ai/llguidance/pull/202)
+- add lark syntax for "any token" and negation of token ranges [`#201`](https://github.com/guidance-ai/llguidance/pull/201)
+- add de-recursion cook book to docs [`#199`](https://github.com/guidance-ai/llguidance/pull/199)
+
 #### [1.0.1](https://github.com/guidance-ai/llguidance/compare/v1.0.0...1.0.1) 2025-07-03
 
 - fix: tokenizers normalizers sequence api changed [`#195`](https://github.com/guidance-ai/llguidance/pull/195)

{llguidance-1.0.1 → llguidance-1.1.1}/Cargo.lock

@@ -711,6 +711,12 @@ version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988"
 
+[[package]]
+name = "futures-timer"
+version = "3.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24"
+
 [[package]]
 name = "futures-util"
 version = "0.3.31"
@@ -760,6 +766,12 @@ version = "0.31.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f"
 
+[[package]]
+name = "glob"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2"
+
 [[package]]
 name = "h2"
 version = "0.4.11"
@@ -1229,7 +1241,7 @@ checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956"
 
 [[package]]
 name = "llguidance"
-version = "1.0.1"
+version = "1.1.1"
 dependencies = [
  "anyhow",
  "derivre",
@@ -1248,7 +1260,7 @@ dependencies = [
 
 [[package]]
 name = "llguidance_py"
-version = "1.0.1"
+version = "1.1.1"
 dependencies = [
  "anyhow",
  "bytemuck",
@@ -1637,6 +1649,15 @@ dependencies = [
  "zerocopy",
 ]
 
+[[package]]
+name = "proc-macro-crate"
+version = "3.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35"
+dependencies = [
+ "toml_edit",
+]
+
 [[package]]
 name = "proc-macro2"
 version = "1.0.95"
@@ -1868,6 +1889,12 @@ version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
 
+[[package]]
+name = "relative-path"
+version = "1.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2"
+
 [[package]]
 name = "reqwest"
 version = "0.12.22"
@@ -1926,6 +1953,36 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "rstest"
+version = "0.25.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6fc39292f8613e913f7df8fa892b8944ceb47c247b78e1b1ae2f09e019be789d"
+dependencies = [
+ "futures-timer",
+ "futures-util",
+ "rstest_macros",
+ "rustc_version",
+]
+
+[[package]]
+name = "rstest_macros"
+version = "0.25.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1f168d99749d307be9de54d23fd226628d99768225ef08f6ffb52e0182a27746"
+dependencies = [
+ "cfg-if",
+ "glob",
+ "proc-macro-crate",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "relative-path",
+ "rustc_version",
+ "syn",
+ "unicode-ident",
+]
+
 [[package]]
 name = "rustc-demangle"
 version = "0.1.25"
@@ -1938,6 +1995,15 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
 
+[[package]]
+name = "rustc_version"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
+dependencies = [
+ "semver",
+]
+
 [[package]]
 name = "rustix"
 version = "1.0.7"
@@ -2006,6 +2072,7 @@ dependencies = [
  "clap",
  "lazy_static",
  "llguidance",
+ "rstest",
  "serde",
  "serde_json",
  "toktrie_hf_downloader",
@@ -2050,6 +2117,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "semver"
+version = "1.0.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0"
+
 [[package]]
 name = "serde"
 version = "1.0.219"
@@ -2405,7 +2478,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie"
-version = "1.0.1"
+version = "1.1.1"
 dependencies = [
  "anyhow",
  "bytemuck",
@@ -2416,7 +2489,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_hf_downloader"
-version = "1.0.1"
+version = "1.1.1"
 dependencies = [
  "anyhow",
  "hf-hub",
@@ -2427,7 +2500,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_hf_tokenizers"
-version = "1.0.1"
+version = "1.1.1"
 dependencies = [
  "anyhow",
  "log",
@@ -2439,7 +2512,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_tiktoken"
-version = "1.0.1"
+version = "1.1.1"
 dependencies = [
  "anyhow",
  "log",
@@ -2449,6 +2522,23 @@ dependencies = [
  "toktrie",
 ]
 
+[[package]]
+name = "toml_datetime"
+version = "0.6.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c"
+
+[[package]]
+name = "toml_edit"
+version = "0.22.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
+dependencies = [
+ "indexmap",
+ "toml_datetime",
+ "winnow",
+]
+
 [[package]]
 name = "tower"
 version = "0.5.2"
@@ -3004,6 +3094,15 @@ version = "0.53.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486"
 
+[[package]]
+name = "winnow"
+version = "0.7.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3edebf492c8125044983378ecb5766203ad3b4c2f7a922bd7dd207f6d443e95"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "wit-bindgen-rt"
 version = "0.39.0"

{llguidance-1.0.1 → llguidance-1.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llguidance
-Version: 1.0.1
+Version: 1.1.1
 License-File: LICENSE
 Summary: Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance
 Author: Michal Moskal

llguidance-1.1.1/docs/de_recursing.md

@@ -0,0 +1,51 @@
+# De-Recursing Grammars
+
+This is a cookbook of examples to help in removing recursion where possible from grammars (see [Syntax](./syntax.md) for more details).
+The examples below will generally already be left-recursive.
+
+## Simple lists
+
+```lark
+item_list : item
+    | item_list item
+```
+can become
+```lark
+item_list : item+
+```
+
+## Lists with Delimiters
+
+```lark
+sep_list : item
+    | item_list SEP item
+```
+becomes
+```lark
+sep_list : item (SEP item)*
+```
+
+## List with alternatives
+
+```lark
+postfix_expression: primary_expression
+    | postfix_expression "[" expression "]"
+    | postfix_expression "(" ")"
+    | postfix_expression "(" argument_expression_list ")"
+    | postfix_expression "." IDENTIFIER
+    | postfix_expression PTR_OP IDENTIFIER
+    | "(" type_name ")" "{" initializer_list "}"
+    | "(" type_name ")" "{" initializer_list "," "}"
+```
+becomes (note the additional rule):
+```lark
+postfix_expression: primary_expression postfix_suffix*
+    | "(" type_name ")" "{" initializer_list "}"
+    | "(" type_name ")" "{" initializer_list "," "}"
+
+postfix_suffix: "[" expression "]"
+    | "(" ")"
+    | "(" argument_expression_list ")"
+    | "." IDENTIFIER
+    | PTR_OP IDENTIFIER
+```

{llguidance-1.0.1 → llguidance-1.1.1}/docs/syntax.md

@@ -70,6 +70,11 @@ You can also use numeric token ids, as in `<[128010]>` (this is `<|python_tag|>`
 Tou can also use ranges like `<[128000-128255]>` for all Llama special tokens, or
 even lists of ranges like `<[128000-128100,128130-128170]>`; ranges are inclusive.
 
+Individual numeric token ids and ranges can be negated with the caret operator, like `<[^128000,128130-128170]>`.
+This is equivalent to `<[0-12799,128001-128129,128171-MAX]>`.
+
+You can also use a *wildcard* token range, `<[*]>`, denoting `<[0-MAX]>`.
+
 For example, this is how to constrain JSON function calling for Meta Llama 3.1,
 according to their [source repo](https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/prompt_format.md#model-response-format-5) (and yes, it's [different](https://github.com/meta-llama/llama-models/issues/266) than the website).
 

{llguidance-1.0.1 → llguidance-1.1.1}/parser/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "llguidance"
-version = "1.0.1"
+version = "1.1.1"
 edition = "2021"
 license = "MIT"
 description = "Super-fast Structured Outputs"

{llguidance-1.0.1 → llguidance-1.1.1}/parser/src/grammar_builder.rs

@@ -282,8 +282,6 @@ impl GrammarBuilder {
     pub fn token_ranges(&mut self, token_ranges: Vec<RangeInclusive<u32>>) -> Result<NodeRef> {
         self.check_limits()?;
 
-        let name = token_ranges_to_string(&token_ranges);
-
         let trie = self.tok_env.as_ref().map(|t| t.tok_trie());
         for r in &token_ranges {
             ensure!(r.start() <= r.end(), "Invalid token range: {:?}", r);
@@ -300,10 +298,69 @@ impl GrammarBuilder {
             self.add_warning("no tokenizer - can't validate <[...]>".to_string());
         }
 
+        let name = token_ranges_to_string(&token_ranges);
         let id = self.regex.spec.add_special_token(name, token_ranges)?;
         Ok(self.lexeme_to_node(id))
     }
 
+    pub fn negated_token_ranges(
+        &mut self,
+        token_ranges: Vec<RangeInclusive<u32>>,
+    ) -> Result<NodeRef> {
+        let negated_ranges = if let Some(te) = &self.tok_env {
+            let trie = te.tok_trie();
+
+            let (min, max) = (0u32, trie.vocab_size() as u32 - 1);
+            ensure!(
+                !token_ranges.is_empty(),
+                "negation of empty token ranges is not supported"
+            );
+
+            let mut sorted = token_ranges.clone();
+            sorted.sort_by_key(|r| *r.start());
+
+            let mut negated = vec![];
+            let mut current = min;
+            for range in sorted {
+                ensure!(
+                    *range.end() < trie.vocab_size() as u32,
+                    "Token range end too large: {:?}",
+                    range.end()
+                );
+                ensure!(
+                    range.start() <= range.end(),
+                    "Invalid token range: {:?}",
+                    range
+                );
+
+                let (&start, &end) = (range.start(), range.end());
+                ensure!(start <= end, "Invalid token range: {:?}", range);
+                if end < current {
+                    // skip this range, it is already covered by the previous one
+                    continue;
+                }
+                if start > current {
+                    // add a range from the current to the start of this one
+                    negated.push(current..=start - 1);
+                }
+                // update the current to the end of this range
+                current = current.max(end + 1);
+            }
+            if current <= max {
+                // add the last range from the current to the max
+                negated.push(current..=max);
+            }
+            negated
+        } else {
+            self.add_warning("no tokenizer - can't validate <[^...]>".to_string());
+            vec![INVALID_TOKEN..=INVALID_TOKEN]
+        };
+
+        let name = token_ranges_to_string(&negated_ranges);
+        let id = self.regex.spec.add_special_token(name, negated_ranges)?;
+        Ok(self.lexeme_to_node(id))
+    }
+
     pub fn special_token(&mut self, token: &str) -> Result<NodeRef> {
         self.check_limits()?;
 
@@ -331,6 +388,22 @@ impl GrammarBuilder {
         Ok(self.lexeme_to_node(idx))
     }
 
+    pub fn any_token(&mut self) -> Result<NodeRef> {
+        self.check_limits()?;
+        let range = if let Some(te) = &self.tok_env {
+            let trie = te.tok_trie();
+            0..=trie.vocab_size() as u32 - 1
+        } else {
+            self.add_warning("no tokenizer - can't validate <any_token>".to_string());
+            INVALID_TOKEN..=INVALID_TOKEN
+        };
+        let idx = self
+            .regex
+            .spec
+            .add_special_token("<[*]>".to_string(), vec![range])?;
+        Ok(self.lexeme_to_node(idx))
+    }
+
     pub fn gen_grammar(&mut self, data: GenGrammarOptions, props: NodeProps) -> NodeRef {
         if props.max_tokens.is_some() {
             self.regex.spec.has_max_tokens = true;

{llguidance-1.0.1 → llguidance-1.1.1}/parser/src/lark/compiler.rs

@@ -313,6 +313,18 @@ impl Compiler {
                     Value::SpecialToken(s) => {
                         if s.starts_with("<[") && s.ends_with("]>") {
                             let s = &s[2..s.len() - 2];
+                            let negate = s.starts_with("^");
+                            let s = if negate { &s[1..] } else { s };
+                            if s == "*" {
+                                if negate {
+                                    bail!("negated wildcard token <[^*]> is not supported");
+                                }
+                                return self.builder.any_token();
+                            } else if s.contains('*') {
+                                bail!(
+                                    "wildcard token range '*' must not contain additional tokens"
+                                );
+                            }
                             let mut ranges = vec![];
                             for range in s.split(",") {
                                 let ends: Vec<&str> = range.split('-').map(|s| s.trim()).collect();
@@ -334,7 +346,11 @@ impl Compiler {
                                 ranges.push(start..=end);
                             }
                             ensure!(!ranges.is_empty(), "empty token range");
-                            return self.builder.token_ranges(ranges);
+                            return if negate {
+                                self.builder.negated_token_ranges(ranges)
+                            } else {
+                                self.builder.token_ranges(ranges)
+                            };
                         }
                         return self.builder.special_token(s);
                     }

{llguidance-1.0.1 → llguidance-1.1.1}/parser/src/matcher.rs

@@ -166,6 +166,7 @@ impl Matcher {
         self.with_inner(|inner| {
             for (idx, &t) in tokens.iter().enumerate() {
                 if !inner.parser.validate_token(t)? {
+                    let _ = inner.parser.check_stop()?;
                     return Ok(idx);
                 }
                 let bt = inner.parser.consume_token(t)?;

{llguidance-1.0.1 → llguidance-1.1.1}/parser/src/tokenparser.rs

@@ -139,6 +139,10 @@ impl TokenParser {
         self.stop_reason
     }
 
+    pub fn stopped(&self) -> bool {
+        self.stop_reason != StopReason::NotStopped
+    }
+
     pub fn is_fresh(&self) -> bool {
         self.is_fresh
     }
@@ -334,7 +338,7 @@ impl TokenParser {
     fn check_initialized(&self, lbl: &str) -> Result<()> {
         ensure!(!self.is_fresh, "process_prompt() not called in {}", lbl);
         ensure!(
-            self.stop_reason == StopReason::NotStopped,
+            !self.stopped(),
             "parser stopped in {}; {}",
             lbl,
             self.error_message()
@@ -344,6 +348,9 @@ impl TokenParser {
     }
 
     pub fn validate_token(&mut self, token: TokenId) -> Result<bool> {
+        if self.stopped() {
+            return Ok(false);
+        }
         self.check_initialized("validate_token")?;
         self.validate_tokens_raw(&[token]).map(|n| n > 0)
     }
@@ -407,6 +414,9 @@ impl TokenParser {
     /// It does not tokenize forced bytes, so will accept non-canonical tokenizations.
     /// If called with more than one token, it may ignore max_tokens constraints.
     pub fn validate_tokens_raw(&mut self, tokens: &[TokenId]) -> Result<usize> {
+        if self.stopped() {
+            return Ok(0);
+        }
         self.check_initialized("validate_tokens_raw")?;
 
         if tokens.is_empty() {

{llguidance-1.0.1 → llguidance-1.1.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "llguidance"
-version = "1.0.1"
+version = "1.1.1"
 description = "Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance"
 requires-python = ">=3.9"
 license = "MIT"

{llguidance-1.0.1 → llguidance-1.1.1}/python/llguidance/hf.py

@@ -1,8 +1,10 @@
+from copy import copy
 from typing import List, Optional
-from ._lib import LLTokenizer
 
 import transformers
 
+from ._lib import LLTokenizer
+
 
 def from_tokenizer(
     hf_tokenizer: transformers.PreTrainedTokenizerFast,
@@ -28,15 +30,18 @@ def from_tokenizer(
         # this will JSON-serialize the Rust impl of the tokenizer,
         # including added tokens from tokenizer_config.json
         # (which may be missing from tokenizer.json)
-        s = hf_tokenizer.backend_tokenizer.to_str() # type: ignore
+        backend_tokenizer = copy(
+            hf_tokenizer.backend_tokenizer  # type: ignore[attr-defined]
+        )
+        # disable padding and truncation on copy before converting to string
+        backend_tokenizer.no_padding()
+        backend_tokenizer.no_truncation()
+        s = backend_tokenizer.to_str()
         # This is probably not needed - it should figure it out by itself
         # if n_vocab is None:
         #     n_vocab = hf_tokenizer.backend_tokenizer.get_vocab_size(with_added_tokens=True)
         if eos_token is None:
-            eos_token = hf_tokenizer.eos_token_id # type: ignore
-        return LLTokenizer(s,
-                           n_vocab=n_vocab,
-                           eos_token=eos_token,
-                           slices=slices)
+            eos_token = hf_tokenizer.eos_token_id  # type: ignore
+        return LLTokenizer(s, n_vocab=n_vocab, eos_token=eos_token, slices=slices)
     else:
         raise ValueError("Only fast tokenizers are supported")

{llguidance-1.0.1 → llguidance-1.1.1}/python/llguidance/llamacpp.py

@@ -44,8 +44,14 @@ def lltokenizer_from_vocab(
         assert n <= buffer_len
         tok = bytes(buffer[:n]) # type: ignore
         attr = llama_cpp.llama_token_get_attr(vocab, token)
-        if attr & llama_cpp.LLAMA_TOKEN_ATTR_CONTROL:
-            tok = b"\xFF" + tok
+        # If the token is a control token or a user-defined token that looks like a control token,
+        # we prefix it with 0xff to indicate that it should be treated as a special token.
+        if attr & llama_cpp.LLAMA_TOKEN_ATTR_CONTROL or (
+            attr & llama_cpp.LLAMA_TOKEN_ATTR_USER_DEFINED
+            and tok.startswith(b"<")
+            and tok.endswith(b">")
+        ):
+            tok = b"\xff" + tok
         tokens.append(tok)
 
     if n_vocab is not None:

{llguidance-1.0.1 → llguidance-1.1.1}/python_ext/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "llguidance_py"
-version = "1.0.1"
+version = "1.1.1"
 edition = "2021"
 license = "MIT"
 description = "Super-fast Structured Outputs"

{llguidance-1.0.1 → llguidance-1.1.1}/sample_parser/Cargo.toml

@@ -14,6 +14,9 @@ serde_json = { version = "1.0.138", features = ["preserve_order"] }
 serde = { version = "1.0.217", features = ["derive"] }
 clap = { version = "4.5.31", features = ["derive"] }
 
+[dev-dependencies]
+rstest = "0.25.0"
+
 [[bin]]
 name = "sample_parser"
 path = "src/sample_parser.rs"