llguidance 1.0.1__tar.gz → 1.1.0__tar.gz

{llguidance-1.0.1 → llguidance-1.1.0}/CHANGELOG.md

@@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file. Dates are d
 
 If a release doesn't introduce any interesting changes (build fixes etc.), it's skipped.
 
+#### [1.1.0](https://github.com/guidance-ai/llguidance/compare/v1.0.1...1.1.0) 2025-07-18
+
+- disable hf tokenizer truncation and padding [`#205`](https://github.com//hudson-ai/llguidance.git/pull/205); fixes [`#1322`](https://github.com/guidance-ai/guidance/issues/1322)
+- llama_cpp tokenizers: infer added tokens starting/ending with < and > to be special tokens [`#202`](https://github.com//hudson-ai/llguidance.git/pull/202)
+- add lark syntax for "any token" and negation of token ranges [`#201`](https://github.com//hudson-ai/llguidance.git/pull/201)
+- add de-recursion cook book to docs [`#199`](https://github.com//hudson-ai/llguidance.git/pull/199)
+
 #### [1.0.1](https://github.com/guidance-ai/llguidance/compare/v1.0.0...1.0.1) 2025-07-03
 
 - fix: tokenizers normalizers sequence api changed [`#195`](https://github.com/guidance-ai/llguidance/pull/195)

{llguidance-1.0.1 → llguidance-1.1.0}/Cargo.lock

@@ -1229,7 +1229,7 @@ checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956"
 
 [[package]]
 name = "llguidance"
-version = "1.0.1"
+version = "1.1.0"
 dependencies = [
  "anyhow",
  "derivre",
@@ -1248,7 +1248,7 @@ dependencies = [
 
 [[package]]
 name = "llguidance_py"
-version = "1.0.1"
+version = "1.1.0"
 dependencies = [
  "anyhow",
  "bytemuck",
@@ -2405,7 +2405,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie"
-version = "1.0.1"
+version = "1.1.0"
 dependencies = [
  "anyhow",
  "bytemuck",
@@ -2416,7 +2416,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_hf_downloader"
-version = "1.0.1"
+version = "1.1.0"
 dependencies = [
  "anyhow",
  "hf-hub",
@@ -2427,7 +2427,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_hf_tokenizers"
-version = "1.0.1"
+version = "1.1.0"
 dependencies = [
  "anyhow",
  "log",
@@ -2439,7 +2439,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_tiktoken"
-version = "1.0.1"
+version = "1.1.0"
 dependencies = [
  "anyhow",
  "log",

{llguidance-1.0.1 → llguidance-1.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llguidance
-Version: 1.0.1
+Version: 1.1.0
 License-File: LICENSE
 Summary: Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance
 Author: Michal Moskal

llguidance-1.1.0/docs/de_recursing.md

@@ -0,0 +1,51 @@
+# De-Recursing Grammars
+
+This is a cookbook of examples to help in removing recursion where possible from grammars (see [Syntax](./syntax.md) for more details).
+The examples below will generally already be left-recursive.
+
+## Simple lists
+
+```lark
+item_list : item
+    | item_list item
+```
+can become
+```lark
+item_list : item+
+```
+
+## Lists with Delimiters
+
+```lark
+sep_list : item
+    | item_list SEP item
+```
+becomes
+```lark
+sep_list : item (SEP item)*
+```
+
+## List with alternatives
+
+```lark
+postfix_expression: primary_expression
+    | postfix_expression "[" expression "]"
+    | postfix_expression "(" ")"
+    | postfix_expression "(" argument_expression_list ")"
+    | postfix_expression "." IDENTIFIER
+    | postfix_expression PTR_OP IDENTIFIER
+    | "(" type_name ")" "{" initializer_list "}"
+    | "(" type_name ")" "{" initializer_list "," "}"
+```
+becomes (note the additional rule):
+```lark
+postfix_expression: primary_expression postfix_suffix*
+    | "(" type_name ")" "{" initializer_list "}"
+    | "(" type_name ")" "{" initializer_list "," "}"
+
+postfix_suffix: "[" expression "]"
+    | "(" ")"
+    | "(" argument_expression_list ")"
+    | "." IDENTIFIER
+    | PTR_OP IDENTIFIER
+```

{llguidance-1.0.1 → llguidance-1.1.0}/docs/syntax.md

@@ -70,6 +70,11 @@ You can also use numeric token ids, as in `<[128010]>` (this is `<|python_tag|>`
 Tou can also use ranges like `<[128000-128255]>` for all Llama special tokens, or
 even lists of ranges like `<[128000-128100,128130-128170]>`; ranges are inclusive.
 
+Individual numeric token ids and ranges can be negated with the caret operator, like `<[^128000,128130-128170]>`.
+This is equivalent to `<[0-12799,128001-128129,128171-MAX]>`.
+
+You can also use a *wildcard* token range, `<[*]>`, denoting `<[0-MAX]>`.
+
 For example, this is how to constrain JSON function calling for Meta Llama 3.1,
 according to their [source repo](https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/prompt_format.md#model-response-format-5) (and yes, it's [different](https://github.com/meta-llama/llama-models/issues/266) than the website).
 

{llguidance-1.0.1 → llguidance-1.1.0}/parser/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "llguidance"
-version = "1.0.1"
+version = "1.1.0"
 edition = "2021"
 license = "MIT"
 description = "Super-fast Structured Outputs"

{llguidance-1.0.1 → llguidance-1.1.0}/parser/src/grammar_builder.rs

@@ -282,8 +282,6 @@ impl GrammarBuilder {
     pub fn token_ranges(&mut self, token_ranges: Vec<RangeInclusive<u32>>) -> Result<NodeRef> {
         self.check_limits()?;
 
-        let name = token_ranges_to_string(&token_ranges);
-
         let trie = self.tok_env.as_ref().map(|t| t.tok_trie());
         for r in &token_ranges {
             ensure!(r.start() <= r.end(), "Invalid token range: {:?}", r);
@@ -300,10 +298,69 @@ impl GrammarBuilder {
             self.add_warning("no tokenizer - can't validate <[...]>".to_string());
         }
 
+        let name = token_ranges_to_string(&token_ranges);
         let id = self.regex.spec.add_special_token(name, token_ranges)?;
         Ok(self.lexeme_to_node(id))
     }
 
+    pub fn negated_token_ranges(
+        &mut self,
+        token_ranges: Vec<RangeInclusive<u32>>,
+    ) -> Result<NodeRef> {
+        let negated_ranges = if let Some(te) = &self.tok_env {
+            let trie = te.tok_trie();
+
+            let (min, max) = (0u32, trie.vocab_size() as u32 - 1);
+            ensure!(
+                !token_ranges.is_empty(),
+                "negation of empty token ranges is not supported"
+            );
+
+            let mut sorted = token_ranges.clone();
+            sorted.sort_by_key(|r| *r.start());
+
+            let mut negated = vec![];
+            let mut current = min;
+            for range in sorted {
+                ensure!(
+                    *range.end() < trie.vocab_size() as u32,
+                    "Token range end too large: {:?}",
+                    range.end()
+                );
+                ensure!(
+                    range.start() <= range.end(),
+                    "Invalid token range: {:?}",
+                    range
+                );
+
+                let (&start, &end) = (range.start(), range.end());
+                ensure!(start <= end, "Invalid token range: {:?}", range);
+                if end < current {
+                    // skip this range, it is already covered by the previous one
+                    continue;
+                }
+                if start > current {
+                    // add a range from the current to the start of this one
+                    negated.push(current..=start - 1);
+                }
+                // update the current to the end of this range
+                current = current.max(end + 1);
+            }
+            if current <= max {
+                // add the last range from the current to the max
+                negated.push(current..=max);
+            }
+            negated
+        } else {
+            self.add_warning("no tokenizer - can't validate <[^...]>".to_string());
+            vec![INVALID_TOKEN..=INVALID_TOKEN]
+        };
+
+        let name = token_ranges_to_string(&negated_ranges);
+        let id = self.regex.spec.add_special_token(name, negated_ranges)?;
+        Ok(self.lexeme_to_node(id))
+    }
+
     pub fn special_token(&mut self, token: &str) -> Result<NodeRef> {
         self.check_limits()?;
 
@@ -331,6 +388,22 @@ impl GrammarBuilder {
         Ok(self.lexeme_to_node(idx))
     }
 
+    pub fn any_token(&mut self) -> Result<NodeRef> {
+        self.check_limits()?;
+        let range = if let Some(te) = &self.tok_env {
+            let trie = te.tok_trie();
+            0..=trie.vocab_size() as u32 - 1
+        } else {
+            self.add_warning("no tokenizer - can't validate <any_token>".to_string());
+            INVALID_TOKEN..=INVALID_TOKEN
+        };
+        let idx = self
+            .regex
+            .spec
+            .add_special_token("<[*]>".to_string(), vec![range])?;
+        Ok(self.lexeme_to_node(idx))
+    }
+
     pub fn gen_grammar(&mut self, data: GenGrammarOptions, props: NodeProps) -> NodeRef {
         if props.max_tokens.is_some() {
             self.regex.spec.has_max_tokens = true;

{llguidance-1.0.1 → llguidance-1.1.0}/parser/src/lark/compiler.rs

@@ -313,6 +313,18 @@ impl Compiler {
                     Value::SpecialToken(s) => {
                         if s.starts_with("<[") && s.ends_with("]>") {
                             let s = &s[2..s.len() - 2];
+                            let negate = s.starts_with("^");
+                            let s = if negate { &s[1..] } else { s };
+                            if s == "*" {
+                                if negate {
+                                    bail!("negated wildcard token <[^*]> is not supported");
+                                }
+                                return self.builder.any_token();
+                            } else if s.contains('*') {
+                                bail!(
+                                    "wildcard token range '*' must not contain additional tokens"
+                                );
+                            }
                             let mut ranges = vec![];
                             for range in s.split(",") {
                                 let ends: Vec<&str> = range.split('-').map(|s| s.trim()).collect();
@@ -334,7 +346,11 @@ impl Compiler {
                                 ranges.push(start..=end);
                             }
                             ensure!(!ranges.is_empty(), "empty token range");
-                            return self.builder.token_ranges(ranges);
+                            return if negate {
+                                self.builder.negated_token_ranges(ranges)
+                            } else {
+                                self.builder.token_ranges(ranges)
+                            };
                         }
                         return self.builder.special_token(s);
                     }

{llguidance-1.0.1 → llguidance-1.1.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "llguidance"
-version = "1.0.1"
+version = "1.1.0"
 description = "Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance"
 requires-python = ">=3.9"
 license = "MIT"

{llguidance-1.0.1 → llguidance-1.1.0}/python/llguidance/hf.py

@@ -1,8 +1,10 @@
+from copy import copy
 from typing import List, Optional
-from ._lib import LLTokenizer
 
 import transformers
 
+from ._lib import LLTokenizer
+
 
 def from_tokenizer(
     hf_tokenizer: transformers.PreTrainedTokenizerFast,
@@ -28,15 +30,18 @@ def from_tokenizer(
         # this will JSON-serialize the Rust impl of the tokenizer,
         # including added tokens from tokenizer_config.json
         # (which may be missing from tokenizer.json)
-        s = hf_tokenizer.backend_tokenizer.to_str() # type: ignore
+        backend_tokenizer = copy(
+            hf_tokenizer.backend_tokenizer  # type: ignore[attr-defined]
+        )
+        # disable padding and truncation on copy before converting to string
+        backend_tokenizer.no_padding()
+        backend_tokenizer.no_truncation()
+        s = backend_tokenizer.to_str()
         # This is probably not needed - it should figure it out by itself
         # if n_vocab is None:
         #     n_vocab = hf_tokenizer.backend_tokenizer.get_vocab_size(with_added_tokens=True)
         if eos_token is None:
-            eos_token = hf_tokenizer.eos_token_id # type: ignore
-        return LLTokenizer(s,
-                           n_vocab=n_vocab,
-                           eos_token=eos_token,
-                           slices=slices)
+            eos_token = hf_tokenizer.eos_token_id  # type: ignore
+        return LLTokenizer(s, n_vocab=n_vocab, eos_token=eos_token, slices=slices)
     else:
         raise ValueError("Only fast tokenizers are supported")

{llguidance-1.0.1 → llguidance-1.1.0}/python/llguidance/llamacpp.py

@@ -44,8 +44,14 @@ def lltokenizer_from_vocab(
         assert n <= buffer_len
         tok = bytes(buffer[:n]) # type: ignore
         attr = llama_cpp.llama_token_get_attr(vocab, token)
-        if attr & llama_cpp.LLAMA_TOKEN_ATTR_CONTROL:
-            tok = b"\xFF" + tok
+        # If the token is a control token or a user-defined token that looks like a control token,
+        # we prefix it with 0xff to indicate that it should be treated as a special token.
+        if attr & llama_cpp.LLAMA_TOKEN_ATTR_CONTROL or (
+            attr & llama_cpp.LLAMA_TOKEN_ATTR_USER_DEFINED
+            and tok.startswith(b"<")
+            and tok.endswith(b">")
+        ):
+            tok = b"\xff" + tok
         tokens.append(tok)
 
     if n_vocab is not None:

{llguidance-1.0.1 → llguidance-1.1.0}/python_ext/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "llguidance_py"
-version = "1.0.1"
+version = "1.1.0"
 edition = "2021"
 license = "MIT"
 description = "Super-fast Structured Outputs"

llguidance-1.1.0/sample_parser/tests/common_lark_utils/mod.rs

@@ -0,0 +1,144 @@
+/*
+Per
+https://doc.rust-lang.org/book/ch11-03-test-organization.html#submodules-in-integration-tests
+we do have an 'old style' mod.rs, so that the test runner doesn't look inside
+ */
+
+use anyhow::Result;
+use llguidance::{
+    api::{GrammarInit, TopLevelGrammar},
+    toktrie::bytes::limit_str,
+    TokenParser,
+};
+use sample_parser::*;
+use serde_json::Value;
+
+pub fn make_parser(lark: &str, quiet: bool) -> Result<TokenParser> {
+    let grm = TopLevelGrammar::from_lark(lark.to_string());
+    let mut parser = get_parser_factory().create_parser_from_init(
+        GrammarInit::Serialized(grm),
+        if quiet { 0 } else { 2 },
+        if quiet { 1 } else { 2 },
+    )?;
+    parser.start_without_prompt();
+    Ok(parser)
+}
+
+pub fn consume(parser: &mut TokenParser, tok: u32) {
+    let n = parser.consume_token(tok).unwrap();
+    assert!(n == 0);
+}
+
+pub fn lark_ok(lark: &str) {
+    if let Err(e) = make_parser(lark, false) {
+        panic!("unexpected error: {e}, grm:\n{lark}")
+    }
+}
+
+pub fn lark_err_test(lark: &str, err: &str) {
+    match make_parser(lark, false) {
+        Err(e) => {
+            let e = format!("{e}");
+            if !e.contains(err) {
+                panic!("unexpected error: {e}, expecting {err:?}; grm:\n{lark}");
+            }
+        }
+        Ok(_) => panic!("expected error: {err}; grm:\n{lark}"),
+    }
+}
+
+pub fn json_err_test(schema: &Value, err: &str) {
+    lark_err_test(
+        &format!(r#"start: %json {}"#, serde_json::to_string(schema).unwrap()),
+        err,
+    );
+}
+
+pub fn lark_str_test(lark: &str, should_accept: bool, input: &str, quiet: bool) {
+    let trie = get_tok_env().tok_trie();
+    let (final_reject, input) = if let Some(input) = input.strip_prefix("FINAL_REJECT:") {
+        (true, input)
+    } else {
+        (false, input)
+    };
+    let tokens = get_tok_env().tokenize(input);
+    let info = format!(
+        "\ninput: {:?}, grm: {:?}",
+        limit_str(input, 500),
+        limit_str(lark, 100)
+    );
+    if !quiet {
+        println!(
+            "\n\ntokens: {}, accpt={}\ngrm:\n{}\n",
+            trie.tokens_dbg(&tokens),
+            should_accept,
+            lark
+        );
+    }
+
+    // let t0 = std::time::Instant::now();
+    let mut p = make_parser(lark, quiet).unwrap();
+    // println!("make_parser: {:?}", t0.elapsed());
+
+    for tok in tokens.iter() {
+        let m = p.compute_mask().unwrap();
+        if m.is_allowed(*tok) {
+            consume(&mut p, *tok);
+        } else {
+            if should_accept {
+                panic!("unexpected token: {}{info}", trie.token_dbg(*tok));
+            }
+            if final_reject {
+                panic!(
+                    "unexpected token: {}; expecting reject only at the end{info}",
+                    trie.token_dbg(*tok)
+                );
+            }
+            return;
+        }
+    }
+
+    if !final_reject && !should_accept {
+        panic!(
+            "expected rejection (in the middle; final accept={})",
+            p.is_accepting()
+        );
+    }
+
+    if p.is_accepting() == final_reject {
+        if p.is_accepting() {
+            panic!("unexpected accept{info}");
+        } else {
+            panic!("unexpected reject{info}");
+        }
+    }
+}
+
+pub fn lark_str_test_many_ext(quiet: bool, lark: &str, passing: &[&str], failing: &[&str]) {
+    for s in passing {
+        lark_str_test(lark, true, s, quiet);
+    }
+    for s in failing {
+        lark_str_test(lark, false, s, quiet);
+    }
+}
+
+pub fn json_test_many(schema: &Value, passing: &[Value], failing: &[Value]) {
+    let lark = format!(r#"start: %json {}"#, serde_json::to_string(schema).unwrap());
+    for s in passing {
+        let s = serde_json::to_string(s).unwrap();
+        lark_str_test(&lark, true, &s, false);
+    }
+    for s in failing {
+        let s = serde_json::to_string(s).unwrap();
+        lark_str_test(&lark, false, &s, false);
+    }
+}
+
+pub fn lark_str_test_many(lark: &str, passing: &[&str], failing: &[&str]) {
+    lark_str_test_many_ext(false, lark, passing, failing);
+}
+
+pub fn lark_str_test_many_quiet(lark: &str, passing: &[&str], failing: &[&str]) {
+    lark_str_test_many_ext(true, lark, passing, failing);
+}

{llguidance-1.0.1 → llguidance-1.1.0}/sample_parser/tests/test_lark.rs

@@ -1,143 +1,8 @@
-use anyhow::Result;
-use llguidance::{
-    api::{GrammarInit, TopLevelGrammar},
-    earley::XorShift,
-    substring::chunk_into_words,
-    toktrie::bytes::limit_str,
-    TokenParser,
-};
-use sample_parser::*;
-use serde_json::{json, Value};
-
-fn make_parser(lark: &str, quiet: bool) -> Result<TokenParser> {
-    let grm = TopLevelGrammar::from_lark(lark.to_string());
-    let mut parser = get_parser_factory().create_parser_from_init(
-        GrammarInit::Serialized(grm),
-        if quiet { 0 } else { 2 },
-        if quiet { 1 } else { 2 },
-    )?;
-    parser.start_without_prompt();
-    Ok(parser)
-}
-
-fn consume(parser: &mut TokenParser, tok: u32) {
-    let n = parser.consume_token(tok).unwrap();
-    assert!(n == 0);
-}
-
-fn lark_ok(lark: &str) {
-    if let Err(e) = make_parser(lark, false) {
-        panic!("unexpected error: {e}, grm:\n{lark}")
-    }
-}
+use llguidance::{earley::XorShift, substring::chunk_into_words};
+use serde_json::json;
 
-fn lark_err_test(lark: &str, err: &str) {
-    match make_parser(lark, false) {
-        Err(e) => {
-            let e = format!("{e}");
-            if !e.contains(err) {
-                panic!("unexpected error: {e}, expecting {err:?}; grm:\n{lark}");
-            }
-        }
-        Ok(_) => panic!("expected error: {err}; grm:\n{lark}"),
-    }
-}
-
-fn json_err_test(schema: &Value, err: &str) {
-    lark_err_test(
-        &format!(r#"start: %json {}"#, serde_json::to_string(schema).unwrap()),
-        err,
-    );
-}
-
-fn lark_str_test(lark: &str, should_accept: bool, input: &str, quiet: bool) {
-    let trie = get_tok_env().tok_trie();
-    let (final_reject, input) = if let Some(input) = input.strip_prefix("FINAL_REJECT:") {
-        (true, input)
-    } else {
-        (false, input)
-    };
-    let tokens = get_tok_env().tokenize(input);
-    let info = format!(
-        "\ninput: {:?}, grm: {:?}",
-        limit_str(input, 500),
-        limit_str(lark, 100)
-    );
-    if !quiet {
-        println!(
-            "\n\ntokens: {}, accpt={}\ngrm:\n{}\n",
-            trie.tokens_dbg(&tokens),
-            should_accept,
-            lark
-        );
-    }
-
-    // let t0 = std::time::Instant::now();
-    let mut p = make_parser(lark, quiet).unwrap();
-    // println!("make_parser: {:?}", t0.elapsed());
-
-    for tok in tokens.iter() {
-        let m = p.compute_mask().unwrap();
-        if m.is_allowed(*tok) {
-            consume(&mut p, *tok);
-        } else {
-            if should_accept {
-                panic!("unexpected token: {}{info}", trie.token_dbg(*tok));
-            }
-            if final_reject {
-                panic!(
-                    "unexpected token: {}; expecting reject only at the end{info}",
-                    trie.token_dbg(*tok)
-                );
-            }
-            return;
-        }
-    }
-
-    if !final_reject && !should_accept {
-        panic!(
-            "expected rejection (in the middle; final accept={})",
-            p.is_accepting()
-        );
-    }
-
-    if p.is_accepting() == final_reject {
-        if p.is_accepting() {
-            panic!("unexpected accept{info}");
-        } else {
-            panic!("unexpected reject{info}");
-        }
-    }
-}
-
-fn lark_str_test_many_ext(quiet: bool, lark: &str, passing: &[&str], failing: &[&str]) {
-    for s in passing {
-        lark_str_test(lark, true, s, quiet);
-    }
-    for s in failing {
-        lark_str_test(lark, false, s, quiet);
-    }
-}
-
-fn json_test_many(schema: &Value, passing: &[Value], failing: &[Value]) {
-    let lark = format!(r#"start: %json {}"#, serde_json::to_string(schema).unwrap());
-    for s in passing {
-        let s = serde_json::to_string(s).unwrap();
-        lark_str_test(&lark, true, &s, false);
-    }
-    for s in failing {
-        let s = serde_json::to_string(s).unwrap();
-        lark_str_test(&lark, false, &s, false);
-    }
-}
-
-fn lark_str_test_many(lark: &str, passing: &[&str], failing: &[&str]) {
-    lark_str_test_many_ext(false, lark, passing, failing);
-}
-
-fn lark_str_test_many_quiet(lark: &str, passing: &[&str], failing: &[&str]) {
-    lark_str_test_many_ext(true, lark, passing, failing);
-}
+mod common_lark_utils;
+use common_lark_utils::*;
 
 #[test]
 fn test_dot_unicode() {
@@ -318,6 +183,17 @@ fn test_lark_syntax_general() {
     lark_err_test(r#"start: <[,]>"#, "empty token range");
     lark_err_test(r#"start: <[200-100]>"#, "invalid token range");
     lark_err_test(r#"start: <[200 - 100]>"#, "lexer error");
+    lark_ok(r#"start: <[*]>"#);
+    lark_err_test(
+        r#"start: <[^*]>"#,
+        "negated wildcard token <[^*]> is not supported",
+    );
+    lark_err_test(
+        r#"start: <[*,100]>"#,
+        "wildcard token range '*' must not contain additional tokens",
+    );
+    lark_ok(r#"start: <[^100,200-300]>"#);
+    lark_ok(r#"start: <[^100-200,100-300]>"#);
 
     lark_err_test(
         r#"