llguidance 0.7.24__tar.gz → 0.7.26__tar.gz

{llguidance-0.7.24 → llguidance-0.7.26}/CHANGELOG.md

@@ -4,6 +4,17 @@ All notable changes to this project will be documented in this file. Dates are d
 
 If a release doesn't introduce any interesting changes (build fixes etc.), it's skipped.
 
+#### [0.7.26](https://github.com/guidance-ai/llguidance/compare/v0.7.25...0.7.26) 2025-05-30
+
+- add support for & and ~ in lark regexes [`96fcee3`](https://github.com/guidance-ai/llguidance/commit/96fcee373697b57bead94d1bc06c17cf1c6134e4)
+- dump grammar in errors in LLInterpreter [`#183`](https://github.com/guidance-ai/llguidance/pull/183)
+- don't check lexer bytes invariant when we cannot rollback [`ec22083`](https://github.com/guidance-ai/llguidance/commit/ec220837051513a70177974ca389b7bf387455f1)
+
+
+#### [0.7.25](https://github.com/guidance-ai/llguidance/compare/v0.7.24...0.7.25) 2025-05-28
+
+- add parse_special=False to tokenize_str/bytes() in python [`#181`](https://github.com/guidance-ai/llguidance/pull/181)
+
 #### [0.7.24](https://github.com/guidance-ai/llguidance/compare/v0.7.23...0.7.24) 2025-05-23
 
 - add the sentinel token hack, fixes #180 [`#180`](https://github.com/guidance-ai/llguidance/issues/180)

{llguidance-0.7.24 → llguidance-0.7.26}/Cargo.lock

@@ -1174,7 +1174,7 @@ checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856"
 
 [[package]]
 name = "llguidance"
-version = "0.7.24"
+version = "0.7.26"
 dependencies = [
  "anyhow",
  "derivre",
@@ -1193,7 +1193,7 @@ dependencies = [
 
 [[package]]
 name = "llguidance_py"
-version = "0.7.24"
+version = "0.7.26"
 dependencies = [
  "anyhow",
  "bytemuck",
@@ -2336,7 +2336,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie"
-version = "0.7.24"
+version = "0.7.26"
 dependencies = [
  "anyhow",
  "bytemuck",
@@ -2347,7 +2347,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_hf_downloader"
-version = "0.7.24"
+version = "0.7.26"
 dependencies = [
  "anyhow",
  "hf-hub",
@@ -2358,7 +2358,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_hf_tokenizers"
-version = "0.7.24"
+version = "0.7.26"
 dependencies = [
  "anyhow",
  "log",

{llguidance-0.7.24 → llguidance-0.7.26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llguidance
-Version: 0.7.24
+Version: 0.7.26
 License-File: LICENSE
 Summary: Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance
 Author: Michal Moskal
@@ -72,9 +72,7 @@ The library is currently integrated in:
 - **vLLM** - [V0 PR](https://github.com/vllm-project/vllm/pull/14589) and [V1 PR](https://github.com/vllm-project/vllm/pull/14779)
 - [LLGTRT](https://github.com/guidance-ai/llgtrt) - OpenAI-compatible REST server using NVIDIA's [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM)
 - [mistral.rs](https://github.com/EricLBuehler/mistral.rs/pull/899)
-
-The integration is ongoing in:
-- **onnxruntime-genai** - [draft PR](https://github.com/microsoft/onnxruntime-genai/pull/1038)
+- [onnxruntime-genai](https://github.com/microsoft/onnxruntime-genai/pull/1381)
 
 ## Technical details
 

{llguidance-0.7.24 → llguidance-0.7.26}/README.md

@@ -60,9 +60,7 @@ The library is currently integrated in:
 - **vLLM** - [V0 PR](https://github.com/vllm-project/vllm/pull/14589) and [V1 PR](https://github.com/vllm-project/vllm/pull/14779)
 - [LLGTRT](https://github.com/guidance-ai/llgtrt) - OpenAI-compatible REST server using NVIDIA's [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM)
 - [mistral.rs](https://github.com/EricLBuehler/mistral.rs/pull/899)
-
-The integration is ongoing in:
-- **onnxruntime-genai** - [draft PR](https://github.com/microsoft/onnxruntime-genai/pull/1038)
+- [onnxruntime-genai](https://github.com/microsoft/onnxruntime-genai/pull/1381)
 
 ## Technical details
 

{llguidance-0.7.24 → llguidance-0.7.26}/docs/optimizations.md

@@ -91,7 +91,6 @@ Walking the trie mostly involves successful lookups in that table,
 and the derivative engine is only used when the table doesn't yet have the
 given transition.
 
-
 ## Earley parser optimizations
 
 - CFG rules are stored in a flat array
@@ -123,7 +122,7 @@ We thus define a series _slices_, under-approximation of such unconstrained cont
 The slices are defined by regular expressions typically of the form `[...]{1,N}`
 (that is a character class repeated up to `N` times).
 
-For example, a good set of slices for JSON schemas is 
+For example, a good set of slices for JSON schemas is
 
 - `[^"\\\x00-\x1F\x7F]{1,10}` (`turtle`, ` turtle`, `)!;`, `żółw`, `🐢`, etc.)
 - `[^"\\\x00-\x1F\x7F]{1,30}` (`/////////////////`, ...)
@@ -169,6 +168,7 @@ Now, the JSON slice is contained in `C*"`,
 and thus we can skip walking the trie for the slice.
 
 Another example:
+
 - assume schemas has `{ "type": "string", "maxLength": 20 }`
 - so after initial quote, the lexer allows `C{0,20}"`
 - the JSON slice `[^"\\\x00-\x1F\x7F]{1,10}` is contained in this lexeme,
@@ -176,17 +176,19 @@ Another example:
 
 This optimization make the mask computation about 10x faster in [MaskBench](https://github.com/guidance-ai/jsonschemabench/tree/main/maskbench).
 
+### Mask density statistics
+
 The reason the optimization works, is that masks tend be either small or sliceable.
 Here are statistics of various kinds of masks, across around 2M masks in MaskBench,
 categorized based on how "full" the mask is and whether the slicer optimization was applied.
 
-| Category            |  % Masks |    % Time | Time/Mask [us] |
-|---------------------|---------:|----------:|---------------:|
-| 0%-2% & !sliced     |    44.6% |     20.7% |             28 |
-| 2%-85% & !sliced    |     1.1% |     11.0% |            576 |
-| 85%+ & !sliced      |     0.5% |     13.0% |           1577 |
-| 85%+ & sliced       |    53.8% |     55.0% |             61 |
-| **Total**           |   100.0% |    100.0% |             60 |
+| Category         | % Masks | % Time | Time/Mask [us] |
+| ---------------- | ------: | -----: | -------------: |
+| 0%-2% & !sliced  |   44.6% |  20.7% |             28 |
+| 2%-85% & !sliced |    1.1% |  11.0% |            576 |
+| 85%+ & !sliced   |    0.5% |  13.0% |           1577 |
+| 85%+ & sliced    |   53.8% |  55.0% |             61 |
+| **Total**        |  100.0% | 100.0% |             60 |
 
 ![Plot of the table above](mask_plot.png)
 
@@ -195,43 +197,43 @@ and in a little over half the slicer optimization can be applied
 (there are no masks under 85% full where the slicer can be applied).
 The remaining sliver of masks are either intermediate size or large, but the slicer optimization can't be applied; they take disproportionately long time to compute.
 
+
 ### Checking regex containment
 
-This is an under-approximation of the containment problem,
-that is it may return false when the containment is actually true.
-If any of the "checks" fail, we return false.
+This is an under-approximation of the containment problem, that is it may return
+false when the containment is actually true. If any of the "checks" fail, we
+return false.
 
 Prefixes of language `R`, are defined as `P(R) = { w | ∃q. wq ∈ R }`.
 
-We need to check if regex `S` (slice) is contained in prefix of regex `L` (lexeme): `S ⊆ P(L)`.
+We need to check if regex `S` (slice) is contained in prefix of regex `L`
+(lexeme): `S ⊆ P(L)`.
 
-We check if `L` is of the form `(X{m,n} & ~E) T`, where 
-`E` is of the form `E0 | E1 | ... | Ek`,
-and both `E` can be `∅` (empty-set/no match) and `T` can be `ε` (empty string).
+We check if `L` is of the form `(X{m,n} & ~E) T`, where `E` is of the form
+`E0 | E1 | ... | Ek`. Note that: `E` can be `∅` (empty-set/no match) and `T` can
+be `ε` (empty string).
 
-Observe that `P(R) ⊆ P(RT)`, ie. making regex longer doesn't remove any prefixes (provided `T ≠ ∅`).
-Thus, we'll be checking containment in `P(X{m,n} & ~E)`.
+Observe that `P(R) ⊆ P(RT)`, ie. making regex longer doesn't remove any prefixes
+(provided `T ≠ ∅`). Thus, we'll be checking containment in `P(X{m,n} & ~E)`.
 
-We (over)estimate maximum length of `E`, let `o >= max { |w| | w ∈ E }`.
-We check that `n > o`, and that `∃v ≠ ε. v ∈ X`.
-In other words, we check that for anything matching `Ei` and `X{m,n}` there is a proper extension of that string in `X{m,n}`.
+We (over)estimate maximum length of `E`, let `o >= max { |w| | w ∈ E }`. We
+check that `n > o`, and that `∃v ≠ ε. v ∈ X`. In other words, we check that for
+anything matching `Ei` and `X{m,n}` there is a proper extension of that string
+in `X{m,n}`.
 
 Now, we prove that `P(X{m,n} & ~E) = P(X{m,n})`.
 
-Consider `w ∈ P(X{m,n})`. We have `wq ∈ X{m,n}` for some `q`.
-If `|wq| > o`, then `wq ∉ E`, and thus `wq ∈ X{m,n} & ~E`.
-Otherwise, `wq ∈ X{p}` for some `p <= o < n`,
-and thus `wqv...v ∈ X{n}` for `n-p` repetitions of `v`.
-We also have `|wqv...v| > o`, and thus `wqv...v ∉ E`,
-and thus `wqv...v ∈ X{m,n} & ~E`,
-and thus `w ∈ P(X{m,n} & ~E)`.
-The other direction is trivial.
+Consider `w ∈ P(X{m,n})`. We have `wq ∈ X{m,n}` for some `q`. If `|wq| > o`,
+then `wq ∉ E`, and thus `wq ∈ X{m,n} & ~E`. Otherwise, `wq ∈ X{p}` for some
+`p <= o < n`, and thus `wqv...v ∈ X{n}` for `n-p` repetitions of `v`. We also
+have `|wqv...v| > o`, and thus `wqv...v ∉ E`, and thus `wqv...v ∈ X{m,n} & ~E`,
+and thus `w ∈ P(X{m,n} & ~E)`. The other direction is trivial.
 
 Now, we just need to check if `S ⊆ P(X{m,n})`.
 
-First, we check if `S` is of the form `Y{m',n'}`.
-Then, we check if `Y` is contained in `X` (this is a cached check using symbolic derivatives; it's typically simple).
-Finally, we check if `n' <= n`.
-Note that we don't care about `m` and `m'`, as we're checking for prefixes.
+First, we check if `S` is of the form `Y{m',n'}`. Then, we check if `Y` is
+contained in `X` (this is a cached check using symbolic derivatives; it's
+typically simple). Finally, we check if `n' <= n`. Note that we don't care about
+`m` and `m'`, as we're checking for prefixes.
 
 Also note that the upper-bound in the above calculations can be infinity.

{llguidance-0.7.24 → llguidance-0.7.26}/docs/syntax.md

@@ -217,6 +217,28 @@ like `<|python_tag|>`, not a string like `<function`.
 
 The `llguidance.StructTag` API, [inspired](https://github.com/mlc-ai/xgrammar/blob/fd9ee31/python/xgrammar/grammar.py#L211) by XGrammar, just compiles down to the above.
 
+### And/Not operators in regexes
+
+The regular expressions in LLGuidance can use additional operators: `&` (and) and `~` (not).
+They can only be used outside of the `/.../` syntax, i.e., in the Lark terminal (token) definitions.
+The `&` operator binds tighter than `|` (alternation), so `A & B | C` means `(A & B) | C`.
+The `~` operator binds tighter than even `+` or `*`, so `~A+` means `(~A)+`.
+
+The negation operator `~` is particularly tricky to use right.
+For example, this is a terminal definition that matches any list of ASCII lines,
+but they cannot have two newlines in a row:
+
+```lark
+ASCII_LINES: /[a-zA-Z \n]*/ & ~/(?s:.*)\n\n(?s:.*)/
+```
+
+Note that `/[a-zA-Z \n]*/ & ~/\n\n/` would mean any list of lines, also with two newlines in a row,
+except for the exact string `\n\n`.
+Also, `/[a-zA-Z \n]*/ & ~/(.*)\n\n(.*)/` would allow double newlines, but if there is at least two of them
+(`/./` doesn't match newline).
+
+These operators are sometimes expensive to use, so you should generally avoid them if alternatives exist.
+
 ### Structured %regex
 
 LLGuidance supports [extended regex syntax](https://docs.rs/regex/latest/regex/#syntax) in `/.../`.
@@ -273,12 +295,6 @@ MULT_NUM: %regex {
 }
 ```
 
-We also plan to add `&` and `~` operators:
-
-```lark
-ASCII_LINES: /[a-zA-Z \n]*/ & ~/.*\n\n.*/
-```
-
 ### Grammar options
 
 Certain grammar options can be set by using `%llguidnace { ... }`,

{llguidance-0.7.24 → llguidance-0.7.26}/parser/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "llguidance"
-version = "0.7.24"
+version = "0.7.26"
 edition = "2021"
 license = "MIT"
 description = "Super-fast Structured Outputs"

{llguidance-0.7.24 → llguidance-0.7.26}/parser/src/earley/parser.rs

@@ -1662,7 +1662,9 @@ impl ParserState {
 
     pub fn scan_eos(&mut self) -> bool {
         self.assert_definitive(); // ???
-        self.check_lexer_bytes_invariant();
+        if self.lexer_spec().can_rollback() {
+            self.check_lexer_bytes_invariant();
+        }
 
         let lexer_eos = self.lexer_allows_eos();
 
@@ -1691,7 +1693,9 @@ impl ParserState {
             self.lexer_stack_top_eos = true;
         }
 
-        self.check_lexer_bytes_invariant();
+        if self.lexer_spec().can_rollback() {
+            self.check_lexer_bytes_invariant();
+        }
 
         false
     }

{llguidance-0.7.24 → llguidance-0.7.26}/parser/src/grammar_builder.rs

@@ -132,6 +132,9 @@ impl RegexBuilder {
     }
 
     pub fn and(&mut self, nodes: Vec<RegexId>) -> RegexId {
+        if nodes.len() == 1 {
+            return nodes[0];
+        }
         self.add_ast(RegexAst::And(map_ids(&nodes))).unwrap()
     }
 

{llguidance-0.7.24 → llguidance-0.7.26}/parser/src/lark/ast.rs

@@ -79,29 +79,41 @@ pub struct RuleParams(pub Vec<String>);
 #[derive(Debug, Clone)]
 pub struct TokenParams(pub Vec<String>);
 
-/// Represents a list of expansions.
+/// Represents an alternative (OR) of productions in a grammar.
 #[derive(Debug)]
 pub struct Expansions(pub Location, pub Vec<Alias>);
 
 impl Expansions {
     pub fn single_atom(&self) -> Option<&Atom> {
-        if self.1.len() == 1 && self.1[0].expansion.0.len() == 1 {
-            Some(&self.1[0].expansion.0[0].atom)
+        if self.1.len() == 1
+            && self.1[0].conjuncts.len() == 1
+            && self.1[0].conjuncts[0].0.len() == 1
+        {
+            Some(&self.1[0].conjuncts[0].0[0].atom)
         } else {
             None
         }
     }
+
+    pub fn take_single_atom(&mut self) -> Option<Atom> {
+        if self.single_atom().is_none() {
+            None
+        } else {
+            Some(self.1[0].conjuncts.pop().unwrap().0.pop().unwrap().atom)
+        }
+    }
 }
 
 /// Represents an alias in the grammar.
+/// Each alias consists of possibly multiple conjuncts (AND).
 #[derive(Debug)]
 pub struct Alias {
-    pub expansion: Expansion,
+    pub conjuncts: Vec<Expansion>,
     #[allow(dead_code)]
     pub alias: Option<String>,
 }
 
-/// Represents an expansion consisting of expressions.
+/// Represents a concatenation of expressions in the grammar.
 #[derive(Debug)]
 pub struct Expansion(pub Vec<Expr>);
 
@@ -119,6 +131,7 @@ pub enum Atom {
     Group(Expansions),
     Maybe(Expansions),
     Value(Value),
+    Not(Box<Atom>),
 }
 
 /// Represents different values in the grammar.

{llguidance-0.7.24 → llguidance-0.7.26}/parser/src/lark/compiler.rs

@@ -111,6 +111,10 @@ impl Compiler {
                 let id = self.do_token_expansions(expansions)?;
                 Ok(self.builder.regex.optional(id))
             }
+            Atom::Not(inner) => {
+                let id = self.do_token_atom(*inner)?;
+                Ok(self.builder.regex.not(id))
+            }
             Atom::Value(value) => match value {
                 Value::LiteralRange(a, b) => {
                     ensure!(
@@ -218,12 +222,18 @@ impl Compiler {
             .into_iter()
             .map(|alias| {
                 let args = alias
-                    .expansion
-                    .0
+                    .conjuncts
                     .into_iter()
-                    .map(|e| self.do_token_expr(e))
+                    .map(|exp| {
+                        let args = exp
+                            .0
+                            .into_iter()
+                            .map(|e| self.do_token_expr(e))
+                            .collect::<Result<Vec<_>>>()?;
+                        Ok(self.builder.regex.concat(args))
+                    })
                     .collect::<Result<Vec<_>>>()?;
-                Ok(self.builder.regex.concat(args))
+                Ok(self.builder.regex.and(args))
             })
             .collect::<Result<Vec<_>>>()
             .map_err(|e| expansions.0.augment(e))?;
@@ -265,6 +275,11 @@ impl Compiler {
                 let id = self.do_expansions(expansions)?;
                 Ok(self.builder.optional(id))
             }
+            Atom::Not(_) => {
+                // treat as token
+                let rx = self.do_token_atom(expr)?;
+                Ok(self.lift_regex(rx)?)
+            }
             Atom::Value(value) => {
                 match &value {
                     Value::Name(n) => {
@@ -363,9 +378,15 @@ impl Compiler {
         let options = expansions
             .1
             .into_iter()
-            .map(|alias| {
+            .map(|mut alias| {
+                ensure!(
+                    alias.conjuncts.len() == 1,
+                    "& is only supported for tokens, not rules; try renaming the rule to UPPERCASE"
+                );
                 let args = alias
-                    .expansion
+                    .conjuncts
+                    .pop()
+                    .unwrap()
                     .0
                     .into_iter()
                     .map(|e| self.do_expr(&loc, e))
@@ -478,8 +499,7 @@ impl Compiler {
                         return self.gen_grammar(g, rule.temperature, props);
                     }
                     Some(Atom::Value(Value::Json(_) | Value::NestedLark(_))) => {
-                        if let Atom::Value(x) = rule.expansions.1[0].expansion.0.pop().unwrap().atom
-                        {
+                        if let Some(Atom::Value(x)) = rule.expansions.take_single_atom() {
                             return self.do_nested(&rule.expansions.0, x, rule.temperature, props);
                         } else {
                             unreachable!();
@@ -580,11 +600,11 @@ impl Grammar {
             expansions: Expansions(
                 loc.clone(),
                 vec![Alias {
-                    expansion: Expansion(vec![Expr {
+                    conjuncts: vec![Expansion(vec![Expr {
                         atom: Atom::Value(Value::LiteralRegex(regex.to_string(), "".to_string())),
                         op: None,
                         range: None,
-                    }]),
+                    }])],
                     alias: None,
                 }],
             ),

{llguidance-0.7.24 → llguidance-0.7.26}/parser/src/lark/lexer.rs

@@ -49,6 +49,7 @@ pub enum Token {
     Number,
     Newline,
     VBar,
+    And,          // &
     SpecialToken, // <something>
     GrammarRef,   // @grammar_id or @7
     // special
@@ -144,6 +145,7 @@ impl Token {
         (Token::RBracket, "]"),
         (Token::Tilde, "~"),
         (Token::VBar, "|"),
+        (Token::And, "&"),
         (Token::Equals, "="),
     ];
 

{llguidance-0.7.24 → llguidance-0.7.26}/parser/src/lark/parser.rs

@@ -357,13 +357,20 @@ impl Parser {
 
     /// Parses an alias.
     fn parse_alias(&mut self) -> Result<Alias> {
-        let expansion = self.parse_expansion()?;
+        let mut conjuncts = Vec::with_capacity(1);
+        loop {
+            let expansion = self.parse_expansion()?;
+            conjuncts.push(expansion);
+            if !self.match_token(Token::And) {
+                break;
+            }
+        }
         let alias = if self.match_token(Token::Arrow) {
             Some(self.expect_token_val(Token::Rule)?)
         } else {
             None
         };
-        Ok(Alias { expansion, alias })
+        Ok(Alias { conjuncts, alias })
     }
 
     /// Parses an expansion.
@@ -376,6 +383,7 @@ impl Parser {
                 || self.has_token(Token::RBrace)
                 || self.has_token(Token::RParen)
                 || self.has_token(Token::RBracket)
+                || self.has_token(Token::And)
             {
                 break;
             }
@@ -391,7 +399,8 @@ impl Parser {
         let mut range = None;
         if let Some(op_token) = self.match_token_with_value(Token::Op) {
             op = Some(Op(op_token.clone()));
-        } else if self.match_token(Token::Tilde) {
+        } else if self.has_tokens(&[Token::Tilde, Token::Number]) {
+            self.expect_token(Token::Tilde)?;
             let start_num = self.expect_token_val(Token::Number)?.parse::<i32>()?;
             let end_num = if self.match_token(Token::DotDot) {
                 Some(self.expect_token_val(Token::Number)?.parse::<i32>()?)
@@ -426,16 +435,27 @@ impl Parser {
 
     /// Parses an atom.
     fn parse_atom(&mut self) -> Result<Atom> {
-        if self.match_token(Token::LParen) {
+        let mut negated = false;
+        if self.match_token(Token::Tilde) {
+            negated = true;
+        }
+
+        let res = if self.match_token(Token::LParen) {
             let expansions = self.parse_expansions()?;
             self.expect_token(Token::RParen)?;
-            Ok(Atom::Group(expansions))
+            Atom::Group(expansions)
         } else if self.match_token(Token::LBracket) {
             let expansions = self.parse_expansions()?;
             self.expect_token(Token::RBracket)?;
-            Ok(Atom::Maybe(expansions))
+            Atom::Maybe(expansions)
+        } else {
+            Atom::Value(self.parse_value()?)
+        };
+
+        if negated {
+            Ok(Atom::Not(Box::new(res)))
         } else {
-            Ok(Atom::Value(self.parse_value()?))
+            Ok(res)
         }
     }
 

{llguidance-0.7.24 → llguidance-0.7.26}/parser/src/tokenparser.rs

@@ -18,6 +18,7 @@ pub struct TokenParser {
     pub logger: Logger,
     pub limits: ParserLimits,
     pub bias_computer: Arc<dyn BiasComputer>,
+    pub dbg_grammar: String,
     last_step_stats: ParserStats,
     max_step_stats: ParserStats,
     eos_token: TokenId,
@@ -106,6 +107,7 @@ impl TokenParser {
             stop_reason: StopReason::NotStopped,
             error_message: None,
             parser,
+            dbg_grammar: String::new(),
             eos_token,
             llm_tokens: Vec::new(),
             llm_bytes: Vec::new(),
@@ -274,7 +276,11 @@ impl TokenParser {
     }
 
     pub fn augment_err(&self, e: impl Display) -> String {
-        format!("{e}\n<state>\n{}\n</state>", self.dump_state())
+        format!(
+            "{e}\n<state>\n{}\n</state><grammar>\n{}\n</grammar>",
+            self.dump_state(),
+            self.dbg_grammar
+        )
     }
 
     pub fn dump_state(&self) -> String {

{llguidance-0.7.24 → llguidance-0.7.26}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "llguidance"
-version = "0.7.24"
+version = "0.7.26"
 description = "Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance"
 requires-python = ">=3.9"
 license = "MIT"

{llguidance-0.7.24 → llguidance-0.7.26}/python/llguidance/_lib.pyi

@@ -51,7 +51,10 @@ class LLTokenizer:
         This will not necessarily match BPE.
         """
 
-    def tokenize_bytes(self, utf8bytes: bytes) -> List[int]:
+    def tokenize_bytes(self,
+                       utf8bytes: bytes,
+                       *,
+                       parse_special: bool = False) -> List[int]:
         """
         Tokenize the text as bytes.
         This will use the underlying Python tokenizer to tokenize valid UTF8
@@ -59,7 +62,10 @@ class LLTokenizer:
         few bytes.
         """
 
-    def tokenize_str(self, text: str) -> List[int]:
+    def tokenize_str(self,
+                     text: str,
+                     *,
+                     parse_special: bool = False) -> List[int]:
         """
         Same as tokenize_bytes, but for strings.
         """

{llguidance-0.7.24 → llguidance-0.7.26}/python/torch_tests/test_llamacpp.py

@@ -40,3 +40,10 @@ def test_llama_cpp(pytestconfig: Any) -> None:
     print(toks)
     assert len(toks) == 1
     assert llt.decode_bytes(toks) == b"\x8b"
+
+    toks1 = llt.tokenize_str("<|eot_id|>")
+    toks0 = llt.tokenize_str("<|eot_id|>", parse_special=False)
+    assert toks1 == toks0
+    assert len(toks0) > 1
+    toks2 = llt.tokenize_str("<|eot_id|>", parse_special=True)
+    assert len(toks2) == 1

{llguidance-0.7.24 → llguidance-0.7.26}/python_ext/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "llguidance_py"
-version = "0.7.24"
+version = "0.7.26"
 edition = "2021"
 license = "MIT"
 description = "Super-fast Structured Outputs"