llguidance 0.7.18__tar.gz → 0.7.19__tar.gz

{llguidance-0.7.18 → llguidance-0.7.19}/CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. Dates are d
 
 If a release doesn't introduce any interesting changes (build fixes etc.), it's skipped.
 
+#### [0.7.19](https://github.com/guidance-ai/llguidance/compare/v0.7.18...0.7.19) 2025-04-24
+
+- fix a numeric token bug [`1f59edf`](https://github.com/guidance-ai/llguidance/commit/1f59edfc49b44cfba74b2380f34874a0778d9441)
+
 #### [0.7.18](https://github.com/guidance-ai/llguidance/compare/v0.7.17...0.7.18) 2025-04-22
 
 - apply x-guidance also in %json{} [`2627891`](https://github.com/guidance-ai/llguidance/commit/2627891c72c7e38062cd3e052f1de146d2e21635)

{llguidance-0.7.18 → llguidance-0.7.19}/Cargo.lock

@@ -1178,7 +1178,7 @@ checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856"
 
 [[package]]
 name = "llguidance"
-version = "0.7.18"
+version = "0.7.19"
 dependencies = [
  "anyhow",
  "derivre",
@@ -1197,7 +1197,7 @@ dependencies = [
 
 [[package]]
 name = "llguidance_py"
-version = "0.7.18"
+version = "0.7.19"
 dependencies = [
  "anyhow",
  "bytemuck",
@@ -2362,7 +2362,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie"
-version = "0.7.18"
+version = "0.7.19"
 dependencies = [
  "anyhow",
  "bytemuck",
@@ -2373,7 +2373,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_hf_downloader"
-version = "0.7.18"
+version = "0.7.19"
 dependencies = [
  "anyhow",
  "hf-hub",
@@ -2384,7 +2384,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_hf_tokenizers"
-version = "0.7.18"
+version = "0.7.19"
 dependencies = [
  "anyhow",
  "log",

{llguidance-0.7.18 → llguidance-0.7.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llguidance
-Version: 0.7.18
+Version: 0.7.19
 License-File: LICENSE
 Summary: Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance
 Author: Michal Moskal

{llguidance-0.7.18 → llguidance-0.7.19}/parser/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "llguidance"
-version = "0.7.18"
+version = "0.7.19"
 edition = "2021"
 license = "MIT"
 description = "Super-fast Structured Outputs"

{llguidance-0.7.18 → llguidance-0.7.19}/parser/src/earley/lexer.rs

@@ -221,8 +221,8 @@ impl Lexer {
         if enable_logging {
             let info = self.state_info(state);
             debug!(
-                "lex: {:?} -{:?}-> {:?}, acpt={:?}",
-                prev, byte as char, state, info.greedy_accepting
+                "lex: {:?} -{:?}-> {:?}, acpt={:?}/{:?}",
+                prev, byte as char, state, info.greedy_accepting, info.lazy_accepting
             );
         }
 

{llguidance-0.7.18 → llguidance-0.7.19}/parser/src/earley/regexvec.rs

@@ -407,14 +407,18 @@ impl RegexVec {
         for (idx, e) in iter_state(&self.rx_sets, desc.state) {
             // If this lexeme is not a match.  (If the derivative at this point is nullable,
             // there is a match, so if it is not nullable, there is no match.)
+            // println!("idx: {:?} e: {:?} {:?}", idx, e,self.special_token_rx);
             if !self.exprs.is_nullable(e) {
                 // No match, so not at end of lexeme
                 all_eoi = false;
                 continue;
-            } else if Some(e) == self.special_token_rx {
+            } else if Some(self.get_rx(idx)) == self.special_token_rx {
                 // the regex is /\xFF\[[0-9]+\]/ so it's guaranteed not to conflict with anything
                 // else (starts with non-unicode byte); thus we ignore the rest of processing
+                // when has_special_token is set, we just need to make sure lazy_accepting is non-empty,
+                // the actual value is not important
                 desc.lazy_accepting = MatchingLexemes::One(idx);
+                desc.has_special_token = true;
                 return;
             }
 
@@ -424,9 +428,6 @@ impl RegexVec {
                 if lazies.is_none() {
                     all_eoi = false;
                     hidden_len = self.exprs.possible_lookahead_len(e) as u32;
-                    if Some(self.get_rx(idx)) == self.special_token_rx {
-                        desc.has_special_token = true;
-                    }
                 }
                 lazies.add(idx);
                 continue;
@@ -711,7 +712,8 @@ impl RegexVec {
 
         self.lowest_match_inner(&mut res);
 
-        // debug!("state {:?} desc: {:?}", state, res);
+        // println!("state {:?} desc: {:?}", state, res);
+
         res
     }
 

{llguidance-0.7.18 → llguidance-0.7.19}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "llguidance"
-version = "0.7.18"
+version = "0.7.19"
 description = "Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance"
 requires-python = ">=3.9"
 license = "MIT"

{llguidance-0.7.18 → llguidance-0.7.19}/python_ext/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "llguidance_py"
-version = "0.7.18"
+version = "0.7.19"
 edition = "2021"
 license = "MIT"
 description = "Super-fast Structured Outputs"

{llguidance-0.7.18 → llguidance-0.7.19}/sample_parser/tests/test_ll.rs

@@ -1262,3 +1262,52 @@ fn test_ll_tool_str_spec() {
 
     check_lark_grammar(TOOL_STR_SPEC_GRAMMAR, tool_chk);
 }
+
+#[test]
+fn test_ll_numeric_bug() {
+    check_lark_grammar(
+        r#"
+            start: text
+            text: (text_tokens)* <[33000]> ap
+            ap: <[33001]> (atok*)
+            atok: <[400-410]>
+            text_tokens: <[300-310]>
+        "#,
+        &[
+            "",
+            "<[300]>‧<[33000]>",
+            "<[33001]>",
+            "<[401]>‧<[402]>‧≺EOS≻",
+        ],
+    );
+
+    check_lark_grammar(
+        r#"
+            start: text
+            text: (text_tokens)* ( (<[33000]> ap) | (<[33002]> (atok*)) )
+            ap: <[33001]> (atok*)
+            atok: <[400-410]>
+            text_tokens: <[300-310]>
+        "#,
+        &[
+            "",
+            "<[300]>‧<[33000]>",
+            "<[33001]>",
+            "<[401]>‧<[402]>‧≺EOS≻",
+        ],
+    );
+
+    check_lark_grammar(
+        r#"
+            start: text
+            text: (text_tokens)* ( (<[33000]> ap) | (<[33002]> (atok*)) )
+            ap: <[33001,33003]> (atok*)
+            atok: <[400-410]>
+            text_tokens: <[300-310]>
+        "#,
+        &[
+            "",
+            "✖<[33001]>‧<[300]>‧<[33000]>✖<[33002]>✖<[300]>‧<[33001]>✖<[33002]>✖<[300]>‧<[401]>‧<[402]>‧≺EOS≻",
+        ],
+    );
+}

{llguidance-0.7.18 → llguidance-0.7.19}/toktrie/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "toktrie"
-version = "0.7.18"
+version = "0.7.19"
 edition = "2021"
 license = "MIT"
 description = "LLM Token Trie library"

{llguidance-0.7.18 → llguidance-0.7.19}/toktrie_hf_downloader/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "toktrie_hf_downloader"
-version = "0.7.18"
+version = "0.7.19"
 edition = "2021"
 license = "MIT"
 description = "HuggingFace Hub download library support for toktrie and llguidance"

{llguidance-0.7.18 → llguidance-0.7.19}/toktrie_hf_tokenizers/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "toktrie_hf_tokenizers"
-version = "0.7.18"
+version = "0.7.19"
 edition = "2021"
 license = "MIT"
 description = "HuggingFace tokenizers library support for toktrie and llguidance"

{llguidance-0.7.18 → llguidance-0.7.19}/toktrie_hf_tokenizers/src/lib.rs

@@ -213,7 +213,9 @@ impl ByteTokenizerEnv {
                 bail!("vocab size too small; {} vs {}", n_vocab, token_bytes.len());
             }
             while n_vocab > token_bytes.len() {
-                token_bytes.push(Vec::new());
+                let mut name = format!(".<[{}]>", token_bytes.len()).into_bytes();
+                name[0] = TokTrie::SPECIAL_TOKEN_MARKER;
+                token_bytes.push(name);
             }
             info.vocab_size = n_vocab as u32;
         }