llguidance 0.7.21__tar.gz → 0.7.22__tar.gz

{llguidance-0.7.21 → llguidance-0.7.22}/CHANGELOG.md

@@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. Dates are d
 
 If a release doesn't introduce any interesting changes (build fixes etc.), it's skipped.
 
+#### [0.7.22](https://github.com/guidance-ai/llguidance/compare/v0.7.21...0.7.22) 2025-05-21
+
+- Keep EOS token bytes in `TokenizerWrapper` [`#178`](https://github.com/guidance-ai/llguidance/pull/178)
+- Stop using prefix/sentinel strings for `TokenizerWrapper` [`#175`](https://github.com/guidance-ai/llguidance/pull/175)
+- avoid taking poisoned locks, see [`#174`](https://github.com/guidance-ai/llguidance/issues/174) [`d41aa9a`](https://github.com/guidance-ai/llguidance/commit/d41aa9a4427967708a951506b2bc0e395871b6c8); thanks [@g-eoj](https://github.com/g-eoj)
 
 #### [0.7.21](https://github.com/guidance-ai/llguidance/compare/v0.7.20...0.7.21) 2025-05-20
 

{llguidance-0.7.21 → llguidance-0.7.22}/Cargo.lock

@@ -1174,7 +1174,7 @@ checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856"
 
 [[package]]
 name = "llguidance"
-version = "0.7.21"
+version = "0.7.22"
 dependencies = [
  "anyhow",
  "derivre",
@@ -1193,7 +1193,7 @@ dependencies = [
 
 [[package]]
 name = "llguidance_py"
-version = "0.7.21"
+version = "0.7.22"
 dependencies = [
  "anyhow",
  "bytemuck",
@@ -2336,7 +2336,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie"
-version = "0.7.21"
+version = "0.7.22"
 dependencies = [
  "anyhow",
  "bytemuck",
@@ -2347,7 +2347,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_hf_downloader"
-version = "0.7.21"
+version = "0.7.22"
 dependencies = [
  "anyhow",
  "hf-hub",
@@ -2358,7 +2358,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_hf_tokenizers"
-version = "0.7.21"
+version = "0.7.22"
 dependencies = [
  "anyhow",
  "log",

{llguidance-0.7.21 → llguidance-0.7.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llguidance
-Version: 0.7.21
+Version: 0.7.22
 License-File: LICENSE
 Summary: Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance
 Author: Michal Moskal

{llguidance-0.7.21 → llguidance-0.7.22}/parser/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "llguidance"
-version = "0.7.21"
+version = "0.7.22"
 edition = "2021"
 license = "MIT"
 description = "Super-fast Structured Outputs"

{llguidance-0.7.21 → llguidance-0.7.22}/parser/src/earley/parser.rs

@@ -2827,4 +2827,10 @@ impl Parser {
         copy.shared = Arc::new(Mutex::new(shared.clone()));
         copy
     }
+
+    pub fn test_trigger_lexer_error(&mut self) -> Result<()> {
+        self.with_shared(|_state| {
+            panic!("synthetic error");
+        })
+    }
 }

{llguidance-0.7.21 → llguidance-0.7.22}/parser/src/matcher.rs

@@ -86,6 +86,10 @@ impl Matcher {
         self.consume_tokens(&[token])
     }
 
+    pub fn test_trigger_lexer_error(&mut self) -> Result<()> {
+        self.with_inner(|inner| inner.parser.parser.test_trigger_lexer_error())
+    }
+
     pub fn rollback(&mut self, num_tokens: usize) -> Result<()> {
         self.with_inner(|inner| inner.parser.rollback(num_tokens))
     }

{llguidance-0.7.21 → llguidance-0.7.22}/parser/src/tokenparser.rs

@@ -278,8 +278,11 @@ impl TokenParser {
     }
 
     pub fn dump_state(&self) -> String {
+        // make sure not take self.parser.shared lock
+        // for example, self.parser.lexer_stats() takes it
+        // if we take it after panic, it will be poisoned
         format!(
-            "Tokens: {}\n{} tokens, {} bytes; grm_prefix: {:?}\nFlags:{}{}\nLexer: {}\nParser: {}\nStop: {}\nError: {}",
+            "Tokens: {}\n{} tokens, {} bytes; grm_prefix: {:?}\nFlags:{}{}\nParser: {}\nStop: {}\nError: {}",
             self.tok_trie().tokens_dbg(&self.llm_tokens),
             self.llm_tokens.len(),
             self.llm_bytes.len(),
@@ -294,7 +297,6 @@ impl TokenParser {
             } else {
                 ""
             },
-            self.parser.lexer_stats(),
             self.parser.stats(),
             self.stop_reason,
             self.error_message.as_deref().unwrap_or("None"),

{llguidance-0.7.21 → llguidance-0.7.22}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "llguidance"
-version = "0.7.21"
+version = "0.7.22"
 description = "Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance"
 requires-python = ">=3.9"
 license = "MIT"

{llguidance-0.7.21 → llguidance-0.7.22}/python/llguidance/_tokenizer.py

@@ -23,10 +23,6 @@ class TokenizerWrapper:
             gtokenizer(b"test")
         except:
             self._accepts_bytes = False
-        # If the tokenizer used bytes, then b"\xff" would be better (since it's invalid UTF-8)
-        # For now, we'll settle for "\x02" as assume it doesn't start any other token
-        self._prefix_string = "\x02"
-        self._prefix_tokens = self._encode_string(self._prefix_string)
 
     def _encode_string(self, s: str) -> List[TokenId]:
         r: List[TokenId]
@@ -37,7 +33,4 @@ class TokenizerWrapper:
         return r
 
     # required by LLTokenizer
-    def __call__(self, s: str) -> List[TokenId]:
-        tokens = self._encode_string(self._prefix_string + s)
-        assert tokens[: len(self._prefix_tokens)] == self._prefix_tokens
-        return tokens[len(self._prefix_tokens) :]
+    __call__ = _encode_string

{llguidance-0.7.21 → llguidance-0.7.22}/python_ext/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "llguidance_py"
-version = "0.7.21"
+version = "0.7.22"
 edition = "2021"
 license = "MIT"
 description = "Super-fast Structured Outputs"

{llguidance-0.7.21 → llguidance-0.7.22}/python_ext/src/py.rs

@@ -241,14 +241,7 @@ impl PyTokenizer {
             }
         }
 
-        // we want decode_bytes([EOS]) etc to be empty
-        tokens[tok_eos as usize] = vec![];
-        // if let Some(t) = tok_bos {
-        //     tokens[t as usize] = vec![];
-        // }
-
         let info = TokRxInfo::new(tokens.len() as u32, tok_eos);
-
         let tok_trie = TokTrie::from(&info, &tokens);
         Ok(PyTokenizer {
             tok_trie: Arc::new(tok_trie),

{llguidance-0.7.21 → llguidance-0.7.22}/sample_parser/tests/test_raw_parser.rs

@@ -235,3 +235,36 @@ fn test_err_state() {
     }
     unreachable!();
 }
+
+#[test]
+fn test_trigger_lexer_error() {
+    let lark = r#"
+        start: /[a-z]*/
+    "#;
+
+    let tokens = get_tok_env().tokenize("fobarbazqu");
+    let mut matcher = Matcher::new(Ok(make_parser(lark)));
+
+    for tok in tokens.iter() {
+        matcher.consume_token(*tok).unwrap();
+    }
+
+    if let Err(e) = matcher.test_trigger_lexer_error() {
+        let e = e.to_string();
+        println!("Error: {}", e);
+        assert!(e.contains("<state>"));
+        assert!(e.contains("synthetic error"));
+    } else {
+        unreachable!();
+    }
+
+    // now all calls should return the same error
+    if let Err(e) = matcher.consume_token(123) {
+        let e = e.to_string();
+        println!("Error: {}", e);
+        assert!(e.contains("<state>"));
+        assert!(e.contains("synthetic error"));
+    } else {
+        unreachable!();
+    }
+}

{llguidance-0.7.21 → llguidance-0.7.22}/toktrie/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "toktrie"
-version = "0.7.21"
+version = "0.7.22"
 edition = "2021"
 license = "MIT"
 description = "LLM Token Trie library"

{llguidance-0.7.21 → llguidance-0.7.22}/toktrie_hf_downloader/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "toktrie_hf_downloader"
-version = "0.7.21"
+version = "0.7.22"
 edition = "2021"
 license = "MIT"
 description = "HuggingFace Hub download library support for toktrie and llguidance"

{llguidance-0.7.21 → llguidance-0.7.22}/toktrie_hf_tokenizers/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "toktrie_hf_tokenizers"
-version = "0.7.21"
+version = "0.7.22"
 edition = "2021"
 license = "MIT"
 description = "HuggingFace tokenizers library support for toktrie and llguidance"