llguidance 0.7.23__tar.gz → 0.7.24__tar.gz

{llguidance-0.7.23 → llguidance-0.7.24}/CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. Dates are d
 
 If a release doesn't introduce any interesting changes (build fixes etc.), it's skipped.
 
+#### [0.7.24](https://github.com/guidance-ai/llguidance/compare/v0.7.23...0.7.24) 2025-05-23
+
+- add the sentinel token hack, fixes #180 [`#180`](https://github.com/guidance-ai/llguidance/issues/180)
+
 #### [0.7.23](https://github.com/guidance-ai/llguidance/compare/v0.7.22...0.7.23) 2025-05-22
 
 - native llama.cpp tokenizer support [`#179`](https://github.com/guidance-ai/llguidance/pull/179)

{llguidance-0.7.23 → llguidance-0.7.24}/Cargo.lock

@@ -1174,7 +1174,7 @@ checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856"
 
 [[package]]
 name = "llguidance"
-version = "0.7.23"
+version = "0.7.24"
 dependencies = [
  "anyhow",
  "derivre",
@@ -1193,7 +1193,7 @@ dependencies = [
 
 [[package]]
 name = "llguidance_py"
-version = "0.7.23"
+version = "0.7.24"
 dependencies = [
  "anyhow",
  "bytemuck",
@@ -2336,7 +2336,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie"
-version = "0.7.23"
+version = "0.7.24"
 dependencies = [
  "anyhow",
  "bytemuck",
@@ -2347,7 +2347,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_hf_downloader"
-version = "0.7.23"
+version = "0.7.24"
 dependencies = [
  "anyhow",
  "hf-hub",
@@ -2358,7 +2358,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_hf_tokenizers"
-version = "0.7.23"
+version = "0.7.24"
 dependencies = [
  "anyhow",
  "log",

{llguidance-0.7.23 → llguidance-0.7.24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llguidance
-Version: 0.7.23
+Version: 0.7.24
 License-File: LICENSE
 Summary: Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance
 Author: Michal Moskal

{llguidance-0.7.23 → llguidance-0.7.24}/parser/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "llguidance"
-version = "0.7.23"
+version = "0.7.24"
 edition = "2021"
 license = "MIT"
 description = "Super-fast Structured Outputs"

{llguidance-0.7.23 → llguidance-0.7.24}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "llguidance"
-version = "0.7.23"
+version = "0.7.24"
 description = "Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance"
 requires-python = ">=3.9"
 license = "MIT"

{llguidance-0.7.23 → llguidance-0.7.24}/python_ext/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "llguidance_py"
-version = "0.7.23"
+version = "0.7.24"
 edition = "2021"
 license = "MIT"
 description = "Super-fast Structured Outputs"

{llguidance-0.7.23 → llguidance-0.7.24}/python_ext/src/lib.rs

@@ -1,5 +1,6 @@
 use pyo3::prelude::*;
 
+mod llamatokenizer;
 mod llinterpreter;
 mod llmatcher;
 mod parserlimits;

llguidance-0.7.24/python_ext/src/llamatokenizer.rs

@@ -0,0 +1,169 @@
+use std::sync::Arc;
+
+use anyhow::{ensure, Result};
+use llguidance::toktrie::{self, TokEnv, TokRxInfo, TokTrie, TokenId, TokenizerEnv};
+
+type LlamaTokenizeFn = unsafe extern "C" fn(
+    vocab: *const std::os::raw::c_void,
+    text: *const std::os::raw::c_char,
+    text_len: i32,
+    tokens: *mut i32,
+    n_tokens_max: i32,
+    add_special: bool,
+    parse_special: bool,
+) -> i32;
+
+struct LlamaTokenizer {
+    trie: TokTrie,
+    tokenize_fn: LlamaTokenizeFn,
+    vocab: *const std::os::raw::c_void,
+    sentinel: Option<u8>,
+    sentinel_tokens: Vec<TokenId>,
+}
+// SAFETY: tokenize_fn is required to be thread-safe
+unsafe impl Send for LlamaTokenizer {}
+unsafe impl Sync for LlamaTokenizer {}
+
+impl LlamaTokenizer {
+    fn tokenize_with_sentinel(&self, s: &[u8]) -> Result<Vec<toktrie::TokenId>> {
+        if s.is_empty() {
+            return Ok(vec![]);
+        }
+
+        if let Some(sentinel) = self.sentinel {
+            let mut b = Vec::with_capacity(s.len() + 1);
+            b.push(sentinel);
+            b.extend_from_slice(s);
+            let mut res = self.raw_tokenize(&b);
+            ensure!(
+                res.len() > self.sentinel_tokens.len(),
+                "tokenize_with_sentinel: res.len() <= sentinel_tokens.len()"
+            );
+            ensure!(
+                res[0..self.sentinel_tokens.len()] == self.sentinel_tokens,
+                "tokenize_with_sentinel: res[0..sentinel_tokens.len()] != sentinel_tokens"
+            );
+            res.splice(0..self.sentinel_tokens.len(), []);
+            Ok(res)
+        } else {
+            Ok(self.raw_tokenize(s))
+        }
+    }
+
+    fn raw_tokenize(&self, s: &[u8]) -> Vec<toktrie::TokenId> {
+        let mut res_toks = vec![0u32; s.len() / 4 + 5];
+        let res = unsafe {
+            (self.tokenize_fn)(
+                self.vocab,
+                s.as_ptr() as *const std::os::raw::c_char,
+                s.len().try_into().unwrap(),
+                res_toks.as_mut_ptr() as *mut i32,
+                res_toks.len().try_into().unwrap(),
+                false,
+                false,
+            )
+        };
+
+        let res = if res < 0 {
+            let n_toks = (-res) as usize;
+            res_toks.resize(n_toks, 0);
+            let res2 = unsafe {
+                (self.tokenize_fn)(
+                    self.vocab,
+                    s.as_ptr() as *const std::os::raw::c_char,
+                    s.len().try_into().unwrap(),
+                    res_toks.as_mut_ptr() as *mut i32,
+                    res_toks.len().try_into().unwrap(),
+                    false,
+                    false,
+                )
+            };
+            assert!(res2 == n_toks as i32);
+            res2
+        } else {
+            res
+        };
+
+        res_toks.truncate(res as usize);
+        res_toks
+    }
+}
+
+impl TokenizerEnv for LlamaTokenizer {
+    fn tok_trie(&self) -> &TokTrie {
+        &self.trie
+    }
+
+    fn tokenize_bytes(&self, s: &[u8]) -> Vec<toktrie::TokenId> {
+        // llama.cpp tokenizer encodes invalid UTF8 as Unicode replacement character U+FFFD,
+        // so we need the greedy fallback
+        self.trie.tokenize_with_greedy_fallback(s, |s| {
+            self.tokenize_with_sentinel(s.as_bytes())
+                .expect("tokenize_with_sentinel failed")
+        })
+    }
+}
+
+pub fn tokenv_from_llamacpp(
+    tokens: Vec<Vec<u8>>,
+    vocab_ptr: usize,
+    tokenize_fptr: usize,
+    eos_token: u32,
+) -> Result<TokEnv> {
+    ensure!(vocab_ptr != 0, "vocab_ptr must be non-null");
+    ensure!(tokenize_fptr != 0, "tokenize_fptr must be non-null");
+
+    let info = TokRxInfo::new(tokens.len() as u32, eos_token);
+    let trie = TokTrie::from(&info, &tokens);
+
+    let mut llama_tok = LlamaTokenizer {
+        trie,
+        tokenize_fn: unsafe { std::mem::transmute::<usize, LlamaTokenizeFn>(tokenize_fptr) },
+        vocab: vocab_ptr as *const std::os::raw::c_void,
+        sentinel: None,
+        sentinel_tokens: vec![],
+    };
+
+    let trie = &llama_tok.trie;
+    let t0 = llama_tok.raw_tokenize(b"a");
+    if trie.decode(&t0) != b"a" {
+        // Now, this likely means that the tokenizer is adding a space in front of the token
+        // (or possibly <BOS> token)
+        // We fill "fix" this by tokenizing [sentinel] + s instead of just s
+        // and then removing tokens corresponding to the sentinel
+
+        // find a good sentinel token - one that doesn't start any other token
+        let sentinel = (1u8..32)
+            .find(|&b| {
+                trie.token_id(&[b]).is_some()
+                    && !trie.has_extensions(&[b])
+                    && !trie.has_extensions(&[b' ', b])
+            })
+            .ok_or_else(|| {
+                anyhow::anyhow!("could not find a good sentinel token in the range 1..32")
+            })?;
+
+        llama_tok.sentinel_tokens = llama_tok.raw_tokenize(&[sentinel]);
+        llama_tok.sentinel = Some(sentinel);
+
+        // now, check if it works
+        let t1 = llama_tok.tokenize_with_sentinel(b"a")?;
+        ensure!(
+            trie.decode(&t1) == b"a",
+            "tokenizer is not working with the sentinel {} {:?}",
+            sentinel,
+            trie.decode(&t1)
+        );
+
+        // make sure we can tokenize double-sentinel
+        let t3 = llama_tok.tokenize_with_sentinel(&[sentinel])?;
+        ensure!(
+            trie.decode(&t3) == [sentinel],
+            "tokenizer is not working with the sentinel (rec) {} {:?}",
+            sentinel,
+            trie.decode(&t3)
+        );
+    }
+
+    Ok(Arc::new(llama_tok))
+}

{llguidance-0.7.23 → llguidance-0.7.24}/python_ext/src/py.rs

@@ -14,6 +14,8 @@ use serde::{Deserialize, Serialize};
 use serde_json::Value;
 use toktrie_hf_tokenizers::ByteTokenizer;
 
+use crate::llamatokenizer::tokenv_from_llamacpp;
+
 struct PyTokenizer {
     tok_trie: Arc<toktrie::TokTrie>,
     tokenizer_fun: Py<PyAny>,
@@ -34,78 +36,6 @@ struct PyMidProcessResult {
     temperature: f32,
 }
 
-type LlamaTokenizeFn = unsafe extern "C" fn(
-    vocab: *const std::os::raw::c_void,
-    text: *const std::os::raw::c_char,
-    text_len: i32,
-    tokens: *mut i32,
-    n_tokens_max: i32,
-    add_special: bool,
-    parse_special: bool,
-) -> i32;
-
-struct LlamaTokenizerInner {
-    trie: TokTrie,
-    tokenize_fn: LlamaTokenizeFn,
-    vocab: *const std::os::raw::c_void,
-}
-// SAFETY: tokenize_fn is required to be thread-safe
-unsafe impl Send for LlamaTokenizerInner {}
-unsafe impl Sync for LlamaTokenizerInner {}
-
-impl LlamaTokenizerInner {
-    fn raw_tokenize(&self, s: &[u8]) -> Vec<toktrie::TokenId> {
-        let mut res_toks = vec![0u32; s.len() / 4 + 5];
-        let res = unsafe {
-            (self.tokenize_fn)(
-                self.vocab,
-                s.as_ptr() as *const std::os::raw::c_char,
-                s.len().try_into().unwrap(),
-                res_toks.as_mut_ptr() as *mut i32,
-                res_toks.len().try_into().unwrap(),
-                false,
-                false,
-            )
-        };
-
-        let res = if res < 0 {
-            let n_toks = (-res) as usize;
-            res_toks.resize(n_toks, 0);
-            let res2 = unsafe {
-                (self.tokenize_fn)(
-                    self.vocab,
-                    s.as_ptr() as *const std::os::raw::c_char,
-                    s.len().try_into().unwrap(),
-                    res_toks.as_mut_ptr() as *mut i32,
-                    res_toks.len().try_into().unwrap(),
-                    false,
-                    false,
-                )
-            };
-            assert!(res2 == n_toks as i32);
-            res2
-        } else {
-            res
-        };
-
-        res_toks.truncate(res as usize);
-        res_toks
-    }
-}
-
-impl TokenizerEnv for LlamaTokenizerInner {
-    fn tok_trie(&self) -> &TokTrie {
-        &self.trie
-    }
-
-    fn tokenize_bytes(&self, s: &[u8]) -> Vec<toktrie::TokenId> {
-        // llama.cpp tokenizer encodes invalid UTF8 as Unicode replacement character U+FFFD,
-        // so we need the greedy fallback
-        self.trie
-            .tokenize_with_greedy_fallback(s, |s| self.raw_tokenize(s.as_bytes()))
-    }
-}
-
 #[pymethods]
 impl LLTokenizer {
     #[new]
@@ -154,22 +84,9 @@ impl LLTokenizer {
         eos_token: u32,
         slices: Option<Vec<String>>,
     ) -> PyResult<Self> {
-        if vocab_ptr == 0 {
-            return Err(PyValueError::new_err("vocab_ptr must be non-null"));
-        }
-        if tokenize_fptr == 0 {
-            return Err(PyValueError::new_err("tokenize_fptr must be non-null"));
-        }
+        let tok_env =
+            tokenv_from_llamacpp(tokens, vocab_ptr, tokenize_fptr, eos_token).map_err(val_error)?;
 
-        let info = TokRxInfo::new(tokens.len() as u32, eos_token);
-        let trie = TokTrie::from(&info, &tokens);
-
-        let llama_tok = LlamaTokenizerInner {
-            trie,
-            tokenize_fn: unsafe { std::mem::transmute::<usize, LlamaTokenizeFn>(tokenize_fptr) },
-            vocab: vocab_ptr as *const std::os::raw::c_void,
-        };
-        let tok_env: TokEnv = Arc::new(llama_tok);
         let factory = ParserFactory::new(
             &tok_env,
             InferenceCapabilities::default(),

{llguidance-0.7.23 → llguidance-0.7.24}/toktrie/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "toktrie"
-version = "0.7.23"
+version = "0.7.24"
 edition = "2021"
 license = "MIT"
 description = "LLM Token Trie library"

{llguidance-0.7.23 → llguidance-0.7.24}/toktrie_hf_downloader/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "toktrie_hf_downloader"
-version = "0.7.23"
+version = "0.7.24"
 edition = "2021"
 license = "MIT"
 description = "HuggingFace Hub download library support for toktrie and llguidance"

{llguidance-0.7.23 → llguidance-0.7.24}/toktrie_hf_tokenizers/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "toktrie_hf_tokenizers"
-version = "0.7.23"
+version = "0.7.24"
 edition = "2021"
 license = "MIT"
 description = "HuggingFace tokenizers library support for toktrie and llguidance"