llguidance 0.7.22__tar.gz → 0.7.24__tar.gz

{llguidance-0.7.22 → llguidance-0.7.24}/CHANGELOG.md

@@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file. Dates are d
 
 If a release doesn't introduce any interesting changes (build fixes etc.), it's skipped.
 
+#### [0.7.24](https://github.com/guidance-ai/llguidance/compare/v0.7.23...0.7.24) 2025-05-23
+
+- add the sentinel token hack, fixes #180 [`#180`](https://github.com/guidance-ai/llguidance/issues/180)
+
+#### [0.7.23](https://github.com/guidance-ai/llguidance/compare/v0.7.22...0.7.23) 2025-05-22
+
+- native llama.cpp tokenizer support [`#179`](https://github.com/guidance-ai/llguidance/pull/179)
+- improve special token detection in HF tokenizers [`6cae393`](https://github.com/guidance-ai/llguidance/commit/6cae393b9c04fe67621615ff22b46beab512d069)
+
 #### [0.7.22](https://github.com/guidance-ai/llguidance/compare/v0.7.21...0.7.22) 2025-05-21
 
 - Keep EOS token bytes in `TokenizerWrapper` [`#178`](https://github.com/guidance-ai/llguidance/pull/178)

{llguidance-0.7.22 → llguidance-0.7.24}/Cargo.lock

@@ -1174,7 +1174,7 @@ checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856"
 
 [[package]]
 name = "llguidance"
-version = "0.7.22"
+version = "0.7.24"
 dependencies = [
  "anyhow",
  "derivre",
@@ -1193,7 +1193,7 @@ dependencies = [
 
 [[package]]
 name = "llguidance_py"
-version = "0.7.22"
+version = "0.7.24"
 dependencies = [
  "anyhow",
  "bytemuck",
@@ -2336,7 +2336,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie"
-version = "0.7.22"
+version = "0.7.24"
 dependencies = [
  "anyhow",
  "bytemuck",
@@ -2347,7 +2347,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_hf_downloader"
-version = "0.7.22"
+version = "0.7.24"
 dependencies = [
  "anyhow",
  "hf-hub",
@@ -2358,7 +2358,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_hf_tokenizers"
-version = "0.7.22"
+version = "0.7.24"
 dependencies = [
  "anyhow",
  "log",

{llguidance-0.7.22 → llguidance-0.7.24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llguidance
-Version: 0.7.22
+Version: 0.7.24
 License-File: LICENSE
 Summary: Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance
 Author: Michal Moskal

{llguidance-0.7.22 → llguidance-0.7.24}/parser/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "llguidance"
-version = "0.7.22"
+version = "0.7.24"
 edition = "2021"
 license = "MIT"
 description = "Super-fast Structured Outputs"

{llguidance-0.7.22 → llguidance-0.7.24}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "llguidance"
-version = "0.7.22"
+version = "0.7.24"
 description = "Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance"
 requires-python = ">=3.9"
 license = "MIT"

llguidance-0.7.24/python/llguidance/llamacpp.py

@@ -0,0 +1,62 @@
+from typing import List, Optional
+
+from ._lib import LLTokenizer
+
+import llama_cpp
+import ctypes
+
+def lltokenizer_from_vocab(
+    vocab: llama_cpp.llama_vocab_p,
+    n_vocab: Optional[int] = None,
+    eos_token: Optional[int] = None,
+    slices: Optional[List[str]] = None,
+) -> LLTokenizer:
+    """
+    Create a new tokenizer from a llama.cpp vocab object.
+    This is an expensive operation (~1s), so the result should be cached.
+
+    Args:
+        vocab: llama_cpp.llama_vocab_p - the vocab object to use
+        n_vocab: int - override the size of the vocabulary
+        eos_token: int - override the EOS token
+        slices: List[str] - configuration for slicer optimization; pass [] to disable,
+            or None to use the default configuration
+    """
+
+    ntok = llama_cpp.llama_vocab_n_tokens(vocab)
+    if eos_token is None:
+        eos_token = llama_cpp.llama_vocab_eos(vocab)
+    buffer_len = 16 * 1024
+    buffer = ctypes.create_string_buffer(buffer_len + 1)
+    tokens: List[bytes] = []
+
+    for token in range(ntok):
+        n = llama_cpp.llama_token_to_piece(
+            vocab,
+            token,
+            buffer,
+            buffer_len,
+            0,
+            True
+        )
+        if n < 0:
+            raise ValueError(f"Error writing token {token} to buffer of size {buffer_len}. Error: {n}")
+        assert n <= buffer_len
+        tok = bytes(buffer[:n]) # type: ignore
+        attr = llama_cpp.llama_token_get_attr(vocab, token)
+        if attr & llama_cpp.LLAMA_TOKEN_ATTR_CONTROL:
+            tok = b"\xFF" + tok
+        tokens.append(tok)
+
+    if n_vocab is not None:
+        while len(tokens) < n_vocab:
+            tokens.append(b"")
+
+    fptr = ctypes.cast(llama_cpp.llama_cpp._lib.llama_tokenize, ctypes.c_void_p).value
+    return LLTokenizer.from_llamacpp( # type: ignore
+        tokens=tokens,
+        vocab_ptr=vocab,
+        tokenize_fptr=fptr,
+        eos_token=eos_token,
+        slices=slices
+    )

{llguidance-0.7.22 → llguidance-0.7.24}/python/torch_tests/test_hf.py

@@ -21,7 +21,8 @@ from transformers import AutoTokenizer
 
 
 def _build_tokenizer() -> LLTokenizer:
-    hf_tok = AutoTokenizer.from_pretrained("unsloth/Meta-Llama-3.1-8B-Instruct")
+    hf_tok = AutoTokenizer.from_pretrained(
+        "unsloth/Meta-Llama-3.1-8B-Instruct")
     return llguidance.hf.from_tokenizer(hf_tok)
 
 
@@ -41,6 +42,22 @@ def lark_matcher(grm: str) -> LLMatcher:
     return interp
 
 
+def test_basic_tokenizer() -> None:
+    llt = tokenizer()
+    for s in [
+            "Hello world!", "Hello world! こんにちは世界！", "wave 👋", "heart 👋💖",
+            "1`a`b`c`d`e`f`g`h`i"
+    ]:
+        toks = llt.tokenize_str(s)
+        print(llt.dbg_tokens(toks))
+        assert llt.decode_str(toks) == s
+    toks = llt.tokenize_bytes(b"\x8b")
+    print(llt.dbg_tokens(toks))
+    print(toks)
+    assert len(toks) == 1
+    assert llt.decode_bytes(toks) == b"\x8b"
+
+
 def test_grammar() -> None:
     t = tokenizer()
     mask = allocate_token_bitmask(2, t.vocab_size)
@@ -66,7 +83,8 @@ def test_grammar() -> None:
 def test_par_grammar() -> None:
     n_gram = 50
     t = tokenizer()
-    grammars = [(lark_matcher(r"start: /[a-zA-Z ]*/"), idx) for idx in range(n_gram)]
+    grammars = [(lark_matcher(r"start: /[a-zA-Z ]*/"), idx)
+                for idx in range(n_gram)]
     mask = allocate_token_bitmask(n_gram, t.vocab_size)
     mask2 = allocate_token_bitmask(n_gram, t.vocab_size)
     exec = LLExecutor()
@@ -88,16 +106,15 @@ def test_tokenize_partial_basic(recent_tokens: List[int]) -> None:
     """Test tokenize_partial with a simple sentence."""
     ll_tok = tokenizer()
     assert ll_tok.is_canonical
-    new_tokens, leftover = ll_tok.tokenize_partial(
-        b" How are you", recent_tokens=recent_tokens
-    )
+    new_tokens, leftover = ll_tok.tokenize_partial(b" How are you",
+                                                   recent_tokens=recent_tokens)
     assert isinstance(new_tokens, list)
     assert isinstance(leftover, bytes)
     assert len(new_tokens) >= 2
     assert ll_tok.decode_bytes(new_tokens) + leftover == b" How are you"
     for suff in ["", "r", "!", " "]:
         tok2 = ll_tok.tokenize_str(" How are you" + suff)
-        assert tok2[0 : len(new_tokens)] == new_tokens
+        assert tok2[0:len(new_tokens)] == new_tokens
 
 
 def test_tokenize_partial_docs() -> None:
@@ -107,16 +124,16 @@ def test_tokenize_partial_docs() -> None:
     assert leftover == b"order"
 
     recent = ll.tokenize_bytes(b'{"')
-    new_tok, leftover = ll.tokenize_partial(
-        b'name_of_the_person"', recent_tokens=recent
-    )
+    new_tok, leftover = ll.tokenize_partial(b'name_of_the_person"',
+                                            recent_tokens=recent)
     print(ll.dbg_tokens(new_tok))
     assert leftover == b'"'
     assert ll.decode_str(new_tok) == "name_of_the_person"
 
 
 def test_incomplete_tokenizer() -> None:
-    hf_tok = AutoTokenizer.from_pretrained("HuggingFaceTB/SmolLM-135M-Instruct")
+    hf_tok = AutoTokenizer.from_pretrained(
+        "HuggingFaceTB/SmolLM-135M-Instruct")
     ll_tok = llguidance.hf.from_tokenizer(hf_tok)
 
     # unknown bytes are to be skipped

llguidance-0.7.24/python/torch_tests/test_llamacpp.py

@@ -0,0 +1,42 @@
+import llguidance.llamacpp
+import llama_cpp
+import os
+import requests # type: ignore
+from typing import Any
+
+def get_llama_vocab_file(pytestconfig: Any) -> str:
+    url = "https://raw.githubusercontent.com/ggml-org/llama.cpp/f4ab2a41476600a98067a9474ea8f9e6db41bcfa/models/ggml-vocab-llama-bpe.gguf"
+    cache_dir = pytestconfig.cache.makedir("llama_vocab")
+    file_name = "vocab.gguf"
+    file_path = os.path.join(cache_dir, file_name)
+
+    if not os.path.exists(file_path):
+        r = requests.get(url)
+        r.raise_for_status()
+        with open(file_path, "wb") as f:
+            f.write(r.content)
+
+    return file_path
+
+
+def test_llama_cpp(pytestconfig: Any) -> None:
+    filepath = get_llama_vocab_file(pytestconfig)
+    p = llama_cpp.llama_model_default_params()
+    p.vocab_only = True
+    model = llama_cpp.llama_model_load_from_file(filepath.encode(), p)
+    assert model is not None
+    vocab = llama_cpp.llama_model_get_vocab(model)
+    assert vocab is not None
+    llt = llguidance.llamacpp.lltokenizer_from_vocab(vocab)
+    for s in [
+            "Hello world!", "Hello world! こんにちは世界！", "wave 👋", "heart 👋💖",
+            "1`a`b`c`d`e`f`g`h`i"
+    ]:
+        toks = llt.tokenize_str(s)
+        print(llt.dbg_tokens(toks))
+        assert llt.decode_str(toks) == s
+    toks = llt.tokenize_bytes(b"\x8b")
+    print(llt.dbg_tokens(toks))
+    print(toks)
+    assert len(toks) == 1
+    assert llt.decode_bytes(toks) == b"\x8b"

{llguidance-0.7.22 → llguidance-0.7.24}/python_ext/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "llguidance_py"
-version = "0.7.22"
+version = "0.7.24"
 edition = "2021"
 license = "MIT"
 description = "Super-fast Structured Outputs"

{llguidance-0.7.22 → llguidance-0.7.24}/python_ext/src/lib.rs

@@ -1,5 +1,6 @@
 use pyo3::prelude::*;
 
+mod llamatokenizer;
 mod llinterpreter;
 mod llmatcher;
 mod parserlimits;

llguidance-0.7.24/python_ext/src/llamatokenizer.rs

@@ -0,0 +1,169 @@
+use std::sync::Arc;
+
+use anyhow::{ensure, Result};
+use llguidance::toktrie::{self, TokEnv, TokRxInfo, TokTrie, TokenId, TokenizerEnv};
+
+type LlamaTokenizeFn = unsafe extern "C" fn(
+    vocab: *const std::os::raw::c_void,
+    text: *const std::os::raw::c_char,
+    text_len: i32,
+    tokens: *mut i32,
+    n_tokens_max: i32,
+    add_special: bool,
+    parse_special: bool,
+) -> i32;
+
+struct LlamaTokenizer {
+    trie: TokTrie,
+    tokenize_fn: LlamaTokenizeFn,
+    vocab: *const std::os::raw::c_void,
+    sentinel: Option<u8>,
+    sentinel_tokens: Vec<TokenId>,
+}
+// SAFETY: tokenize_fn is required to be thread-safe
+unsafe impl Send for LlamaTokenizer {}
+unsafe impl Sync for LlamaTokenizer {}
+
+impl LlamaTokenizer {
+    fn tokenize_with_sentinel(&self, s: &[u8]) -> Result<Vec<toktrie::TokenId>> {
+        if s.is_empty() {
+            return Ok(vec![]);
+        }
+
+        if let Some(sentinel) = self.sentinel {
+            let mut b = Vec::with_capacity(s.len() + 1);
+            b.push(sentinel);
+            b.extend_from_slice(s);
+            let mut res = self.raw_tokenize(&b);
+            ensure!(
+                res.len() > self.sentinel_tokens.len(),
+                "tokenize_with_sentinel: res.len() <= sentinel_tokens.len()"
+            );
+            ensure!(
+                res[0..self.sentinel_tokens.len()] == self.sentinel_tokens,
+                "tokenize_with_sentinel: res[0..sentinel_tokens.len()] != sentinel_tokens"
+            );
+            res.splice(0..self.sentinel_tokens.len(), []);
+            Ok(res)
+        } else {
+            Ok(self.raw_tokenize(s))
+        }
+    }
+
+    fn raw_tokenize(&self, s: &[u8]) -> Vec<toktrie::TokenId> {
+        let mut res_toks = vec![0u32; s.len() / 4 + 5];
+        let res = unsafe {
+            (self.tokenize_fn)(
+                self.vocab,
+                s.as_ptr() as *const std::os::raw::c_char,
+                s.len().try_into().unwrap(),
+                res_toks.as_mut_ptr() as *mut i32,
+                res_toks.len().try_into().unwrap(),
+                false,
+                false,
+            )
+        };
+
+        let res = if res < 0 {
+            let n_toks = (-res) as usize;
+            res_toks.resize(n_toks, 0);
+            let res2 = unsafe {
+                (self.tokenize_fn)(
+                    self.vocab,
+                    s.as_ptr() as *const std::os::raw::c_char,
+                    s.len().try_into().unwrap(),
+                    res_toks.as_mut_ptr() as *mut i32,
+                    res_toks.len().try_into().unwrap(),
+                    false,
+                    false,
+                )
+            };
+            assert!(res2 == n_toks as i32);
+            res2
+        } else {
+            res
+        };
+
+        res_toks.truncate(res as usize);
+        res_toks
+    }
+}
+
+impl TokenizerEnv for LlamaTokenizer {
+    fn tok_trie(&self) -> &TokTrie {
+        &self.trie
+    }
+
+    fn tokenize_bytes(&self, s: &[u8]) -> Vec<toktrie::TokenId> {
+        // llama.cpp tokenizer encodes invalid UTF8 as Unicode replacement character U+FFFD,
+        // so we need the greedy fallback
+        self.trie.tokenize_with_greedy_fallback(s, |s| {
+            self.tokenize_with_sentinel(s.as_bytes())
+                .expect("tokenize_with_sentinel failed")
+        })
+    }
+}
+
+pub fn tokenv_from_llamacpp(
+    tokens: Vec<Vec<u8>>,
+    vocab_ptr: usize,
+    tokenize_fptr: usize,
+    eos_token: u32,
+) -> Result<TokEnv> {
+    ensure!(vocab_ptr != 0, "vocab_ptr must be non-null");
+    ensure!(tokenize_fptr != 0, "tokenize_fptr must be non-null");
+
+    let info = TokRxInfo::new(tokens.len() as u32, eos_token);
+    let trie = TokTrie::from(&info, &tokens);
+
+    let mut llama_tok = LlamaTokenizer {
+        trie,
+        tokenize_fn: unsafe { std::mem::transmute::<usize, LlamaTokenizeFn>(tokenize_fptr) },
+        vocab: vocab_ptr as *const std::os::raw::c_void,
+        sentinel: None,
+        sentinel_tokens: vec![],
+    };
+
+    let trie = &llama_tok.trie;
+    let t0 = llama_tok.raw_tokenize(b"a");
+    if trie.decode(&t0) != b"a" {
+        // Now, this likely means that the tokenizer is adding a space in front of the token
+        // (or possibly <BOS> token)
+        // We fill "fix" this by tokenizing [sentinel] + s instead of just s
+        // and then removing tokens corresponding to the sentinel
+
+        // find a good sentinel token - one that doesn't start any other token
+        let sentinel = (1u8..32)
+            .find(|&b| {
+                trie.token_id(&[b]).is_some()
+                    && !trie.has_extensions(&[b])
+                    && !trie.has_extensions(&[b' ', b])
+            })
+            .ok_or_else(|| {
+                anyhow::anyhow!("could not find a good sentinel token in the range 1..32")
+            })?;
+
+        llama_tok.sentinel_tokens = llama_tok.raw_tokenize(&[sentinel]);
+        llama_tok.sentinel = Some(sentinel);
+
+        // now, check if it works
+        let t1 = llama_tok.tokenize_with_sentinel(b"a")?;
+        ensure!(
+            trie.decode(&t1) == b"a",
+            "tokenizer is not working with the sentinel {} {:?}",
+            sentinel,
+            trie.decode(&t1)
+        );
+
+        // make sure we can tokenize double-sentinel
+        let t3 = llama_tok.tokenize_with_sentinel(&[sentinel])?;
+        ensure!(
+            trie.decode(&t3) == [sentinel],
+            "tokenizer is not working with the sentinel (rec) {} {:?}",
+            sentinel,
+            trie.decode(&t3)
+        );
+    }
+
+    Ok(Arc::new(llama_tok))
+}

{llguidance-0.7.22 → llguidance-0.7.24}/python_ext/src/py.rs

@@ -14,6 +14,8 @@ use serde::{Deserialize, Serialize};
 use serde_json::Value;
 use toktrie_hf_tokenizers::ByteTokenizer;
 
+use crate::llamatokenizer::tokenv_from_llamacpp;
+
 struct PyTokenizer {
     tok_trie: Arc<toktrie::TokTrie>,
     tokenizer_fun: Py<PyAny>,
@@ -73,6 +75,29 @@ impl LLTokenizer {
         })
     }
 
+    #[staticmethod]
+    #[pyo3(signature = (*, tokens, vocab_ptr, tokenize_fptr, eos_token, slices=None))]
+    fn from_llamacpp(
+        tokens: Vec<Vec<u8>>,
+        vocab_ptr: usize,
+        tokenize_fptr: usize,
+        eos_token: u32,
+        slices: Option<Vec<String>>,
+    ) -> PyResult<Self> {
+        let tok_env =
+            tokenv_from_llamacpp(tokens, vocab_ptr, tokenize_fptr, eos_token).map_err(val_error)?;
+
+        let factory = ParserFactory::new(
+            &tok_env,
+            InferenceCapabilities::default(),
+            &slices.unwrap_or_else(SlicedBiasComputer::general_slices),
+        )
+        .map_err(val_error)?;
+        Ok(LLTokenizer {
+            factory: Arc::new(factory),
+        })
+    }
+
     fn with_slices(&self, slices: Vec<String>) -> PyResult<Self> {
         let factory = self.factory.with_slices(&slices)?;
         Ok(LLTokenizer {

{llguidance-0.7.22 → llguidance-0.7.24}/scripts/install-deps.sh

@@ -2,7 +2,7 @@
 
 # installing guidance for deps
 pip install pytest guidance huggingface_hub tokenizers jsonschema maturin[zig] \
-    torch transformers==4.52.1 bitsandbytes ipython psutil mypy
+    torch transformers==4.52.1 bitsandbytes ipython psutil mypy llama_cpp_python
 pip uninstall -y guidance
 
 # print out versions

{llguidance-0.7.22 → llguidance-0.7.24}/toktrie/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "toktrie"
-version = "0.7.22"
+version = "0.7.24"
 edition = "2021"
 license = "MIT"
 description = "LLM Token Trie library"

{llguidance-0.7.22 → llguidance-0.7.24}/toktrie_hf_downloader/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "toktrie_hf_downloader"
-version = "0.7.22"
+version = "0.7.24"
 edition = "2021"
 license = "MIT"
 description = "HuggingFace Hub download library support for toktrie and llguidance"

{llguidance-0.7.22 → llguidance-0.7.24}/toktrie_hf_tokenizers/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "toktrie_hf_tokenizers"
-version = "0.7.22"
+version = "0.7.24"
 edition = "2021"
 license = "MIT"
 description = "HuggingFace tokenizers library support for toktrie and llguidance"

{llguidance-0.7.22 → llguidance-0.7.24}/toktrie_hf_tokenizers/src/lib.rs

@@ -102,7 +102,11 @@ impl ByteTokenizer {
         }
 
         let vocab_size = hft.get_vocab_size(true) as u32;
-        let added = hft.get_added_tokens_decoder();
+        let mut added = hft
+            .get_added_tokens_decoder()
+            .into_iter()
+            .collect::<Vec<_>>();
+        added.sort_by_key(|(id, _)| *id);
 
         let mut res = ByteTokenizer {
             hf_model: "foobar".to_string(),
@@ -114,7 +118,8 @@ impl ByteTokenizer {
         let mut specials = HashSet::new();
 
         for (id, info) in added.iter() {
-            if info.special {
+            // we treat all added tokens of the form <...> as special tokens
+            if info.special || (info.content.starts_with("<") && info.content.ends_with(">")) {
                 match info.content.as_str() {
                     "</s>"
                     | "<|endoftext|>"