llguidance 0.7.26__tar.gz → 0.7.27__tar.gz

{llguidance-0.7.26 → llguidance-0.7.27}/CHANGELOG.md

@@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. Dates are d
 
 If a release doesn't introduce any interesting changes (build fixes etc.), it's skipped.
 
+#### [0.7.27](https://github.com/guidance-ai/llguidance/compare/v0.7.26...0.7.27) 2025-06-04
+
+- add toktrie_tiktoken and llguidance.tiktoken.lltokenizer_from_encoding [`#154`](https://github.com/guidance-ai/llguidance/issues/154)
+- implement clone on StopController [`#185`](https://github.com/guidance-ai/llguidance/issues/185)
+
 #### [0.7.26](https://github.com/guidance-ai/llguidance/compare/v0.7.25...0.7.26) 2025-05-30
 
 - add support for & and ~ in lark regexes [`96fcee3`](https://github.com/guidance-ai/llguidance/commit/96fcee373697b57bead94d1bc06c17cf1c6134e4)

{llguidance-0.7.26 → llguidance-0.7.27}/Cargo.lock

@@ -135,15 +135,30 @@ version = "0.22.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
 
+[[package]]
+name = "bit-set"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1"
+dependencies = [
+ "bit-vec 0.6.3",
+]
+
 [[package]]
 name = "bit-set"
 version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3"
 dependencies = [
- "bit-vec",
+ "bit-vec 0.8.0",
 ]
 
+[[package]]
+name = "bit-vec"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb"
+
 [[package]]
 name = "bit-vec"
 version = "0.8.0"
@@ -162,6 +177,17 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3eeab4423108c5d7c744f4d234de88d18d636100093ae04caf4825134b9c3a32"
 
+[[package]]
+name = "bstr"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4"
+dependencies = [
+ "memchr",
+ "regex-automata",
+ "serde",
+]
+
 [[package]]
 name = "bumpalo"
 version = "3.17.0"
@@ -492,13 +518,24 @@ version = "0.1.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6"
 
+[[package]]
+name = "fancy-regex"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2"
+dependencies = [
+ "bit-set 0.5.3",
+ "regex-automata",
+ "regex-syntax",
+]
+
 [[package]]
 name = "fancy-regex"
 version = "0.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298"
 dependencies = [
- "bit-set",
+ "bit-set 0.8.0",
  "regex-automata",
  "regex-syntax",
 ]
@@ -1123,7 +1160,7 @@ dependencies = [
  "base64 0.22.1",
  "bytecount",
  "email_address",
- "fancy-regex",
+ "fancy-regex 0.14.0",
  "fraction",
  "idna",
  "itoa",
@@ -1174,7 +1211,7 @@ checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856"
 
 [[package]]
 name = "llguidance"
-version = "0.7.26"
+version = "0.7.27"
 dependencies = [
  "anyhow",
  "derivre",
@@ -1193,7 +1230,7 @@ dependencies = [
 
 [[package]]
 name = "llguidance_py"
-version = "0.7.26"
+version = "0.7.27"
 dependencies = [
  "anyhow",
  "bytemuck",
@@ -1203,6 +1240,7 @@ dependencies = [
  "serde",
  "serde_json",
  "toktrie_hf_tokenizers",
+ "toktrie_tiktoken",
 ]
 
 [[package]]
@@ -1865,6 +1903,12 @@ version = "0.1.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
 
+[[package]]
+name = "rustc-hash"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
+
 [[package]]
 name = "rustix"
 version = "1.0.5"
@@ -2233,6 +2277,21 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "tiktoken-rs"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "25563eeba904d770acf527e8b370fe9a5547bacd20ff84a0b6c3bc41288e5625"
+dependencies = [
+ "anyhow",
+ "base64 0.22.1",
+ "bstr",
+ "fancy-regex 0.13.0",
+ "lazy_static",
+ "regex",
+ "rustc-hash",
+]
+
 [[package]]
 name = "tinystr"
 version = "0.7.6"
@@ -2252,7 +2311,7 @@ dependencies = [
  "aho-corasick",
  "derive_builder",
  "esaxx-rs",
- "fancy-regex",
+ "fancy-regex 0.14.0",
  "getrandom 0.2.15",
  "itertools 0.13.0",
  "lazy_static",
@@ -2336,7 +2395,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie"
-version = "0.7.26"
+version = "0.7.27"
 dependencies = [
  "anyhow",
  "bytemuck",
@@ -2347,7 +2406,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_hf_downloader"
-version = "0.7.26"
+version = "0.7.27"
 dependencies = [
  "anyhow",
  "hf-hub",
@@ -2358,7 +2417,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_hf_tokenizers"
-version = "0.7.26"
+version = "0.7.27"
 dependencies = [
  "anyhow",
  "log",
@@ -2368,6 +2427,18 @@ dependencies = [
  "toktrie",
 ]
 
+[[package]]
+name = "toktrie_tiktoken"
+version = "0.7.27"
+dependencies = [
+ "anyhow",
+ "log",
+ "serde",
+ "serde_json",
+ "tiktoken-rs",
+ "toktrie",
+]
+
 [[package]]
 name = "tower"
 version = "0.5.2"

{llguidance-0.7.26 → llguidance-0.7.27}/Cargo.toml

@@ -7,6 +7,7 @@ members = [
     "toktrie",
     "toktrie_hf_tokenizers",
     "toktrie_hf_downloader",
+    "toktrie_tiktoken",
 ]
 # just exclude python_ext since it doesn't build without maturin
 default-members = [
@@ -16,6 +17,7 @@ default-members = [
     "toktrie",
     "toktrie_hf_tokenizers",
     "toktrie_hf_downloader",
+    "toktrie_tiktoken",
 ]
 resolver = "2"
 
@@ -36,4 +38,5 @@ opt-level = 3
 toktrie = { path = "toktrie" }
 llguidance = { path = "parser" }
 toktrie_hf_tokenizers = { path = "toktrie_hf_tokenizers" }
-toktrie_hf_downloader = { path = "toktrie_hf_downloader" }
+toktrie_hf_downloader = { path = "toktrie_hf_downloader" }
+toktrie_tiktoken = { path = "toktrie_tiktoken" }

{llguidance-0.7.26 → llguidance-0.7.27}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llguidance
-Version: 0.7.26
+Version: 0.7.27
 License-File: LICENSE
 Summary: Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance
 Author: Michal Moskal

{llguidance-0.7.26 → llguidance-0.7.27}/parser/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "llguidance"
-version = "0.7.26"
+version = "0.7.27"
 edition = "2021"
 license = "MIT"
 description = "Super-fast Structured Outputs"

{llguidance-0.7.26 → llguidance-0.7.27}/parser/llguidance.h

@@ -433,6 +433,13 @@ const char *llg_stop_commit_token(struct LlgStopController *stop_ctrl,
                                   size_t *output_len_p,
                                   bool *is_stopped_p);
 
+/**
+ * Clone the stop-sequence controller.
+ * The cloned controller shares (under mutex) regex caches if any, so that
+ * cloning is cheap.
+ */
+struct LlgStopController *llg_clone_stop_controller(const struct LlgStopController *stop_ctrl);
+
 /**
  * Free the stop-sequence controller
  */

{llguidance-0.7.26 → llguidance-0.7.27}/parser/src/earley/parser.rs

@@ -53,6 +53,14 @@ macro_rules! debug {
     }
 }
 
+macro_rules! debug_def {
+    ($s:expr, $($arg:tt)*) => {
+        if cfg!(feature = "logging") && DEBUG && $s.scratch.log_enabled() {
+            eprintln!($($arg)*);
+        }
+    }
+}
+
 macro_rules! item_trace {
     ($($arg:tt)*) => {
         if ITEM_TRACE {
@@ -322,6 +330,8 @@ struct Scratch {
     // mode, which is used for computing the token mask on the
     // pre-lexemes.
     definitive: bool,
+
+    log_override: bool,
 }
 
 #[derive(Clone)]
@@ -488,9 +498,14 @@ impl Scratch {
             items: vec![],
             grammar_stack: vec![],
             definitive: true,
+            log_override: false,
         }
     }
 
+    fn log_enabled(&self) -> bool {
+        self.definitive || self.log_override
+    }
+
     // Set current working Earley to empty set
     // The set backing data is at `pos`
     fn new_row(&mut self, pos: usize) {
@@ -523,7 +538,7 @@ impl Scratch {
     }
 
     fn push_grammar_stack(&mut self, node: GrammarStackNode) {
-        if self.definitive {
+        if self.log_enabled() {
             debug!("push_grammar_stack: {:?}", node);
         }
         let ptr = GrammarStackPtr::new(self.grammar_stack.len());
@@ -543,7 +558,7 @@ impl Scratch {
         } else {
             self.items[self.row_end] = item;
         }
-        if self.definitive {
+        if self.log_enabled() {
             debug!(
                 "      addu: {} ({})",
                 self.item_to_string(self.row_end),
@@ -1003,6 +1018,7 @@ impl ParserState {
     pub fn validate_tokens(&mut self, tokens: &[TokenId]) -> usize {
         self.assert_definitive();
         self.run_speculative("validate_tokens", |state| {
+            state.scratch.log_override = true;
             let mut applied_idx = state.byte_to_token_idx.len();
             let tok_env = state.tok_env.clone();
             let trie = tok_env.tok_trie();
@@ -1497,6 +1513,7 @@ impl ParserState {
         self.scratch.definitive = true;
         self.assert_definitive();
         self.rows_valid_end = self.num_rows();
+        self.scratch.log_override = false; // reset
     }
 
     fn run_speculative<T>(&mut self, lbl: &str, f: impl FnOnce(&mut Self) -> T) -> T {
@@ -1762,14 +1779,13 @@ impl ParserState {
         self.scratch.new_row(items.end);
         self.scratch.push_lexeme_idx = lexeme.idx;
 
-        if self.scratch.definitive {
-            debug!(
-                "  scan: {} at row={} token={}",
-                self.lexer().dbg_lexeme(lexeme),
-                row_idx,
-                self.token_idx,
-            );
-        }
+        debug_def!(
+            self,
+            "  scan: {} at row={} token={}",
+            self.lexer().dbg_lexeme(lexeme),
+            row_idx,
+            self.token_idx,
+        );
 
         // This loop performs the scan inference rule
         // (slide 21 of Kallmeyer 2018).  It is an
@@ -1890,9 +1906,7 @@ impl ParserState {
             let item_idx = agenda_ptr;
             let item = self.scratch.items[agenda_ptr];
             agenda_ptr += 1;
-            if self.scratch.definitive {
-                debug!("    agenda: {}", self.item_to_string(item_idx));
-            }
+            debug_def!(self, "    agenda: {}", self.item_to_string(item_idx));
 
             let rule = item.rhs_ptr();
             let after_dot = self.grammar.sym_idx_dot(rule);
@@ -1988,13 +2002,12 @@ impl ParserState {
                     .start_state(&self.scratch.push_allowed_lexemes)
             };
 
-            if self.scratch.definitive {
-                debug!(
-                    "  push row: {} {:?}",
-                    self.allowed_lexemes_dbg(lex_start),
-                    grammar_id
-                );
-            }
+            debug_def!(
+                self,
+                "  push row: {} {:?}",
+                self.allowed_lexemes_dbg(lex_start),
+                grammar_id
+            );
 
             // Add the working row to the parser state
             let idx = self.num_rows();
@@ -2042,9 +2055,7 @@ impl ParserState {
     }
 
     fn process_max_tokens(&mut self, ptr: GrammarStackPtr, lexeme: &Lexeme) {
-        if self.scratch.definitive {
-            debug!("  process_max_tokens");
-        }
+        debug_def!(self, "  process_max_tokens");
         let curr_idx = self.num_rows();
         let top = &self.scratch.grammar_stack[ptr.as_usize()];
         self.scratch.push_grm_top = top.back_ptr;
@@ -2118,12 +2129,13 @@ impl ParserState {
 
         while grm_stack_top.as_usize() > 0 {
             let grm_top = &self.scratch.grammar_stack[grm_stack_top.as_usize()];
-            if self.scratch.definitive {
-                debug!(
-                    "  pop grammar_stack: top={:?}, curr={:?}, #{}",
-                    grm_top.grammar_id, grammar_ids, self.token_idx
-                );
-            }
+            debug_def!(
+                self,
+                "  pop grammar_stack: top={:?}, curr={:?}, #{}",
+                grm_top.grammar_id,
+                grammar_ids,
+                self.token_idx
+            );
             if grammar_ids.contains(&grm_top.grammar_id) {
                 // token_idx is one behind
                 if grm_top.token_horizon <= self.token_idx as u32 {
@@ -2132,12 +2144,12 @@ impl ParserState {
                     // We only pop one grammar off the stack.
                     // If more grammars have the same token horizon, they will get popped
                     // in the next step - we might overrun a bit.
-                    if self.scratch.definitive {
-                        debug!(
-                            "  hit token limit horizon={} token_idx={}",
-                            grm_top.token_horizon, self.token_idx
-                        );
-                    }
+                    debug_def!(
+                        self,
+                        "  hit token limit horizon={} token_idx={}",
+                        grm_top.token_horizon,
+                        self.token_idx
+                    );
                     max_token_ptr = Some(grm_stack_top);
                 }
                 break;
@@ -2252,13 +2264,14 @@ impl ParserState {
                     .saturating_sub(1);
                 self.row_infos[added_row].start_byte_idx -= new_start;
             }
-            debug!(
-                "lex: re-start {:?} (via {:?}); allowed: {}",
-                no_hidden.lexer_state,
-                transition_byte.map(|b| b as char),
-                self.allowed_lexemes_dbg(added_row_start_state)
-            );
         }
+        debug_def!(
+            self,
+            "lex: re-start {:?} (via {:?}); allowed: {}",
+            no_hidden.lexer_state,
+            transition_byte.map(|b| b as char),
+            self.allowed_lexemes_dbg(added_row_start_state)
+        );
 
         no_hidden
     }
@@ -2277,7 +2290,7 @@ impl ParserState {
 
         let hidden_bytes = lexeme.hidden_bytes();
 
-        let trace_here = self.scratch.definitive;
+        let trace_here = self.scratch.log_enabled();
 
         if trace_here {
             trace!(
@@ -2467,9 +2480,7 @@ impl ParserState {
                         .lexer_mut()
                         .check_for_single_byte_lexeme(no_hidden.lexer_state, b);
                     if let Some(second_lexeme) = single {
-                        if self.scratch.definitive {
-                            debug!("single byte lexeme: {:?}", second_lexeme);
-                        }
+                        debug_def!(self, "single byte lexeme: {:?}", second_lexeme);
                         no_hidden.byte = None;
                         self.lexer_stack.push(no_hidden);
 
@@ -2495,9 +2506,7 @@ impl ParserState {
             }
             true
         } else {
-            if self.scratch.definitive {
-                debug!("  scan failed");
-            }
+            debug_def!(self, "  scan failed");
             false
         }
     }

{llguidance-0.7.26 → llguidance-0.7.27}/parser/src/ffi.rs

@@ -330,6 +330,7 @@ pub struct LlgConstraint {
     last_commit_result: CommitResult,
 }
 
+#[derive(Clone)]
 pub struct LlgStopController {
     stop_controller: StopController,
     last_result: String,
@@ -923,6 +924,16 @@ pub extern "C" fn llg_stop_commit_token(
     stop_ctrl.last_result.as_ptr() as *const c_char
 }
 
+/// Clone the stop-sequence controller.
+/// The cloned controller shares (under mutex) regex caches if any, so that
+/// cloning is cheap.
+#[no_mangle]
+pub extern "C" fn llg_clone_stop_controller(
+    stop_ctrl: &LlgStopController,
+) -> *mut LlgStopController {
+    Box::into_raw(Box::new(stop_ctrl.clone()))
+}
+
 /// Free the stop-sequence controller
 /// # Safety
 /// This function should only be called from C code.

{llguidance-0.7.26 → llguidance-0.7.27}/parser/src/stop_controller.rs

@@ -1,3 +1,5 @@
+use std::sync::{Arc, Mutex};
+
 use anyhow::Result;
 use derivre::{RegexAst, RegexBuilder, StateID};
 use toktrie::{TokEnv, TokTrie, TokenId};
@@ -10,12 +12,14 @@ use crate::{
     },
 };
 
+#[derive(Clone)]
 struct StopRegex {
-    dfa: RegexVec,
+    dfa: Arc<Mutex<RegexVec>>,
     state: StateID,
     initial_state: StateID,
 }
 
+#[derive(Clone)]
 pub struct StopController {
     tok_env: TokEnv,
     is_stopped: bool,
@@ -73,7 +77,7 @@ impl StopController {
             )?;
             let initial_state = dfa.initial_state(&all_regex);
             res.regex = Some(StopRegex {
-                dfa,
+                dfa: Arc::new(Mutex::new(dfa)),
                 state: initial_state,
                 initial_state,
             });
@@ -104,23 +108,24 @@ impl StopController {
                 buf.extend_from_slice(format!("<[{}]>", tok_id).as_bytes());
             } else if let Some(rx) = self.regex.as_mut() {
                 let mut state = rx.state;
+                let mut dfa = rx.dfa.lock().unwrap();
                 for &b in bytes {
                     buf.push(b);
-                    let state2 = rx.dfa.transition(state, b);
+                    let state2 = dfa.transition(state, b);
                     // println!("state: {:?} -{:?}-> {:?}", state, b as char, state2);
                     state = state2;
                     assert!(!state.is_dead());
                     if state.has_lowest_match() {
                         self.is_stopped = true;
                         rx.state = state;
-                        let stop_len = rx.dfa.lookahead_len_for_state(state).unwrap_or(0);
+                        let stop_len = dfa.lookahead_len_for_state(state).unwrap_or(0);
                         buf.truncate(buf.len().saturating_sub(stop_len));
                         return buf;
                     }
                 }
 
                 rx.state = state;
-                let chop = rx.dfa.possible_lookahead_len(state);
+                let chop = dfa.possible_lookahead_len(state);
                 let to_return = buf.len().saturating_sub(chop);
                 // println!("chop: {:?} {}", String::from_utf8_lossy(&buf), chop);
                 let valid_len = valid_utf8_len(&buf[..to_return]);

{llguidance-0.7.26 → llguidance-0.7.27}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "llguidance"
-version = "0.7.26"
+version = "0.7.27"
 description = "Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance"
 requires-python = ">=3.9"
 license = "MIT"

llguidance-0.7.27/python/llguidance/tiktoken.py

@@ -0,0 +1,34 @@
+from typing import List, Optional, TYPE_CHECKING
+
+from ._lib import LLTokenizer
+
+if TYPE_CHECKING:
+    import tiktoken
+
+
+def lltokenizer_from_encoding(
+    encoding: 'tiktoken.Encoding',
+    *,
+    n_vocab: Optional[int] = None,
+    eos_token: Optional[int] = None,
+    slices: Optional[List[str]] = None,
+) -> LLTokenizer:
+    """
+    Create a new tokenizer from a tiktoken Encoding object.
+    This is an expensive operation (~1s), so the result should be cached.
+
+    Args:
+        encoding: tiktoken.Encoding - the encoding object to use
+        n_vocab: int - override the size of the vocabulary
+        eos_token: int - override the EOS token
+        slices: List[str] - configuration for slicer optimization; pass [] to disable,
+            or None to use the default configuration
+    """
+
+    return LLTokenizer.from_tiktoken(  # type: ignore
+        encoder=encoding._mergeable_ranks,
+        special_tokens=encoding._special_tokens,
+        pattern=encoding._pat_str,
+        eos_token=encoding.eot_token if eos_token is None else eos_token,
+        n_vocab=n_vocab,
+        slices=slices)

{llguidance-0.7.26 → llguidance-0.7.27}/python/torch_tests/test_llamacpp.py

@@ -47,3 +47,7 @@ def test_llama_cpp(pytestconfig: Any) -> None:
     assert len(toks0) > 1
     toks2 = llt.tokenize_str("<|eot_id|>", parse_special=True)
     assert len(toks2) == 1
+
+    toks3 = llt.tokenize_str("a<|eot_id|>b", parse_special=True)
+    print(llt.dbg_tokens(toks3))
+    assert len(toks3) == 3

llguidance-0.7.27/python/torch_tests/test_tiktoken.py

@@ -0,0 +1,30 @@
+import llguidance.tiktoken
+import tiktoken
+
+
+def test_tiktoken() -> None:
+    enc = tiktoken.get_encoding("o200k_base")
+    llt = llguidance.tiktoken.lltokenizer_from_encoding(enc)
+    for s in [
+            "Hello world!", "Hello world! こんにちは世界！", "wave 👋", "heart 👋💖",
+            "1`a`b`c`d`e`f`g`h`i"
+    ]:
+        toks = llt.tokenize_str(s)
+        print(llt.dbg_tokens(toks))
+        assert llt.decode_str(toks) == s
+    toks = llt.tokenize_bytes(b"\x8b")
+    print(llt.dbg_tokens(toks))
+    print(toks)
+    assert len(toks) == 1
+    assert llt.decode_bytes(toks) == b"\x8b"
+
+    toks1 = llt.tokenize_str("<|endoftext|>")
+    toks0 = llt.tokenize_str("<|endoftext|>", parse_special=False)
+    assert toks1 == toks0
+    assert len(toks0) > 1
+    toks2 = llt.tokenize_str("<|endoftext|>", parse_special=True)
+    assert len(toks2) == 1
+
+    toks3 = llt.tokenize_str("a<|endoftext|>b", parse_special=True)
+    print(llt.dbg_tokens(toks3))
+    assert len(toks3) == 3

{llguidance-0.7.26 → llguidance-0.7.27}/python_ext/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "llguidance_py"
-version = "0.7.26"
+version = "0.7.27"
 edition = "2021"
 license = "MIT"
 description = "Super-fast Structured Outputs"
@@ -9,6 +9,7 @@ repository = "https://github.com/guidance-ai/llguidance"
 [dependencies]
 llguidance = { workspace = true }
 toktrie_hf_tokenizers = { workspace = true }
+toktrie_tiktoken = { workspace = true }
 bytemuck = "1.21.0"
 pyo3 = {version = "0.24.1", features = ["extension-module", "abi3-py39", "anyhow"]}
 serde = { version = "1.0.217", features = ["derive"] }