llguidance 0.7.25__tar.gz → 0.7.27__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (182) hide show
  1. {llguidance-0.7.25 → llguidance-0.7.27}/CHANGELOG.md +12 -0
  2. {llguidance-0.7.25 → llguidance-0.7.27}/Cargo.lock +80 -9
  3. {llguidance-0.7.25 → llguidance-0.7.27}/Cargo.toml +4 -1
  4. {llguidance-0.7.25 → llguidance-0.7.27}/PKG-INFO +2 -4
  5. {llguidance-0.7.25 → llguidance-0.7.27}/README.md +1 -3
  6. {llguidance-0.7.25 → llguidance-0.7.27}/docs/optimizations.md +35 -33
  7. {llguidance-0.7.25 → llguidance-0.7.27}/docs/syntax.md +22 -6
  8. {llguidance-0.7.25 → llguidance-0.7.27}/parser/Cargo.toml +1 -1
  9. {llguidance-0.7.25 → llguidance-0.7.27}/parser/llguidance.h +7 -0
  10. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/earley/parser.rs +63 -50
  11. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/ffi.rs +11 -0
  12. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/grammar_builder.rs +3 -0
  13. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/lark/ast.rs +18 -5
  14. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/lark/compiler.rs +30 -10
  15. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/lark/lexer.rs +2 -0
  16. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/lark/parser.rs +27 -7
  17. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/stop_controller.rs +10 -5
  18. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/tokenparser.rs +7 -1
  19. {llguidance-0.7.25 → llguidance-0.7.27}/pyproject.toml +1 -1
  20. llguidance-0.7.27/python/llguidance/tiktoken.py +34 -0
  21. {llguidance-0.7.25 → llguidance-0.7.27}/python/torch_tests/test_llamacpp.py +4 -0
  22. llguidance-0.7.27/python/torch_tests/test_tiktoken.py +30 -0
  23. {llguidance-0.7.25 → llguidance-0.7.27}/python_ext/Cargo.toml +2 -1
  24. {llguidance-0.7.25 → llguidance-0.7.27}/python_ext/src/llinterpreter.rs +2 -1
  25. {llguidance-0.7.25 → llguidance-0.7.27}/python_ext/src/llmatcher.rs +17 -8
  26. {llguidance-0.7.25 → llguidance-0.7.27}/python_ext/src/py.rs +36 -1
  27. {llguidance-0.7.25 → llguidance-0.7.27}/sample_parser/tests/test_lark.rs +134 -0
  28. {llguidance-0.7.25 → llguidance-0.7.27}/sample_parser/tests/test_ll.rs +22 -0
  29. {llguidance-0.7.25 → llguidance-0.7.27}/scripts/bump.py +1 -1
  30. {llguidance-0.7.25 → llguidance-0.7.27}/scripts/ci-publish.py +18 -16
  31. {llguidance-0.7.25 → llguidance-0.7.27}/toktrie/Cargo.toml +1 -1
  32. {llguidance-0.7.25 → llguidance-0.7.27}/toktrie/src/toktree.rs +62 -0
  33. {llguidance-0.7.25 → llguidance-0.7.27}/toktrie_hf_downloader/Cargo.toml +1 -1
  34. {llguidance-0.7.25 → llguidance-0.7.27}/toktrie_hf_tokenizers/Cargo.toml +1 -1
  35. llguidance-0.7.27/toktrie_tiktoken/Cargo.toml +15 -0
  36. llguidance-0.7.27/toktrie_tiktoken/LICENSE +21 -0
  37. llguidance-0.7.27/toktrie_tiktoken/src/lib.rs +103 -0
  38. {llguidance-0.7.25 → llguidance-0.7.27}/.github/workflows/rust.yml +0 -0
  39. {llguidance-0.7.25 → llguidance-0.7.27}/.github/workflows/wheels.yml +0 -0
  40. {llguidance-0.7.25 → llguidance-0.7.27}/.gitignore +0 -0
  41. {llguidance-0.7.25 → llguidance-0.7.27}/CODE_OF_CONDUCT.md +0 -0
  42. {llguidance-0.7.25 → llguidance-0.7.27}/LICENSE +0 -0
  43. {llguidance-0.7.25 → llguidance-0.7.27}/SECURITY.md +0 -0
  44. {llguidance-0.7.25 → llguidance-0.7.27}/SUPPORT.md +0 -0
  45. {llguidance-0.7.25 → llguidance-0.7.27}/c_sample/Makefile +0 -0
  46. {llguidance-0.7.25 → llguidance-0.7.27}/c_sample/README.md +0 -0
  47. {llguidance-0.7.25 → llguidance-0.7.27}/c_sample/c_sample.cpp +0 -0
  48. {llguidance-0.7.25 → llguidance-0.7.27}/docs/fast_forward.md +0 -0
  49. {llguidance-0.7.25 → llguidance-0.7.27}/docs/json_schema.md +0 -0
  50. {llguidance-0.7.25 → llguidance-0.7.27}/docs/mask_plot.png +0 -0
  51. {llguidance-0.7.25 → llguidance-0.7.27}/docs/special_tokens.md +0 -0
  52. {llguidance-0.7.25 → llguidance-0.7.27}/docs/toktrie.md +0 -0
  53. {llguidance-0.7.25 → llguidance-0.7.27}/json_stats/Cargo.toml +0 -0
  54. {llguidance-0.7.25 → llguidance-0.7.27}/json_stats/expected_maskbench.json +0 -0
  55. {llguidance-0.7.25 → llguidance-0.7.27}/json_stats/jstats.sh +0 -0
  56. {llguidance-0.7.25 → llguidance-0.7.27}/json_stats/scripts/split-stats.sh +0 -0
  57. {llguidance-0.7.25 → llguidance-0.7.27}/json_stats/scripts/split_plot.py +0 -0
  58. {llguidance-0.7.25 → llguidance-0.7.27}/json_stats/src/json_stats.rs +0 -0
  59. {llguidance-0.7.25 → llguidance-0.7.27}/json_stats/src/lib.rs +0 -0
  60. {llguidance-0.7.25 → llguidance-0.7.27}/json_stats/src/stats.rs +0 -0
  61. {llguidance-0.7.25 → llguidance-0.7.27}/parser/LICENSE +0 -0
  62. {llguidance-0.7.25 → llguidance-0.7.27}/parser/README.md +0 -0
  63. {llguidance-0.7.25 → llguidance-0.7.27}/parser/build.rs +0 -0
  64. {llguidance-0.7.25 → llguidance-0.7.27}/parser/cbindgen.toml +0 -0
  65. {llguidance-0.7.25 → llguidance-0.7.27}/parser/grammars/character.json +0 -0
  66. {llguidance-0.7.25 → llguidance-0.7.27}/parser/grammars/json.json +0 -0
  67. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/api.rs +0 -0
  68. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/constraint.rs +0 -0
  69. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/earley/from_guidance.rs +0 -0
  70. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/earley/grammar.rs +0 -0
  71. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/earley/lexer.rs +0 -0
  72. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/earley/lexerspec.rs +0 -0
  73. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/earley/mod.rs +0 -0
  74. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/earley/perf.rs +0 -0
  75. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/earley/regexvec.rs +0 -0
  76. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/earley/slicer.rs +0 -0
  77. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/factory.rs +0 -0
  78. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/ffi_par.rs +0 -0
  79. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/json/README.md +0 -0
  80. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/json/compiler.rs +0 -0
  81. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/json/context_ref.rs +0 -0
  82. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/json/context_simple/context.rs +0 -0
  83. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/json/context_simple/draft.rs +0 -0
  84. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/json/context_simple/mod.rs +0 -0
  85. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/json/formats.rs +0 -0
  86. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/json/mod.rs +0 -0
  87. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/json/numeric.rs +0 -0
  88. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/json/schema.rs +0 -0
  89. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/json/shared_context.rs +0 -0
  90. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/json_validation.rs +0 -0
  91. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/lark/README.md +0 -0
  92. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/lark/common.rs +0 -0
  93. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/lark/mod.rs +0 -0
  94. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/lib.rs +0 -0
  95. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/logging.rs +0 -0
  96. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/matcher.rs +0 -0
  97. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/output.rs +0 -0
  98. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/panic_utils.rs +0 -0
  99. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/regex_rewrite.rs +0 -0
  100. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/substring.rs +0 -0
  101. {llguidance-0.7.25 → llguidance-0.7.27}/parser/src/tokenizer_json.rs +0 -0
  102. {llguidance-0.7.25 → llguidance-0.7.27}/plan.md +0 -0
  103. {llguidance-0.7.25 → llguidance-0.7.27}/python/llguidance/__init__.py +0 -0
  104. {llguidance-0.7.25 → llguidance-0.7.27}/python/llguidance/_grammar_from.py +0 -0
  105. {llguidance-0.7.25 → llguidance-0.7.27}/python/llguidance/_lib.pyi +0 -0
  106. {llguidance-0.7.25 → llguidance-0.7.27}/python/llguidance/_struct_tag.py +0 -0
  107. {llguidance-0.7.25 → llguidance-0.7.27}/python/llguidance/_tokenizer.py +0 -0
  108. {llguidance-0.7.25 → llguidance-0.7.27}/python/llguidance/_util.py +0 -0
  109. {llguidance-0.7.25 → llguidance-0.7.27}/python/llguidance/cli.py +0 -0
  110. {llguidance-0.7.25 → llguidance-0.7.27}/python/llguidance/gbnf_to_lark.py +0 -0
  111. {llguidance-0.7.25 → llguidance-0.7.27}/python/llguidance/hf.py +0 -0
  112. {llguidance-0.7.25 → llguidance-0.7.27}/python/llguidance/llamacpp.py +0 -0
  113. {llguidance-0.7.25 → llguidance-0.7.27}/python/llguidance/mlx.py +0 -0
  114. {llguidance-0.7.25 → llguidance-0.7.27}/python/llguidance/numpy.py +0 -0
  115. {llguidance-0.7.25 → llguidance-0.7.27}/python/llguidance/py.typed +0 -0
  116. {llguidance-0.7.25 → llguidance-0.7.27}/python/llguidance/torch.py +0 -0
  117. {llguidance-0.7.25 → llguidance-0.7.27}/python/mypy.ini +0 -0
  118. {llguidance-0.7.25 → llguidance-0.7.27}/python/torch_tests/__init__.py +0 -0
  119. {llguidance-0.7.25 → llguidance-0.7.27}/python/torch_tests/test_bitmask.py +0 -0
  120. {llguidance-0.7.25 → llguidance-0.7.27}/python/torch_tests/test_hf.py +0 -0
  121. {llguidance-0.7.25 → llguidance-0.7.27}/python/torch_tests/test_matcher.py +0 -0
  122. {llguidance-0.7.25 → llguidance-0.7.27}/python_ext/src/lib.rs +0 -0
  123. {llguidance-0.7.25 → llguidance-0.7.27}/python_ext/src/llamatokenizer.rs +0 -0
  124. {llguidance-0.7.25 → llguidance-0.7.27}/python_ext/src/parserlimits.rs +0 -0
  125. {llguidance-0.7.25 → llguidance-0.7.27}/python_ext/src/pyjson.rs +0 -0
  126. {llguidance-0.7.25 → llguidance-0.7.27}/sample_parser/Cargo.toml +0 -0
  127. {llguidance-0.7.25 → llguidance-0.7.27}/sample_parser/README.md +0 -0
  128. {llguidance-0.7.25 → llguidance-0.7.27}/sample_parser/cli.sh +0 -0
  129. {llguidance-0.7.25 → llguidance-0.7.27}/sample_parser/data/blog.sample.json +0 -0
  130. {llguidance-0.7.25 → llguidance-0.7.27}/sample_parser/data/blog.schema.json +0 -0
  131. {llguidance-0.7.25 → llguidance-0.7.27}/sample_parser/data/blog.schema.ll.json +0 -0
  132. {llguidance-0.7.25 → llguidance-0.7.27}/sample_parser/data/from-llama.cpp/README.md +0 -0
  133. {llguidance-0.7.25 → llguidance-0.7.27}/sample_parser/data/from-llama.cpp/arithmetic.gbnf +0 -0
  134. {llguidance-0.7.25 → llguidance-0.7.27}/sample_parser/data/from-llama.cpp/c.gbnf +0 -0
  135. {llguidance-0.7.25 → llguidance-0.7.27}/sample_parser/data/from-llama.cpp/chess.gbnf +0 -0
  136. {llguidance-0.7.25 → llguidance-0.7.27}/sample_parser/data/from-llama.cpp/english.gbnf +0 -0
  137. {llguidance-0.7.25 → llguidance-0.7.27}/sample_parser/data/from-llama.cpp/japanese.gbnf +0 -0
  138. {llguidance-0.7.25 → llguidance-0.7.27}/sample_parser/data/from-llama.cpp/json.gbnf +0 -0
  139. {llguidance-0.7.25 → llguidance-0.7.27}/sample_parser/data/from-llama.cpp/json_arr.gbnf +0 -0
  140. {llguidance-0.7.25 → llguidance-0.7.27}/sample_parser/data/from-llama.cpp/list.gbnf +0 -0
  141. {llguidance-0.7.25 → llguidance-0.7.27}/sample_parser/data/from-llama.cpp/vllm-sql.gbnf +0 -0
  142. {llguidance-0.7.25 → llguidance-0.7.27}/sample_parser/data/lark.lark +0 -0
  143. {llguidance-0.7.25 → llguidance-0.7.27}/sample_parser/data/rfc.lark +0 -0
  144. {llguidance-0.7.25 → llguidance-0.7.27}/sample_parser/data/rfc.xml +0 -0
  145. {llguidance-0.7.25 → llguidance-0.7.27}/sample_parser/data/ulysses.md +0 -0
  146. {llguidance-0.7.25 → llguidance-0.7.27}/sample_parser/gtest.sh +0 -0
  147. {llguidance-0.7.25 → llguidance-0.7.27}/sample_parser/lark.sh +0 -0
  148. {llguidance-0.7.25 → llguidance-0.7.27}/sample_parser/run.sh +0 -0
  149. {llguidance-0.7.25 → llguidance-0.7.27}/sample_parser/src/lib.rs +0 -0
  150. {llguidance-0.7.25 → llguidance-0.7.27}/sample_parser/src/minimal.rs +0 -0
  151. {llguidance-0.7.25 → llguidance-0.7.27}/sample_parser/src/sample_parser.rs +0 -0
  152. {llguidance-0.7.25 → llguidance-0.7.27}/sample_parser/tests/test_raw_parser.rs +0 -0
  153. {llguidance-0.7.25 → llguidance-0.7.27}/sample_parser/tests/test_stop.rs +0 -0
  154. {llguidance-0.7.25 → llguidance-0.7.27}/scripts/annotate_asm.js +0 -0
  155. {llguidance-0.7.25 → llguidance-0.7.27}/scripts/cbindgen.sh +0 -0
  156. {llguidance-0.7.25 → llguidance-0.7.27}/scripts/checklinks.py +0 -0
  157. {llguidance-0.7.25 → llguidance-0.7.27}/scripts/checklinks.sh +0 -0
  158. {llguidance-0.7.25 → llguidance-0.7.27}/scripts/disasm.sh +0 -0
  159. {llguidance-0.7.25 → llguidance-0.7.27}/scripts/gbnf_to_lark.py +0 -0
  160. {llguidance-0.7.25 → llguidance-0.7.27}/scripts/gen-testcase.py +0 -0
  161. {llguidance-0.7.25 → llguidance-0.7.27}/scripts/git-version.sh +0 -0
  162. {llguidance-0.7.25 → llguidance-0.7.27}/scripts/install-deps.sh +0 -0
  163. {llguidance-0.7.25 → llguidance-0.7.27}/scripts/jsonschema-stats.js +0 -0
  164. {llguidance-0.7.25 → llguidance-0.7.27}/scripts/remote-guidance-test.sh +0 -0
  165. {llguidance-0.7.25 → llguidance-0.7.27}/scripts/rust-size.js +0 -0
  166. {llguidance-0.7.25 → llguidance-0.7.27}/scripts/rust_size.py +0 -0
  167. {llguidance-0.7.25 → llguidance-0.7.27}/scripts/test-guidance.sh +0 -0
  168. {llguidance-0.7.25 → llguidance-0.7.27}/scripts/tokenizer_test.py +0 -0
  169. {llguidance-0.7.25 → llguidance-0.7.27}/scripts/update-git.py +0 -0
  170. {llguidance-0.7.25 → llguidance-0.7.27}/toktrie/LICENSE +0 -0
  171. {llguidance-0.7.25 → llguidance-0.7.27}/toktrie/README.md +0 -0
  172. {llguidance-0.7.25 → llguidance-0.7.27}/toktrie/src/bytes.rs +0 -0
  173. {llguidance-0.7.25 → llguidance-0.7.27}/toktrie/src/lib.rs +0 -0
  174. {llguidance-0.7.25 → llguidance-0.7.27}/toktrie/src/recognizer.rs +0 -0
  175. {llguidance-0.7.25 → llguidance-0.7.27}/toktrie/src/rng.rs +0 -0
  176. {llguidance-0.7.25 → llguidance-0.7.27}/toktrie/src/svob.rs +0 -0
  177. {llguidance-0.7.25 → llguidance-0.7.27}/toktrie/src/tokenv.rs +0 -0
  178. {llguidance-0.7.25 → llguidance-0.7.27}/toktrie/tests/test_svob.rs +0 -0
  179. {llguidance-0.7.25 → llguidance-0.7.27}/toktrie_hf_downloader/LICENSE +0 -0
  180. {llguidance-0.7.25 → llguidance-0.7.27}/toktrie_hf_downloader/src/lib.rs +0 -0
  181. {llguidance-0.7.25 → llguidance-0.7.27}/toktrie_hf_tokenizers/LICENSE +0 -0
  182. {llguidance-0.7.25 → llguidance-0.7.27}/toktrie_hf_tokenizers/src/lib.rs +0 -0
{llguidance-0.7.25 → llguidance-0.7.27}/CHANGELOG.md RENAMED
@@ -4,6 +4,18 @@ All notable changes to this project will be documented in this file. Dates are d
4
4
 
5
5
  If a release doesn't introduce any interesting changes (build fixes etc.), it's skipped.
6
6
 
7
+ #### [0.7.27](https://github.com/guidance-ai/llguidance/compare/v0.7.26...0.7.27) 2025-06-04
8
+
9
+ - add toktrie_tiktoken and llguidance.tiktoken.lltokenizer_from_encoding [`#154`](https://github.com/guidance-ai/llguidance/issues/154)
10
+ - implement clone on StopController [`#185`](https://github.com/guidance-ai/llguidance/issues/185)
11
+
12
+ #### [0.7.26](https://github.com/guidance-ai/llguidance/compare/v0.7.25...0.7.26) 2025-05-30
13
+
14
+ - add support for & and ~ in lark regexes [`96fcee3`](https://github.com/guidance-ai/llguidance/commit/96fcee373697b57bead94d1bc06c17cf1c6134e4)
15
+ - dump grammar in errors in LLInterpreter [`#183`](https://github.com/guidance-ai/llguidance/pull/183)
16
+ - don't check lexer bytes invariant when we cannot rollback [`ec22083`](https://github.com/guidance-ai/llguidance/commit/ec220837051513a70177974ca389b7bf387455f1)
17
+
18
+
7
19
  #### [0.7.25](https://github.com/guidance-ai/llguidance/compare/v0.7.24...0.7.25) 2025-05-28
8
20
 
9
21
  - add parse_special=False to tokenize_str/bytes() in python [`#181`](https://github.com/guidance-ai/llguidance/pull/181)
{llguidance-0.7.25 → llguidance-0.7.27}/Cargo.lock RENAMED
@@ -135,15 +135,30 @@ version = "0.22.1"
135
135
  source = "registry+https://github.com/rust-lang/crates.io-index"
136
136
  checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
137
137
 
138
+ [[package]]
139
+ name = "bit-set"
140
+ version = "0.5.3"
141
+ source = "registry+https://github.com/rust-lang/crates.io-index"
142
+ checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1"
143
+ dependencies = [
144
+ "bit-vec 0.6.3",
145
+ ]
146
+
138
147
  [[package]]
139
148
  name = "bit-set"
140
149
  version = "0.8.0"
141
150
  source = "registry+https://github.com/rust-lang/crates.io-index"
142
151
  checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3"
143
152
  dependencies = [
144
- "bit-vec",
153
+ "bit-vec 0.8.0",
145
154
  ]
146
155
 
156
+ [[package]]
157
+ name = "bit-vec"
158
+ version = "0.6.3"
159
+ source = "registry+https://github.com/rust-lang/crates.io-index"
160
+ checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb"
161
+
147
162
  [[package]]
148
163
  name = "bit-vec"
149
164
  version = "0.8.0"
@@ -162,6 +177,17 @@ version = "0.2.2"
162
177
  source = "registry+https://github.com/rust-lang/crates.io-index"
163
178
  checksum = "3eeab4423108c5d7c744f4d234de88d18d636100093ae04caf4825134b9c3a32"
164
179
 
180
+ [[package]]
181
+ name = "bstr"
182
+ version = "1.12.0"
183
+ source = "registry+https://github.com/rust-lang/crates.io-index"
184
+ checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4"
185
+ dependencies = [
186
+ "memchr",
187
+ "regex-automata",
188
+ "serde",
189
+ ]
190
+
165
191
  [[package]]
166
192
  name = "bumpalo"
167
193
  version = "3.17.0"
@@ -492,13 +518,24 @@ version = "0.1.10"
492
518
  source = "registry+https://github.com/rust-lang/crates.io-index"
493
519
  checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6"
494
520
 
521
+ [[package]]
522
+ name = "fancy-regex"
523
+ version = "0.13.0"
524
+ source = "registry+https://github.com/rust-lang/crates.io-index"
525
+ checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2"
526
+ dependencies = [
527
+ "bit-set 0.5.3",
528
+ "regex-automata",
529
+ "regex-syntax",
530
+ ]
531
+
495
532
  [[package]]
496
533
  name = "fancy-regex"
497
534
  version = "0.14.0"
498
535
  source = "registry+https://github.com/rust-lang/crates.io-index"
499
536
  checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298"
500
537
  dependencies = [
501
- "bit-set",
538
+ "bit-set 0.8.0",
502
539
  "regex-automata",
503
540
  "regex-syntax",
504
541
  ]
@@ -1123,7 +1160,7 @@ dependencies = [
1123
1160
  "base64 0.22.1",
1124
1161
  "bytecount",
1125
1162
  "email_address",
1126
- "fancy-regex",
1163
+ "fancy-regex 0.14.0",
1127
1164
  "fraction",
1128
1165
  "idna",
1129
1166
  "itoa",
@@ -1174,7 +1211,7 @@ checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856"
1174
1211
 
1175
1212
  [[package]]
1176
1213
  name = "llguidance"
1177
- version = "0.7.25"
1214
+ version = "0.7.27"
1178
1215
  dependencies = [
1179
1216
  "anyhow",
1180
1217
  "derivre",
@@ -1193,7 +1230,7 @@ dependencies = [
1193
1230
 
1194
1231
  [[package]]
1195
1232
  name = "llguidance_py"
1196
- version = "0.7.25"
1233
+ version = "0.7.27"
1197
1234
  dependencies = [
1198
1235
  "anyhow",
1199
1236
  "bytemuck",
@@ -1203,6 +1240,7 @@ dependencies = [
1203
1240
  "serde",
1204
1241
  "serde_json",
1205
1242
  "toktrie_hf_tokenizers",
1243
+ "toktrie_tiktoken",
1206
1244
  ]
1207
1245
 
1208
1246
  [[package]]
@@ -1865,6 +1903,12 @@ version = "0.1.24"
1865
1903
  source = "registry+https://github.com/rust-lang/crates.io-index"
1866
1904
  checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
1867
1905
 
1906
+ [[package]]
1907
+ name = "rustc-hash"
1908
+ version = "1.1.0"
1909
+ source = "registry+https://github.com/rust-lang/crates.io-index"
1910
+ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
1911
+
1868
1912
  [[package]]
1869
1913
  name = "rustix"
1870
1914
  version = "1.0.5"
@@ -2233,6 +2277,21 @@ dependencies = [
2233
2277
  "syn",
2234
2278
  ]
2235
2279
 
2280
+ [[package]]
2281
+ name = "tiktoken-rs"
2282
+ version = "0.7.0"
2283
+ source = "registry+https://github.com/rust-lang/crates.io-index"
2284
+ checksum = "25563eeba904d770acf527e8b370fe9a5547bacd20ff84a0b6c3bc41288e5625"
2285
+ dependencies = [
2286
+ "anyhow",
2287
+ "base64 0.22.1",
2288
+ "bstr",
2289
+ "fancy-regex 0.13.0",
2290
+ "lazy_static",
2291
+ "regex",
2292
+ "rustc-hash",
2293
+ ]
2294
+
2236
2295
  [[package]]
2237
2296
  name = "tinystr"
2238
2297
  version = "0.7.6"
@@ -2252,7 +2311,7 @@ dependencies = [
2252
2311
  "aho-corasick",
2253
2312
  "derive_builder",
2254
2313
  "esaxx-rs",
2255
- "fancy-regex",
2314
+ "fancy-regex 0.14.0",
2256
2315
  "getrandom 0.2.15",
2257
2316
  "itertools 0.13.0",
2258
2317
  "lazy_static",
@@ -2336,7 +2395,7 @@ dependencies = [
2336
2395
 
2337
2396
  [[package]]
2338
2397
  name = "toktrie"
2339
- version = "0.7.25"
2398
+ version = "0.7.27"
2340
2399
  dependencies = [
2341
2400
  "anyhow",
2342
2401
  "bytemuck",
@@ -2347,7 +2406,7 @@ dependencies = [
2347
2406
 
2348
2407
  [[package]]
2349
2408
  name = "toktrie_hf_downloader"
2350
- version = "0.7.25"
2409
+ version = "0.7.27"
2351
2410
  dependencies = [
2352
2411
  "anyhow",
2353
2412
  "hf-hub",
@@ -2358,7 +2417,7 @@ dependencies = [
2358
2417
 
2359
2418
  [[package]]
2360
2419
  name = "toktrie_hf_tokenizers"
2361
- version = "0.7.25"
2420
+ version = "0.7.27"
2362
2421
  dependencies = [
2363
2422
  "anyhow",
2364
2423
  "log",
@@ -2368,6 +2427,18 @@ dependencies = [
2368
2427
  "toktrie",
2369
2428
  ]
2370
2429
 
2430
+ [[package]]
2431
+ name = "toktrie_tiktoken"
2432
+ version = "0.7.27"
2433
+ dependencies = [
2434
+ "anyhow",
2435
+ "log",
2436
+ "serde",
2437
+ "serde_json",
2438
+ "tiktoken-rs",
2439
+ "toktrie",
2440
+ ]
2441
+
2371
2442
  [[package]]
2372
2443
  name = "tower"
2373
2444
  version = "0.5.2"
{llguidance-0.7.25 → llguidance-0.7.27}/Cargo.toml RENAMED
@@ -7,6 +7,7 @@ members = [
7
7
  "toktrie",
8
8
  "toktrie_hf_tokenizers",
9
9
  "toktrie_hf_downloader",
10
+ "toktrie_tiktoken",
10
11
  ]
11
12
  # just exclude python_ext since it doesn't build without maturin
12
13
  default-members = [
@@ -16,6 +17,7 @@ default-members = [
16
17
  "toktrie",
17
18
  "toktrie_hf_tokenizers",
18
19
  "toktrie_hf_downloader",
20
+ "toktrie_tiktoken",
19
21
  ]
20
22
  resolver = "2"
21
23
 
@@ -36,4 +38,5 @@ opt-level = 3
36
38
  toktrie = { path = "toktrie" }
37
39
  llguidance = { path = "parser" }
38
40
  toktrie_hf_tokenizers = { path = "toktrie_hf_tokenizers" }
39
- toktrie_hf_downloader = { path = "toktrie_hf_downloader" }
41
+ toktrie_hf_downloader = { path = "toktrie_hf_downloader" }
42
+ toktrie_tiktoken = { path = "toktrie_tiktoken" }
{llguidance-0.7.25 → llguidance-0.7.27}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: llguidance
3
- Version: 0.7.25
3
+ Version: 0.7.27
4
4
  License-File: LICENSE
5
5
  Summary: Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance
6
6
  Author: Michal Moskal
@@ -72,9 +72,7 @@ The library is currently integrated in:
72
72
  - **vLLM** - [V0 PR](https://github.com/vllm-project/vllm/pull/14589) and [V1 PR](https://github.com/vllm-project/vllm/pull/14779)
73
73
  - [LLGTRT](https://github.com/guidance-ai/llgtrt) - OpenAI-compatible REST server using NVIDIA's [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM)
74
74
  - [mistral.rs](https://github.com/EricLBuehler/mistral.rs/pull/899)
75
-
76
- The integration is ongoing in:
77
- - **onnxruntime-genai** - [draft PR](https://github.com/microsoft/onnxruntime-genai/pull/1038)
75
+ - [onnxruntime-genai](https://github.com/microsoft/onnxruntime-genai/pull/1381)
78
76
 
79
77
  ## Technical details
80
78
 
{llguidance-0.7.25 → llguidance-0.7.27}/README.md RENAMED
@@ -60,9 +60,7 @@ The library is currently integrated in:
60
60
  - **vLLM** - [V0 PR](https://github.com/vllm-project/vllm/pull/14589) and [V1 PR](https://github.com/vllm-project/vllm/pull/14779)
61
61
  - [LLGTRT](https://github.com/guidance-ai/llgtrt) - OpenAI-compatible REST server using NVIDIA's [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM)
62
62
  - [mistral.rs](https://github.com/EricLBuehler/mistral.rs/pull/899)
63
-
64
- The integration is ongoing in:
65
- - **onnxruntime-genai** - [draft PR](https://github.com/microsoft/onnxruntime-genai/pull/1038)
63
+ - [onnxruntime-genai](https://github.com/microsoft/onnxruntime-genai/pull/1381)
66
64
 
67
65
  ## Technical details
68
66
 
{llguidance-0.7.25 → llguidance-0.7.27}/docs/optimizations.md RENAMED
@@ -91,7 +91,6 @@ Walking the trie mostly involves successful lookups in that table,
91
91
  and the derivative engine is only used when the table doesn't yet have the
92
92
  given transition.
93
93
 
94
-
95
94
  ## Earley parser optimizations
96
95
 
97
96
  - CFG rules are stored in a flat array
@@ -123,7 +122,7 @@ We thus define a series _slices_, under-approximation of such unconstrained cont
123
122
  The slices are defined by regular expressions typically of the form `[...]{1,N}`
124
123
  (that is a character class repeated up to `N` times).
125
124
 
126
- For example, a good set of slices for JSON schemas is
125
+ For example, a good set of slices for JSON schemas is
127
126
 
128
127
  - `[^"\\\x00-\x1F\x7F]{1,10}` (`turtle`, ` turtle`, `)!;`, `żółw`, `🐢`, etc.)
129
128
  - `[^"\\\x00-\x1F\x7F]{1,30}` (`/////////////////`, ...)
@@ -169,6 +168,7 @@ Now, the JSON slice is contained in `C*"`,
169
168
  and thus we can skip walking the trie for the slice.
170
169
 
171
170
  Another example:
171
+
172
172
  - assume schemas has `{ "type": "string", "maxLength": 20 }`
173
173
  - so after initial quote, the lexer allows `C{0,20}"`
174
174
  - the JSON slice `[^"\\\x00-\x1F\x7F]{1,10}` is contained in this lexeme,
@@ -176,17 +176,19 @@ Another example:
176
176
 
177
177
  This optimization make the mask computation about 10x faster in [MaskBench](https://github.com/guidance-ai/jsonschemabench/tree/main/maskbench).
178
178
 
179
+ ### Mask density statistics
180
+
179
181
  The reason the optimization works, is that masks tend be either small or sliceable.
180
182
  Here are statistics of various kinds of masks, across around 2M masks in MaskBench,
181
183
  categorized based on how "full" the mask is and whether the slicer optimization was applied.
182
184
 
183
- | Category | % Masks | % Time | Time/Mask [us] |
184
- |---------------------|---------:|----------:|---------------:|
185
- | 0%-2% & !sliced | 44.6% | 20.7% | 28 |
186
- | 2%-85% & !sliced | 1.1% | 11.0% | 576 |
187
- | 85%+ & !sliced | 0.5% | 13.0% | 1577 |
188
- | 85%+ & sliced | 53.8% | 55.0% | 61 |
189
- | **Total** | 100.0% | 100.0% | 60 |
185
+ | Category | % Masks | % Time | Time/Mask [us] |
186
+ | ---------------- | ------: | -----: | -------------: |
187
+ | 0%-2% & !sliced | 44.6% | 20.7% | 28 |
188
+ | 2%-85% & !sliced | 1.1% | 11.0% | 576 |
189
+ | 85%+ & !sliced | 0.5% | 13.0% | 1577 |
190
+ | 85%+ & sliced | 53.8% | 55.0% | 61 |
191
+ | **Total** | 100.0% | 100.0% | 60 |
190
192
 
191
193
  ![Plot of the table above](mask_plot.png)
192
194
 
@@ -195,43 +197,43 @@ and in a little over half the slicer optimization can be applied
195
197
  (there are no masks under 85% full where the slicer can be applied).
196
198
  The remaining sliver of masks are either intermediate size or large, but the slicer optimization can't be applied; they take disproportionately long time to compute.
197
199
 
200
+
198
201
  ### Checking regex containment
199
202
 
200
- This is an under-approximation of the containment problem,
201
- that is it may return false when the containment is actually true.
202
- If any of the "checks" fail, we return false.
203
+ This is an under-approximation of the containment problem, that is it may return
204
+ false when the containment is actually true. If any of the "checks" fail, we
205
+ return false.
203
206
 
204
207
  Prefixes of language `R`, are defined as `P(R) = { w | ∃q. wq ∈ R }`.
205
208
 
206
- We need to check if regex `S` (slice) is contained in prefix of regex `L` (lexeme): `S ⊆ P(L)`.
209
+ We need to check if regex `S` (slice) is contained in prefix of regex `L`
210
+ (lexeme): `S ⊆ P(L)`.
207
211
 
208
- We check if `L` is of the form `(X{m,n} & ~E) T`, where
209
- `E` is of the form `E0 | E1 | ... | Ek`,
210
- and both `E` can be `∅` (empty-set/no match) and `T` can be `ε` (empty string).
212
+ We check if `L` is of the form `(X{m,n} & ~E) T`, where `E` is of the form
213
+ `E0 | E1 | ... | Ek`. Note that: `E` can be `∅` (empty-set/no match) and `T` can
214
+ be `ε` (empty string).
211
215
 
212
- Observe that `P(R) ⊆ P(RT)`, ie. making regex longer doesn't remove any prefixes (provided `T ≠ ∅`).
213
- Thus, we'll be checking containment in `P(X{m,n} & ~E)`.
216
+ Observe that `P(R) ⊆ P(RT)`, ie. making regex longer doesn't remove any prefixes
217
+ (provided `T ≠ ∅`). Thus, we'll be checking containment in `P(X{m,n} & ~E)`.
214
218
 
215
- We (over)estimate maximum length of `E`, let `o >= max { |w| | w ∈ E }`.
216
- We check that `n > o`, and that `∃v ≠ ε. v ∈ X`.
217
- In other words, we check that for anything matching `Ei` and `X{m,n}` there is a proper extension of that string in `X{m,n}`.
219
+ We (over)estimate maximum length of `E`, let `o >= max { |w| | w ∈ E }`. We
220
+ check that `n > o`, and that `∃v ≠ ε. v ∈ X`. In other words, we check that for
221
+ anything matching `Ei` and `X{m,n}` there is a proper extension of that string
222
+ in `X{m,n}`.
218
223
 
219
224
  Now, we prove that `P(X{m,n} & ~E) = P(X{m,n})`.
220
225
 
221
- Consider `w ∈ P(X{m,n})`. We have `wq ∈ X{m,n}` for some `q`.
222
- If `|wq| > o`, then `wq ∉ E`, and thus `wq ∈ X{m,n} & ~E`.
223
- Otherwise, `wq ∈ X{p}` for some `p <= o < n`,
224
- and thus `wqv...v ∈ X{n}` for `n-p` repetitions of `v`.
225
- We also have `|wqv...v| > o`, and thus `wqv...v ∉ E`,
226
- and thus `wqv...v ∈ X{m,n} & ~E`,
227
- and thus `w ∈ P(X{m,n} & ~E)`.
228
- The other direction is trivial.
226
+ Consider `w ∈ P(X{m,n})`. We have `wq ∈ X{m,n}` for some `q`. If `|wq| > o`,
227
+ then `wq ∉ E`, and thus `wq ∈ X{m,n} & ~E`. Otherwise, `wq ∈ X{p}` for some
228
+ `p <= o < n`, and thus `wqv...v ∈ X{n}` for `n-p` repetitions of `v`. We also
229
+ have `|wqv...v| > o`, and thus `wqv...v ∉ E`, and thus `wqv...v ∈ X{m,n} & ~E`,
230
+ and thus `w ∈ P(X{m,n} & ~E)`. The other direction is trivial.
229
231
 
230
232
  Now, we just need to check if `S ⊆ P(X{m,n})`.
231
233
 
232
- First, we check if `S` is of the form `Y{m',n'}`.
233
- Then, we check if `Y` is contained in `X` (this is a cached check using symbolic derivatives; it's typically simple).
234
- Finally, we check if `n' <= n`.
235
- Note that we don't care about `m` and `m'`, as we're checking for prefixes.
234
+ First, we check if `S` is of the form `Y{m',n'}`. Then, we check if `Y` is
235
+ contained in `X` (this is a cached check using symbolic derivatives; it's
236
+ typically simple). Finally, we check if `n' <= n`. Note that we don't care about
237
+ `m` and `m'`, as we're checking for prefixes.
236
238
 
237
239
  Also note that the upper-bound in the above calculations can be infinity.
{llguidance-0.7.25 → llguidance-0.7.27}/docs/syntax.md RENAMED
@@ -217,6 +217,28 @@ like `<|python_tag|>`, not a string like `<function`.
217
217
 
218
218
  The `llguidance.StructTag` API, [inspired](https://github.com/mlc-ai/xgrammar/blob/fd9ee31/python/xgrammar/grammar.py#L211) by XGrammar, just compiles down to the above.
219
219
 
220
+ ### And/Not operators in regexes
221
+
222
+ The regular expressions in LLGuidance can use additional operators: `&` (and) and `~` (not).
223
+ They can only be used outside of the `/.../` syntax, i.e., in the Lark terminal (token) definitions.
224
+ The `&` operator binds tighter than `|` (alternation), so `A & B | C` means `(A & B) | C`.
225
+ The `~` operator binds tighter than even `+` or `*`, so `~A+` means `(~A)+`.
226
+
227
+ The negation operator `~` is particularly tricky to use right.
228
+ For example, this is a terminal definition that matches any list of ASCII lines,
229
+ but they cannot have two newlines in a row:
230
+
231
+ ```lark
232
+ ASCII_LINES: /[a-zA-Z \n]*/ & ~/(?s:.*)\n\n(?s:.*)/
233
+ ```
234
+
235
+ Note that `/[a-zA-Z \n]*/ & ~/\n\n/` would mean any list of lines, also with two newlines in a row,
236
+ except for the exact string `\n\n`.
237
+ Also, `/[a-zA-Z \n]*/ & ~/(.*)\n\n(.*)/` would allow double newlines, but if there is at least two of them
238
+ (`/./` doesn't match newline).
239
+
240
+ These operators are sometimes expensive to use, so you should generally avoid them if alternatives exist.
241
+
220
242
  ### Structured %regex
221
243
 
222
244
  LLGuidance supports [extended regex syntax](https://docs.rs/regex/latest/regex/#syntax) in `/.../`.
@@ -273,12 +295,6 @@ MULT_NUM: %regex {
273
295
  }
274
296
  ```
275
297
 
276
- We also plan to add `&` and `~` operators:
277
-
278
- ```lark
279
- ASCII_LINES: /[a-zA-Z \n]*/ & ~/.*\n\n.*/
280
- ```
281
-
282
298
  ### Grammar options
283
299
 
284
300
  Certain grammar options can be set by using `%llguidnace { ... }`,
{llguidance-0.7.25 → llguidance-0.7.27}/parser/Cargo.toml RENAMED
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "llguidance"
3
- version = "0.7.25"
3
+ version = "0.7.27"
4
4
  edition = "2021"
5
5
  license = "MIT"
6
6
  description = "Super-fast Structured Outputs"
{llguidance-0.7.25 → llguidance-0.7.27}/parser/llguidance.h RENAMED
@@ -433,6 +433,13 @@ const char *llg_stop_commit_token(struct LlgStopController *stop_ctrl,
433
433
  size_t *output_len_p,
434
434
  bool *is_stopped_p);
435
435
 
436
+ /**
437
+ * Clone the stop-sequence controller.
438
+ * The cloned controller shares (under mutex) regex caches if any, so that
439
+ * cloning is cheap.
440
+ */
441
+ struct LlgStopController *llg_clone_stop_controller(const struct LlgStopController *stop_ctrl);
442
+
436
443
  /**
437
444
  * Free the stop-sequence controller
438
445
  */