llguidance 0.7.18__tar.gz → 0.7.20__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (174) hide show
  1. {llguidance-0.7.18 → llguidance-0.7.20}/.github/workflows/rust.yml +17 -0
  2. {llguidance-0.7.18 → llguidance-0.7.20}/CHANGELOG.md +11 -0
  3. {llguidance-0.7.18 → llguidance-0.7.20}/Cargo.lock +17 -43
  4. {llguidance-0.7.18 → llguidance-0.7.20}/PKG-INFO +1 -1
  5. {llguidance-0.7.18 → llguidance-0.7.20}/parser/Cargo.toml +2 -2
  6. {llguidance-0.7.18 → llguidance-0.7.20}/parser/build.rs +1 -1
  7. {llguidance-0.7.18 → llguidance-0.7.20}/parser/llguidance.h +1 -1
  8. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/earley/lexer.rs +2 -2
  9. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/earley/regexvec.rs +7 -5
  10. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/ffi.rs +7 -4
  11. {llguidance-0.7.18 → llguidance-0.7.20}/pyproject.toml +1 -1
  12. {llguidance-0.7.18 → llguidance-0.7.20}/python_ext/Cargo.toml +1 -1
  13. {llguidance-0.7.18 → llguidance-0.7.20}/sample_parser/tests/test_ll.rs +49 -0
  14. {llguidance-0.7.18 → llguidance-0.7.20}/toktrie/Cargo.toml +1 -1
  15. {llguidance-0.7.18 → llguidance-0.7.20}/toktrie_hf_downloader/Cargo.toml +1 -1
  16. {llguidance-0.7.18 → llguidance-0.7.20}/toktrie_hf_tokenizers/Cargo.toml +5 -2
  17. {llguidance-0.7.18 → llguidance-0.7.20}/toktrie_hf_tokenizers/src/lib.rs +3 -1
  18. {llguidance-0.7.18 → llguidance-0.7.20}/.github/workflows/wheels.yml +0 -0
  19. {llguidance-0.7.18 → llguidance-0.7.20}/.gitignore +0 -0
  20. {llguidance-0.7.18 → llguidance-0.7.20}/CODE_OF_CONDUCT.md +0 -0
  21. {llguidance-0.7.18 → llguidance-0.7.20}/Cargo.toml +0 -0
  22. {llguidance-0.7.18 → llguidance-0.7.20}/LICENSE +0 -0
  23. {llguidance-0.7.18 → llguidance-0.7.20}/README.md +0 -0
  24. {llguidance-0.7.18 → llguidance-0.7.20}/SECURITY.md +0 -0
  25. {llguidance-0.7.18 → llguidance-0.7.20}/SUPPORT.md +0 -0
  26. {llguidance-0.7.18 → llguidance-0.7.20}/c_sample/Makefile +0 -0
  27. {llguidance-0.7.18 → llguidance-0.7.20}/c_sample/README.md +0 -0
  28. {llguidance-0.7.18 → llguidance-0.7.20}/c_sample/c_sample.cpp +0 -0
  29. {llguidance-0.7.18 → llguidance-0.7.20}/docs/fast_forward.md +0 -0
  30. {llguidance-0.7.18 → llguidance-0.7.20}/docs/json_schema.md +0 -0
  31. {llguidance-0.7.18 → llguidance-0.7.20}/docs/mask_plot.png +0 -0
  32. {llguidance-0.7.18 → llguidance-0.7.20}/docs/optimizations.md +0 -0
  33. {llguidance-0.7.18 → llguidance-0.7.20}/docs/special_tokens.md +0 -0
  34. {llguidance-0.7.18 → llguidance-0.7.20}/docs/syntax.md +0 -0
  35. {llguidance-0.7.18 → llguidance-0.7.20}/docs/toktrie.md +0 -0
  36. {llguidance-0.7.18 → llguidance-0.7.20}/json_stats/Cargo.toml +0 -0
  37. {llguidance-0.7.18 → llguidance-0.7.20}/json_stats/expected_maskbench.json +0 -0
  38. {llguidance-0.7.18 → llguidance-0.7.20}/json_stats/jstats.sh +0 -0
  39. {llguidance-0.7.18 → llguidance-0.7.20}/json_stats/scripts/split-stats.sh +0 -0
  40. {llguidance-0.7.18 → llguidance-0.7.20}/json_stats/scripts/split_plot.py +0 -0
  41. {llguidance-0.7.18 → llguidance-0.7.20}/json_stats/src/json_stats.rs +0 -0
  42. {llguidance-0.7.18 → llguidance-0.7.20}/json_stats/src/lib.rs +0 -0
  43. {llguidance-0.7.18 → llguidance-0.7.20}/json_stats/src/stats.rs +0 -0
  44. {llguidance-0.7.18 → llguidance-0.7.20}/parser/LICENSE +0 -0
  45. {llguidance-0.7.18 → llguidance-0.7.20}/parser/README.md +0 -0
  46. {llguidance-0.7.18 → llguidance-0.7.20}/parser/cbindgen.toml +0 -0
  47. {llguidance-0.7.18 → llguidance-0.7.20}/parser/grammars/character.json +0 -0
  48. {llguidance-0.7.18 → llguidance-0.7.20}/parser/grammars/json.json +0 -0
  49. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/api.rs +0 -0
  50. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/constraint.rs +0 -0
  51. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/earley/from_guidance.rs +0 -0
  52. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/earley/grammar.rs +0 -0
  53. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/earley/lexerspec.rs +0 -0
  54. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/earley/mod.rs +0 -0
  55. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/earley/parser.rs +0 -0
  56. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/earley/perf.rs +0 -0
  57. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/earley/slicer.rs +0 -0
  58. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/factory.rs +0 -0
  59. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/ffi_par.rs +0 -0
  60. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/grammar_builder.rs +0 -0
  61. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/json/README.md +0 -0
  62. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/json/compiler.rs +0 -0
  63. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/json/context_ref.rs +0 -0
  64. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/json/context_simple/context.rs +0 -0
  65. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/json/context_simple/draft.rs +0 -0
  66. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/json/context_simple/mod.rs +0 -0
  67. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/json/formats.rs +0 -0
  68. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/json/mod.rs +0 -0
  69. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/json/numeric.rs +0 -0
  70. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/json/schema.rs +0 -0
  71. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/json/shared_context.rs +0 -0
  72. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/json_validation.rs +0 -0
  73. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/lark/README.md +0 -0
  74. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/lark/ast.rs +0 -0
  75. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/lark/common.rs +0 -0
  76. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/lark/compiler.rs +0 -0
  77. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/lark/lexer.rs +0 -0
  78. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/lark/mod.rs +0 -0
  79. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/lark/parser.rs +0 -0
  80. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/lib.rs +0 -0
  81. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/logging.rs +0 -0
  82. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/matcher.rs +0 -0
  83. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/output.rs +0 -0
  84. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/panic_utils.rs +0 -0
  85. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/regex_rewrite.rs +0 -0
  86. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/stop_controller.rs +0 -0
  87. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/substring.rs +0 -0
  88. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/tokenizer_json.rs +0 -0
  89. {llguidance-0.7.18 → llguidance-0.7.20}/parser/src/tokenparser.rs +0 -0
  90. {llguidance-0.7.18 → llguidance-0.7.20}/plan.md +0 -0
  91. {llguidance-0.7.18 → llguidance-0.7.20}/python/llguidance/__init__.py +0 -0
  92. {llguidance-0.7.18 → llguidance-0.7.20}/python/llguidance/_grammar_from.py +0 -0
  93. {llguidance-0.7.18 → llguidance-0.7.20}/python/llguidance/_lib.pyi +0 -0
  94. {llguidance-0.7.18 → llguidance-0.7.20}/python/llguidance/_struct_tag.py +0 -0
  95. {llguidance-0.7.18 → llguidance-0.7.20}/python/llguidance/_tokenizer.py +0 -0
  96. {llguidance-0.7.18 → llguidance-0.7.20}/python/llguidance/_util.py +0 -0
  97. {llguidance-0.7.18 → llguidance-0.7.20}/python/llguidance/cli.py +0 -0
  98. {llguidance-0.7.18 → llguidance-0.7.20}/python/llguidance/gbnf_to_lark.py +0 -0
  99. {llguidance-0.7.18 → llguidance-0.7.20}/python/llguidance/hf.py +0 -0
  100. {llguidance-0.7.18 → llguidance-0.7.20}/python/llguidance/mlx.py +0 -0
  101. {llguidance-0.7.18 → llguidance-0.7.20}/python/llguidance/numpy.py +0 -0
  102. {llguidance-0.7.18 → llguidance-0.7.20}/python/llguidance/py.typed +0 -0
  103. {llguidance-0.7.18 → llguidance-0.7.20}/python/llguidance/torch.py +0 -0
  104. {llguidance-0.7.18 → llguidance-0.7.20}/python/mypy.ini +0 -0
  105. {llguidance-0.7.18 → llguidance-0.7.20}/python/torch_tests/__init__.py +0 -0
  106. {llguidance-0.7.18 → llguidance-0.7.20}/python/torch_tests/test_bitmask.py +0 -0
  107. {llguidance-0.7.18 → llguidance-0.7.20}/python/torch_tests/test_hf.py +0 -0
  108. {llguidance-0.7.18 → llguidance-0.7.20}/python/torch_tests/test_matcher.py +0 -0
  109. {llguidance-0.7.18 → llguidance-0.7.20}/python_ext/src/lib.rs +0 -0
  110. {llguidance-0.7.18 → llguidance-0.7.20}/python_ext/src/llinterpreter.rs +0 -0
  111. {llguidance-0.7.18 → llguidance-0.7.20}/python_ext/src/llmatcher.rs +0 -0
  112. {llguidance-0.7.18 → llguidance-0.7.20}/python_ext/src/parserlimits.rs +0 -0
  113. {llguidance-0.7.18 → llguidance-0.7.20}/python_ext/src/py.rs +0 -0
  114. {llguidance-0.7.18 → llguidance-0.7.20}/python_ext/src/pyjson.rs +0 -0
  115. {llguidance-0.7.18 → llguidance-0.7.20}/sample_parser/Cargo.toml +0 -0
  116. {llguidance-0.7.18 → llguidance-0.7.20}/sample_parser/README.md +0 -0
  117. {llguidance-0.7.18 → llguidance-0.7.20}/sample_parser/cli.sh +0 -0
  118. {llguidance-0.7.18 → llguidance-0.7.20}/sample_parser/data/blog.sample.json +0 -0
  119. {llguidance-0.7.18 → llguidance-0.7.20}/sample_parser/data/blog.schema.json +0 -0
  120. {llguidance-0.7.18 → llguidance-0.7.20}/sample_parser/data/blog.schema.ll.json +0 -0
  121. {llguidance-0.7.18 → llguidance-0.7.20}/sample_parser/data/from-llama.cpp/README.md +0 -0
  122. {llguidance-0.7.18 → llguidance-0.7.20}/sample_parser/data/from-llama.cpp/arithmetic.gbnf +0 -0
  123. {llguidance-0.7.18 → llguidance-0.7.20}/sample_parser/data/from-llama.cpp/c.gbnf +0 -0
  124. {llguidance-0.7.18 → llguidance-0.7.20}/sample_parser/data/from-llama.cpp/chess.gbnf +0 -0
  125. {llguidance-0.7.18 → llguidance-0.7.20}/sample_parser/data/from-llama.cpp/english.gbnf +0 -0
  126. {llguidance-0.7.18 → llguidance-0.7.20}/sample_parser/data/from-llama.cpp/japanese.gbnf +0 -0
  127. {llguidance-0.7.18 → llguidance-0.7.20}/sample_parser/data/from-llama.cpp/json.gbnf +0 -0
  128. {llguidance-0.7.18 → llguidance-0.7.20}/sample_parser/data/from-llama.cpp/json_arr.gbnf +0 -0
  129. {llguidance-0.7.18 → llguidance-0.7.20}/sample_parser/data/from-llama.cpp/list.gbnf +0 -0
  130. {llguidance-0.7.18 → llguidance-0.7.20}/sample_parser/data/from-llama.cpp/vllm-sql.gbnf +0 -0
  131. {llguidance-0.7.18 → llguidance-0.7.20}/sample_parser/data/lark.lark +0 -0
  132. {llguidance-0.7.18 → llguidance-0.7.20}/sample_parser/data/rfc.lark +0 -0
  133. {llguidance-0.7.18 → llguidance-0.7.20}/sample_parser/data/rfc.xml +0 -0
  134. {llguidance-0.7.18 → llguidance-0.7.20}/sample_parser/data/ulysses.md +0 -0
  135. {llguidance-0.7.18 → llguidance-0.7.20}/sample_parser/gtest.sh +0 -0
  136. {llguidance-0.7.18 → llguidance-0.7.20}/sample_parser/lark.sh +0 -0
  137. {llguidance-0.7.18 → llguidance-0.7.20}/sample_parser/run.sh +0 -0
  138. {llguidance-0.7.18 → llguidance-0.7.20}/sample_parser/src/lib.rs +0 -0
  139. {llguidance-0.7.18 → llguidance-0.7.20}/sample_parser/src/minimal.rs +0 -0
  140. {llguidance-0.7.18 → llguidance-0.7.20}/sample_parser/src/sample_parser.rs +0 -0
  141. {llguidance-0.7.18 → llguidance-0.7.20}/sample_parser/tests/test_lark.rs +0 -0
  142. {llguidance-0.7.18 → llguidance-0.7.20}/sample_parser/tests/test_raw_parser.rs +0 -0
  143. {llguidance-0.7.18 → llguidance-0.7.20}/sample_parser/tests/test_stop.rs +0 -0
  144. {llguidance-0.7.18 → llguidance-0.7.20}/scripts/annotate_asm.js +0 -0
  145. {llguidance-0.7.18 → llguidance-0.7.20}/scripts/bump.py +0 -0
  146. {llguidance-0.7.18 → llguidance-0.7.20}/scripts/cbindgen.sh +0 -0
  147. {llguidance-0.7.18 → llguidance-0.7.20}/scripts/checklinks.py +0 -0
  148. {llguidance-0.7.18 → llguidance-0.7.20}/scripts/checklinks.sh +0 -0
  149. {llguidance-0.7.18 → llguidance-0.7.20}/scripts/ci-publish.py +0 -0
  150. {llguidance-0.7.18 → llguidance-0.7.20}/scripts/disasm.sh +0 -0
  151. {llguidance-0.7.18 → llguidance-0.7.20}/scripts/gbnf_to_lark.py +0 -0
  152. {llguidance-0.7.18 → llguidance-0.7.20}/scripts/gen-testcase.py +0 -0
  153. {llguidance-0.7.18 → llguidance-0.7.20}/scripts/git-version.sh +0 -0
  154. {llguidance-0.7.18 → llguidance-0.7.20}/scripts/install-deps.sh +0 -0
  155. {llguidance-0.7.18 → llguidance-0.7.20}/scripts/jsonschema-stats.js +0 -0
  156. {llguidance-0.7.18 → llguidance-0.7.20}/scripts/remote-guidance-test.sh +0 -0
  157. {llguidance-0.7.18 → llguidance-0.7.20}/scripts/rust-size.js +0 -0
  158. {llguidance-0.7.18 → llguidance-0.7.20}/scripts/rust_size.py +0 -0
  159. {llguidance-0.7.18 → llguidance-0.7.20}/scripts/test-guidance.sh +0 -0
  160. {llguidance-0.7.18 → llguidance-0.7.20}/scripts/tokenizer_test.py +0 -0
  161. {llguidance-0.7.18 → llguidance-0.7.20}/scripts/update-git.py +0 -0
  162. {llguidance-0.7.18 → llguidance-0.7.20}/toktrie/LICENSE +0 -0
  163. {llguidance-0.7.18 → llguidance-0.7.20}/toktrie/README.md +0 -0
  164. {llguidance-0.7.18 → llguidance-0.7.20}/toktrie/src/bytes.rs +0 -0
  165. {llguidance-0.7.18 → llguidance-0.7.20}/toktrie/src/lib.rs +0 -0
  166. {llguidance-0.7.18 → llguidance-0.7.20}/toktrie/src/recognizer.rs +0 -0
  167. {llguidance-0.7.18 → llguidance-0.7.20}/toktrie/src/rng.rs +0 -0
  168. {llguidance-0.7.18 → llguidance-0.7.20}/toktrie/src/svob.rs +0 -0
  169. {llguidance-0.7.18 → llguidance-0.7.20}/toktrie/src/tokenv.rs +0 -0
  170. {llguidance-0.7.18 → llguidance-0.7.20}/toktrie/src/toktree.rs +0 -0
  171. {llguidance-0.7.18 → llguidance-0.7.20}/toktrie/tests/test_svob.rs +0 -0
  172. {llguidance-0.7.18 → llguidance-0.7.20}/toktrie_hf_downloader/LICENSE +0 -0
  173. {llguidance-0.7.18 → llguidance-0.7.20}/toktrie_hf_downloader/src/lib.rs +0 -0
  174. {llguidance-0.7.18 → llguidance-0.7.20}/toktrie_hf_tokenizers/LICENSE +0 -0
{llguidance-0.7.18 → llguidance-0.7.20}/.github/workflows/rust.yml RENAMED
@@ -52,3 +52,20 @@ jobs:
52
52
  with:
53
53
  name: wheels
54
54
  path: target/wheels/*
55
+
56
+ msrv:
57
+ name: MSRV Check
58
+
59
+ runs-on: ubuntu-latest
60
+
61
+ steps:
62
+ - uses: actions/checkout@v4
63
+
64
+ - name: Set up Rust
65
+ uses: dtolnay/rust-toolchain@1.80.0
66
+ with:
67
+ components: clippy
68
+
69
+ - name: Build parser
70
+ run: cargo build --verbose --locked
71
+ working-directory: parser
{llguidance-0.7.18 → llguidance-0.7.20}/CHANGELOG.md RENAMED
@@ -4,6 +4,17 @@ All notable changes to this project will be documented in this file. Dates are d
4
4
 
5
5
  If a release doesn't introduce any interesting changes (build fixes etc.), it's skipped.
6
6
 
7
+
8
+ #### [0.7.20](https://github.com/guidance-ai/llguidance/compare/v0.7.19...0.7.20) 2025-05-15
9
+
10
+ - use fancy-regex instead of onig as tokenizers regex library [`#172`](https://github.com/guidance-ai/llguidance/pull/172)
11
+ - fixes compilation on GCC 15, thanks [@Slowki](https://github.com/Slowki)
12
+ - msrv 1.80 support (incl. derivre bump) [`c89e386`](https://github.com/guidance-ai/llguidance/commit/c89e386685cd911a89fd47df225de88f88c10883), thank you [@nteodosio](https://github.com/nteodosio) for initial [PR](https://github.com/guidance-ai/llguidance/pull/170)!
13
+
14
+ #### [0.7.19](https://github.com/guidance-ai/llguidance/compare/v0.7.18...0.7.19) 2025-04-24
15
+
16
+ - fix a numeric token bug [`1f59edf`](https://github.com/guidance-ai/llguidance/commit/1f59edfc49b44cfba74b2380f34874a0778d9441)
17
+
7
18
  #### [0.7.18](https://github.com/guidance-ai/llguidance/compare/v0.7.17...0.7.18) 2025-04-22
8
19
 
9
20
  - apply x-guidance also in %json{} [`2627891`](https://github.com/guidance-ai/llguidance/commit/2627891c72c7e38062cd3e052f1de146d2e21635)
{llguidance-0.7.18 → llguidance-0.7.20}/Cargo.lock RENAMED
@@ -150,12 +150,6 @@ version = "0.8.0"
150
150
  source = "registry+https://github.com/rust-lang/crates.io-index"
151
151
  checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"
152
152
 
153
- [[package]]
154
- name = "bitflags"
155
- version = "1.3.2"
156
- source = "registry+https://github.com/rust-lang/crates.io-index"
157
- checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
158
-
159
153
  [[package]]
160
154
  name = "bitflags"
161
155
  version = "2.9.0"
@@ -401,9 +395,9 @@ dependencies = [
401
395
 
402
396
  [[package]]
403
397
  name = "derivre"
404
- version = "0.3.7"
398
+ version = "0.3.8"
405
399
  source = "registry+https://github.com/rust-lang/crates.io-index"
406
- checksum = "4a605f30e6a1460a323cc4de7bc62dea81df1d9d67eb92194d3a983a8a9601c4"
400
+ checksum = "786c7c65c4ef0c7deb05de3005e01991612a8f09fe0844fc0969c68b90468ba8"
407
401
  dependencies = [
408
402
  "ahash",
409
403
  "anyhow",
@@ -672,8 +666,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
672
666
  checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
673
667
  dependencies = [
674
668
  "cfg-if",
669
+ "js-sys",
675
670
  "libc",
676
671
  "wasi 0.11.0+wasi-snapshot-preview1",
672
+ "wasm-bindgen",
677
673
  ]
678
674
 
679
675
  [[package]]
@@ -1160,7 +1156,7 @@ version = "0.1.3"
1160
1156
  source = "registry+https://github.com/rust-lang/crates.io-index"
1161
1157
  checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d"
1162
1158
  dependencies = [
1163
- "bitflags 2.9.0",
1159
+ "bitflags",
1164
1160
  "libc",
1165
1161
  ]
1166
1162
 
@@ -1178,7 +1174,7 @@ checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856"
1178
1174
 
1179
1175
  [[package]]
1180
1176
  name = "llguidance"
1181
- version = "0.7.18"
1177
+ version = "0.7.20"
1182
1178
  dependencies = [
1183
1179
  "anyhow",
1184
1180
  "derivre",
@@ -1197,7 +1193,7 @@ dependencies = [
1197
1193
 
1198
1194
  [[package]]
1199
1195
  name = "llguidance_py"
1200
- version = "0.7.18"
1196
+ version = "0.7.20"
1201
1197
  dependencies = [
1202
1198
  "anyhow",
1203
1199
  "bytemuck",
@@ -1446,35 +1442,13 @@ version = "1.21.3"
1446
1442
  source = "registry+https://github.com/rust-lang/crates.io-index"
1447
1443
  checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
1448
1444
 
1449
- [[package]]
1450
- name = "onig"
1451
- version = "6.4.0"
1452
- source = "registry+https://github.com/rust-lang/crates.io-index"
1453
- checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f"
1454
- dependencies = [
1455
- "bitflags 1.3.2",
1456
- "libc",
1457
- "once_cell",
1458
- "onig_sys",
1459
- ]
1460
-
1461
- [[package]]
1462
- name = "onig_sys"
1463
- version = "69.8.1"
1464
- source = "registry+https://github.com/rust-lang/crates.io-index"
1465
- checksum = "7b829e3d7e9cc74c7e315ee8edb185bf4190da5acde74afd7fc59c35b1f086e7"
1466
- dependencies = [
1467
- "cc",
1468
- "pkg-config",
1469
- ]
1470
-
1471
1445
  [[package]]
1472
1446
  name = "openssl"
1473
1447
  version = "0.10.72"
1474
1448
  source = "registry+https://github.com/rust-lang/crates.io-index"
1475
1449
  checksum = "fedfea7d58a1f73118430a55da6a286e7b044961736ce96a16a17068ea25e5da"
1476
1450
  dependencies = [
1477
- "bitflags 2.9.0",
1451
+ "bitflags",
1478
1452
  "cfg-if",
1479
1453
  "foreign-types",
1480
1454
  "libc",
@@ -1747,7 +1721,7 @@ version = "0.5.11"
1747
1721
  source = "registry+https://github.com/rust-lang/crates.io-index"
1748
1722
  checksum = "d2f103c6d277498fbceb16e84d317e2a400f160f46904d5f5410848c829511a3"
1749
1723
  dependencies = [
1750
- "bitflags 2.9.0",
1724
+ "bitflags",
1751
1725
  ]
1752
1726
 
1753
1727
  [[package]]
@@ -1897,7 +1871,7 @@ version = "1.0.5"
1897
1871
  source = "registry+https://github.com/rust-lang/crates.io-index"
1898
1872
  checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf"
1899
1873
  dependencies = [
1900
- "bitflags 2.9.0",
1874
+ "bitflags",
1901
1875
  "errno",
1902
1876
  "libc",
1903
1877
  "linux-raw-sys",
@@ -1992,7 +1966,7 @@ version = "2.11.1"
1992
1966
  source = "registry+https://github.com/rust-lang/crates.io-index"
1993
1967
  checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02"
1994
1968
  dependencies = [
1995
- "bitflags 2.9.0",
1969
+ "bitflags",
1996
1970
  "core-foundation",
1997
1971
  "core-foundation-sys",
1998
1972
  "libc",
@@ -2185,7 +2159,7 @@ version = "0.6.1"
2185
2159
  source = "registry+https://github.com/rust-lang/crates.io-index"
2186
2160
  checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b"
2187
2161
  dependencies = [
2188
- "bitflags 2.9.0",
2162
+ "bitflags",
2189
2163
  "core-foundation",
2190
2164
  "system-configuration-sys",
2191
2165
  ]
@@ -2278,13 +2252,13 @@ dependencies = [
2278
2252
  "aho-corasick",
2279
2253
  "derive_builder",
2280
2254
  "esaxx-rs",
2255
+ "fancy-regex",
2281
2256
  "getrandom 0.2.15",
2282
2257
  "itertools 0.13.0",
2283
2258
  "lazy_static",
2284
2259
  "log",
2285
2260
  "macro_rules_attribute",
2286
2261
  "monostate",
2287
- "onig",
2288
2262
  "paste",
2289
2263
  "rand",
2290
2264
  "rayon",
@@ -2362,7 +2336,7 @@ dependencies = [
2362
2336
 
2363
2337
  [[package]]
2364
2338
  name = "toktrie"
2365
- version = "0.7.18"
2339
+ version = "0.7.20"
2366
2340
  dependencies = [
2367
2341
  "anyhow",
2368
2342
  "bytemuck",
@@ -2373,7 +2347,7 @@ dependencies = [
2373
2347
 
2374
2348
  [[package]]
2375
2349
  name = "toktrie_hf_downloader"
2376
- version = "0.7.18"
2350
+ version = "0.7.20"
2377
2351
  dependencies = [
2378
2352
  "anyhow",
2379
2353
  "hf-hub",
@@ -2384,7 +2358,7 @@ dependencies = [
2384
2358
 
2385
2359
  [[package]]
2386
2360
  name = "toktrie_hf_tokenizers"
2387
- version = "0.7.18"
2361
+ version = "0.7.20"
2388
2362
  dependencies = [
2389
2363
  "anyhow",
2390
2364
  "log",
@@ -2987,7 +2961,7 @@ version = "0.39.0"
2987
2961
  source = "registry+https://github.com/rust-lang/crates.io-index"
2988
2962
  checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"
2989
2963
  dependencies = [
2990
- "bitflags 2.9.0",
2964
+ "bitflags",
2991
2965
  ]
2992
2966
 
2993
2967
  [[package]]
{llguidance-0.7.18 → llguidance-0.7.20}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: llguidance
3
- Version: 0.7.18
3
+ Version: 0.7.20
4
4
  License-File: LICENSE
5
5
  Summary: Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance
6
6
  Author: Michal Moskal
{llguidance-0.7.18 → llguidance-0.7.20}/parser/Cargo.toml RENAMED
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "llguidance"
3
- version = "0.7.18"
3
+ version = "0.7.20"
4
4
  edition = "2021"
5
5
  license = "MIT"
6
6
  description = "Super-fast Structured Outputs"
@@ -8,7 +8,7 @@ repository = "https://github.com/guidance-ai/llguidance"
8
8
 
9
9
  [dependencies]
10
10
  toktrie = { workspace = true }
11
- derivre = { version = "=0.3.7", default-features = false, features = ["compress"] }
11
+ derivre = { version = "=0.3.8", default-features = false, features = ["compress"] }
12
12
  serde = { version = "1.0.217", features = ["derive"] }
13
13
  serde_json = { version = "1.0.138", features = ["preserve_order"] }
14
14
  anyhow = "1.0.95"
{llguidance-0.7.18 → llguidance-0.7.20}/parser/build.rs RENAMED
@@ -5,7 +5,7 @@ fn main() {
5
5
 
6
6
  let required_vars = [
7
7
  "CARGO",
8
- "CARGO_MANIFEST_PATH",
8
+ // "CARGO_MANIFEST_PATH",
9
9
  "CARGO_PKG_NAME",
10
10
  "CARGO_PKG_VERSION",
11
11
  "OUT_DIR",
{llguidance-0.7.18 → llguidance-0.7.20}/parser/llguidance.h RENAMED
@@ -494,7 +494,7 @@ int32_t llg_matcher_compute_mask(struct LlgMatcher *matcher);
494
494
  const uint32_t *llg_matcher_get_mask(struct LlgMatcher *matcher);
495
495
 
496
496
  /**
497
- * Return pointer to the mask computed by llg_matcher_compute_mask(), if any.
497
+ * Return the size of the mask in bytes.
498
498
  */
499
499
  size_t llg_matcher_get_mask_byte_size(struct LlgMatcher *matcher);
500
500
 
{llguidance-0.7.18 → llguidance-0.7.20}/parser/src/earley/lexer.rs RENAMED
@@ -221,8 +221,8 @@ impl Lexer {
221
221
  if enable_logging {
222
222
  let info = self.state_info(state);
223
223
  debug!(
224
- "lex: {:?} -{:?}-> {:?}, acpt={:?}",
225
- prev, byte as char, state, info.greedy_accepting
224
+ "lex: {:?} -{:?}-> {:?}, acpt={:?}/{:?}",
225
+ prev, byte as char, state, info.greedy_accepting, info.lazy_accepting
226
226
  );
227
227
  }
228
228
 
{llguidance-0.7.18 → llguidance-0.7.20}/parser/src/earley/regexvec.rs RENAMED
@@ -407,14 +407,18 @@ impl RegexVec {
407
407
  for (idx, e) in iter_state(&self.rx_sets, desc.state) {
408
408
  // If this lexeme is not a match. (If the derivative at this point is nullable,
409
409
  // there is a match, so if it is not nullable, there is no match.)
410
+ // println!("idx: {:?} e: {:?} {:?}", idx, e,self.special_token_rx);
410
411
  if !self.exprs.is_nullable(e) {
411
412
  // No match, so not at end of lexeme
412
413
  all_eoi = false;
413
414
  continue;
414
- } else if Some(e) == self.special_token_rx {
415
+ } else if Some(self.get_rx(idx)) == self.special_token_rx {
415
416
  // the regex is /\xFF\[[0-9]+\]/ so it's guaranteed not to conflict with anything
416
417
  // else (starts with non-unicode byte); thus we ignore the rest of processing
418
+ // when has_special_token is set, we just need to make sure lazy_accepting is non-empty,
419
+ // the actual value is not important
417
420
  desc.lazy_accepting = MatchingLexemes::One(idx);
421
+ desc.has_special_token = true;
418
422
  return;
419
423
  }
420
424
 
@@ -424,9 +428,6 @@ impl RegexVec {
424
428
  if lazies.is_none() {
425
429
  all_eoi = false;
426
430
  hidden_len = self.exprs.possible_lookahead_len(e) as u32;
427
- if Some(self.get_rx(idx)) == self.special_token_rx {
428
- desc.has_special_token = true;
429
- }
430
431
  }
431
432
  lazies.add(idx);
432
433
  continue;
@@ -711,7 +712,8 @@ impl RegexVec {
711
712
 
712
713
  self.lowest_match_inner(&mut res);
713
714
 
714
- // debug!("state {:?} desc: {:?}", state, res);
715
+ // println!("state {:?} desc: {:?}", state, res);
716
+
715
717
  res
716
718
  }
717
719
 
{llguidance-0.7.18 → llguidance-0.7.20}/parser/src/ffi.rs RENAMED
@@ -554,9 +554,12 @@ pub extern "C" fn llg_get_temperature(cc: &LlgConstraint) -> f32 {
554
554
  /// Check if constraint is stopped (cannot be extended further).
555
555
  #[no_mangle]
556
556
  pub extern "C" fn llg_is_stopped(cc: &LlgConstraint) -> bool {
557
- cc.constraint
558
- .as_ref()
559
- .is_none_or(|c| c.step_result().is_stop())
557
+ if let Some(c) = &cc.constraint {
558
+ c.step_result().is_stop()
559
+ } else {
560
+ // if there is no constraint, we consider it stopped
561
+ true
562
+ }
560
563
  }
561
564
 
562
565
  /// Compute mask for the next token sampling
@@ -1102,7 +1105,7 @@ pub extern "C" fn llg_matcher_get_mask(matcher: &mut LlgMatcher) -> *const u32 {
1102
1105
  .map_or(std::ptr::null(), |m| m.as_ptr())
1103
1106
  }
1104
1107
 
1105
- /// Return pointer to the mask computed by llg_matcher_compute_mask(), if any.
1108
+ /// Return the size of the mask in bytes.
1106
1109
  #[no_mangle]
1107
1110
  pub extern "C" fn llg_matcher_get_mask_byte_size(matcher: &mut LlgMatcher) -> usize {
1108
1111
  matcher.mask_elts() * 4
{llguidance-0.7.18 → llguidance-0.7.20}/pyproject.toml RENAMED
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "llguidance"
3
- version = "0.7.18"
3
+ version = "0.7.20"
4
4
  description = "Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance"
5
5
  requires-python = ">=3.9"
6
6
  license = "MIT"
{llguidance-0.7.18 → llguidance-0.7.20}/python_ext/Cargo.toml RENAMED
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "llguidance_py"
3
- version = "0.7.18"
3
+ version = "0.7.20"
4
4
  edition = "2021"
5
5
  license = "MIT"
6
6
  description = "Super-fast Structured Outputs"
{llguidance-0.7.18 → llguidance-0.7.20}/sample_parser/tests/test_ll.rs RENAMED
@@ -1262,3 +1262,52 @@ fn test_ll_tool_str_spec() {
1262
1262
 
1263
1263
  check_lark_grammar(TOOL_STR_SPEC_GRAMMAR, tool_chk);
1264
1264
  }
1265
+
1266
+ #[test]
1267
+ fn test_ll_numeric_bug() {
1268
+ check_lark_grammar(
1269
+ r#"
1270
+ start: text
1271
+ text: (text_tokens)* <[33000]> ap
1272
+ ap: <[33001]> (atok*)
1273
+ atok: <[400-410]>
1274
+ text_tokens: <[300-310]>
1275
+ "#,
1276
+ &[
1277
+ "",
1278
+ "<[300]>‧<[33000]>",
1279
+ "<[33001]>",
1280
+ "<[401]>‧<[402]>‧≺EOS≻",
1281
+ ],
1282
+ );
1283
+
1284
+ check_lark_grammar(
1285
+ r#"
1286
+ start: text
1287
+ text: (text_tokens)* ( (<[33000]> ap) | (<[33002]> (atok*)) )
1288
+ ap: <[33001]> (atok*)
1289
+ atok: <[400-410]>
1290
+ text_tokens: <[300-310]>
1291
+ "#,
1292
+ &[
1293
+ "",
1294
+ "<[300]>‧<[33000]>",
1295
+ "<[33001]>",
1296
+ "<[401]>‧<[402]>‧≺EOS≻",
1297
+ ],
1298
+ );
1299
+
1300
+ check_lark_grammar(
1301
+ r#"
1302
+ start: text
1303
+ text: (text_tokens)* ( (<[33000]> ap) | (<[33002]> (atok*)) )
1304
+ ap: <[33001,33003]> (atok*)
1305
+ atok: <[400-410]>
1306
+ text_tokens: <[300-310]>
1307
+ "#,
1308
+ &[
1309
+ "",
1310
+ "✖<[33001]>‧<[300]>‧<[33000]>✖<[33002]>✖<[300]>‧<[33001]>✖<[33002]>✖<[300]>‧<[401]>‧<[402]>‧≺EOS≻",
1311
+ ],
1312
+ );
1313
+ }
{llguidance-0.7.18 → llguidance-0.7.20}/toktrie/Cargo.toml RENAMED
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "toktrie"
3
- version = "0.7.18"
3
+ version = "0.7.20"
4
4
  edition = "2021"
5
5
  license = "MIT"
6
6
  description = "LLM Token Trie library"
{llguidance-0.7.18 → llguidance-0.7.20}/toktrie_hf_downloader/Cargo.toml RENAMED
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "toktrie_hf_downloader"
3
- version = "0.7.18"
3
+ version = "0.7.20"
4
4
  edition = "2021"
5
5
  license = "MIT"
6
6
  description = "HuggingFace Hub download library support for toktrie and llguidance"
{llguidance-0.7.18 → llguidance-0.7.20}/toktrie_hf_tokenizers/Cargo.toml RENAMED
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "toktrie_hf_tokenizers"
3
- version = "0.7.18"
3
+ version = "0.7.20"
4
4
  edition = "2021"
5
5
  license = "MIT"
6
6
  description = "HuggingFace tokenizers library support for toktrie and llguidance"
@@ -11,5 +11,8 @@ toktrie = { workspace = true }
11
11
  serde = { version = "1.0.217", features = ["derive"] }
12
12
  serde_json = "1.0.138"
13
13
  anyhow = "1.0.95"
14
- tokenizers = { version = ">=0.20.0, <1.0.0", default-features = false, features = ["onig"] }
14
+ tokenizers = { version = ">=0.20.0, <1.0.0", default-features = false, features = [
15
+ "unstable_wasm",
16
+ "fancy-regex",
17
+ ] }
15
18
  log = "0.4.25"
{llguidance-0.7.18 → llguidance-0.7.20}/toktrie_hf_tokenizers/src/lib.rs RENAMED
@@ -213,7 +213,9 @@ impl ByteTokenizerEnv {
213
213
  bail!("vocab size too small; {} vs {}", n_vocab, token_bytes.len());
214
214
  }
215
215
  while n_vocab > token_bytes.len() {
216
- token_bytes.push(Vec::new());
216
+ let mut name = format!(".<[{}]>", token_bytes.len()).into_bytes();
217
+ name[0] = TokTrie::SPECIAL_TOKEN_MARKER;
218
+ token_bytes.push(name);
217
219
  }
218
220
  info.vocab_size = n_vocab as u32;
219
221
  }
{llguidance-0.7.18 → llguidance-0.7.20}/.gitignore RENAMED
File without changes
{llguidance-0.7.18 → llguidance-0.7.20}/Cargo.toml RENAMED
File without changes
{llguidance-0.7.18 → llguidance-0.7.20}/LICENSE RENAMED
File without changes
{llguidance-0.7.18 → llguidance-0.7.20}/README.md RENAMED
File without changes
{llguidance-0.7.18 → llguidance-0.7.20}/SECURITY.md RENAMED
File without changes
{llguidance-0.7.18 → llguidance-0.7.20}/SUPPORT.md RENAMED
File without changes
{llguidance-0.7.18 → llguidance-0.7.20}/docs/syntax.md RENAMED
File without changes
{llguidance-0.7.18 → llguidance-0.7.20}/docs/toktrie.md RENAMED
File without changes
{llguidance-0.7.18 → llguidance-0.7.20}/parser/LICENSE RENAMED
File without changes
{llguidance-0.7.18 → llguidance-0.7.20}/plan.md RENAMED
File without changes
{llguidance-0.7.18 → llguidance-0.7.20}/python/mypy.ini RENAMED
File without changes
{llguidance-0.7.18 → llguidance-0.7.20}/scripts/bump.py RENAMED
File without changes
{llguidance-0.7.18 → llguidance-0.7.20}/toktrie/LICENSE RENAMED
File without changes