llguidance 1.0.0__tar.gz → 1.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. {llguidance-1.0.0 → llguidance-1.1.0}/CHANGELOG.md +14 -0
  2. {llguidance-1.0.0 → llguidance-1.1.0}/Cargo.lock +387 -424
  3. {llguidance-1.0.0 → llguidance-1.1.0}/Cargo.toml +2 -1
  4. {llguidance-1.0.0 → llguidance-1.1.0}/PKG-INFO +1 -1
  5. llguidance-1.1.0/docs/de_recursing.md +51 -0
  6. {llguidance-1.0.0 → llguidance-1.1.0}/docs/syntax.md +5 -0
  7. {llguidance-1.0.0 → llguidance-1.1.0}/json_stats/src/json_stats.rs +15 -16
  8. {llguidance-1.0.0 → llguidance-1.1.0}/json_stats/src/stats.rs +7 -7
  9. {llguidance-1.0.0 → llguidance-1.1.0}/parser/Cargo.toml +1 -1
  10. {llguidance-1.0.0 → llguidance-1.1.0}/parser/build.rs +1 -1
  11. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/api.rs +2 -2
  12. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/earley/from_guidance.rs +2 -2
  13. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/earley/grammar.rs +29 -30
  14. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/earley/lexer.rs +1 -1
  15. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/earley/lexerspec.rs +3 -3
  16. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/earley/parser.rs +7 -9
  17. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/earley/regexvec.rs +2 -2
  18. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/earley/slicer.rs +1 -1
  19. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/grammar_builder.rs +75 -2
  20. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/json/compiler.rs +9 -20
  21. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/json/numeric.rs +29 -37
  22. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/json/schema.rs +4 -4
  23. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/lark/compiler.rs +20 -4
  24. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/lark/lexer.rs +6 -6
  25. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/logging.rs +1 -1
  26. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/stop_controller.rs +1 -1
  27. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/tokenparser.rs +4 -4
  28. {llguidance-1.0.0 → llguidance-1.1.0}/pyproject.toml +1 -2
  29. {llguidance-1.0.0 → llguidance-1.1.0}/python/llguidance/hf.py +12 -7
  30. {llguidance-1.0.0 → llguidance-1.1.0}/python/llguidance/llamacpp.py +8 -2
  31. {llguidance-1.0.0 → llguidance-1.1.0}/python_ext/Cargo.toml +1 -1
  32. {llguidance-1.0.0 → llguidance-1.1.0}/python_ext/src/pyjson.rs +7 -9
  33. {llguidance-1.0.0 → llguidance-1.1.0}/sample_parser/src/lib.rs +18 -36
  34. {llguidance-1.0.0 → llguidance-1.1.0}/sample_parser/src/sample_parser.rs +4 -4
  35. llguidance-1.1.0/sample_parser/tests/common_lark_utils/mod.rs +144 -0
  36. {llguidance-1.0.0 → llguidance-1.1.0}/sample_parser/tests/test_lark.rs +24 -151
  37. {llguidance-1.0.0 → llguidance-1.1.0}/sample_parser/tests/test_ll.rs +26 -4
  38. {llguidance-1.0.0 → llguidance-1.1.0}/sample_parser/tests/test_raw_parser.rs +3 -3
  39. {llguidance-1.0.0 → llguidance-1.1.0}/toktrie/Cargo.toml +1 -1
  40. {llguidance-1.0.0 → llguidance-1.1.0}/toktrie/src/bytes.rs +3 -3
  41. {llguidance-1.0.0 → llguidance-1.1.0}/toktrie/src/toktree.rs +12 -13
  42. {llguidance-1.0.0 → llguidance-1.1.0}/toktrie_hf_downloader/Cargo.toml +1 -1
  43. {llguidance-1.0.0 → llguidance-1.1.0}/toktrie_hf_tokenizers/Cargo.toml +2 -2
  44. {llguidance-1.0.0 → llguidance-1.1.0}/toktrie_hf_tokenizers/src/lib.rs +3 -3
  45. {llguidance-1.0.0 → llguidance-1.1.0}/toktrie_tiktoken/Cargo.toml +1 -1
  46. {llguidance-1.0.0 → llguidance-1.1.0}/toktrie_tiktoken/src/lib.rs +1 -1
  47. {llguidance-1.0.0 → llguidance-1.1.0}/.github/workflows/rust.yml +0 -0
  48. {llguidance-1.0.0 → llguidance-1.1.0}/.github/workflows/wheels.yml +0 -0
  49. {llguidance-1.0.0 → llguidance-1.1.0}/.gitignore +0 -0
  50. {llguidance-1.0.0 → llguidance-1.1.0}/CODE_OF_CONDUCT.md +0 -0
  51. {llguidance-1.0.0 → llguidance-1.1.0}/LICENSE +0 -0
  52. {llguidance-1.0.0 → llguidance-1.1.0}/README.md +0 -0
  53. {llguidance-1.0.0 → llguidance-1.1.0}/SECURITY.md +0 -0
  54. {llguidance-1.0.0 → llguidance-1.1.0}/SUPPORT.md +0 -0
  55. {llguidance-1.0.0 → llguidance-1.1.0}/c_sample/Makefile +0 -0
  56. {llguidance-1.0.0 → llguidance-1.1.0}/c_sample/README.md +0 -0
  57. {llguidance-1.0.0 → llguidance-1.1.0}/c_sample/c_sample.cpp +0 -0
  58. {llguidance-1.0.0 → llguidance-1.1.0}/docs/fast_forward.md +0 -0
  59. {llguidance-1.0.0 → llguidance-1.1.0}/docs/json_schema.md +0 -0
  60. {llguidance-1.0.0 → llguidance-1.1.0}/docs/mask_plot.png +0 -0
  61. {llguidance-1.0.0 → llguidance-1.1.0}/docs/optimizations.md +0 -0
  62. {llguidance-1.0.0 → llguidance-1.1.0}/docs/parametric.md +0 -0
  63. {llguidance-1.0.0 → llguidance-1.1.0}/docs/special_tokens.md +0 -0
  64. {llguidance-1.0.0 → llguidance-1.1.0}/docs/toktrie.md +0 -0
  65. {llguidance-1.0.0 → llguidance-1.1.0}/json_stats/Cargo.toml +0 -0
  66. {llguidance-1.0.0 → llguidance-1.1.0}/json_stats/expected_maskbench.json +0 -0
  67. {llguidance-1.0.0 → llguidance-1.1.0}/json_stats/jstats.sh +0 -0
  68. {llguidance-1.0.0 → llguidance-1.1.0}/json_stats/scripts/split-stats.sh +0 -0
  69. {llguidance-1.0.0 → llguidance-1.1.0}/json_stats/scripts/split_plot.py +0 -0
  70. {llguidance-1.0.0 → llguidance-1.1.0}/json_stats/src/lib.rs +0 -0
  71. {llguidance-1.0.0 → llguidance-1.1.0}/parser/LICENSE +0 -0
  72. {llguidance-1.0.0 → llguidance-1.1.0}/parser/README.md +0 -0
  73. {llguidance-1.0.0 → llguidance-1.1.0}/parser/cbindgen.toml +0 -0
  74. {llguidance-1.0.0 → llguidance-1.1.0}/parser/grammars/character.json +0 -0
  75. {llguidance-1.0.0 → llguidance-1.1.0}/parser/grammars/json.json +0 -0
  76. {llguidance-1.0.0 → llguidance-1.1.0}/parser/llguidance.h +0 -0
  77. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/constraint.rs +0 -0
  78. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/earley/mod.rs +0 -0
  79. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/earley/perf.rs +0 -0
  80. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/factory.rs +0 -0
  81. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/ffi.rs +0 -0
  82. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/ffi_par.rs +0 -0
  83. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/hashcons.rs +0 -0
  84. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/json/README.md +0 -0
  85. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/json/context_ref.rs +0 -0
  86. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/json/context_simple/context.rs +0 -0
  87. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/json/context_simple/draft.rs +0 -0
  88. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/json/context_simple/mod.rs +0 -0
  89. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/json/formats.rs +0 -0
  90. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/json/mod.rs +0 -0
  91. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/json/shared_context.rs +0 -0
  92. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/json_validation.rs +0 -0
  93. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/lark/README.md +0 -0
  94. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/lark/ast.rs +0 -0
  95. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/lark/common.rs +0 -0
  96. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/lark/mod.rs +0 -0
  97. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/lark/parser.rs +0 -0
  98. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/lib.rs +0 -0
  99. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/matcher.rs +0 -0
  100. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/output.rs +0 -0
  101. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/panic_utils.rs +0 -0
  102. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/regex_rewrite.rs +0 -0
  103. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/substring.rs +0 -0
  104. {llguidance-1.0.0 → llguidance-1.1.0}/parser/src/tokenizer_json.rs +0 -0
  105. {llguidance-1.0.0 → llguidance-1.1.0}/plan.md +0 -0
  106. {llguidance-1.0.0 → llguidance-1.1.0}/python/llguidance/__init__.py +0 -0
  107. {llguidance-1.0.0 → llguidance-1.1.0}/python/llguidance/_grammar_from.py +0 -0
  108. {llguidance-1.0.0 → llguidance-1.1.0}/python/llguidance/_lib.pyi +0 -0
  109. {llguidance-1.0.0 → llguidance-1.1.0}/python/llguidance/_struct_tag.py +0 -0
  110. {llguidance-1.0.0 → llguidance-1.1.0}/python/llguidance/_tokenizer.py +0 -0
  111. {llguidance-1.0.0 → llguidance-1.1.0}/python/llguidance/_util.py +0 -0
  112. {llguidance-1.0.0 → llguidance-1.1.0}/python/llguidance/cli.py +0 -0
  113. {llguidance-1.0.0 → llguidance-1.1.0}/python/llguidance/gbnf_to_lark.py +0 -0
  114. {llguidance-1.0.0 → llguidance-1.1.0}/python/llguidance/mlx.py +0 -0
  115. {llguidance-1.0.0 → llguidance-1.1.0}/python/llguidance/numpy.py +0 -0
  116. {llguidance-1.0.0 → llguidance-1.1.0}/python/llguidance/py.typed +0 -0
  117. {llguidance-1.0.0 → llguidance-1.1.0}/python/llguidance/tiktoken.py +0 -0
  118. {llguidance-1.0.0 → llguidance-1.1.0}/python/llguidance/torch.py +0 -0
  119. {llguidance-1.0.0 → llguidance-1.1.0}/python/mypy.ini +0 -0
  120. {llguidance-1.0.0 → llguidance-1.1.0}/python/torch_tests/__init__.py +0 -0
  121. {llguidance-1.0.0 → llguidance-1.1.0}/python/torch_tests/test_bitmask.py +0 -0
  122. {llguidance-1.0.0 → llguidance-1.1.0}/python/torch_tests/test_hf.py +0 -0
  123. {llguidance-1.0.0 → llguidance-1.1.0}/python/torch_tests/test_llamacpp.py +0 -0
  124. {llguidance-1.0.0 → llguidance-1.1.0}/python/torch_tests/test_matcher.py +0 -0
  125. {llguidance-1.0.0 → llguidance-1.1.0}/python/torch_tests/test_tiktoken.py +0 -0
  126. {llguidance-1.0.0 → llguidance-1.1.0}/python_ext/src/lib.rs +0 -0
  127. {llguidance-1.0.0 → llguidance-1.1.0}/python_ext/src/llamatokenizer.rs +0 -0
  128. {llguidance-1.0.0 → llguidance-1.1.0}/python_ext/src/llinterpreter.rs +0 -0
  129. {llguidance-1.0.0 → llguidance-1.1.0}/python_ext/src/llmatcher.rs +0 -0
  130. {llguidance-1.0.0 → llguidance-1.1.0}/python_ext/src/parserlimits.rs +0 -0
  131. {llguidance-1.0.0 → llguidance-1.1.0}/python_ext/src/py.rs +0 -0
  132. {llguidance-1.0.0 → llguidance-1.1.0}/sample_parser/Cargo.toml +0 -0
  133. {llguidance-1.0.0 → llguidance-1.1.0}/sample_parser/README.md +0 -0
  134. {llguidance-1.0.0 → llguidance-1.1.0}/sample_parser/cli.sh +0 -0
  135. {llguidance-1.0.0 → llguidance-1.1.0}/sample_parser/data/blog.sample.json +0 -0
  136. {llguidance-1.0.0 → llguidance-1.1.0}/sample_parser/data/blog.schema.json +0 -0
  137. {llguidance-1.0.0 → llguidance-1.1.0}/sample_parser/data/blog.schema.ll.json +0 -0
  138. {llguidance-1.0.0 → llguidance-1.1.0}/sample_parser/data/from-llama.cpp/README.md +0 -0
  139. {llguidance-1.0.0 → llguidance-1.1.0}/sample_parser/data/from-llama.cpp/arithmetic.gbnf +0 -0
  140. {llguidance-1.0.0 → llguidance-1.1.0}/sample_parser/data/from-llama.cpp/c.gbnf +0 -0
  141. {llguidance-1.0.0 → llguidance-1.1.0}/sample_parser/data/from-llama.cpp/chess.gbnf +0 -0
  142. {llguidance-1.0.0 → llguidance-1.1.0}/sample_parser/data/from-llama.cpp/english.gbnf +0 -0
  143. {llguidance-1.0.0 → llguidance-1.1.0}/sample_parser/data/from-llama.cpp/japanese.gbnf +0 -0
  144. {llguidance-1.0.0 → llguidance-1.1.0}/sample_parser/data/from-llama.cpp/json.gbnf +0 -0
  145. {llguidance-1.0.0 → llguidance-1.1.0}/sample_parser/data/from-llama.cpp/json_arr.gbnf +0 -0
  146. {llguidance-1.0.0 → llguidance-1.1.0}/sample_parser/data/from-llama.cpp/list.gbnf +0 -0
  147. {llguidance-1.0.0 → llguidance-1.1.0}/sample_parser/data/from-llama.cpp/vllm-sql.gbnf +0 -0
  148. {llguidance-1.0.0 → llguidance-1.1.0}/sample_parser/data/lark.lark +0 -0
  149. {llguidance-1.0.0 → llguidance-1.1.0}/sample_parser/data/rfc.lark +0 -0
  150. {llguidance-1.0.0 → llguidance-1.1.0}/sample_parser/data/rfc.xml +0 -0
  151. {llguidance-1.0.0 → llguidance-1.1.0}/sample_parser/data/ulysses.md +0 -0
  152. {llguidance-1.0.0 → llguidance-1.1.0}/sample_parser/gtest.sh +0 -0
  153. {llguidance-1.0.0 → llguidance-1.1.0}/sample_parser/lark.sh +0 -0
  154. {llguidance-1.0.0 → llguidance-1.1.0}/sample_parser/run.sh +0 -0
  155. {llguidance-1.0.0 → llguidance-1.1.0}/sample_parser/src/minimal.rs +0 -0
  156. {llguidance-1.0.0 → llguidance-1.1.0}/sample_parser/tests/test_stop.rs +0 -0
  157. {llguidance-1.0.0 → llguidance-1.1.0}/scripts/annotate_asm.js +0 -0
  158. {llguidance-1.0.0 → llguidance-1.1.0}/scripts/bump.py +0 -0
  159. {llguidance-1.0.0 → llguidance-1.1.0}/scripts/cbindgen.sh +0 -0
  160. {llguidance-1.0.0 → llguidance-1.1.0}/scripts/checklinks.py +0 -0
  161. {llguidance-1.0.0 → llguidance-1.1.0}/scripts/checklinks.sh +0 -0
  162. {llguidance-1.0.0 → llguidance-1.1.0}/scripts/ci-publish.py +0 -0
  163. {llguidance-1.0.0 → llguidance-1.1.0}/scripts/disasm.sh +0 -0
  164. {llguidance-1.0.0 → llguidance-1.1.0}/scripts/gbnf_to_lark.py +0 -0
  165. {llguidance-1.0.0 → llguidance-1.1.0}/scripts/gen-testcase.py +0 -0
  166. {llguidance-1.0.0 → llguidance-1.1.0}/scripts/git-version.sh +0 -0
  167. {llguidance-1.0.0 → llguidance-1.1.0}/scripts/install-deps.sh +0 -0
  168. {llguidance-1.0.0 → llguidance-1.1.0}/scripts/jsonschema-stats.js +0 -0
  169. {llguidance-1.0.0 → llguidance-1.1.0}/scripts/remote-guidance-test.sh +0 -0
  170. {llguidance-1.0.0 → llguidance-1.1.0}/scripts/rust-size.js +0 -0
  171. {llguidance-1.0.0 → llguidance-1.1.0}/scripts/rust_size.py +0 -0
  172. {llguidance-1.0.0 → llguidance-1.1.0}/scripts/test-guidance.sh +0 -0
  173. {llguidance-1.0.0 → llguidance-1.1.0}/scripts/tokenizer_test.py +0 -0
  174. {llguidance-1.0.0 → llguidance-1.1.0}/scripts/update-git.py +0 -0
  175. {llguidance-1.0.0 → llguidance-1.1.0}/toktrie/LICENSE +0 -0
  176. {llguidance-1.0.0 → llguidance-1.1.0}/toktrie/README.md +0 -0
  177. {llguidance-1.0.0 → llguidance-1.1.0}/toktrie/src/lib.rs +0 -0
  178. {llguidance-1.0.0 → llguidance-1.1.0}/toktrie/src/recognizer.rs +0 -0
  179. {llguidance-1.0.0 → llguidance-1.1.0}/toktrie/src/rng.rs +0 -0
  180. {llguidance-1.0.0 → llguidance-1.1.0}/toktrie/src/svob.rs +0 -0
  181. {llguidance-1.0.0 → llguidance-1.1.0}/toktrie/src/tokenv.rs +0 -0
  182. {llguidance-1.0.0 → llguidance-1.1.0}/toktrie/tests/test_svob.rs +0 -0
  183. {llguidance-1.0.0 → llguidance-1.1.0}/toktrie_hf_downloader/LICENSE +0 -0
  184. {llguidance-1.0.0 → llguidance-1.1.0}/toktrie_hf_downloader/src/lib.rs +0 -0
  185. {llguidance-1.0.0 → llguidance-1.1.0}/toktrie_hf_tokenizers/LICENSE +0 -0
  186. {llguidance-1.0.0 → llguidance-1.1.0}/toktrie_tiktoken/LICENSE +0 -0
{llguidance-1.0.0 → llguidance-1.1.0}/CHANGELOG.md RENAMED
@@ -4,6 +4,20 @@ All notable changes to this project will be documented in this file. Dates are d
4
4
 
5
5
  If a release doesn't introduce any interesting changes (build fixes etc.), it's skipped.
6
6
 
7
+ #### [1.1.0](https://github.com/guidance-ai/llguidance/compare/v1.0.1...1.1.0) 2025-07-18
8
+
9
+ - disable hf tokenizer truncation and padding [`#205`](https://github.com//hudson-ai/llguidance.git/pull/205); fixes [`#1322`](https://github.com/guidance-ai/guidance/issues/1322)
10
+ - llama_cpp tokenizers: infer added tokens starting/ending with < and > to be special tokens [`#202`](https://github.com//hudson-ai/llguidance.git/pull/202)
11
+ - add lark syntax for "any token" and negation of token ranges [`#201`](https://github.com//hudson-ai/llguidance.git/pull/201)
12
+ - add de-recursion cook book to docs [`#199`](https://github.com//hudson-ai/llguidance.git/pull/199)
13
+
14
+ #### [1.0.1](https://github.com/guidance-ai/llguidance/compare/v1.0.0...1.0.1) 2025-07-03
15
+
16
+ - fix: tokenizers normalizers sequence api changed [`#195`](https://github.com/guidance-ai/llguidance/pull/195)
17
+ - Strip debug info from the wheels [`#194`](https://github.com/guidance-ai/llguidance/pull/194)
18
+
19
+ Thank you @ammar-elsabe and @Ahajha!
20
+
7
21
  #### [1.0.0](https://github.com/guidance-ai/llguidance/compare/v0.7.30...1.0.0) 2025-06-23
8
22
 
9
23
  This is identical to `0.7.30`, but indicates intended stability and from now on we'll try to follow semver.