llguidance 1.1.1__tar.gz → 1.1.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (192) hide show
  1. llguidance-1.1.2/.github/workflows/code-coverage.yml +62 -0
  2. {llguidance-1.1.1 → llguidance-1.1.2}/CHANGELOG.md +29 -24
  3. {llguidance-1.1.1 → llguidance-1.1.2}/Cargo.lock +6 -6
  4. {llguidance-1.1.1 → llguidance-1.1.2}/PKG-INFO +2 -2
  5. {llguidance-1.1.1 → llguidance-1.1.2}/parser/Cargo.toml +1 -1
  6. {llguidance-1.1.1 → llguidance-1.1.2}/parser/llguidance.h +6 -0
  7. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/api.rs +6 -0
  8. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/panic_utils.rs +1 -1
  9. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/tokenparser.rs +9 -5
  10. {llguidance-1.1.1 → llguidance-1.1.2}/pyproject.toml +1 -1
  11. {llguidance-1.1.1 → llguidance-1.1.2}/python/llguidance/_lib.pyi +9 -0
  12. {llguidance-1.1.1 → llguidance-1.1.2}/python_ext/Cargo.toml +1 -1
  13. {llguidance-1.1.1 → llguidance-1.1.2}/python_ext/src/parserlimits.rs +11 -1
  14. {llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/tests/common_lark_utils/mod.rs +49 -3
  15. {llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/tests/test_json_arrays.rs +86 -0
  16. llguidance-1.1.2/sample_parser/tests/test_json_enum_const.rs +92 -0
  17. {llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/tests/test_json_primitives.rs +238 -74
  18. llguidance-1.1.2/sample_parser/tests/test_json_schema_combinations.rs +154 -0
  19. {llguidance-1.1.1 → llguidance-1.1.2}/scripts/bump.py +2 -3
  20. {llguidance-1.1.1 → llguidance-1.1.2}/toktrie/Cargo.toml +1 -1
  21. {llguidance-1.1.1 → llguidance-1.1.2}/toktrie_hf_downloader/Cargo.toml +1 -1
  22. {llguidance-1.1.1 → llguidance-1.1.2}/toktrie_hf_tokenizers/Cargo.toml +1 -1
  23. {llguidance-1.1.1 → llguidance-1.1.2}/toktrie_tiktoken/Cargo.toml +1 -1
  24. {llguidance-1.1.1 → llguidance-1.1.2}/.github/workflows/rust.yml +0 -0
  25. {llguidance-1.1.1 → llguidance-1.1.2}/.github/workflows/wheels.yml +0 -0
  26. {llguidance-1.1.1 → llguidance-1.1.2}/.gitignore +0 -0
  27. {llguidance-1.1.1 → llguidance-1.1.2}/CODE_OF_CONDUCT.md +0 -0
  28. {llguidance-1.1.1 → llguidance-1.1.2}/Cargo.toml +0 -0
  29. {llguidance-1.1.1 → llguidance-1.1.2}/LICENSE +0 -0
  30. {llguidance-1.1.1 → llguidance-1.1.2}/README.md +0 -0
  31. {llguidance-1.1.1 → llguidance-1.1.2}/SECURITY.md +0 -0
  32. {llguidance-1.1.1 → llguidance-1.1.2}/SUPPORT.md +0 -0
  33. {llguidance-1.1.1 → llguidance-1.1.2}/c_sample/Makefile +0 -0
  34. {llguidance-1.1.1 → llguidance-1.1.2}/c_sample/README.md +0 -0
  35. {llguidance-1.1.1 → llguidance-1.1.2}/c_sample/c_sample.cpp +0 -0
  36. {llguidance-1.1.1 → llguidance-1.1.2}/docs/de_recursing.md +0 -0
  37. {llguidance-1.1.1 → llguidance-1.1.2}/docs/fast_forward.md +0 -0
  38. {llguidance-1.1.1 → llguidance-1.1.2}/docs/json_schema.md +0 -0
  39. {llguidance-1.1.1 → llguidance-1.1.2}/docs/mask_plot.png +0 -0
  40. {llguidance-1.1.1 → llguidance-1.1.2}/docs/optimizations.md +0 -0
  41. {llguidance-1.1.1 → llguidance-1.1.2}/docs/parametric.md +0 -0
  42. {llguidance-1.1.1 → llguidance-1.1.2}/docs/special_tokens.md +0 -0
  43. {llguidance-1.1.1 → llguidance-1.1.2}/docs/syntax.md +0 -0
  44. {llguidance-1.1.1 → llguidance-1.1.2}/docs/toktrie.md +0 -0
  45. {llguidance-1.1.1 → llguidance-1.1.2}/json_stats/Cargo.toml +0 -0
  46. {llguidance-1.1.1 → llguidance-1.1.2}/json_stats/expected_maskbench.json +0 -0
  47. {llguidance-1.1.1 → llguidance-1.1.2}/json_stats/jstats.sh +0 -0
  48. {llguidance-1.1.1 → llguidance-1.1.2}/json_stats/scripts/split-stats.sh +0 -0
  49. {llguidance-1.1.1 → llguidance-1.1.2}/json_stats/scripts/split_plot.py +0 -0
  50. {llguidance-1.1.1 → llguidance-1.1.2}/json_stats/src/json_stats.rs +0 -0
  51. {llguidance-1.1.1 → llguidance-1.1.2}/json_stats/src/lib.rs +0 -0
  52. {llguidance-1.1.1 → llguidance-1.1.2}/json_stats/src/stats.rs +0 -0
  53. {llguidance-1.1.1 → llguidance-1.1.2}/parser/LICENSE +0 -0
  54. {llguidance-1.1.1 → llguidance-1.1.2}/parser/README.md +0 -0
  55. {llguidance-1.1.1 → llguidance-1.1.2}/parser/build.rs +0 -0
  56. {llguidance-1.1.1 → llguidance-1.1.2}/parser/cbindgen.toml +0 -0
  57. {llguidance-1.1.1 → llguidance-1.1.2}/parser/grammars/character.json +0 -0
  58. {llguidance-1.1.1 → llguidance-1.1.2}/parser/grammars/json.json +0 -0
  59. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/constraint.rs +0 -0
  60. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/earley/from_guidance.rs +0 -0
  61. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/earley/grammar.rs +0 -0
  62. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/earley/lexer.rs +0 -0
  63. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/earley/lexerspec.rs +0 -0
  64. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/earley/mod.rs +0 -0
  65. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/earley/parser.rs +0 -0
  66. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/earley/perf.rs +0 -0
  67. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/earley/regexvec.rs +0 -0
  68. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/earley/slicer.rs +0 -0
  69. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/factory.rs +0 -0
  70. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/ffi.rs +0 -0
  71. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/ffi_par.rs +0 -0
  72. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/grammar_builder.rs +0 -0
  73. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/hashcons.rs +0 -0
  74. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/json/README.md +0 -0
  75. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/json/compiler.rs +0 -0
  76. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/json/context_ref.rs +0 -0
  77. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/json/context_simple/context.rs +0 -0
  78. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/json/context_simple/draft.rs +0 -0
  79. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/json/context_simple/mod.rs +0 -0
  80. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/json/formats.rs +0 -0
  81. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/json/mod.rs +0 -0
  82. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/json/numeric.rs +0 -0
  83. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/json/schema.rs +0 -0
  84. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/json/shared_context.rs +0 -0
  85. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/json_validation.rs +0 -0
  86. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/lark/README.md +0 -0
  87. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/lark/ast.rs +0 -0
  88. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/lark/common.rs +0 -0
  89. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/lark/compiler.rs +0 -0
  90. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/lark/lexer.rs +0 -0
  91. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/lark/mod.rs +0 -0
  92. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/lark/parser.rs +0 -0
  93. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/lib.rs +0 -0
  94. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/logging.rs +0 -0
  95. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/matcher.rs +0 -0
  96. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/output.rs +0 -0
  97. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/regex_rewrite.rs +0 -0
  98. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/stop_controller.rs +0 -0
  99. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/substring.rs +0 -0
  100. {llguidance-1.1.1 → llguidance-1.1.2}/parser/src/tokenizer_json.rs +0 -0
  101. {llguidance-1.1.1 → llguidance-1.1.2}/plan.md +0 -0
  102. {llguidance-1.1.1 → llguidance-1.1.2}/python/llguidance/__init__.py +0 -0
  103. {llguidance-1.1.1 → llguidance-1.1.2}/python/llguidance/_grammar_from.py +0 -0
  104. {llguidance-1.1.1 → llguidance-1.1.2}/python/llguidance/_struct_tag.py +0 -0
  105. {llguidance-1.1.1 → llguidance-1.1.2}/python/llguidance/_tokenizer.py +0 -0
  106. {llguidance-1.1.1 → llguidance-1.1.2}/python/llguidance/_util.py +0 -0
  107. {llguidance-1.1.1 → llguidance-1.1.2}/python/llguidance/cli.py +0 -0
  108. {llguidance-1.1.1 → llguidance-1.1.2}/python/llguidance/gbnf_to_lark.py +0 -0
  109. {llguidance-1.1.1 → llguidance-1.1.2}/python/llguidance/hf.py +0 -0
  110. {llguidance-1.1.1 → llguidance-1.1.2}/python/llguidance/llamacpp.py +0 -0
  111. {llguidance-1.1.1 → llguidance-1.1.2}/python/llguidance/mlx.py +0 -0
  112. {llguidance-1.1.1 → llguidance-1.1.2}/python/llguidance/numpy.py +0 -0
  113. {llguidance-1.1.1 → llguidance-1.1.2}/python/llguidance/py.typed +0 -0
  114. {llguidance-1.1.1 → llguidance-1.1.2}/python/llguidance/tiktoken.py +0 -0
  115. {llguidance-1.1.1 → llguidance-1.1.2}/python/llguidance/torch.py +0 -0
  116. {llguidance-1.1.1 → llguidance-1.1.2}/python/mypy.ini +0 -0
  117. {llguidance-1.1.1 → llguidance-1.1.2}/python/torch_tests/__init__.py +0 -0
  118. {llguidance-1.1.1 → llguidance-1.1.2}/python/torch_tests/test_bitmask.py +0 -0
  119. {llguidance-1.1.1 → llguidance-1.1.2}/python/torch_tests/test_hf.py +0 -0
  120. {llguidance-1.1.1 → llguidance-1.1.2}/python/torch_tests/test_llamacpp.py +0 -0
  121. {llguidance-1.1.1 → llguidance-1.1.2}/python/torch_tests/test_matcher.py +0 -0
  122. {llguidance-1.1.1 → llguidance-1.1.2}/python/torch_tests/test_tiktoken.py +0 -0
  123. {llguidance-1.1.1 → llguidance-1.1.2}/python_ext/src/lib.rs +0 -0
  124. {llguidance-1.1.1 → llguidance-1.1.2}/python_ext/src/llamatokenizer.rs +0 -0
  125. {llguidance-1.1.1 → llguidance-1.1.2}/python_ext/src/llinterpreter.rs +0 -0
  126. {llguidance-1.1.1 → llguidance-1.1.2}/python_ext/src/llmatcher.rs +0 -0
  127. {llguidance-1.1.1 → llguidance-1.1.2}/python_ext/src/py.rs +0 -0
  128. {llguidance-1.1.1 → llguidance-1.1.2}/python_ext/src/pyjson.rs +0 -0
  129. {llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/Cargo.toml +0 -0
  130. {llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/README.md +0 -0
  131. {llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/cli.sh +0 -0
  132. {llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/data/blog.sample.json +0 -0
  133. {llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/data/blog.schema.json +0 -0
  134. {llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/data/blog.schema.ll.json +0 -0
  135. {llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/data/from-llama.cpp/README.md +0 -0
  136. {llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/data/from-llama.cpp/arithmetic.gbnf +0 -0
  137. {llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/data/from-llama.cpp/c.gbnf +0 -0
  138. {llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/data/from-llama.cpp/chess.gbnf +0 -0
  139. {llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/data/from-llama.cpp/english.gbnf +0 -0
  140. {llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/data/from-llama.cpp/japanese.gbnf +0 -0
  141. {llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/data/from-llama.cpp/json.gbnf +0 -0
  142. {llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/data/from-llama.cpp/json_arr.gbnf +0 -0
  143. {llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/data/from-llama.cpp/list.gbnf +0 -0
  144. {llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/data/from-llama.cpp/vllm-sql.gbnf +0 -0
  145. {llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/data/lark.lark +0 -0
  146. {llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/data/rfc.lark +0 -0
  147. {llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/data/rfc.xml +0 -0
  148. {llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/data/ulysses.md +0 -0
  149. {llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/gtest.sh +0 -0
  150. {llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/lark.sh +0 -0
  151. {llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/run.sh +0 -0
  152. {llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/src/lib.rs +0 -0
  153. {llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/src/minimal.rs +0 -0
  154. {llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/src/sample_parser.rs +0 -0
  155. {llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/tests/test_json_objects.rs +0 -0
  156. {llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/tests/test_lark.rs +0 -0
  157. {llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/tests/test_ll.rs +0 -0
  158. {llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/tests/test_raw_parser.rs +0 -0
  159. {llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/tests/test_stop.rs +0 -0
  160. {llguidance-1.1.1 → llguidance-1.1.2}/scripts/annotate_asm.js +0 -0
  161. {llguidance-1.1.1 → llguidance-1.1.2}/scripts/cbindgen.sh +0 -0
  162. {llguidance-1.1.1 → llguidance-1.1.2}/scripts/checklinks.py +0 -0
  163. {llguidance-1.1.1 → llguidance-1.1.2}/scripts/checklinks.sh +0 -0
  164. {llguidance-1.1.1 → llguidance-1.1.2}/scripts/ci-publish.py +0 -0
  165. {llguidance-1.1.1 → llguidance-1.1.2}/scripts/disasm.sh +0 -0
  166. {llguidance-1.1.1 → llguidance-1.1.2}/scripts/gbnf_to_lark.py +0 -0
  167. {llguidance-1.1.1 → llguidance-1.1.2}/scripts/gen-testcase.py +0 -0
  168. {llguidance-1.1.1 → llguidance-1.1.2}/scripts/git-version.sh +0 -0
  169. {llguidance-1.1.1 → llguidance-1.1.2}/scripts/install-deps.sh +0 -0
  170. {llguidance-1.1.1 → llguidance-1.1.2}/scripts/jsonschema-stats.js +0 -0
  171. {llguidance-1.1.1 → llguidance-1.1.2}/scripts/remote-guidance-test.sh +0 -0
  172. {llguidance-1.1.1 → llguidance-1.1.2}/scripts/rust-size.js +0 -0
  173. {llguidance-1.1.1 → llguidance-1.1.2}/scripts/rust_size.py +0 -0
  174. {llguidance-1.1.1 → llguidance-1.1.2}/scripts/test-guidance.sh +0 -0
  175. {llguidance-1.1.1 → llguidance-1.1.2}/scripts/tokenizer_test.py +0 -0
  176. {llguidance-1.1.1 → llguidance-1.1.2}/scripts/update-git.py +0 -0
  177. {llguidance-1.1.1 → llguidance-1.1.2}/toktrie/LICENSE +0 -0
  178. {llguidance-1.1.1 → llguidance-1.1.2}/toktrie/README.md +0 -0
  179. {llguidance-1.1.1 → llguidance-1.1.2}/toktrie/src/bytes.rs +0 -0
  180. {llguidance-1.1.1 → llguidance-1.1.2}/toktrie/src/lib.rs +0 -0
  181. {llguidance-1.1.1 → llguidance-1.1.2}/toktrie/src/recognizer.rs +0 -0
  182. {llguidance-1.1.1 → llguidance-1.1.2}/toktrie/src/rng.rs +0 -0
  183. {llguidance-1.1.1 → llguidance-1.1.2}/toktrie/src/svob.rs +0 -0
  184. {llguidance-1.1.1 → llguidance-1.1.2}/toktrie/src/tokenv.rs +0 -0
  185. {llguidance-1.1.1 → llguidance-1.1.2}/toktrie/src/toktree.rs +0 -0
  186. {llguidance-1.1.1 → llguidance-1.1.2}/toktrie/tests/test_svob.rs +0 -0
  187. {llguidance-1.1.1 → llguidance-1.1.2}/toktrie_hf_downloader/LICENSE +0 -0
  188. {llguidance-1.1.1 → llguidance-1.1.2}/toktrie_hf_downloader/src/lib.rs +0 -0
  189. {llguidance-1.1.1 → llguidance-1.1.2}/toktrie_hf_tokenizers/LICENSE +0 -0
  190. {llguidance-1.1.1 → llguidance-1.1.2}/toktrie_hf_tokenizers/src/lib.rs +0 -0
  191. {llguidance-1.1.1 → llguidance-1.1.2}/toktrie_tiktoken/LICENSE +0 -0
  192. {llguidance-1.1.1 → llguidance-1.1.2}/toktrie_tiktoken/src/lib.rs +0 -0
llguidance-1.1.2/.github/workflows/code-coverage.yml ADDED
@@ -0,0 +1,62 @@
1
+ name: Code Coverage
2
+ permissions:
3
+ contents: read
4
+
5
+
6
+ on:
7
+ pull_request:
8
+ push:
9
+ branches: [ "main" ]
10
+ workflow_dispatch:
11
+ inputs:
12
+ commit_id:
13
+ description: 'Branch or Commit ID (optional)'
14
+ required: false
15
+ type: string
16
+
17
+
18
+ env:
19
+ CARGO_TERM_COLOR: always
20
+ RUSTFLAGS: "-Cinstrument-coverage"
21
+ LLVM_PROFILE_FILE: "llg-%p-%m.profraw"
22
+
23
+
24
+ jobs:
25
+ code-cov:
26
+ runs-on: ubuntu-latest
27
+
28
+ steps:
29
+ - name: Checkout repo at ${{ github.event_name == 'workflow_dispatch' && inputs.commit_id || github.sha }}
30
+ uses: actions/checkout@v4
31
+ with:
32
+ ref: ${{ github.event_name == 'workflow_dispatch' && inputs.commit_id || github.sha }}
33
+
34
+ - name: Update toolchain
35
+ run: rustup component add llvm-tools
36
+
37
+ - name: Install grcov
38
+ run: cargo install grcov
39
+
40
+ - name: Build everything
41
+ run: cargo build
42
+
43
+ - name: Run tests
44
+ run: cargo test
45
+
46
+ - name: Check environment
47
+ run: |
48
+ echo "CARGO_TERM_COLOR: $CARGO_TERM_COLOR"
49
+ echo "RUSTFLAGS: $RUSTFLAGS"
50
+ echo "LLVM_PROFILE_FILE: $LLVM_PROFILE_FILE"
51
+
52
+ - name: Generate coverage report
53
+ run: |
54
+ grcov . -s . --binary-path target/debug/ -t html --branch --ignore-not-existing -o target/debug/coverage/
55
+
56
+ - name: Check output
57
+ run: ls target/debug/coverage/
58
+
59
+ - uses: actions/upload-artifact@v4
60
+ with:
61
+ name: coverage-report
62
+ path: target/debug/coverage/
{llguidance-1.1.1 → llguidance-1.1.2}/CHANGELOG.md RENAMED
@@ -4,109 +4,114 @@ All notable changes to this project will be documented in this file. Dates are d
4
4
 
5
5
  If a release doesn't introduce any interesting changes (build fixes etc.), it's skipped.
6
6
 
7
- #### [1.1.1](https://github.com/guidance-ai/llguidance/compare/v1.1.0...1.1.1) 2025-07-23
7
+ #### [v1.1.2](https://github.com/guidance-ai/llguidance/compare/v1.1.1...v1.1.2) 2025-08-08
8
+
9
+ - add flag in ParserLimits to disable verbose errors [`#227`](https://github.com/guidance-ai/llguidance/pull/227)
10
+ - new tests and cleanups
11
+
12
+ #### [v1.1.1](https://github.com/guidance-ai/llguidance/compare/v1.1.0...v1.1.1) 2025-07-23
8
13
 
9
14
  - prevent error state when calling `try_consume_tokens` after parser is stopped [`#213`](https://github.com/guidance-ai/llguidance/pull/213); fixes [`#211`](https://github.com/guidance-ai/llguidance/issues/211)
10
15
  - set parser stop condition in `try_consume_tokens` even when some tokens are rejected [`#212`](https://github.com/guidance-ai/llguidance/pull/212)
11
16
 
12
- #### [1.1.0](https://github.com/guidance-ai/llguidance/compare/v1.0.1...1.1.0) 2025-07-18
17
+ #### [v1.1.0](https://github.com/guidance-ai/llguidance/compare/v1.0.1...v1.1.0) 2025-07-18
13
18
 
14
19
  - disable hf tokenizer truncation and padding [`#205`](https://github.com/guidance-ai/llguidance/pull/205); fixes [`#1322`](https://github.com/guidance-ai/guidance/issues/1322)
15
20
  - llama_cpp tokenizers: infer added tokens starting/ending with < and > to be special tokens [`#202`](https://github.com/guidance-ai/llguidance/pull/202)
16
21
  - add lark syntax for "any token" and negation of token ranges [`#201`](https://github.com/guidance-ai/llguidance/pull/201)
17
22
  - add de-recursion cook book to docs [`#199`](https://github.com/guidance-ai/llguidance/pull/199)
18
23
 
19
- #### [1.0.1](https://github.com/guidance-ai/llguidance/compare/v1.0.0...1.0.1) 2025-07-03
24
+ #### [v1.0.1](https://github.com/guidance-ai/llguidance/compare/v1.0.0...v1.0.1) 2025-07-03
20
25
 
21
26
  - fix: tokenizers normalizers sequence api changed [`#195`](https://github.com/guidance-ai/llguidance/pull/195)
22
27
  - Strip debug info from the wheels [`#194`](https://github.com/guidance-ai/llguidance/pull/194)
23
28
 
24
29
  Thank you @ammar-elsabe and @Ahajha!
25
30
 
26
- #### [1.0.0](https://github.com/guidance-ai/llguidance/compare/v0.7.30...1.0.0) 2025-06-23
31
+ #### [v1.0.0](https://github.com/guidance-ai/llguidance/compare/v0.7.30...v1.0.0) 2025-06-23
27
32
 
28
33
  This is identical to `0.7.30`, but indicates intended stability and from now on we'll try to follow semver.
29
34
 
30
- #### [0.7.30](https://github.com/guidance-ai/llguidance/compare/v0.7.29...0.7.30) 2025-06-23
35
+ #### [v0.7.30](https://github.com/guidance-ai/llguidance/compare/v0.7.29...v0.7.30) 2025-06-23
31
36
 
32
37
  - parametric grammars [`#192`](https://github.com/guidance-ai/llguidance/pull/192)
33
38
  - allow for tokens up to ~2k bytes; fixes #188 [`#188`](https://github.com/guidance-ai/llguidance/issues/188)
34
39
 
35
- #### [0.7.29](https://github.com/guidance-ai/llguidance/compare/v0.7.28...0.7.29) 2025-06-06
40
+ #### [v0.7.29](https://github.com/guidance-ai/llguidance/compare/v0.7.28...v0.7.29) 2025-06-06
36
41
 
37
42
  - cargo fmt
38
43
 
39
- #### [0.7.28](https://github.com/guidance-ai/llguidance/compare/v0.7.27...0.7.28) 2025-06-06
44
+ #### [v0.7.28](https://github.com/guidance-ai/llguidance/compare/v0.7.27...v0.7.28) 2025-06-06
40
45
 
41
46
  - fix lexer_stack=... panic with numeric tokens [`4e91b0f`](https://github.com/guidance-ai/llguidance/commit/4e91b0fa0c03572a5fc221ac0e0b05035af9dcfa)
42
47
 
43
- #### [0.7.27](https://github.com/guidance-ai/llguidance/compare/v0.7.26...0.7.27) 2025-06-04
48
+ #### [v0.7.27](https://github.com/guidance-ai/llguidance/compare/v0.7.26...v0.7.27) 2025-06-04
44
49
 
45
50
  - add toktrie_tiktoken and llguidance.tiktoken.lltokenizer_from_encoding [`#154`](https://github.com/guidance-ai/llguidance/issues/154)
46
51
  - implement clone on StopController [`#185`](https://github.com/guidance-ai/llguidance/issues/185)
47
52
 
48
- #### [0.7.26](https://github.com/guidance-ai/llguidance/compare/v0.7.25...0.7.26) 2025-05-30
53
+ #### [v0.7.26](https://github.com/guidance-ai/llguidance/compare/v0.7.25...v0.7.26) 2025-05-30
49
54
 
50
55
  - add support for & and ~ in lark regexes [`96fcee3`](https://github.com/guidance-ai/llguidance/commit/96fcee373697b57bead94d1bc06c17cf1c6134e4)
51
56
  - dump grammar in errors in LLInterpreter [`#183`](https://github.com/guidance-ai/llguidance/pull/183)
52
57
  - don't check lexer bytes invariant when we cannot rollback [`ec22083`](https://github.com/guidance-ai/llguidance/commit/ec220837051513a70177974ca389b7bf387455f1)
53
58
 
54
59
 
55
- #### [0.7.25](https://github.com/guidance-ai/llguidance/compare/v0.7.24...0.7.25) 2025-05-28
60
+ #### [v0.7.25](https://github.com/guidance-ai/llguidance/compare/v0.7.24...v0.7.25) 2025-05-28
56
61
 
57
62
  - add parse_special=False to tokenize_str/bytes() in python [`#181`](https://github.com/guidance-ai/llguidance/pull/181)
58
63
 
59
- #### [0.7.24](https://github.com/guidance-ai/llguidance/compare/v0.7.23...0.7.24) 2025-05-23
64
+ #### [v0.7.24](https://github.com/guidance-ai/llguidance/compare/v0.7.23...v0.7.24) 2025-05-23
60
65
 
61
66
  - add the sentinel token hack, fixes #180 [`#180`](https://github.com/guidance-ai/llguidance/issues/180)
62
67
 
63
- #### [0.7.23](https://github.com/guidance-ai/llguidance/compare/v0.7.22...0.7.23) 2025-05-22
68
+ #### [v0.7.23](https://github.com/guidance-ai/llguidance/compare/v0.7.22...v0.7.23) 2025-05-22
64
69
 
65
70
  - native llama.cpp tokenizer support [`#179`](https://github.com/guidance-ai/llguidance/pull/179)
66
71
  - improve special token detection in HF tokenizers [`6cae393`](https://github.com/guidance-ai/llguidance/commit/6cae393b9c04fe67621615ff22b46beab512d069)
67
72
 
68
- #### [0.7.22](https://github.com/guidance-ai/llguidance/compare/v0.7.21...0.7.22) 2025-05-21
73
+ #### [v0.7.22](https://github.com/guidance-ai/llguidance/compare/v0.7.21...v0.7.22) 2025-05-21
69
74
 
70
75
  - Keep EOS token bytes in `TokenizerWrapper` [`#178`](https://github.com/guidance-ai/llguidance/pull/178)
71
76
  - Stop using prefix/sentinel strings for `TokenizerWrapper` [`#175`](https://github.com/guidance-ai/llguidance/pull/175)
72
77
  - avoid taking poisoned locks, see [`#174`](https://github.com/guidance-ai/llguidance/issues/174) [`d41aa9a`](https://github.com/guidance-ai/llguidance/commit/d41aa9a4427967708a951506b2bc0e395871b6c8); thanks [@g-eoj](https://github.com/g-eoj)
73
78
 
74
- #### [0.7.21](https://github.com/guidance-ai/llguidance/compare/v0.7.20...0.7.21) 2025-05-20
79
+ #### [v0.7.21](https://github.com/guidance-ai/llguidance/compare/v0.7.20...v0.7.21) 2025-05-20
75
80
 
76
81
  - include parser state in errors [`82e34da`](https://github.com/guidance-ai/llguidance/commit/82e34da704d22f04979d8cbc54a0ac00885a277d)
77
82
  - tighten email format in JSON schema [`7454ea9`](https://github.com/guidance-ai/llguidance/commit/7454ea9df958f8bcc42e6bb986d6de397de65b3e)
78
83
 
79
- #### [0.7.20](https://github.com/guidance-ai/llguidance/compare/v0.7.19...0.7.20) 2025-05-15
84
+ #### [v0.7.20](https://github.com/guidance-ai/llguidance/compare/v0.7.19...v0.7.20) 2025-05-15
80
85
 
81
86
  - use fancy-regex instead of onig as tokenizers regex library [`#172`](https://github.com/guidance-ai/llguidance/pull/172)
82
87
  - fixes compilation on GCC 15, thanks [@Slowki](https://github.com/Slowki)
83
88
  - msrv 1.80 support (incl. derivre bump) [`c89e386`](https://github.com/guidance-ai/llguidance/commit/c89e386685cd911a89fd47df225de88f88c10883), thank you [@nteodosio](https://github.com/nteodosio) for initial [PR](https://github.com/guidance-ai/llguidance/pull/170)!
84
89
 
85
- #### [0.7.19](https://github.com/guidance-ai/llguidance/compare/v0.7.18...0.7.19) 2025-04-24
90
+ #### [v0.7.19](https://github.com/guidance-ai/llguidance/compare/v0.7.18...v0.7.19) 2025-04-24
86
91
 
87
92
  - fix a numeric token bug [`1f59edf`](https://github.com/guidance-ai/llguidance/commit/1f59edfc49b44cfba74b2380f34874a0778d9441)
88
93
 
89
- #### [0.7.18](https://github.com/guidance-ai/llguidance/compare/v0.7.17...0.7.18) 2025-04-22
94
+ #### [v0.7.18](https://github.com/guidance-ai/llguidance/compare/v0.7.17...v0.7.18) 2025-04-22
90
95
 
91
96
  - apply x-guidance also in %json{} [`2627891`](https://github.com/guidance-ai/llguidance/commit/2627891c72c7e38062cd3e052f1de146d2e21635)
92
97
  - more sensible llg_validate_grammar() signature [`41928c0`](https://github.com/guidance-ai/llguidance/commit/41928c07298e69e3c8adc4a3c1f43ef9b1cc1c6b)
93
98
 
94
- #### [0.7.17](https://github.com/guidance-ai/llguidance/compare/v0.7.16...0.7.17) 2025-04-22
99
+ #### [v0.7.17](https://github.com/guidance-ai/llguidance/compare/v0.7.16...v0.7.17) 2025-04-22
95
100
 
96
101
  - support for min/maxProperties in JSON Schema [`#168`](https://github.com/guidance-ai/llguidance/issues/168)
97
102
  - give priority to <[123]> over "foo" in grammar [`3e9f3b5`](https://github.com/guidance-ai/llguidance/commit/3e9f3b5e8c1cac92daab6e9709f01ebccc20342b)
98
103
 
99
- #### [0.7.16](https://github.com/guidance-ai/llguidance/compare/v0.7.15...0.7.16) 2025-04-17
104
+ #### [v0.7.16](https://github.com/guidance-ai/llguidance/compare/v0.7.15...v0.7.16) 2025-04-17
100
105
 
101
106
  - fix special token tokenization [`ae7870f`](https://github.com/guidance-ai/llguidance/commit/ae7870f05ca0de68599088607ba742b7071f92ad)
102
107
 
103
- #### [0.7.15](https://github.com/guidance-ai/llguidance/compare/v0.7.14...0.7.15) 2025-04-16
108
+ #### [v0.7.15](https://github.com/guidance-ai/llguidance/compare/v0.7.14...v0.7.15) 2025-04-16
104
109
 
105
110
  - support for patternProperties in JSON schema [`#167`](https://github.com/guidance-ai/llguidance/pull/167)
106
111
  - add lenient option to JSON schemas [`#163`](https://github.com/guidance-ai/llguidance/pull/163) [`#136`](https://github.com/guidance-ai/llguidance/issues/136)
107
112
  - Add llg_validate_grammar() in C FFI [`e5c21cf`](https://github.com/guidance-ai/llguidance/commit/e5c21cf480a17e6b310e46b24b272576cfd9c4c6)
108
113
 
109
- #### [0.7.14](https://github.com/guidance-ai/llguidance/compare/v0.7.13...0.7.14) 2025-04-11
114
+ #### [v0.7.14](https://github.com/guidance-ai/llguidance/compare/v0.7.13...v0.7.14) 2025-04-11
110
115
 
111
116
  - support %lark { ... } syntax for nested grammars [`#157`](https://github.com/guidance-ai/llguidance/pull/157)
112
117
  - treat \d and \w in json schema as ASCII; fix ^$ anchors [`#158`](https://github.com/guidance-ai/llguidance/issues/158)
@@ -115,19 +120,19 @@ This is identical to `0.7.30`, but indicates intended stability and from now on
115
120
  - expose regex_to_lark() in Rust and Python; add \d\w\s replacement [`78fb32f`](https://github.com/guidance-ai/llguidance/commit/78fb32fe2745d30ca94a62b00e5a7299750d80b0)
116
121
  - fix usage of / vs \* in python signatures [`ca73c2a`](https://github.com/guidance-ai/llguidance/commit/ca73c2abd44e75d569230b942f53c72b052ed2ab)
117
122
 
118
- #### [0.7.13](https://github.com/guidance-ai/llguidance/compare/v0.7.12...0.7.13) 2025-04-05
123
+ #### [v0.7.13](https://github.com/guidance-ai/llguidance/compare/v0.7.12...v0.7.13) 2025-04-05
119
124
 
120
125
  - expose LLParserLimits in Python API [`598dc8f`](https://github.com/guidance-ai/llguidance/commit/598dc8f37f69f51244e54d9885445abf02a515a7)
121
126
  - pre-compute lexer states for particularly large regexes (can be disabled in ParserLimits)
122
127
 
123
- #### [0.7.12](https://github.com/guidance-ai/llguidance/compare/v0.7.11...0.7.12) 2025-04-04
128
+ #### [v0.7.12](https://github.com/guidance-ai/llguidance/compare/v0.7.11...v0.7.12) 2025-04-04
124
129
 
125
130
  - performance optimizations
126
131
  - use factory in C FFI (otherwise slicer was not used)
127
132
  - add some null checks and safety comments in C FFI
128
133
  - implement subgrammar lexeme class merging; fixes [`#113`](https://github.com/guidance-ai/llguidance/issues/113)
129
134
 
130
- #### [0.7.11](https://github.com/guidance-ai/llguidance/compare/v0.7.10...0.7.11) 2025-03-27
135
+ #### [v0.7.11](https://github.com/guidance-ai/llguidance/compare/v0.7.10...v0.7.11) 2025-03-27
131
136
 
132
137
  - add StructTag python API; fixes [`#146`](https://github.com/guidance-ai/llguidance/issues/146)
133
138
  - fix handling of AddedToken.special (gemma tokenizer, fixes [`#147`](https://github.com/guidance-ai/llguidance/issues/147))
{llguidance-1.1.1 → llguidance-1.1.2}/Cargo.lock RENAMED
@@ -1241,7 +1241,7 @@ checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956"
1241
1241
 
1242
1242
  [[package]]
1243
1243
  name = "llguidance"
1244
- version = "1.1.1"
1244
+ version = "1.1.2"
1245
1245
  dependencies = [
1246
1246
  "anyhow",
1247
1247
  "derivre",
@@ -1260,7 +1260,7 @@ dependencies = [
1260
1260
 
1261
1261
  [[package]]
1262
1262
  name = "llguidance_py"
1263
- version = "1.1.1"
1263
+ version = "1.1.2"
1264
1264
  dependencies = [
1265
1265
  "anyhow",
1266
1266
  "bytemuck",
@@ -2478,7 +2478,7 @@ dependencies = [
2478
2478
 
2479
2479
  [[package]]
2480
2480
  name = "toktrie"
2481
- version = "1.1.1"
2481
+ version = "1.1.2"
2482
2482
  dependencies = [
2483
2483
  "anyhow",
2484
2484
  "bytemuck",
@@ -2489,7 +2489,7 @@ dependencies = [
2489
2489
 
2490
2490
  [[package]]
2491
2491
  name = "toktrie_hf_downloader"
2492
- version = "1.1.1"
2492
+ version = "1.1.2"
2493
2493
  dependencies = [
2494
2494
  "anyhow",
2495
2495
  "hf-hub",
@@ -2500,7 +2500,7 @@ dependencies = [
2500
2500
 
2501
2501
  [[package]]
2502
2502
  name = "toktrie_hf_tokenizers"
2503
- version = "1.1.1"
2503
+ version = "1.1.2"
2504
2504
  dependencies = [
2505
2505
  "anyhow",
2506
2506
  "log",
@@ -2512,7 +2512,7 @@ dependencies = [
2512
2512
 
2513
2513
  [[package]]
2514
2514
  name = "toktrie_tiktoken"
2515
- version = "1.1.1"
2515
+ version = "1.1.2"
2516
2516
  dependencies = [
2517
2517
  "anyhow",
2518
2518
  "log",
{llguidance-1.1.1 → llguidance-1.1.2}/PKG-INFO RENAMED
@@ -1,10 +1,10 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: llguidance
3
- Version: 1.1.1
3
+ Version: 1.1.2
4
4
  License-File: LICENSE
5
5
  Summary: Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance
6
6
  Author: Michal Moskal
7
- License: MIT
7
+ License-Expression: MIT
8
8
  Requires-Python: >=3.9
9
9
  Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
10
10
  Project-URL: repository, https://github.com/microsoft/llguidance
{llguidance-1.1.1 → llguidance-1.1.2}/parser/Cargo.toml RENAMED
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "llguidance"
3
- version = "1.1.1"
3
+ version = "1.1.2"
4
4
  edition = "2021"
5
5
  license = "MIT"
6
6
  description = "Super-fast Structured Outputs"
{llguidance-1.1.1 → llguidance-1.1.2}/parser/llguidance.h RENAMED
@@ -78,6 +78,12 @@ typedef struct LlgParserLimits {
78
78
  * Default: true
79
79
  */
80
80
  bool precompute_large_lexemes;
81
+ /**
82
+ * If true, include parser state (including tokens so far) and grammar in
83
+ * errors.
84
+ * Default: true
85
+ */
86
+ bool verbose_errors;
81
87
  } LlgParserLimits;
82
88
 
83
89
  typedef struct LlgConstraintInit {
{llguidance-1.1.1 → llguidance-1.1.2}/parser/src/api.rs RENAMED
@@ -258,6 +258,11 @@ pub struct ParserLimits {
258
258
  /// the time it takes to construct the lexer.
259
259
  /// Default: true
260
260
  pub precompute_large_lexemes: bool,
261
+
262
+ /// If true, include parser state (including tokens so far) and grammar in
263
+ /// errors.
264
+ /// Default: true
265
+ pub verbose_errors: bool,
261
266
  }
262
267
 
263
268
  impl Default for ParserLimits {
@@ -270,6 +275,7 @@ impl Default for ParserLimits {
270
275
  max_grammar_size: 500_000, // fhir schema => 200k
271
276
  step_max_items: 50_000, //
272
277
  precompute_large_lexemes: true,
278
+ verbose_errors: true,
273
279
  }
274
280
  }
275
281
  }
{llguidance-1.1.1 → llguidance-1.1.2}/parser/src/panic_utils.rs RENAMED
@@ -24,7 +24,7 @@ pub fn mk_panic_error(info: &Box<dyn Any + Send>) -> String {
24
24
  let b = BACKTRACE.with(|b| b.take());
25
25
 
26
26
  if let Some(b) = b {
27
- format!("panic: {msg}\n{b}")
27
+ format!("panic: {msg}\n<backtrace>\n{b}\n</backtrace>")
28
28
  } else {
29
29
  format!("panic: {msg}")
30
30
  }
{llguidance-1.1.1 → llguidance-1.1.2}/parser/src/tokenparser.rs RENAMED
@@ -280,11 +280,15 @@ impl TokenParser {
280
280
  }
281
281
 
282
282
  pub fn augment_err(&self, e: impl Display) -> String {
283
- format!(
284
- "{e}\n<state>\n{}\n</state><grammar>\n{}\n</grammar>",
285
- self.dump_state(),
286
- self.dbg_grammar
287
- )
283
+ if self.limits.verbose_errors {
284
+ format!(
285
+ "{e}\n<state>\n{}\n</state><grammar>\n{}\n</grammar>",
286
+ self.dump_state(),
287
+ self.dbg_grammar
288
+ )
289
+ } else {
290
+ format!("{e}\n<non-verbose/>")
291
+ }
288
292
  }
289
293
 
290
294
  pub fn dump_state(&self) -> String {
{llguidance-1.1.1 → llguidance-1.1.2}/pyproject.toml RENAMED
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "llguidance"
3
- version = "1.1.1"
3
+ version = "1.1.2"
4
4
  description = "Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance"
5
5
  requires-python = ">=3.9"
6
6
  license = "MIT"
{llguidance-1.1.1 → llguidance-1.1.2}/python/llguidance/_lib.pyi RENAMED
@@ -565,6 +565,7 @@ class LLParserLimits:
565
565
  max_lexer_states: Optional[int] = None,
566
566
  max_grammar_size: Optional[int] = None,
567
567
  precompute_large_lexemes: Optional[bool] = None,
568
+ verbose_errors: Optional[bool] = None,
568
569
  ) -> None:
569
570
  """
570
571
  ParserLimits configuration for controlling parser and lexer resource usage.
@@ -597,6 +598,10 @@ class LLParserLimits:
597
598
  precompute_large_lexemes (Optional[bool]):
598
599
  Whether to run large regexes eagerly on the entire token trie during lexer build.
599
600
  Increases lexer construction time, but speeds up mask computation. Default: True.
601
+
602
+ verbose_errors (Optional[bool]):
603
+ If true, include parser state and grammar details in error messages.
604
+ Useful for debugging; may leak schema/state in logs. Default: True.
600
605
  """
601
606
 
602
607
  @property
@@ -627,6 +632,10 @@ class LLParserLimits:
627
632
  def precompute_large_lexemes(self) -> bool:
628
633
  """Precompute large regexes during lexer construction. Default: True"""
629
634
 
635
+ @property
636
+ def verbose_errors(self) -> bool:
637
+ """Include parser state and grammar in errors. Default: True"""
638
+
630
639
 
631
640
  def regex_to_lark(regex: str, use_ascii: str = "d") -> str:
632
641
  r"""
{llguidance-1.1.1 → llguidance-1.1.2}/python_ext/Cargo.toml RENAMED
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "llguidance_py"
3
- version = "1.1.1"
3
+ version = "1.1.2"
4
4
  edition = "2021"
5
5
  license = "MIT"
6
6
  description = "Super-fast Structured Outputs"
{llguidance-1.1.1 → llguidance-1.1.2}/python_ext/src/parserlimits.rs RENAMED
@@ -21,7 +21,8 @@ impl LLParserLimits {
21
21
  #[new]
22
22
  #[allow(clippy::too_many_arguments)]
23
23
  #[pyo3(signature = (*, max_items_in_row=None, initial_lexer_fuel=None, step_lexer_fuel=None,
24
- step_max_items=None, max_lexer_states=None, max_grammar_size=None, precompute_large_lexemes=None))]
24
+ step_max_items=None, max_lexer_states=None, max_grammar_size=None, precompute_large_lexemes=None,
25
+ verbose_errors=None))]
25
26
  fn new(
26
27
  max_items_in_row: Option<usize>,
27
28
  initial_lexer_fuel: Option<u64>,
@@ -30,6 +31,7 @@ impl LLParserLimits {
30
31
  max_lexer_states: Option<usize>,
31
32
  max_grammar_size: Option<usize>,
32
33
  precompute_large_lexemes: Option<bool>,
34
+ verbose_errors: Option<bool>,
33
35
  ) -> Self {
34
36
  let mut inner = ParserLimits::default();
35
37
  if let Some(v) = max_items_in_row {
@@ -53,6 +55,9 @@ impl LLParserLimits {
53
55
  if let Some(v) = precompute_large_lexemes {
54
56
  inner.precompute_large_lexemes = v;
55
57
  }
58
+ if let Some(v) = verbose_errors {
59
+ inner.verbose_errors = v;
60
+ }
56
61
  Self { inner }
57
62
  }
58
63
 
@@ -91,6 +96,11 @@ impl LLParserLimits {
91
96
  self.inner.precompute_large_lexemes
92
97
  }
93
98
 
99
+ #[getter]
100
+ fn verbose_errors(&self) -> bool {
101
+ self.inner.verbose_errors
102
+ }
103
+
94
104
  fn __str__(&self) -> String {
95
105
  format!("{:?}", self.inner)
96
106
  }
{llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/tests/common_lark_utils/mod.rs RENAMED
@@ -36,6 +36,7 @@ pub fn lark_ok(lark: &str) {
36
36
  }
37
37
  }
38
38
 
39
+ #[allow(dead_code)]
39
40
  pub fn lark_err_test(lark: &str, err: &str) {
40
41
  match make_parser(lark, false) {
41
42
  Err(e) => {
@@ -48,6 +49,7 @@ pub fn lark_err_test(lark: &str, err: &str) {
48
49
  }
49
50
  }
50
51
 
52
+ #[allow(dead_code)]
51
53
  pub fn json_err_test(schema: &Value, err: &str) {
52
54
  lark_err_test(
53
55
  &format!(r#"start: %json {}"#, serde_json::to_string(schema).unwrap()),
@@ -126,9 +128,53 @@ pub fn lark_str_test_many_ext(quiet: bool, lark: &str, passing: &[&str], failing
126
128
  }
127
129
 
128
130
  pub fn json_schema_check(schema: &Value, json_obj: &Value, expect_valid: bool) {
129
- let lark = format!(r#"start: %json {}"#, serde_json::to_string(schema).unwrap());
130
- let s = serde_json::to_string(json_obj).unwrap();
131
- lark_str_test(&lark, expect_valid, &s, false);
131
+ /*
132
+ This is a modification of the lark_str_test function, which makes the
133
+ assumption that the input Value completely satifies the schema.
134
+
135
+ The subtlety is that a string of tokens might not match a grammar _yet_
136
+ but could with the addition of more tokens. For example, if we're trying
137
+ to construct an integer which is greater than 2, then the string "1"
138
+ is not yet a match, but it could become one if we add more tokens.
139
+ lark_str_test uses the magic 'FINAL_REJECT:' rule to handle this,
140
+ but we can write something a little simpler here.
141
+ */
142
+ let lark_grammar = format!(r#"start: %json {}"#, serde_json::to_string(schema).unwrap());
143
+ let json_string = serde_json::to_string(json_obj).unwrap();
144
+
145
+ // Tokenize the string representation of the JSON object
146
+ let tokens = get_tok_env().tokenize(&json_string);
147
+
148
+ // Create the parser
149
+ let mut p = make_parser(&lark_grammar, false).unwrap();
150
+
151
+ // Work through token by token
152
+ for (i, tok) in tokens.iter().enumerate() {
153
+ // Compute the mask of allowed tokens
154
+ let m = p.compute_mask().unwrap();
155
+
156
+ if m.is_allowed(*tok) {
157
+ // Consume the token
158
+ consume(&mut p, *tok);
159
+ } else {
160
+ // Token isn't allowed, so check if we expect this
161
+ let curr_tok_str = get_tok_env().tok_trie().token_dbg(*tok);
162
+ assert!(
163
+ !expect_valid,
164
+ "Unexpected token: {curr_tok_str} at token index {i}",
165
+ );
166
+ // We were expecting this to fail, so return early
167
+ return;
168
+ }
169
+ }
170
+
171
+ /*
172
+ Note that p.is_accepting() will be true if the parser has reached a valid end state.
173
+ It does not mean that we couldn't add more tokens and remain valid.
174
+ For example, if we have a schema for any integer, then we can always add more digits
175
+ to a valid integer string.
176
+ */
177
+ assert_eq!(p.is_accepting(), expect_valid, "Final state mismatch");
132
178
  }
133
179
 
134
180
  #[allow(dead_code)]
{llguidance-1.1.1 → llguidance-1.1.2}/sample_parser/tests/test_json_arrays.rs RENAMED
@@ -128,3 +128,89 @@ fn array_of_objects(#[case] sample_array: &Value) {
128
128
  fn array_of_objects_failures(#[case] sample_array: &Value) {
129
129
  json_schema_check(&ARRAY_OF_OBJECTS, sample_array, false);
130
130
  }
131
+
132
+ lazy_static! {
133
+ static ref SIMPLE_PREFIXED_ARRAY: Value = json!({ "type": "array",
134
+ "prefixItems": [
135
+ { "type": "string" }, // First item must be a string
136
+ { "type": "number" } // Second item must be a number
137
+ ],
138
+ "items": { "type": "boolean" } // Remaining items must be booleans
139
+ });
140
+ }
141
+
142
+ #[rstest]
143
+ #[case::only_prefix(&json!(["Hello", 42]))]
144
+ #[case::prefix_one_item(&json!(["Cruel", 817.2, true]))]
145
+ #[case::prefix_multiple_items(&json!(["World", 41.3, false, true, false]))]
146
+ fn array_with_prefix_items(#[case] sample_array: &Value) {
147
+ json_schema_check(&SIMPLE_PREFIXED_ARRAY, sample_array, true);
148
+ }
149
+
150
+ #[rstest]
151
+ #[case(&json!([41.3]))]
152
+ #[case(&json!(["Hello", 42, 41.3]))]
153
+ #[case(&json!(["Hello", 42, true, "Not a boolean"]))]
154
+ fn array_with_prefix_items_failures(#[case] sample_array: &Value) {
155
+ json_schema_check(&SIMPLE_PREFIXED_ARRAY, sample_array, false);
156
+ }
157
+
158
+ lazy_static! {
159
+ static ref SIMPLE_PREFIXED_ARRAY_FIXED: Value = json!({ "type": "array",
160
+ "prefixItems": [
161
+ { "type": "string" }, // First item must be a string
162
+ { "type": "number" } // Second item must be a number
163
+ ],
164
+ "items": false // No additional items allowed
165
+ });
166
+ }
167
+
168
+ #[rstest]
169
+ #[case(&json!(["A"]))]
170
+ #[case(&json!(["B", 42]))]
171
+ fn array_with_prefix_items_fixed(#[case] sample_array: &Value) {
172
+ json_schema_check(&SIMPLE_PREFIXED_ARRAY_FIXED, sample_array, true);
173
+ }
174
+
175
+ #[rstest]
176
+ #[case(&json!(["Hello", 42, true]))]
177
+ #[case(&json!([41.3]))]
178
+ fn array_with_prefix_items_fixed_failures(#[case] sample_array: &Value) {
179
+ json_schema_check(&SIMPLE_PREFIXED_ARRAY_FIXED, sample_array, false);
180
+ }
181
+
182
+ lazy_static! {
183
+ static ref SIMPLE_PREFIXED_ARRAY_LENGTH_CONSTRAINED: Value = json!({ "type": "array",
184
+ "prefixItems": [
185
+ { "type": "string" }, // First item must be a string
186
+ { "type": "number" } // Second item must be a number
187
+ ],
188
+ "items": { "type": "boolean" }, // Remaining items must be booleans
189
+ "minItems": 2,
190
+ "maxItems": 4
191
+ });
192
+ }
193
+
194
+ #[rstest]
195
+ #[case(&json!(["Hello", 3.13]))]
196
+ #[case(&json!(["World", 817.2, false]))]
197
+ #[case(&json!(["Test", 1.0, true, false]))]
198
+ fn array_with_prefix_items_length_constrained(#[case] sample_array: &Value) {
199
+ json_schema_check(
200
+ &SIMPLE_PREFIXED_ARRAY_LENGTH_CONSTRAINED,
201
+ sample_array,
202
+ true,
203
+ );
204
+ }
205
+
206
+ #[rstest]
207
+ #[case::too_short(&json!(["Hello"]))]
208
+ #[case::too_long(&json!(["Hello", 41.3, false, true, true]))]
209
+ #[case::prefix_schema_violation(&json!([817.2, false]))]
210
+ fn array_with_prefix_items_length_constrained_failures(#[case] sample_array: &Value) {
211
+ json_schema_check(
212
+ &SIMPLE_PREFIXED_ARRAY_LENGTH_CONSTRAINED,
213
+ sample_array,
214
+ false,
215
+ );
216
+ }