llguidance 0.7.15__tar.gz → 0.7.16__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (174) hide show
  1. {llguidance-0.7.15 → llguidance-0.7.16}/CHANGELOG.md +4 -0
  2. {llguidance-0.7.15 → llguidance-0.7.16}/Cargo.lock +5 -5
  3. {llguidance-0.7.15 → llguidance-0.7.16}/PKG-INFO +1 -1
  4. {llguidance-0.7.15 → llguidance-0.7.16}/parser/Cargo.toml +1 -1
  5. {llguidance-0.7.15 → llguidance-0.7.16}/pyproject.toml +1 -1
  6. {llguidance-0.7.15 → llguidance-0.7.16}/python_ext/Cargo.toml +1 -1
  7. {llguidance-0.7.15 → llguidance-0.7.16}/toktrie/Cargo.toml +1 -1
  8. {llguidance-0.7.15 → llguidance-0.7.16}/toktrie/src/tokenv.rs +7 -1
  9. {llguidance-0.7.15 → llguidance-0.7.16}/toktrie_hf_downloader/Cargo.toml +1 -1
  10. {llguidance-0.7.15 → llguidance-0.7.16}/toktrie_hf_tokenizers/Cargo.toml +1 -1
  11. {llguidance-0.7.15 → llguidance-0.7.16}/.github/workflows/rust.yml +0 -0
  12. {llguidance-0.7.15 → llguidance-0.7.16}/.github/workflows/wheels.yml +0 -0
  13. {llguidance-0.7.15 → llguidance-0.7.16}/.gitignore +0 -0
  14. {llguidance-0.7.15 → llguidance-0.7.16}/CODE_OF_CONDUCT.md +0 -0
  15. {llguidance-0.7.15 → llguidance-0.7.16}/Cargo.toml +0 -0
  16. {llguidance-0.7.15 → llguidance-0.7.16}/LICENSE +0 -0
  17. {llguidance-0.7.15 → llguidance-0.7.16}/README.md +0 -0
  18. {llguidance-0.7.15 → llguidance-0.7.16}/SECURITY.md +0 -0
  19. {llguidance-0.7.15 → llguidance-0.7.16}/SUPPORT.md +0 -0
  20. {llguidance-0.7.15 → llguidance-0.7.16}/c_sample/Makefile +0 -0
  21. {llguidance-0.7.15 → llguidance-0.7.16}/c_sample/README.md +0 -0
  22. {llguidance-0.7.15 → llguidance-0.7.16}/c_sample/c_sample.cpp +0 -0
  23. {llguidance-0.7.15 → llguidance-0.7.16}/docs/fast_forward.md +0 -0
  24. {llguidance-0.7.15 → llguidance-0.7.16}/docs/json_schema.md +0 -0
  25. {llguidance-0.7.15 → llguidance-0.7.16}/docs/mask_plot.png +0 -0
  26. {llguidance-0.7.15 → llguidance-0.7.16}/docs/optimizations.md +0 -0
  27. {llguidance-0.7.15 → llguidance-0.7.16}/docs/special_tokens.md +0 -0
  28. {llguidance-0.7.15 → llguidance-0.7.16}/docs/syntax.md +0 -0
  29. {llguidance-0.7.15 → llguidance-0.7.16}/docs/toktrie.md +0 -0
  30. {llguidance-0.7.15 → llguidance-0.7.16}/json_stats/Cargo.toml +0 -0
  31. {llguidance-0.7.15 → llguidance-0.7.16}/json_stats/expected_maskbench.json +0 -0
  32. {llguidance-0.7.15 → llguidance-0.7.16}/json_stats/jstats.sh +0 -0
  33. {llguidance-0.7.15 → llguidance-0.7.16}/json_stats/scripts/split-stats.sh +0 -0
  34. {llguidance-0.7.15 → llguidance-0.7.16}/json_stats/scripts/split_plot.py +0 -0
  35. {llguidance-0.7.15 → llguidance-0.7.16}/json_stats/src/json_stats.rs +0 -0
  36. {llguidance-0.7.15 → llguidance-0.7.16}/json_stats/src/lib.rs +0 -0
  37. {llguidance-0.7.15 → llguidance-0.7.16}/json_stats/src/stats.rs +0 -0
  38. {llguidance-0.7.15 → llguidance-0.7.16}/parser/LICENSE +0 -0
  39. {llguidance-0.7.15 → llguidance-0.7.16}/parser/README.md +0 -0
  40. {llguidance-0.7.15 → llguidance-0.7.16}/parser/build.rs +0 -0
  41. {llguidance-0.7.15 → llguidance-0.7.16}/parser/cbindgen.toml +0 -0
  42. {llguidance-0.7.15 → llguidance-0.7.16}/parser/grammars/character.json +0 -0
  43. {llguidance-0.7.15 → llguidance-0.7.16}/parser/grammars/json.json +0 -0
  44. {llguidance-0.7.15 → llguidance-0.7.16}/parser/llguidance.h +0 -0
  45. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/api.rs +0 -0
  46. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/constraint.rs +0 -0
  47. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/earley/from_guidance.rs +0 -0
  48. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/earley/grammar.rs +0 -0
  49. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/earley/lexer.rs +0 -0
  50. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/earley/lexerspec.rs +0 -0
  51. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/earley/mod.rs +0 -0
  52. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/earley/parser.rs +0 -0
  53. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/earley/perf.rs +0 -0
  54. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/earley/regexvec.rs +0 -0
  55. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/earley/slicer.rs +0 -0
  56. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/factory.rs +0 -0
  57. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/ffi.rs +0 -0
  58. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/ffi_par.rs +0 -0
  59. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/grammar_builder.rs +0 -0
  60. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/json/README.md +0 -0
  61. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/json/compiler.rs +0 -0
  62. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/json/context_ref.rs +0 -0
  63. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/json/context_simple/context.rs +0 -0
  64. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/json/context_simple/draft.rs +0 -0
  65. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/json/context_simple/mod.rs +0 -0
  66. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/json/formats.rs +0 -0
  67. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/json/mod.rs +0 -0
  68. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/json/numeric.rs +0 -0
  69. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/json/schema.rs +0 -0
  70. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/json/shared_context.rs +0 -0
  71. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/json_validation.rs +0 -0
  72. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/lark/README.md +0 -0
  73. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/lark/ast.rs +0 -0
  74. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/lark/common.rs +0 -0
  75. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/lark/compiler.rs +0 -0
  76. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/lark/lexer.rs +0 -0
  77. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/lark/mod.rs +0 -0
  78. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/lark/parser.rs +0 -0
  79. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/lib.rs +0 -0
  80. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/logging.rs +0 -0
  81. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/matcher.rs +0 -0
  82. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/output.rs +0 -0
  83. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/panic_utils.rs +0 -0
  84. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/regex_rewrite.rs +0 -0
  85. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/stop_controller.rs +0 -0
  86. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/substring.rs +0 -0
  87. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/tokenizer_json.rs +0 -0
  88. {llguidance-0.7.15 → llguidance-0.7.16}/parser/src/tokenparser.rs +0 -0
  89. {llguidance-0.7.15 → llguidance-0.7.16}/plan.md +0 -0
  90. {llguidance-0.7.15 → llguidance-0.7.16}/python/llguidance/__init__.py +0 -0
  91. {llguidance-0.7.15 → llguidance-0.7.16}/python/llguidance/_grammar_from.py +0 -0
  92. {llguidance-0.7.15 → llguidance-0.7.16}/python/llguidance/_lib.pyi +0 -0
  93. {llguidance-0.7.15 → llguidance-0.7.16}/python/llguidance/_struct_tag.py +0 -0
  94. {llguidance-0.7.15 → llguidance-0.7.16}/python/llguidance/_tokenizer.py +0 -0
  95. {llguidance-0.7.15 → llguidance-0.7.16}/python/llguidance/_util.py +0 -0
  96. {llguidance-0.7.15 → llguidance-0.7.16}/python/llguidance/cli.py +0 -0
  97. {llguidance-0.7.15 → llguidance-0.7.16}/python/llguidance/gbnf_to_lark.py +0 -0
  98. {llguidance-0.7.15 → llguidance-0.7.16}/python/llguidance/hf.py +0 -0
  99. {llguidance-0.7.15 → llguidance-0.7.16}/python/llguidance/mlx.py +0 -0
  100. {llguidance-0.7.15 → llguidance-0.7.16}/python/llguidance/numpy.py +0 -0
  101. {llguidance-0.7.15 → llguidance-0.7.16}/python/llguidance/py.typed +0 -0
  102. {llguidance-0.7.15 → llguidance-0.7.16}/python/llguidance/torch.py +0 -0
  103. {llguidance-0.7.15 → llguidance-0.7.16}/python/mypy.ini +0 -0
  104. {llguidance-0.7.15 → llguidance-0.7.16}/python/torch_tests/__init__.py +0 -0
  105. {llguidance-0.7.15 → llguidance-0.7.16}/python/torch_tests/test_bitmask.py +0 -0
  106. {llguidance-0.7.15 → llguidance-0.7.16}/python/torch_tests/test_hf.py +0 -0
  107. {llguidance-0.7.15 → llguidance-0.7.16}/python/torch_tests/test_matcher.py +0 -0
  108. {llguidance-0.7.15 → llguidance-0.7.16}/python_ext/src/lib.rs +0 -0
  109. {llguidance-0.7.15 → llguidance-0.7.16}/python_ext/src/llinterpreter.rs +0 -0
  110. {llguidance-0.7.15 → llguidance-0.7.16}/python_ext/src/llmatcher.rs +0 -0
  111. {llguidance-0.7.15 → llguidance-0.7.16}/python_ext/src/parserlimits.rs +0 -0
  112. {llguidance-0.7.15 → llguidance-0.7.16}/python_ext/src/py.rs +0 -0
  113. {llguidance-0.7.15 → llguidance-0.7.16}/python_ext/src/pyjson.rs +0 -0
  114. {llguidance-0.7.15 → llguidance-0.7.16}/sample_parser/Cargo.toml +0 -0
  115. {llguidance-0.7.15 → llguidance-0.7.16}/sample_parser/README.md +0 -0
  116. {llguidance-0.7.15 → llguidance-0.7.16}/sample_parser/cli.sh +0 -0
  117. {llguidance-0.7.15 → llguidance-0.7.16}/sample_parser/data/blog.sample.json +0 -0
  118. {llguidance-0.7.15 → llguidance-0.7.16}/sample_parser/data/blog.schema.json +0 -0
  119. {llguidance-0.7.15 → llguidance-0.7.16}/sample_parser/data/blog.schema.ll.json +0 -0
  120. {llguidance-0.7.15 → llguidance-0.7.16}/sample_parser/data/from-llama.cpp/README.md +0 -0
  121. {llguidance-0.7.15 → llguidance-0.7.16}/sample_parser/data/from-llama.cpp/arithmetic.gbnf +0 -0
  122. {llguidance-0.7.15 → llguidance-0.7.16}/sample_parser/data/from-llama.cpp/c.gbnf +0 -0
  123. {llguidance-0.7.15 → llguidance-0.7.16}/sample_parser/data/from-llama.cpp/chess.gbnf +0 -0
  124. {llguidance-0.7.15 → llguidance-0.7.16}/sample_parser/data/from-llama.cpp/english.gbnf +0 -0
  125. {llguidance-0.7.15 → llguidance-0.7.16}/sample_parser/data/from-llama.cpp/japanese.gbnf +0 -0
  126. {llguidance-0.7.15 → llguidance-0.7.16}/sample_parser/data/from-llama.cpp/json.gbnf +0 -0
  127. {llguidance-0.7.15 → llguidance-0.7.16}/sample_parser/data/from-llama.cpp/json_arr.gbnf +0 -0
  128. {llguidance-0.7.15 → llguidance-0.7.16}/sample_parser/data/from-llama.cpp/list.gbnf +0 -0
  129. {llguidance-0.7.15 → llguidance-0.7.16}/sample_parser/data/from-llama.cpp/vllm-sql.gbnf +0 -0
  130. {llguidance-0.7.15 → llguidance-0.7.16}/sample_parser/data/lark.lark +0 -0
  131. {llguidance-0.7.15 → llguidance-0.7.16}/sample_parser/data/rfc.lark +0 -0
  132. {llguidance-0.7.15 → llguidance-0.7.16}/sample_parser/data/rfc.xml +0 -0
  133. {llguidance-0.7.15 → llguidance-0.7.16}/sample_parser/data/ulysses.md +0 -0
  134. {llguidance-0.7.15 → llguidance-0.7.16}/sample_parser/gtest.sh +0 -0
  135. {llguidance-0.7.15 → llguidance-0.7.16}/sample_parser/lark.sh +0 -0
  136. {llguidance-0.7.15 → llguidance-0.7.16}/sample_parser/run.sh +0 -0
  137. {llguidance-0.7.15 → llguidance-0.7.16}/sample_parser/src/lib.rs +0 -0
  138. {llguidance-0.7.15 → llguidance-0.7.16}/sample_parser/src/minimal.rs +0 -0
  139. {llguidance-0.7.15 → llguidance-0.7.16}/sample_parser/src/sample_parser.rs +0 -0
  140. {llguidance-0.7.15 → llguidance-0.7.16}/sample_parser/tests/test_lark.rs +0 -0
  141. {llguidance-0.7.15 → llguidance-0.7.16}/sample_parser/tests/test_ll.rs +0 -0
  142. {llguidance-0.7.15 → llguidance-0.7.16}/sample_parser/tests/test_raw_parser.rs +0 -0
  143. {llguidance-0.7.15 → llguidance-0.7.16}/sample_parser/tests/test_stop.rs +0 -0
  144. {llguidance-0.7.15 → llguidance-0.7.16}/scripts/annotate_asm.js +0 -0
  145. {llguidance-0.7.15 → llguidance-0.7.16}/scripts/bump.py +0 -0
  146. {llguidance-0.7.15 → llguidance-0.7.16}/scripts/cbindgen.sh +0 -0
  147. {llguidance-0.7.15 → llguidance-0.7.16}/scripts/checklinks.py +0 -0
  148. {llguidance-0.7.15 → llguidance-0.7.16}/scripts/checklinks.sh +0 -0
  149. {llguidance-0.7.15 → llguidance-0.7.16}/scripts/ci-publish.py +0 -0
  150. {llguidance-0.7.15 → llguidance-0.7.16}/scripts/disasm.sh +0 -0
  151. {llguidance-0.7.15 → llguidance-0.7.16}/scripts/gbnf_to_lark.py +0 -0
  152. {llguidance-0.7.15 → llguidance-0.7.16}/scripts/gen-testcase.py +0 -0
  153. {llguidance-0.7.15 → llguidance-0.7.16}/scripts/git-version.sh +0 -0
  154. {llguidance-0.7.15 → llguidance-0.7.16}/scripts/install-deps.sh +0 -0
  155. {llguidance-0.7.15 → llguidance-0.7.16}/scripts/jsonschema-stats.js +0 -0
  156. {llguidance-0.7.15 → llguidance-0.7.16}/scripts/remote-guidance-test.sh +0 -0
  157. {llguidance-0.7.15 → llguidance-0.7.16}/scripts/rust-size.js +0 -0
  158. {llguidance-0.7.15 → llguidance-0.7.16}/scripts/rust_size.py +0 -0
  159. {llguidance-0.7.15 → llguidance-0.7.16}/scripts/test-guidance.sh +0 -0
  160. {llguidance-0.7.15 → llguidance-0.7.16}/scripts/tokenizer_test.py +0 -0
  161. {llguidance-0.7.15 → llguidance-0.7.16}/scripts/update-git.py +0 -0
  162. {llguidance-0.7.15 → llguidance-0.7.16}/toktrie/LICENSE +0 -0
  163. {llguidance-0.7.15 → llguidance-0.7.16}/toktrie/README.md +0 -0
  164. {llguidance-0.7.15 → llguidance-0.7.16}/toktrie/src/bytes.rs +0 -0
  165. {llguidance-0.7.15 → llguidance-0.7.16}/toktrie/src/lib.rs +0 -0
  166. {llguidance-0.7.15 → llguidance-0.7.16}/toktrie/src/recognizer.rs +0 -0
  167. {llguidance-0.7.15 → llguidance-0.7.16}/toktrie/src/rng.rs +0 -0
  168. {llguidance-0.7.15 → llguidance-0.7.16}/toktrie/src/svob.rs +0 -0
  169. {llguidance-0.7.15 → llguidance-0.7.16}/toktrie/src/toktree.rs +0 -0
  170. {llguidance-0.7.15 → llguidance-0.7.16}/toktrie/tests/test_svob.rs +0 -0
  171. {llguidance-0.7.15 → llguidance-0.7.16}/toktrie_hf_downloader/LICENSE +0 -0
  172. {llguidance-0.7.15 → llguidance-0.7.16}/toktrie_hf_downloader/src/lib.rs +0 -0
  173. {llguidance-0.7.15 → llguidance-0.7.16}/toktrie_hf_tokenizers/LICENSE +0 -0
  174. {llguidance-0.7.15 → llguidance-0.7.16}/toktrie_hf_tokenizers/src/lib.rs +0 -0
{llguidance-0.7.15 → llguidance-0.7.16}/CHANGELOG.md RENAMED
@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. Dates are d
4
4
 
5
5
  If a release doesn't introduce any interesting changes (build fixes etc.), it's skipped.
6
6
 
7
+ #### [0.7.16](https://github.com/guidance-ai/llguidance/compare/v0.7.15...0.7.16) 2025-04-17
8
+
9
+ - fix special token tokenization [`ae7870f`](https://github.com/guidance-ai/llguidance/commit/ae7870f05ca0de68599088607ba742b7071f92ad)
10
+
7
11
  #### [0.7.15](https://github.com/guidance-ai/llguidance/compare/v0.7.14...0.7.15) 2025-04-16
8
12
 
9
13
  - support for patternProperties in JSON schema [`#167`](https://github.com/guidance-ai/llguidance/pull/167)
{llguidance-0.7.15 → llguidance-0.7.16}/Cargo.lock RENAMED
@@ -1178,7 +1178,7 @@ checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856"
1178
1178
 
1179
1179
  [[package]]
1180
1180
  name = "llguidance"
1181
- version = "0.7.15"
1181
+ version = "0.7.16"
1182
1182
  dependencies = [
1183
1183
  "anyhow",
1184
1184
  "derivre",
@@ -1197,7 +1197,7 @@ dependencies = [
1197
1197
 
1198
1198
  [[package]]
1199
1199
  name = "llguidance_py"
1200
- version = "0.7.15"
1200
+ version = "0.7.16"
1201
1201
  dependencies = [
1202
1202
  "anyhow",
1203
1203
  "bytemuck",
@@ -2362,7 +2362,7 @@ dependencies = [
2362
2362
 
2363
2363
  [[package]]
2364
2364
  name = "toktrie"
2365
- version = "0.7.15"
2365
+ version = "0.7.16"
2366
2366
  dependencies = [
2367
2367
  "anyhow",
2368
2368
  "bytemuck",
@@ -2373,7 +2373,7 @@ dependencies = [
2373
2373
 
2374
2374
  [[package]]
2375
2375
  name = "toktrie_hf_downloader"
2376
- version = "0.7.15"
2376
+ version = "0.7.16"
2377
2377
  dependencies = [
2378
2378
  "anyhow",
2379
2379
  "hf-hub",
@@ -2384,7 +2384,7 @@ dependencies = [
2384
2384
 
2385
2385
  [[package]]
2386
2386
  name = "toktrie_hf_tokenizers"
2387
- version = "0.7.15"
2387
+ version = "0.7.16"
2388
2388
  dependencies = [
2389
2389
  "anyhow",
2390
2390
  "log",
{llguidance-0.7.15 → llguidance-0.7.16}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: llguidance
3
- Version: 0.7.15
3
+ Version: 0.7.16
4
4
  License-File: LICENSE
5
5
  Summary: Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance
6
6
  Author: Michal Moskal
{llguidance-0.7.15 → llguidance-0.7.16}/parser/Cargo.toml RENAMED
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "llguidance"
3
- version = "0.7.15"
3
+ version = "0.7.16"
4
4
  edition = "2021"
5
5
  license = "MIT"
6
6
  description = "Super-fast Structured Outputs"
{llguidance-0.7.15 → llguidance-0.7.16}/pyproject.toml RENAMED
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "llguidance"
3
- version = "0.7.15"
3
+ version = "0.7.16"
4
4
  description = "Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance"
5
5
  requires-python = ">=3.9"
6
6
  license = "MIT"
{llguidance-0.7.15 → llguidance-0.7.16}/python_ext/Cargo.toml RENAMED
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "llguidance_py"
3
- version = "0.7.15"
3
+ version = "0.7.16"
4
4
  edition = "2021"
5
5
  license = "MIT"
6
6
  description = "Super-fast Structured Outputs"
{llguidance-0.7.15 → llguidance-0.7.16}/toktrie/Cargo.toml RENAMED
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "toktrie"
3
- version = "0.7.15"
3
+ version = "0.7.16"
4
4
  edition = "2021"
5
5
  license = "MIT"
6
6
  description = "LLM Token Trie library"
{llguidance-0.7.15 → llguidance-0.7.16}/toktrie/src/tokenv.rs RENAMED
@@ -26,7 +26,13 @@ pub trait TokenizerEnv: Send {
26
26
  .position(|&x| x == ff)
27
27
  .unwrap_or(s.len() - idx);
28
28
  if normal_len != 0 {
29
- result.extend_from_slice(&self.tokenize_bytes(&s[idx..idx + normal_len]));
29
+ let new_tokens = self.tokenize_bytes(&s[idx..idx + normal_len]);
30
+ for (idx, t) in new_tokens.iter().enumerate() {
31
+ if trie.is_special_token(*t) {
32
+ num_fixed_tokens = result.len() + idx + 1;
33
+ }
34
+ }
35
+ result.extend_from_slice(&new_tokens);
30
36
  idx += normal_len;
31
37
  }
32
38
  idx += 1; // skip ff
{llguidance-0.7.15 → llguidance-0.7.16}/toktrie_hf_downloader/Cargo.toml RENAMED
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "toktrie_hf_downloader"
3
- version = "0.7.15"
3
+ version = "0.7.16"
4
4
  edition = "2021"
5
5
  license = "MIT"
6
6
  description = "HuggingFace Hub download library support for toktrie and llguidance"
{llguidance-0.7.15 → llguidance-0.7.16}/toktrie_hf_tokenizers/Cargo.toml RENAMED
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "toktrie_hf_tokenizers"
3
- version = "0.7.15"
3
+ version = "0.7.16"
4
4
  edition = "2021"
5
5
  license = "MIT"
6
6
  description = "HuggingFace tokenizers library support for toktrie and llguidance"
{llguidance-0.7.15 → llguidance-0.7.16}/.gitignore RENAMED
File without changes
{llguidance-0.7.15 → llguidance-0.7.16}/Cargo.toml RENAMED
File without changes
{llguidance-0.7.15 → llguidance-0.7.16}/LICENSE RENAMED
File without changes
{llguidance-0.7.15 → llguidance-0.7.16}/README.md RENAMED
File without changes
{llguidance-0.7.15 → llguidance-0.7.16}/SECURITY.md RENAMED
File without changes
{llguidance-0.7.15 → llguidance-0.7.16}/SUPPORT.md RENAMED
File without changes
{llguidance-0.7.15 → llguidance-0.7.16}/docs/syntax.md RENAMED
File without changes
{llguidance-0.7.15 → llguidance-0.7.16}/docs/toktrie.md RENAMED
File without changes
{llguidance-0.7.15 → llguidance-0.7.16}/parser/LICENSE RENAMED
File without changes
{llguidance-0.7.15 → llguidance-0.7.16}/parser/build.rs RENAMED
File without changes
{llguidance-0.7.15 → llguidance-0.7.16}/plan.md RENAMED
File without changes
{llguidance-0.7.15 → llguidance-0.7.16}/python/mypy.ini RENAMED
File without changes
{llguidance-0.7.15 → llguidance-0.7.16}/scripts/bump.py RENAMED
File without changes
{llguidance-0.7.15 → llguidance-0.7.16}/toktrie/LICENSE RENAMED
File without changes