renderers 0.1.8.dev45__tar.gz → 0.1.8.dev50__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. renderers-0.1.8.dev50/.github/workflows/publish-dev.yml +67 -0
  2. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/PKG-INFO +1 -1
  3. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/pyproject.toml +16 -0
  4. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/renderers/_version.py +2 -2
  5. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/renderers/base.py +85 -20
  6. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/tests/conftest.py +5 -6
  7. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/tests/test_llama_3.py +13 -5
  8. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/tests/test_load_tokenizer.py +67 -1
  9. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/tests/test_preserve_thinking.py +4 -0
  10. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/uv.lock +2 -2
  11. renderers-0.1.8.dev45/.github/workflows/publish-dev.yml +0 -104
  12. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/.github/workflows/publish.yml +0 -0
  13. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/.github/workflows/style.yml +0 -0
  14. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/.github/workflows/test.yml +0 -0
  15. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/.gitignore +0 -0
  16. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/.pre-commit-config.yaml +0 -0
  17. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/LICENSE +0 -0
  18. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/README.md +0 -0
  19. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/docs/renderer-config.md +0 -0
  20. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/examples/README.md +0 -0
  21. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/examples/sglang/multiturn_generate_sglang.py +0 -0
  22. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/examples/sglang/online_multiturn_sglang.py +0 -0
  23. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/examples/tinker/multiturn_generate_tinker.py +0 -0
  24. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/examples/transformers/multiturn_generate_transformers.py +0 -0
  25. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/examples/vllm/multiturn_generate_vllm.py +0 -0
  26. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/renderers/__init__.py +0 -0
  27. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/renderers/client.py +0 -0
  28. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/renderers/configs.py +0 -0
  29. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/renderers/deepseek_r1.py +0 -0
  30. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/renderers/deepseek_v3.py +0 -0
  31. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/renderers/default.py +0 -0
  32. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/renderers/glm45.py +0 -0
  33. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/renderers/glm5.py +0 -0
  34. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/renderers/gpt_oss.py +0 -0
  35. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/renderers/kimi_k2.py +0 -0
  36. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/renderers/kimi_k25.py +0 -0
  37. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/renderers/laguna_xs2.py +0 -0
  38. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/renderers/llama_3.py +0 -0
  39. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/renderers/minimax_m2.py +0 -0
  40. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/renderers/nemotron3.py +0 -0
  41. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/renderers/parsers.py +0 -0
  42. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/renderers/parsing.py +0 -0
  43. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/renderers/qwen3.py +0 -0
  44. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/renderers/qwen35.py +0 -0
  45. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/renderers/qwen36.py +0 -0
  46. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/renderers/qwen3_vl.py +0 -0
  47. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/tests/test_bridge.py +0 -0
  48. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/tests/test_build_helpers.py +0 -0
  49. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/tests/test_client.py +0 -0
  50. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/tests/test_deepseek_r1.py +0 -0
  51. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/tests/test_gpt_oss_harmony_parity.py +0 -0
  52. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/tests/test_incremental.py +0 -0
  53. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/tests/test_is_content.py +0 -0
  54. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/tests/test_kimi_k25_tool_schema.py +0 -0
  55. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/tests/test_load_tokenizer_fastokens.py +0 -0
  56. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/tests/test_message_indices.py +0 -0
  57. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/tests/test_message_tool_names.py +0 -0
  58. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/tests/test_multimodal.py +0 -0
  59. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/tests/test_nemotron3_parity.py +0 -0
  60. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/tests/test_nemotron3_ultra.py +0 -0
  61. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/tests/test_parse_response.py +0 -0
  62. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/tests/test_parse_response_robustness.py +0 -0
  63. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/tests/test_parsers.py +0 -0
  64. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/tests/test_qwen35_size_coverage.py +0 -0
  65. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/tests/test_render_ids.py +0 -0
  66. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/tests/test_renderer_config.py +0 -0
  67. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/tests/test_renderer_config_parity.py +0 -0
  68. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/tests/test_roundtrip.py +0 -0
  69. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/tests/test_sampled_mask.py +0 -0
  70. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/tests/test_tokens_per_message.py +0 -0
  71. {renderers-0.1.8.dev45 → renderers-0.1.8.dev50}/tests/test_tool_arg_type_preservation.py +0 -0
@@ -0,0 +1,67 @@
1
+ name: Publish Dev
2
+
3
+ # Every push to main publishes a pre-release wheel to PyPI. The version
4
+ # is derived at build time by hatch-vcs from ``git describe`` distance
5
+ # to the latest stable ``renderers-v<MAJOR>.<MINOR>.<PATCH>`` tag (see
6
+ # ``[tool.hatch.version.raw-options]`` in pyproject.toml) — no per-commit
7
+ # tag is created. ``local_scheme = "no-local-version"`` strips the
8
+ # ``+gHASH`` segment so PyPI accepts the wheel directly.
9
+
10
+ on:
11
+ push:
12
+ branches: [main]
13
+
14
+ concurrency:
15
+ group: publish-dev-${{ github.ref }}
16
+ cancel-in-progress: false
17
+
18
+ jobs:
19
+ # Build (no OIDC) → publish (OIDC only). The build job runs ``uv build``
20
+ # with ``contents: read`` only so a poisoned build-time dep cannot mint
21
+ # the OIDC token. The publish job has ``id-token: write`` and the
22
+ # pypi-prod environment but no source checkout — it only downloads the
23
+ # prebuilt artifact and runs the SHA-pinned pypa publish action.
24
+ build:
25
+ runs-on: ubuntu-latest
26
+ permissions:
27
+ contents: read
28
+ steps:
29
+ - uses: actions/checkout@v4
30
+ with:
31
+ # hatch-vcs needs the full tag history to resolve the version
32
+ # from the latest stable tag's distance.
33
+ fetch-depth: 0
34
+
35
+ - uses: astral-sh/setup-uv@v7
36
+
37
+ - name: Build renderers
38
+ run: uv build
39
+
40
+ - name: Show derived version
41
+ run: ls -1 dist/
42
+
43
+ - name: Upload dist artifacts
44
+ uses: actions/upload-artifact@v4
45
+ with:
46
+ name: dist-dev
47
+ path: dist/
48
+ if-no-files-found: error
49
+ retention-days: 7
50
+
51
+ publish:
52
+ needs: build
53
+ runs-on: ubuntu-latest
54
+ environment: pypi-prod
55
+ permissions:
56
+ id-token: write
57
+ steps:
58
+ - name: Download dist artifacts
59
+ uses: actions/download-artifact@v4
60
+ with:
61
+ name: dist-dev
62
+ path: dist/
63
+
64
+ - name: Publish to PyPI
65
+ uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0
66
+ with:
67
+ skip-existing: true
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: renderers
3
- Version: 0.1.8.dev45
3
+ Version: 0.1.8.dev50
4
4
  Summary: Chat template renderers — deterministic message-to-token conversion for LLM training
5
5
  License-Expression: Apache-2.0
6
6
  License-File: LICENSE
@@ -57,6 +57,22 @@ tag-pattern = '^renderers-v(?P<version>.+)$'
57
57
  # only fires when the resolver has nothing to go on.
58
58
  fallback-version = "0.0.0"
59
59
 
60
+ [tool.hatch.version.raw-options]
61
+ # Only stable ``renderers-v<MAJOR>.<MINOR>.<PATCH>`` tags participate in
62
+ # version resolution; legacy per-commit ``.devN`` tags are excluded.
63
+ # Untagged commits get a clean PEP 440 dev version derived from distance
64
+ # to the latest stable tag (e.g. ``0.1.8.dev46``), with no ``+gHASH``
65
+ # local segment so PyPI accepts the wheel directly. setuptools-scm
66
+ # refuses to bump a pre-existing ``.devN`` tag, so matching only stable
67
+ # tags also unblocks local ``uv lock`` / ``uv build`` on untagged
68
+ # branches. Mirrors verifiers' setup.
69
+ local_scheme = "no-local-version"
70
+ git_describe_command = [
71
+ "git", "describe", "--tags", "--long",
72
+ "--match", "renderers-v[0-9]*.[0-9]*.[0-9]*",
73
+ "--exclude", "*dev*",
74
+ ]
75
+
60
76
  [tool.hatch.build.hooks.vcs]
61
77
  # Write the resolved version to a Python file so it can be inspected
62
78
  # at runtime via ``renderers.__version__`` without re-parsing the
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
18
18
  commit_id: str | None
19
19
  __commit_id__: str | None
20
20
 
21
- __version__ = version = '0.1.8.dev45'
22
- __version_tuple__ = version_tuple = (0, 1, 8, 'dev45')
21
+ __version__ = version = '0.1.8.dev50'
22
+ __version_tuple__ = version_tuple = (0, 1, 8, 'dev50')
23
23
 
24
24
  __commit_id__ = commit_id = None
@@ -1152,6 +1152,17 @@ TRUSTED_REVISIONS: dict[str, str] = {
1152
1152
  }
1153
1153
 
1154
1154
 
1155
+ # Tokenizer repos to use when a canonical model repo is gated but an
1156
+ # audited unrestricted mirror ships byte-identical tokenizer files and
1157
+ # chat_template. The returned tokenizer keeps the caller's original
1158
+ # ``name_or_path`` so exact-match renderer resolution still uses
1159
+ # ``MODEL_RENDERER_MAP``.
1160
+ TOKENIZER_SOURCE_OVERRIDES: dict[str, str] = {
1161
+ "meta-llama/Llama-3.2-1B-Instruct": "unsloth/Llama-3.2-1B-Instruct",
1162
+ "meta-llama/Llama-3.2-3B-Instruct": "unsloth/Llama-3.2-3B-Instruct",
1163
+ }
1164
+
1165
+
1155
1166
  # Models for which ``fastokens`` is known to diverge from vanilla
1156
1167
  # ``transformers.AutoTokenizer`` and therefore must NOT be patched.
1157
1168
  # Empirical audit ran each entry of ``MODEL_RENDERER_MAP`` through both
@@ -1175,6 +1186,42 @@ _FASTOKENS_PATCH_LOCK = threading.Lock()
1175
1186
  _FASTOKENS_ANNOUNCED = False
1176
1187
 
1177
1188
 
1189
+ def _tokenizer_source_for(model_name_or_path: str) -> str:
1190
+ return TOKENIZER_SOURCE_OVERRIDES.get(model_name_or_path, model_name_or_path)
1191
+
1192
+
1193
+ def _tokenizer_load_kwargs(model_name_or_path: str) -> dict[str, Any]:
1194
+ revision = TRUSTED_REVISIONS.get(model_name_or_path)
1195
+ if revision is not None:
1196
+ return {"trust_remote_code": True, "revision": revision}
1197
+ return {"trust_remote_code": False}
1198
+
1199
+
1200
+ def _preserve_requested_tokenizer_name(
1201
+ tokenizer,
1202
+ *,
1203
+ requested_name_or_path: str,
1204
+ loaded_name_or_path: str,
1205
+ ):
1206
+ if requested_name_or_path == loaded_name_or_path:
1207
+ return tokenizer
1208
+
1209
+ try:
1210
+ tokenizer.name_or_path = requested_name_or_path
1211
+ except Exception:
1212
+ init_kwargs = getattr(tokenizer, "init_kwargs", None)
1213
+ if isinstance(init_kwargs, dict):
1214
+ init_kwargs["name_or_path"] = requested_name_or_path
1215
+
1216
+ if getattr(tokenizer, "name_or_path", "") != requested_name_or_path:
1217
+ raise RuntimeError(
1218
+ f"Loaded tokenizer for {requested_name_or_path!r} from "
1219
+ f"{loaded_name_or_path!r}, but could not preserve the requested "
1220
+ "name_or_path for renderer auto-resolution."
1221
+ )
1222
+ return tokenizer
1223
+
1224
+
1178
1225
  def _patched_load(model_name_or_path: str, **kwargs):
1179
1226
  """Run ``AutoTokenizer.from_pretrained`` with fastokens patched in
1180
1227
  process-locally — patch around the load, unpatch right after.
@@ -1312,29 +1359,41 @@ def load_tokenizer(
1312
1359
  validation for configs with nested ``rope_parameters``), we fall
1313
1360
  back to loading the repo's self-contained ``tokenizer.json``
1314
1361
  directly — see ``_load_tokenizer_via_auto``.
1315
- """
1316
- kwargs: dict[str, Any] = {}
1317
- revision = TRUSTED_REVISIONS.get(model_name_or_path)
1318
- if revision is not None:
1319
- kwargs = {"trust_remote_code": True, "revision": revision}
1320
- else:
1321
- kwargs = {"trust_remote_code": False}
1322
1362
 
1323
- if not use_fastokens or model_name_or_path in FASTOKENS_INCOMPATIBLE:
1324
- return _load_tokenizer_via_auto(model_name_or_path, **kwargs)
1363
+ Canonical Meta Llama-3.2 Instruct repos are gated on HuggingFace. For
1364
+ those exact IDs we load tokenizer files from the audited unrestricted
1365
+ ``unsloth`` mirrors instead, then restore ``tokenizer.name_or_path`` to
1366
+ the requested Meta ID so auto-resolution still selects ``Llama3Renderer``.
1367
+ """
1368
+ load_name_or_path = _tokenizer_source_for(model_name_or_path)
1369
+ kwargs = _tokenizer_load_kwargs(load_name_or_path)
1370
+
1371
+ if not use_fastokens or load_name_or_path in FASTOKENS_INCOMPATIBLE:
1372
+ tok = _load_tokenizer_via_auto(load_name_or_path, **kwargs)
1373
+ return _preserve_requested_tokenizer_name(
1374
+ tok,
1375
+ requested_name_or_path=model_name_or_path,
1376
+ loaded_name_or_path=load_name_or_path,
1377
+ )
1325
1378
 
1326
1379
  try:
1327
- return _patched_load(model_name_or_path, **kwargs)
1380
+ tok = _patched_load(load_name_or_path, **kwargs)
1328
1381
  except Exception as exc:
1329
1382
  logger.info(
1330
1383
  "fastokens could not load %r (%s: %s); falling back to vanilla "
1331
1384
  "AutoTokenizer. Add this model to FASTOKENS_INCOMPATIBLE in "
1332
1385
  "renderers.base to suppress the retry.",
1333
- model_name_or_path,
1386
+ load_name_or_path,
1334
1387
  type(exc).__name__,
1335
1388
  str(exc)[:160],
1336
1389
  )
1337
- return _load_tokenizer_via_auto(model_name_or_path, **kwargs)
1390
+ tok = _load_tokenizer_via_auto(load_name_or_path, **kwargs)
1391
+
1392
+ return _preserve_requested_tokenizer_name(
1393
+ tok,
1394
+ requested_name_or_path=model_name_or_path,
1395
+ loaded_name_or_path=load_name_or_path,
1396
+ )
1338
1397
 
1339
1398
 
1340
1399
  def _populate_registry():
@@ -1702,12 +1761,8 @@ def _get_offset_tokenizer(tokenizer):
1702
1761
  if cached is not None:
1703
1762
  return cached
1704
1763
 
1705
- kwargs: dict[str, Any] = {}
1706
- revision = TRUSTED_REVISIONS.get(name_or_path)
1707
- if revision is not None:
1708
- kwargs = {"trust_remote_code": True, "revision": revision}
1709
- else:
1710
- kwargs = {"trust_remote_code": False}
1764
+ load_name_or_path = _tokenizer_source_for(name_or_path)
1765
+ kwargs = _tokenizer_load_kwargs(load_name_or_path)
1711
1766
 
1712
1767
  def _has_offsets(tok) -> bool:
1713
1768
  if not getattr(tok, "is_fast", False):
@@ -1727,7 +1782,12 @@ def _get_offset_tokenizer(tokenizer):
1727
1782
  # off — serialized against pool patch/unpatch via ``_FASTOKENS_PATCH_LOCK``
1728
1783
  # so no concurrent window can swap the shim back in mid-load — then
1729
1784
  # restore the prior patch state. Never cache a non-offset tokenizer.
1730
- offset_tok = _load_tokenizer_via_auto(name_or_path, **kwargs)
1785
+ offset_tok = _load_tokenizer_via_auto(load_name_or_path, **kwargs)
1786
+ offset_tok = _preserve_requested_tokenizer_name(
1787
+ offset_tok,
1788
+ requested_name_or_path=name_or_path,
1789
+ loaded_name_or_path=load_name_or_path,
1790
+ )
1731
1791
  if not _has_offsets(offset_tok):
1732
1792
  import fastokens
1733
1793
 
@@ -1737,7 +1797,12 @@ def _get_offset_tokenizer(tokenizer):
1737
1797
  with contextlib.redirect_stdout(io.StringIO()):
1738
1798
  fastokens.unpatch_transformers()
1739
1799
  try:
1740
- offset_tok = _load_tokenizer_via_auto(name_or_path, **kwargs)
1800
+ offset_tok = _load_tokenizer_via_auto(load_name_or_path, **kwargs)
1801
+ offset_tok = _preserve_requested_tokenizer_name(
1802
+ offset_tok,
1803
+ requested_name_or_path=name_or_path,
1804
+ loaded_name_or_path=load_name_or_path,
1805
+ )
1741
1806
  finally:
1742
1807
  if was_patched:
1743
1808
  with contextlib.redirect_stdout(io.StringIO()):
@@ -47,11 +47,10 @@ RENDERER_MODELS = [
47
47
  # there's just no byte-output to parity-check against. Split-specific
48
48
  # parity (V3 bare prompt vs R1 <think>+history-strip) is covered in
49
49
  # tests/test_deepseek_r1.py.
50
- # Llama-3 loads via the unrestricted unsloth mirror (byte-identical
51
- # chat template) so CI needs no Meta-gated HF token. Pinned to the
52
- # explicit "llama-3" config because the mirror name isn't in
53
- # MODEL_RENDERER_MAP (so "auto" would fall back to DefaultRenderer).
54
- ("unsloth/Llama-3.2-1B-Instruct", "llama-3"),
50
+ # Llama-3 uses the canonical Meta ID for renderer auto-resolution, while
51
+ # load_tokenizer fetches the tokenizer/chat_template from the unrestricted
52
+ # unsloth mirror so CI needs no Meta-gated HF token.
53
+ ("meta-llama/Llama-3.2-1B-Instruct", "auto"),
55
54
  ("openai/gpt-oss-20b", "gpt-oss"),
56
55
  ("Qwen/Qwen2.5-0.5B-Instruct", "default"),
57
56
  ]
@@ -139,7 +138,7 @@ _LLAMA_HF_PARITY_TEST_FILES = {
139
138
  def _skip_llama_for_hf_parity_tests(request):
140
139
  callspec = getattr(request.node, "callspec", None)
141
140
  model_name = callspec.params.get("model_name") if callspec else None
142
- if model_name != "unsloth/Llama-3.2-1B-Instruct":
141
+ if model_name != "meta-llama/Llama-3.2-1B-Instruct":
143
142
  return
144
143
  test_file = os.path.basename(str(request.node.fspath))
145
144
  if test_file in _LLAMA_HF_PARITY_TEST_FILES:
@@ -1,10 +1,10 @@
1
1
  """Llama-3 renderer coverage.
2
2
 
3
3
  Covers ``Llama3Renderer`` and the ``meta-llama/Llama-3.2-{1B,3B}-Instruct``
4
- entries in ``MODEL_RENDERER_MAP``. Tokenizers are loaded via the
5
- unrestricted ``unsloth/Llama-3.2-{1B,3B}-Instruct`` mirrors (verified
6
- byte-identical chat templates) so CI doesn't need an HF token with Meta
7
- license access.
4
+ entries in ``MODEL_RENDERER_MAP``. ``load_tokenizer`` uses the
5
+ unrestricted ``unsloth/Llama-3.2-{1B,3B}-Instruct`` mirrors underneath
6
+ (verified byte-identical chat templates) so CI doesn't need an HF token
7
+ with Meta license access.
8
8
  """
9
9
 
10
10
  from __future__ import annotations
@@ -34,7 +34,7 @@ _MODEL_PAIRS = [
34
34
  @pytest.fixture(scope="module", params=_MODEL_PAIRS, ids=[m for m, _ in _MODEL_PAIRS])
35
35
  def llama_pair(request):
36
36
  canonical, mirror = request.param
37
- tok = load_tokenizer(mirror)
37
+ tok = load_tokenizer(canonical)
38
38
  renderer = Llama3Renderer(tok, Llama3RendererConfig(date_string=_PINNED_DATE))
39
39
  return canonical, mirror, tok, renderer
40
40
 
@@ -58,6 +58,14 @@ def test_create_renderer_via_explicit_config(llama_pair):
58
58
  assert isinstance(r, Llama3Renderer)
59
59
 
60
60
 
61
+ def test_create_renderer_auto_resolves_after_mirror_load(llama_pair):
62
+ """``load_tokenizer(canonical_meta_id)`` loads from the unrestricted
63
+ mirror but preserves the canonical name needed for auto-resolution."""
64
+ canonical, _, tok, _ = llama_pair
65
+ assert tok.name_or_path == canonical
66
+ assert isinstance(create_renderer(tok), Llama3Renderer)
67
+
68
+
61
69
  # ---------------------------------------------------------------------------
62
70
  # Constructor contract
63
71
  # ---------------------------------------------------------------------------
@@ -9,9 +9,11 @@ repo doesn't auto-propagate.
9
9
  from __future__ import annotations
10
10
 
11
11
  import re
12
+ from types import SimpleNamespace
12
13
  from unittest.mock import patch
13
14
 
14
- from renderers.base import TRUSTED_REVISIONS, load_tokenizer
15
+ from renderers import base
16
+ from renderers.base import TOKENIZER_SOURCE_OVERRIDES, TRUSTED_REVISIONS, load_tokenizer
15
17
 
16
18
 
17
19
  # ---------------------------------------------------------------------------
@@ -70,6 +72,23 @@ def test_kimi_loads_with_pinned_revision(mock_from_pretrained):
70
72
  }
71
73
 
72
74
 
75
+ @patch("transformers.AutoTokenizer.from_pretrained")
76
+ def test_meta_llama_loads_tokenizer_from_unsloth_mirror(mock_from_pretrained):
77
+ """Canonical Meta Llama repos are gated; load their tokenizer/chat
78
+ template from the audited unrestricted mirror while preserving the
79
+ canonical name for renderer auto-resolution."""
80
+ canonical = "meta-llama/Llama-3.2-1B-Instruct"
81
+ mirror = "unsloth/Llama-3.2-1B-Instruct"
82
+ mock_from_pretrained.return_value = SimpleNamespace(name_or_path=mirror)
83
+
84
+ tok = load_tokenizer(canonical, use_fastokens=False)
85
+
86
+ args, kwargs = mock_from_pretrained.call_args
87
+ assert args == (mirror,)
88
+ assert kwargs == {"trust_remote_code": False}
89
+ assert tok.name_or_path == canonical
90
+
91
+
73
92
  @patch("transformers.AutoTokenizer.from_pretrained")
74
93
  def test_unknown_path_falls_through_to_no_remote_code(mock_from_pretrained):
75
94
  """Unknown / fine-tuned model paths — including ``moonshotai/Kimi-K2*``
@@ -92,6 +111,53 @@ def test_unknown_path_falls_through_to_no_remote_code(mock_from_pretrained):
92
111
  )
93
112
 
94
113
 
114
+ def test_tokenizer_source_overrides_are_exact_llama_mirrors():
115
+ """Mirror overrides are intentionally narrow: only verified
116
+ byte-identical Llama tokenizer/template mirrors should live here."""
117
+ assert TOKENIZER_SOURCE_OVERRIDES == {
118
+ "meta-llama/Llama-3.2-1B-Instruct": "unsloth/Llama-3.2-1B-Instruct",
119
+ "meta-llama/Llama-3.2-3B-Instruct": "unsloth/Llama-3.2-3B-Instruct",
120
+ }
121
+
122
+
123
+ def test_offset_tokenizer_uses_unsloth_mirror_for_meta_llama(monkeypatch):
124
+ """Offset-tokenizer reloads must use the same unrestricted source
125
+ override, otherwise Llama rendering can hit the gated Meta repo after
126
+ the initial tokenizer load succeeds."""
127
+
128
+ class _NoOffsets:
129
+ name_or_path = "meta-llama/Llama-3.2-1B-Instruct"
130
+
131
+ def __call__(self, *args, **kwargs):
132
+ raise NotImplementedError("fastokens shim has no offsets")
133
+
134
+ class _OffsetTokenizer:
135
+ is_fast = True
136
+
137
+ def __init__(self, name_or_path: str):
138
+ self.name_or_path = name_or_path
139
+
140
+ def __call__(self, *args, **kwargs):
141
+ return {"offset_mapping": [(0, 1)]}
142
+
143
+ calls = []
144
+
145
+ def _fake_load(name_or_path, **kwargs):
146
+ calls.append((name_or_path, kwargs))
147
+ return _OffsetTokenizer(name_or_path)
148
+
149
+ base._offset_tokenizers.clear()
150
+ monkeypatch.setattr(base, "_load_tokenizer_via_auto", _fake_load)
151
+
152
+ try:
153
+ tok = base._get_offset_tokenizer(_NoOffsets())
154
+ finally:
155
+ base._offset_tokenizers.clear()
156
+
157
+ assert calls == [("unsloth/Llama-3.2-1B-Instruct", {"trust_remote_code": False})]
158
+ assert tok.name_or_path == "meta-llama/Llama-3.2-1B-Instruct"
159
+
160
+
95
161
  # ---------------------------------------------------------------------------
96
162
  # Smoke: real tokenizer loads behave as expected
97
163
  # ---------------------------------------------------------------------------
@@ -53,6 +53,8 @@ NO_OP_MODELS = {
53
53
  "poolside/Laguna-XS.2",
54
54
  # Llama-3 has no reasoning channel at all — preserve flags can't add
55
55
  # or drop anything, so they're pure no-ops.
56
+ "meta-llama/Llama-3.2-1B-Instruct",
57
+ "meta-llama/Llama-3.2-3B-Instruct",
56
58
  "unsloth/Llama-3.2-1B-Instruct",
57
59
  }
58
60
 
@@ -324,6 +326,8 @@ NEVER_PRESERVES_MODELS = {
324
326
  "Qwen/Qwen3-VL-30B-A3B-Instruct",
325
327
  # Llama-3 ships no <think> rendering path, so reasoning_content never
326
328
  # surfaces in the output regardless of the preserve flags.
329
+ "meta-llama/Llama-3.2-1B-Instruct",
330
+ "meta-llama/Llama-3.2-3B-Instruct",
327
331
  "unsloth/Llama-3.2-1B-Instruct",
328
332
  }
329
333
 
@@ -9,7 +9,7 @@ resolution-markers = [
9
9
  ]
10
10
 
11
11
  [options]
12
- exclude-newer = "2026-05-18T21:42:54.18041997Z"
12
+ exclude-newer = "2026-06-19T02:55:59.889910839Z"
13
13
  exclude-newer-span = "P7D"
14
14
 
15
15
  [options.exclude-newer-package]
@@ -1409,7 +1409,7 @@ requires-dist = [
1409
1409
  { name = "jinja2" },
1410
1410
  { name = "numpy" },
1411
1411
  { name = "openai", specifier = ">=1.108.1" },
1412
- { name = "openai-harmony", specifier = ">=0.0.8" },
1412
+ { name = "openai-harmony", specifier = ">=0.0.4" },
1413
1413
  { name = "prime-pydantic-config", specifier = ">=0.3.0.dev83" },
1414
1414
  { name = "tiktoken" },
1415
1415
  { name = "transformers", specifier = ">=4.50.0" },
@@ -1,104 +0,0 @@
1
- name: Publish Dev
2
-
3
- # Tag every commit on main as ``renderers-v<next>.dev<N>`` and publish the
4
- # wheel to PyPI as a pre-release. ``<next>`` is the latest release tag with
5
- # its patch bumped; ``<N>`` is the number of commits since that release so
6
- # each main commit maps to a unique PEP 440 dev version.
7
- #
8
- # Building from the freshly-created tag means hatch-vcs resolves the version
9
- # cleanly (no ``+gHASH`` local segment), which PyPI requires.
10
-
11
- on:
12
- push:
13
- branches: [main]
14
-
15
- concurrency:
16
- group: publish-dev-${{ github.ref }}
17
- cancel-in-progress: false
18
-
19
- jobs:
20
- tag:
21
- runs-on: ubuntu-latest
22
- permissions:
23
- contents: write
24
- outputs:
25
- tag: ${{ steps.compute.outputs.tag }}
26
- steps:
27
- - uses: actions/checkout@v4
28
- with:
29
- fetch-depth: 0
30
-
31
- - name: Compute next dev tag
32
- id: compute
33
- run: |
34
- set -euo pipefail
35
- LATEST_RELEASE=$(git tag --list 'renderers-v*' --sort=-v:refname \
36
- | grep -Ev '(dev|rc|a[0-9]|b[0-9])' \
37
- | head -1)
38
- if [ -z "$LATEST_RELEASE" ]; then
39
- echo "No release tag matching 'renderers-v<MAJOR.MINOR.PATCH>' found" >&2
40
- exit 1
41
- fi
42
- BASE=${LATEST_RELEASE#renderers-v}
43
- MAJOR=$(echo "$BASE" | cut -d. -f1)
44
- MINOR=$(echo "$BASE" | cut -d. -f2)
45
- PATCH=$(echo "$BASE" | cut -d. -f3)
46
- NEXT="${MAJOR}.${MINOR}.$((PATCH + 1))"
47
- N=$(git rev-list --count "${LATEST_RELEASE}..HEAD")
48
- TAG="renderers-v${NEXT}.dev${N}"
49
- echo "tag=${TAG}" >> "$GITHUB_OUTPUT"
50
- echo "Computed tag: ${TAG} (base=${LATEST_RELEASE}, commits=${N})"
51
-
52
- - name: Create and push tag
53
- env:
54
- TAG: ${{ steps.compute.outputs.tag }}
55
- run: |
56
- set -euo pipefail
57
- if git ls-remote --exit-code --tags origin "refs/tags/${TAG}" >/dev/null 2>&1; then
58
- echo "Tag ${TAG} already exists on origin — nothing to do" >&2
59
- exit 0
60
- fi
61
- git config user.name 'github-actions[bot]'
62
- git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
63
- git tag -a "$TAG" -m "Automated dev release ${TAG}"
64
- git push origin "$TAG"
65
-
66
- build:
67
- needs: tag
68
- runs-on: ubuntu-latest
69
- permissions:
70
- contents: read
71
- steps:
72
- - uses: actions/checkout@v4
73
- with:
74
- fetch-depth: 0
75
- ref: refs/tags/${{ needs.tag.outputs.tag }}
76
-
77
- - uses: astral-sh/setup-uv@v7
78
-
79
- - name: Build renderers
80
- run: uv build
81
-
82
- - name: Upload dist artifacts
83
- uses: actions/upload-artifact@v4
84
- with:
85
- name: dist-dev
86
- path: dist/
87
- if-no-files-found: error
88
- retention-days: 7
89
-
90
- publish:
91
- needs: build
92
- runs-on: ubuntu-latest
93
- environment: pypi-prod
94
- permissions:
95
- id-token: write
96
- steps:
97
- - name: Download dist artifacts
98
- uses: actions/download-artifact@v4
99
- with:
100
- name: dist-dev
101
- path: dist/
102
-
103
- - name: Publish to PyPI
104
- uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0
File without changes