renderers 0.1.8.dev36__tar.gz → 0.1.8.dev37__tar.gz

{renderers-0.1.8.dev36 → renderers-0.1.8.dev37}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: renderers
-Version: 0.1.8.dev36
+Version: 0.1.8.dev37
 Summary: Chat template renderers — deterministic message-to-token conversion for LLM training
 License-Expression: Apache-2.0
 License-File: LICENSE

{renderers-0.1.8.dev36 → renderers-0.1.8.dev37}/renderers/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.1.8.dev36'
-__version_tuple__ = version_tuple = (0, 1, 8, 'dev36')
+__version__ = version = '0.1.8.dev37'
+__version_tuple__ = version_tuple = (0, 1, 8, 'dev37')
 
 __commit_id__ = commit_id = None

{renderers-0.1.8.dev36 → renderers-0.1.8.dev37}/renderers/base.py

@@ -1677,7 +1677,6 @@ def _get_offset_tokenizer(tokenizer):
         cached = _offset_tokenizers.get(name_or_path)
         if cached is not None:
             return cached
-        from transformers import AutoTokenizer
 
         kwargs: dict[str, Any] = {}
         revision = TRUSTED_REVISIONS.get(name_or_path)
@@ -1687,10 +1686,12 @@ def _get_offset_tokenizer(tokenizer):
             kwargs = {"trust_remote_code": False}
         # Explicitly vanilla — we want HF's Rust tokenizer with offset
         # tracking, not the fastokens shim. ``load_tokenizer`` would
-        # patch fastokens in by default; calling
-        # ``AutoTokenizer.from_pretrained`` directly here keeps the
-        # fastokens patch out of this code path entirely.
-        offset_tok = AutoTokenizer.from_pretrained(name_or_path, **kwargs)
+        # patch fastokens in by default; routing through
+        # ``_load_tokenizer_via_auto`` keeps the fastokens patch out
+        # of this code path while still applying the config-build
+        # fallback (RoPE-validation failures on nested
+        # ``rope_parameters``, etc.).
+        offset_tok = _load_tokenizer_via_auto(name_or_path, **kwargs)
         if not getattr(offset_tok, "is_fast", False):
             raise RuntimeError(
                 f"Vanilla tokenizer for {name_or_path!r} is not a fast "