renderers 0.1.8.dev33__tar.gz → 0.1.8.dev35__tar.gz

{renderers-0.1.8.dev33 → renderers-0.1.8.dev35}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: renderers
-Version: 0.1.8.dev33
+Version: 0.1.8.dev35
 Summary: Chat template renderers — deterministic message-to-token conversion for LLM training
 License-Expression: Apache-2.0
 License-File: LICENSE

{renderers-0.1.8.dev33 → renderers-0.1.8.dev35}/renderers/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.1.8.dev33'
-__version_tuple__ = version_tuple = (0, 1, 8, 'dev33')
+__version__ = version = '0.1.8.dev35'
+__version_tuple__ = version_tuple = (0, 1, 8, 'dev35')
 
 __commit_id__ = commit_id = None

{renderers-0.1.8.dev33 → renderers-0.1.8.dev35}/renderers/base.py

@@ -911,8 +911,8 @@ MODEL_RENDERER_MAP: dict[str, str] = {
     # ``enable_thinking=true`` (open ``<think>\n`` at the gen prompt);
     # the smaller 0.8B / 2B variants flip the polarity (default
     # ``enable_thinking=false``, empty ``<think>\n\n</think>\n\n``).
-    # ``Qwen35Renderer`` auto-detects polarity from the tokenizer's
-    # chat_template at construction, so all seven sizes are
+    # ``Qwen35Renderer`` hard-codes this polarity per model
+    # (``_ENABLE_THINKING_DEFAULTS``), so all seven sizes are
     # token-for-token parity-tested against their own
     # ``apply_chat_template`` — including with
     # ``add_generation_prompt=True``.
@@ -1089,7 +1089,6 @@ def _patched_load(model_name_or_path: str, **kwargs):
     path is still discoverable in logs.
     """
     import fastokens
-    from transformers import AutoTokenizer
 
     global _FASTOKENS_ANNOUNCED
 
@@ -1102,13 +1101,72 @@ def _patched_load(model_name_or_path: str, **kwargs):
             )
             _FASTOKENS_ANNOUNCED = True
     try:
-        return AutoTokenizer.from_pretrained(model_name_or_path, **kwargs)
+        return _load_tokenizer_via_auto(model_name_or_path, **kwargs)
     finally:
         with _FASTOKENS_PATCH_LOCK:
             with contextlib.redirect_stdout(io.StringIO()):
                 fastokens.unpatch_transformers()
 
 
+def _load_fast_tokenizer_directly(
+    model_name_or_path: str, revision: str | None
+) -> Any | None:
+    """Load a self-contained fast tokenizer without building the model config.
+
+    ``AutoTokenizer.from_pretrained`` eagerly constructs the *model* config to
+    resolve the tokenizer class — even for a plain ``PreTrainedTokenizerFast``.
+    That construction can raise on modeling-only concerns the tokenizer never
+    needs (e.g. RoPE parameter validation for configs that carry nested
+    ``rope_parameters``). When the repo ships a complete ``tokenizer.json`` and
+    declares no custom tokenizer, the tokenizer is fully self-describing, so we
+    load it directly and skip the config detour.
+
+    Returns ``None`` when there's nothing safe to load this way — a custom
+    ``auto_map`` tokenizer (which must run through ``AutoTokenizer`` with
+    ``trust_remote_code``) or no fast tokenizer at all — so the caller can
+    surface its original error instead.
+    """
+    from transformers import PreTrainedTokenizerFast
+    from transformers.models.auto.tokenization_auto import get_tokenizer_config
+
+    try:
+        if "auto_map" in get_tokenizer_config(model_name_or_path, revision=revision):
+            return None
+        return PreTrainedTokenizerFast.from_pretrained(
+            model_name_or_path, revision=revision
+        )
+    except Exception:
+        return None
+
+
+def _load_tokenizer_via_auto(model_name_or_path: str, **kwargs) -> Any:
+    """``AutoTokenizer.from_pretrained`` with a config-free fallback.
+
+    renderers needs the tokenizer, not the model. If ``AutoTokenizer`` fails
+    while building the model config it loads to resolve the tokenizer class,
+    retry by loading the repo's self-contained ``tokenizer.json`` directly. The
+    original error is re-raised if the repo has no such tokenizer.
+    """
+    from transformers import AutoTokenizer
+
+    try:
+        return AutoTokenizer.from_pretrained(model_name_or_path, **kwargs)
+    except Exception as exc:
+        tok = _load_fast_tokenizer_directly(
+            model_name_or_path, revision=kwargs.get("revision")
+        )
+        if tok is None:
+            raise
+        logger.debug(
+            "AutoTokenizer.from_pretrained(%r) failed building the model config "
+            "(%s: %s); loaded the tokenizer directly from tokenizer.json.",
+            model_name_or_path,
+            type(exc).__name__,
+            str(exc)[:160],
+        )
+        return tok
+
+
 def load_tokenizer(
     model_name_or_path: str,
     *,
@@ -1138,9 +1196,14 @@ def load_tokenizer(
     fastokens raises during the patched load (e.g. an unknown
     pre-tokenizer type), we automatically retry with the vanilla
     backend and emit an INFO log.
-    """
-    from transformers import AutoTokenizer
 
+    ``AutoTokenizer.from_pretrained`` eagerly builds the model config to
+    resolve the tokenizer class. If that construction raises on a
+    modeling-only concern the tokenizer doesn't need (e.g. RoPE
+    validation for configs with nested ``rope_parameters``), we fall
+    back to loading the repo's self-contained ``tokenizer.json``
+    directly — see ``_load_tokenizer_via_auto``.
+    """
     kwargs: dict[str, Any] = {}
     revision = TRUSTED_REVISIONS.get(model_name_or_path)
     if revision is not None:
@@ -1149,7 +1212,7 @@ def load_tokenizer(
         kwargs = {"trust_remote_code": False}
 
     if not use_fastokens or model_name_or_path in FASTOKENS_INCOMPATIBLE:
-        return AutoTokenizer.from_pretrained(model_name_or_path, **kwargs)
+        return _load_tokenizer_via_auto(model_name_or_path, **kwargs)
 
     try:
         return _patched_load(model_name_or_path, **kwargs)
@@ -1162,7 +1225,7 @@ def load_tokenizer(
             type(exc).__name__,
             str(exc)[:160],
         )
-        return AutoTokenizer.from_pretrained(model_name_or_path, **kwargs)
+        return _load_tokenizer_via_auto(model_name_or_path, **kwargs)
 
 
 def _populate_registry():

{renderers-0.1.8.dev33 → renderers-0.1.8.dev35}/renderers/qwen35.py

@@ -66,39 +66,44 @@ _TOOLS_INSTRUCTIONS = (
 )
 
 
-def _detect_enable_thinking_default(tokenizer: PreTrainedTokenizer) -> bool:
-    """Probe the tokenizer's chat template to learn its ``enable_thinking``
-    default polarity at the generation-prompt boundary.
-
-    The Qwen3.5 family ships two template variants that differ only in the
-    polarity of the gated branch:
-
-    * Big sizes (4B / 9B / 35B-A3B / 122B-A10B / 397B-A17B) emit an open
-      ``<think>\\n`` by default and the empty ``<think>\\n\\n</think>\\n\\n``
-      block when ``enable_thinking`` is explicitly false.
-    * Small sizes (0.8B / 2B) flip the polarity — they emit the empty
-      block by default and the open ``<think>\\n`` only when
-      ``enable_thinking`` is explicitly true.
-
-    A one-shot ``apply_chat_template`` call with no flag and a minimal
-    user message reveals which variant is in use: the empty-block tail
-    ends with ``</think>``, the open-think tail does not. Failing the
-    probe (no chat_template, exotic config) falls back to the big-model
-    default of True, which matches every entry in
-    ``MODEL_RENDERER_MAP`` that routes to ``qwen3.5`` without explicit
-    polarity awareness.
+# Per-model ``enable_thinking`` default, applied when the renderer config
+# leaves it ``None``. The Qwen3.5 family ships two chat-template variants
+# that differ only in the polarity of the gated thinking branch:
+#
+#   * Big sizes (4B / 9B / 35B-A3B / 122B-A10B / 397B-A17B) default
+#     ``enable_thinking=true`` — an open ``<think>\n`` at the gen prompt.
+#   * Small sizes (0.8B / 2B) flip it — default ``false``, emitting the
+#     empty ``<think>\n\n</think>\n\n`` block.
+#
+# These are hard-coded (keyed by ``tokenizer.name_or_path``) rather than
+# probed from the live ``chat_template``: probing meant calling
+# ``apply_chat_template`` at construction, which pulls ``transformers`` onto
+# the hot path and breaks bring-your-own-tokenizer use. The values are the
+# ground truth pinned by ``tests/test_qwen35_size_coverage.py`` — both the
+# polarity assertions and byte-parity against each size's own
+# ``apply_chat_template``.
+_ENABLE_THINKING_DEFAULTS: dict[str, bool] = {
+    "Qwen/Qwen3.5-0.8B": False,
+    "Qwen/Qwen3.5-2B": False,
+    "Qwen/Qwen3.5-4B": True,
+    "Qwen/Qwen3.5-9B": True,
+    "Qwen/Qwen3.5-35B-A3B": True,
+    "Qwen/Qwen3.5-122B-A10B": True,
+    "Qwen/Qwen3.5-397B-A17B": True,
+    # Qwen3.6 extends the Qwen3.5 template; same big-size polarity.
+    "Qwen/Qwen3.6-35B-A3B": True,
+}
+
+
+def _default_enable_thinking(tokenizer) -> bool:
+    """Hard-coded ``enable_thinking`` default for ``tokenizer``'s model.
+
+    Falls back to ``True`` (the big-model default, and the majority of the
+    family) for unknown / fine-tuned checkpoints whose ``name_or_path`` isn't
+    in ``_ENABLE_THINKING_DEFAULTS``; pass an explicit ``enable_thinking=`` to
+    a small-size fine-tune that needs ``False``.
     """
-    try:
-        out = tokenizer.apply_chat_template(
-            [{"role": "user", "content": "x"}],
-            tokenize=False,
-            add_generation_prompt=True,
-        )
-    except Exception:
-        return True
-    if not isinstance(out, str):
-        return True
-    return not out.rstrip().endswith("</think>")
+    return _ENABLE_THINKING_DEFAULTS.get(getattr(tokenizer, "name_or_path", ""), True)
 
 
 class Qwen35Renderer:
@@ -116,13 +121,13 @@ class Qwen35Renderer:
         self._tokenizer = tokenizer
         self._processor = processor
         cfg = config or type(self)._config_cls()
-        # ``enable_thinking=None`` defers to the tokenizer's chat-template
-        # default (Instruct → off, Thinking → on). Materialise here so
-        # downstream reads see a concrete bool; rebind the config with
-        # the resolved value so introspection sees the same.
+        # ``enable_thinking=None`` defers to the model's known default (see
+        # ``_ENABLE_THINKING_DEFAULTS``). Materialise here so downstream reads
+        # see a concrete bool; rebind the config with the resolved value so
+        # introspection sees the same.
         if cfg.enable_thinking is None:
             cfg = cfg.model_copy(
-                update={"enable_thinking": _detect_enable_thinking_default(tokenizer)}
+                update={"enable_thinking": _default_enable_thinking(tokenizer)}
             )
         self.config = cfg
 

{renderers-0.1.8.dev33 → renderers-0.1.8.dev35}/tests/test_qwen35_size_coverage.py

@@ -5,9 +5,8 @@ Seven Qwen3.5 sizes route to ``Qwen35Renderer``. The 4B / 9B / 35B-A3B /
 ``enable_thinking=true``); the smaller 0.8B / 2B sizes ship the polarity-
 flipped variant (default ``enable_thinking=false`` → empty
 ``<think>\\n\\n</think>\\n\\n`` at the gen-prompt boundary). The renderer
-detects polarity from the tokenizer's chat_template at construction, so
-both variants render byte-identical to their own
-``apply_chat_template``.
+hard-codes this polarity per model (``_ENABLE_THINKING_DEFAULTS``), so
+both variants render byte-identical to their own ``apply_chat_template``.
 
 These tests lock in (a) the exact set of Qwen3.5 sizes in the map and
 (b) byte parity for every one of them across representative
@@ -57,7 +56,7 @@ def test_no_other_qwen35_sizes_silently_added():
 
 
 # ---------------------------------------------------------------------------
-# Polarity auto-detection: 0.8B / 2B flip ``enable_thinking`` default.
+# Polarity defaults: 0.8B / 2B flip ``enable_thinking`` default.
 # ---------------------------------------------------------------------------
 
 
@@ -73,10 +72,10 @@ def test_no_other_qwen35_sizes_silently_added():
         ("Qwen/Qwen3.5-397B-A17B", True),
     ],
 )
-def test_qwen35_enable_thinking_polarity_autodetected(qwen35_model, expected_default):
-    """The renderer's ``_enable_thinking`` resolves to the chat template's
-    own default when no explicit flag is passed — so big / small sizes
-    each match their own template at the gen-prompt boundary."""
+def test_qwen35_enable_thinking_polarity_default(qwen35_model, expected_default):
+    """With no explicit flag, the renderer resolves ``enable_thinking`` from
+    the hard-coded per-model default — so big / small sizes each match their
+    own template at the gen-prompt boundary."""
     tok = load_tokenizer(qwen35_model)
     renderer = create_renderer(tok, Qwen35RendererConfig())
     assert isinstance(renderer, Qwen35Renderer)
@@ -86,6 +85,30 @@ def test_qwen35_enable_thinking_polarity_autodetected(qwen35_model, expected_def
     )
 
 
+def test_construction_does_not_call_apply_chat_template():
+    """The ``enable_thinking`` default is hard-coded per model, so building a
+    ``Qwen35Renderer`` must not probe ``apply_chat_template`` — a
+    bring-your-own tokenizer with no chat-template support still works."""
+
+    class _Stub:
+        name_or_path = "Qwen/Qwen3.5-0.8B"
+        unk_token_id = -1
+
+        def convert_tokens_to_ids(self, token):
+            # Any stable non-unk id per token; the renderer only needs the
+            # special tokens to resolve to distinct, in-vocab ids.
+            return abs(hash(token)) % 1_000_000 + 1
+
+        def apply_chat_template(self, *args, **kwargs):
+            raise AssertionError(
+                "apply_chat_template must not be called at construction"
+            )
+
+    renderer = Qwen35Renderer(_Stub())
+    # 0.8B is a small size → thinking defaults off, from the hard-coded table.
+    assert renderer.config.enable_thinking is False
+
+
 # ---------------------------------------------------------------------------
 # Byte parity for each in-map Qwen3.5 size.
 # ---------------------------------------------------------------------------
@@ -146,7 +169,7 @@ def test_qwen35_size_parity_with_apply_chat_template(
     """Each in-map Qwen3.5 size renders byte-identical to its own
     ``apply_chat_template`` output. Locks in the property that lets us
     share ``Qwen35Renderer`` across all seven sizes — the polarity
-    flip on 0.8B / 2B is absorbed by the constructor's auto-detect."""
+    flip on 0.8B / 2B is absorbed by the per-model default."""
     tok = load_tokenizer(qwen35_model)
     renderer = create_renderer(tok, Qwen35RendererConfig())
     assert isinstance(renderer, Qwen35Renderer)