interp-engine 1.2.2__tar.gz → 1.2.4__tar.gz

{interp_engine-1.2.2 → interp_engine-1.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: interp-engine
-Version: 1.2.2
+Version: 1.2.4
 Summary: A fast, standardized interpretability engine that supports most modern models and architectures. Powers Neuronpedia.
 Project-URL: Homepage, https://github.com/decoderesearch/interp-engine
 Project-URL: Repository, https://github.com/decoderesearch/interp-engine
@@ -26,7 +26,7 @@ Provides-Extra: quant
 Requires-Dist: accelerate>=1.0; extra == 'quant'
 Requires-Dist: kernels<0.17.0,>=0.15.2; extra == 'quant'
 Provides-Extra: vllm
-Requires-Dist: vllm>=0.25.1; (sys_platform == 'linux') and extra == 'vllm'
+Requires-Dist: vllm>=0.27.1; (sys_platform == 'linux') and extra == 'vllm'
 Description-Content-Type: text/markdown
 
 # interp-engine
@@ -38,6 +38,7 @@ Description-Content-Type: text/markdown
   🔗 <a href="https://interp-engine.org"><strong>interp-engine.org</strong></a>
 </p>
 
+
 `interp-engine` is an interpretability engine that is fast (>40x tok/s vs HF eager), standardized (34 'points'/addresses across architectures), and easy to use and debug. It powers all of [Neuronpedia](https://neuronpedia.org)'s inference and is checked for accuracy against HF Transformers and other engines.
 
 <p align="center">

{interp_engine-1.2.2 → interp_engine-1.2.4}/README.md

@@ -7,6 +7,7 @@
   🔗 <a href="https://interp-engine.org"><strong>interp-engine.org</strong></a>
 </p>
 
+
 `interp-engine` is an interpretability engine that is fast (>40x tok/s vs HF eager), standardized (34 'points'/addresses across architectures), and easy to use and debug. It powers all of [Neuronpedia](https://neuronpedia.org)'s inference and is checked for accuracy against HF Transformers and other engines.
 
 <p align="center">

{interp_engine-1.2.2 → interp_engine-1.2.4}/docs/PERFORMANCE.md

@@ -42,13 +42,17 @@ every new upstream flag into an engine change.
 
 ### The vLLM version is a floor, not a ceiling
 
-`interp-engine[vllm]` declares `vllm>=0.25.1` on Linux and no upper bound. Read that as **the oldest
+`interp-engine[vllm]` declares `vllm>=0.27.1` on Linux and no upper bound. Read that as **the oldest
 version we have evidence for**, not the version to run:
 
-- **The floor is measured, not guessed.** 0.25.1 is the oldest version the engine has been exercised
-  on. The current sweep in [`benchmarks/results-latest.md`](../benchmarks/results-latest.md) ran on
-  **0.26.0**, which is also what `apps/inference` and `apps/nla` resolve to — being ahead of the floor
-  is the normal state, not drift.
+- **The floor is measured, not guessed.** 0.27.1 is the version the DeepSeek-V4-Flash-0731
+  cross-engine comparison scored, and the oldest the engine is supported on. The performance sweep in
+  [`benchmarks/results-latest.md`](../benchmarks/results-latest.md) ran on **0.26.0**, one below —
+  being ahead of the floor is the normal state, not drift.
+- **This floor is load-bearing, not hygiene.** It was 0.25.1 until `vllm_capture/mhc.py` needed
+  `mhc_pre_broadcast_tilelang`, which arrived in 0.26.0. Below that, every DeepSeek-V4
+  hyper-connection point is refused at install, and since `FREEZE_POINTS=auto` selects
+  `resid_streams` on such a trunk, the pod does not load at all.
 - **There is deliberately no cap.** vLLM moves fast and a `<` bound would make every upstream release
   an engine release; a lock file is the right place to pin a deployment, and both apps have one. The
   version an app runs is whatever its `uv.lock` resolved, which is normally newer than this floor.

{interp_engine-1.2.2 → interp_engine-1.2.4}/interp_engine/__init__.py

@@ -58,6 +58,13 @@ from interp_engine.chat_conventions import (
     detect_reasoning_tags,
     is_harmony,
 )
+from interp_engine.chat_formatters import (
+    CODE_CHAT_FORMATS,
+    ChatFormatter,
+    ChatFormatterUnavailable,
+    RenderedChat,
+    resolve_chat_formatter,
+)
 from interp_engine.cuda_preflight import check_cuda_driver
 from interp_engine.dispatch import CAPABILITIES, Capability, CapabilityUnsupported, TokensLike
 from interp_engine.facts import rms_norm_eps_for_model
@@ -265,11 +272,15 @@ __all__ = [
     "resolve_arch",
     "unsupported_attn_config",
     # Tokenization and chat templates.
+    "CODE_CHAT_FORMATS",
+    "ChatFormatter",
+    "ChatFormatterUnavailable",
     "ChatTurn",
     "GeneratedTurnSpans",
     "NoChatTemplateError",
     "REASONING_TAGS",
     "ReasoningTags",
+    "RenderedChat",
     "TURN_END_TOKENS",
     "TokenSpan",
     "Tokenize",
@@ -277,6 +288,7 @@ __all__ = [
     "compose_assistant_turns",
     "detect_reasoning_tags",
     "is_harmony",
+    "resolve_chat_formatter",
     "special_token_ids",
     "special_token_positions",
     "strip_wire_reasoning",

{interp_engine-1.2.2 → interp_engine-1.2.4}/interp_engine/chat_conventions.py

@@ -50,6 +50,9 @@ TURN_END_TOKENS = frozenset(
         "<|end_of_text|>",
         "<|endoftext|>",
         "<|eom_id|>",
+        # DeepSeek (V3 and V4 lines). Note the full-width bars and the U+2581 word separators:
+        # these are the checkpoint's actual added-token strings, not the ASCII lookalikes.
+        "<｜end▁of▁sentence｜>",
     }
 )
 

interp_engine-1.2.4/interp_engine/chat_formatters.py

@@ -0,0 +1,451 @@
+"""Chat formats that live in Python instead of in a Jinja ``chat_template``.
+
+Almost every chat model describes its own prompt format in the tokenizer's ``chat_template``,
+which is why :meth:`Tokenize.message_spans` needs no per-model knowledge: it renders the real
+template and diffs. A few checkpoints ship no template at all and define the format in code
+instead, because the thing they need to express is not a render. DeepSeek-V4 is the case this
+module exists for: its ``encoding/`` folder carries an encoder *and* a parser
+(``parse_message_from_completion_text``), and a template can only ever do the first half.
+
+**The reference implementation ships inside the checkpoint**, at
+``encoding/encoding_dsv4.py``, beside the weights and versioned with them. So this module
+downloads and imports it rather than carrying a copy: a vendored fork is a second source of
+truth for a format whose first source of truth we already have on disk. That the risk is real
+rather than theoretical is easy to check -- vLLM's fork of the same file
+(``vllm/tokenizers/deepseek_v4_encoding.py``) renamed the assistant field
+``reasoning_content`` to ``reasoning``, so feeding it upstream-shaped messages silently drops
+every thinking block. :func:`_reasoning_field` reads the field name back out of whichever copy
+got loaded rather than assuming either spelling.
+
+Loading it is remote code execution, so it is gated on the same ``trust_remote_code`` flag the
+weights are. When the file cannot be fetched the engine keeps loading and simply reports no
+chat support, which lands the caller on the existing raw-text refusal rather than on a failed
+load -- chat is one endpoint, not the model.
+
+Adding a family means one entry in :data:`CODE_CHAT_FORMATS` and a class satisfying
+:class:`ChatFormatter`. Generation-side structure (reasoning delimiters, harmony channels)
+stays in ``chat_conventions``; this module is only about rendering the prompt.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import inspect
+import logging
+import re
+import sys
+from collections.abc import Callable, Mapping, Sequence
+from dataclasses import dataclass
+from types import ModuleType
+from typing import Any, Protocol
+
+logger = logging.getLogger(__name__)
+
+Message = Mapping[str, Any]
+
+
+class ChatFormatterUnavailable(RuntimeError):
+    """The architecture needs a code formatter and the engine could not load one.
+
+    Raised by the loaders, caught by :func:`resolve_chat_formatter`, which downgrades it to a
+    warning: a model whose chat format is unreachable still captures, steers and completes raw
+    text, so this must not gate loading.
+    """
+
+
+@dataclass(frozen=True)
+class RenderedChat:
+    """A chat render, kept split at the boundaries span metadata is derived from.
+
+    ``blocks`` is 1:1 with the input messages, in order, so ``blocks[k]`` is exactly what
+    message ``k`` contributed to the prompt. ``prefix`` holds what precedes message 0 (a BOS
+    token, an injected tools preamble) and ``suffix`` the trailing generation scaffold.
+
+    The split is the point. A Jinja template can only be rendered whole, so ``message_spans``
+    has to recover boundaries by re-rendering growing message prefixes and diffing tokens --
+    which assumes that growing the message list only appends, and DeepSeek-V4 breaks that
+    assumption (whether a turn is the *last* user turn changes how earlier turns render). A
+    formatter that reports its own boundaries sidesteps the diff entirely: ``upto`` returns a
+    genuine string prefix of ``text``, so the token prefix is exact.
+    """
+
+    prefix: str
+    blocks: tuple[str, ...]
+    suffix: str
+
+    @property
+    def text(self) -> str:
+        """The whole prompt, exactly as it would be tokenized."""
+        return self.prefix + "".join(self.blocks) + self.suffix
+
+    def upto(self, count: int) -> str:
+        """``text`` truncated to the first ``count`` messages, generation scaffold excluded."""
+        return self.prefix + "".join(self.blocks[:count])
+
+
+class ChatFormatter(Protocol):
+    """What :class:`~interp_engine.tokenize.Tokenize` needs from a code-defined chat format.
+
+    ``template_kwargs`` is the set of keyword arguments the format actually reads. Callers use
+    it the way they used to grep the Jinja source for a variable name: to pass
+    ``enable_thinking`` only where it means something. A formatter refuses an unknown kwarg
+    rather than ignoring it, since a silently dropped ``enable_thinking`` renders the wrong
+    prompt and returns 200.
+    """
+
+    name: str
+    template_kwargs: frozenset[str]
+
+    def render(
+        self,
+        messages: Sequence[Message],
+        *,
+        add_generation_prompt: bool = True,
+        continue_final_message: bool = False,
+        **template_kwargs: Any,
+    ) -> RenderedChat: ...
+
+
+# --------------------------------------------------------------------------- #
+# DeepSeek-V4
+# --------------------------------------------------------------------------- #
+
+# Where the reference encoder sits in every DeepSeek-V4 repo (V4-Flash, V4-Pro). The folder
+# also carries README.md (the format spec) and tests/ (input/expected-output fixtures, which
+# validator/ uses as golden cases).
+DEEPSEEK_V4_ENCODER_FILE = "encoding/encoding_dsv4.py"
+
+# vLLM vendors the same file. Only consulted when the checkpoint's own copy cannot be fetched,
+# and it is a fork rather than a mirror -- see the module docstring.
+DEEPSEEK_V4_FALLBACK_MODULE = "vllm.tokenizers.deepseek_v4_encoding"
+
+
+class DeepseekV4Formatter:
+    """Renders DeepSeek-V4 prompts through the encoder shipped with the checkpoint.
+
+    Three of the encoder's conventions differ from the ``apply_chat_template`` vocabulary the
+    rest of the engine speaks, and each is translated here rather than at the call sites:
+
+    - **There is no ``add_generation_prompt`` flag.** The encoder appends
+      ``<｜Assistant｜>`` plus a thinking delimiter whenever the conversation ends on a user or
+      developer turn, unconditionally. So the scaffold is *identified* after the fact and
+      reported as :attr:`RenderedChat.suffix`, which callers keep or drop. Note this makes
+      ``add_generation_prompt=True`` a no-op after a closed assistant turn, where a Jinja
+      template would open a fresh one: the encoder's own answer is that a transcript ending in
+      a completed answer is a transcript, not a prompt, and synthesizing the opener anyway
+      would put two tokens in front of the model that the checkpoint's own reference encoder
+      never emits. The fixtures in ``encoding/tests/`` are exactly such transcripts.
+    - **There is no ``continue_final_message`` flag.** A prefill is spelled per message, as
+      ``wo_eos``, which suppresses that turn's end-of-sentence token. That is a better fit
+      than the flag it replaces: transformers implements ``continue_final_message`` by
+      rendering a sentinel into the content and cutting the string, and here nothing is cut.
+    - **Historical reasoning is dropped by default** (``drop_thinking``), which is what makes
+      the render non-monotonic and is why this class reports message blocks itself.
+    """
+
+    name = "deepseek_v4"
+    # `thinking` is the encoder's own spelling and `enable_thinking` the one every other family
+    # in this engine uses; both are accepted so a caller need not know which model it has.
+    template_kwargs = frozenset({"enable_thinking", "thinking", "reasoning_effort", "drop_thinking", "tools"})
+
+    def __init__(self, module: ModuleType):
+        self._module = module
+        self._encode: Callable[..., str] = getattr(module, "encode_messages")  # noqa: B009 - module attr
+        self._reasoning_key = _reasoning_field(module)
+        self._assistant_token = str(getattr(module, "ASSISTANT_SP_TOKEN", "<｜Assistant｜>"))
+        self._think_open = str(getattr(module, "thinking_start_token", "<think>"))
+        self._think_close = str(getattr(module, "thinking_end_token", "</think>"))
+        self._bos = str(getattr(module, "bos_token", ""))
+        self._encode_params = _accepted_parameters(self._encode)
+
+    # --- rendering ----------------------------------------------------------
+    def render(
+        self,
+        messages: Sequence[Message],
+        *,
+        add_generation_prompt: bool = True,
+        continue_final_message: bool = False,
+        **template_kwargs: Any,
+    ) -> RenderedChat:
+        config, tools = self._encode_config(template_kwargs)
+        turns = [self._normalize(m) for m in messages]
+
+        if continue_final_message:
+            if not turns or turns[-1].get("role") != "assistant":
+                raise ValueError(
+                    "continue_final_message=True keeps a trailing ASSISTANT turn open, but the last "
+                    f"message is {turns[-1].get('role') if turns else 'absent'!r}."
+                )
+            turns[-1] = {**turns[-1], "wo_eos": True}
+
+        # Tools ride on a system message in this format. Injecting one shifts every index, so
+        # its render is folded into `prefix` and `blocks` stays 1:1 with the input messages.
+        lead: list[dict[str, Any]] = [{"role": "system", "tools": list(tools)}] if tools else []
+        conversation = lead + turns
+
+        tails = self._tails(conversation, config)
+        blocks = [tails[k][: len(tails[k]) - len(tails[k + 1])] for k in range(len(conversation))]
+        prefix = self._bos + "".join(blocks[: len(lead)])
+        blocks = blocks[len(lead) :]
+
+        scaffold = self._split_scaffold(blocks, turns)
+        return RenderedChat(prefix=prefix, blocks=tuple(blocks), suffix=scaffold if add_generation_prompt else "")
+
+    def _tails(self, conversation: Sequence[Message], config: dict[str, Any]) -> list[str]:
+        """``tails[k]`` = messages ``k..end`` rendered *in the context of the whole conversation*.
+
+        The encoder's own ``context`` argument is what makes per-message blocks exact. Passing
+        ``messages[k:]`` with ``context=messages[:k]`` renders only the tail, but computes
+        ``last_user_index`` -- the fact that makes this format non-monotonic -- over the whole
+        list, so ``tails[k] == block_k + tails[k + 1]`` holds by construction and subtracting
+        one from the next recovers each block exactly.
+        """
+        tails = [""] * (len(conversation) + 1)
+        for k in range(len(conversation) - 1, -1, -1):
+            tails[k] = self._call_encoder(
+                list(conversation[k:]),
+                context=list(conversation[:k]),
+                **config,
+            )
+            if not tails[k].endswith(tails[k + 1]):
+                # Would mean the encoder's `context` no longer decomposes the render this way,
+                # which is the assumption every span position rests on.
+                raise ChatFormatterUnavailable(
+                    f"{DEEPSEEK_V4_ENCODER_FILE} did not render message {k} as a prefix of the "
+                    "messages after it, so message boundaries cannot be located. The encoder's "
+                    "`context` argument may have changed meaning."
+                )
+        return tails
+
+    def _call_encoder(self, messages: list[Any], **kwargs: Any) -> str:
+        """Call ``encode_messages``, passing only arguments this copy of it declares.
+
+        The checkpoint's encoder and vLLM's fork of it do not have identical signatures, and a
+        future revision may add or drop a knob. Filtering here means an unknown argument
+        degrades to the encoder's own default instead of raising ``TypeError`` at render time.
+        """
+        kwargs.setdefault("add_default_bos_token", False)
+        if self._encode_params is not None:
+            kwargs = {k: v for k, v in kwargs.items() if k in self._encode_params}
+        return str(self._encode(messages, **kwargs))
+
+    # --- translation --------------------------------------------------------
+    def _encode_config(self, template_kwargs: Mapping[str, Any]) -> tuple[dict[str, Any], Sequence[Any]]:
+        """Map the engine's template kwargs onto ``encode_messages`` arguments."""
+        unknown = sorted(set(template_kwargs) - self.template_kwargs)
+        if unknown:
+            raise ValueError(
+                f"{self.name} does not read {', '.join(unknown)}. It accepts: "
+                f"{', '.join(sorted(self.template_kwargs))}."
+            )
+        thinking = bool(template_kwargs.get("enable_thinking") or template_kwargs.get("thinking"))
+        effort = template_kwargs.get("reasoning_effort")
+        effort = effort if isinstance(effort, str) else None
+        if effort == "none":
+            thinking, effort = False, None
+        elif effort in ("max", "xhigh"):
+            effort = "max"
+        elif effort == "low":
+            # The encoder only branches on "max"; it asserts on anything outside
+            # {"max", "high", None}, and treats "high" and None identically. So "low" is None
+            # rather than "high" -- same render either way, but it does not claim otherwise.
+            effort = None
+        elif effort is not None:
+            effort = "high"
+        return (
+            {
+                "thinking_mode": "thinking" if thinking else "chat",
+                "drop_thinking": bool(template_kwargs.get("drop_thinking", True)),
+                "reasoning_effort": effort,
+            },
+            template_kwargs.get("tools") or (),
+        )
+
+    def _normalize(self, message: Message) -> dict[str, Any]:
+        """A message dict spelling reasoning the way the loaded encoder reads it."""
+        turn = dict(message)
+        alias = "reasoning" if self._reasoning_key == "reasoning_content" else "reasoning_content"
+        if alias in turn:
+            value = turn.pop(alias)
+            turn.setdefault(self._reasoning_key, value)
+        return turn
+
+    def _split_scaffold(self, blocks: list[str], turns: Sequence[Message]) -> str:
+        """Remove and return the trailing generation scaffold from ``blocks``, or ``""``.
+
+        The encoder emits it only after a user or developer turn that carries no ``task``, so
+        that is the only case checked. Narrowing it matters twice over: assistant content that
+        happened to end in these two tokens would otherwise be mistaken for scaffolding, and a
+        ``task`` turn puts its own task token *after* the scaffold, so the suffix match would
+        miss it and the strip would cut in the wrong place.
+        """
+        if not blocks or not turns:
+            return ""
+        last = turns[-1]
+        if last.get("role") not in ("user", "developer") or last.get("task") is not None:
+            return ""
+        for delimiter in (self._think_open, self._think_close):
+            scaffold = self._assistant_token + delimiter
+            if blocks[-1].endswith(scaffold):
+                blocks[-1] = blocks[-1][: -len(scaffold)]
+                return scaffold
+        return ""
+
+
+def _reasoning_field(module: ModuleType) -> str:
+    """The message key this copy of the encoder reads a thinking block from.
+
+    Read out of ``thinking_template`` -- ``"{reasoning_content}"`` upstream,
+    ``"{reasoning}"`` in vLLM's fork -- rather than assumed, because guessing wrong drops
+    every thinking block without raising.
+    """
+    template = str(getattr(module, "thinking_template", "") or "")
+    match = re.fullmatch(r"\s*\{(\w+)\}\s*", template)
+    if match is None:
+        logger.warning("Could not read the reasoning field name from %r; assuming reasoning_content", template)
+        return "reasoning_content"
+    return match.group(1)
+
+
+def _accepted_parameters(func: Callable[..., Any]) -> frozenset[str] | None:
+    """The keyword arguments ``func`` declares. ``None`` means "anything" (it takes ``**kwargs``)."""
+    try:
+        parameters = inspect.signature(func).parameters
+    except (TypeError, ValueError):  # pragma: no cover - C-implemented callable
+        return None
+    if any(p.kind is inspect.Parameter.VAR_KEYWORD for p in parameters.values()):
+        return None
+    return frozenset(parameters)
+
+
+# --------------------------------------------------------------------------- #
+# Loading the checkpoint's own encoder
+# --------------------------------------------------------------------------- #
+
+
+def _download_repo_file(hf_model_id: str, filename: str) -> str | None:
+    """Local path to ``filename`` from the checkpoint repo, cache first, then the hub."""
+    try:
+        from huggingface_hub import hf_hub_download
+    except ImportError:  # pragma: no cover - transformers depends on it, so this is belt-and-braces
+        logger.warning("huggingface_hub is not installed, so %s cannot be fetched", filename)
+        return None
+    last: Exception | None = None
+    for local_files_only in (True, False):
+        try:
+            return hf_hub_download(hf_model_id, filename, local_files_only=local_files_only)
+        except Exception as exc:  # noqa: BLE001 - offline, gated, or absent: all mean "no file"
+            last = exc
+    logger.info("Could not fetch %s from %s (%s)", filename, hf_model_id, last)
+    return None
+
+
+def _import_file(path: str, module_name: str) -> ModuleType | None:
+    """Import a standalone ``.py`` file under ``module_name``."""
+    existing = sys.modules.get(module_name)
+    if existing is not None:
+        return existing
+    spec = importlib.util.spec_from_file_location(module_name, path)
+    if spec is None or spec.loader is None:
+        return None
+    module = importlib.util.module_from_spec(spec)
+    # Registered before exec so the module can import itself by name if it ever needs to; also
+    # what makes the check above a cache rather than a repeated exec.
+    sys.modules[module_name] = module
+    try:
+        spec.loader.exec_module(module)
+    except Exception:
+        sys.modules.pop(module_name, None)
+        raise
+    return module
+
+
+def _module_alias(hf_model_id: str, suffix: str) -> str:
+    """A ``sys.modules`` key unique to this checkpoint, so two revisions cannot collide."""
+    return f"interp_engine._chat_encoders.{re.sub(r'[^0-9A-Za-z_]+', '_', hf_model_id)}_{suffix}"
+
+
+def load_deepseek_v4_formatter(hf_model_id: str, *, trust_remote_code: bool = True) -> DeepseekV4Formatter:
+    """Load DeepSeek-V4's reference encoder and wrap it as a :class:`ChatFormatter`.
+
+    Prefers the copy inside the checkpoint, which is the format's source of truth and travels
+    with the weights. Falls back to vLLM's vendored fork only when that file cannot be
+    fetched -- it is a fork, and the module docstring lists how it has diverged.
+    """
+    if not trust_remote_code:
+        raise ChatFormatterUnavailable(
+            f"{hf_model_id} defines its chat format in {DEEPSEEK_V4_ENCODER_FILE} rather than in a "
+            "chat template, and importing it is remote code execution. Load with "
+            "trust_remote_code=True to render chat messages for this model."
+        )
+
+    path = _download_repo_file(hf_model_id, DEEPSEEK_V4_ENCODER_FILE)
+    if path is not None:
+        module = _import_file(path, _module_alias(hf_model_id, "dsv4"))
+        if module is not None and hasattr(module, "encode_messages"):
+            logger.info("Loaded the DeepSeek-V4 chat encoder from %s", path)
+            return DeepseekV4Formatter(module)
+
+    try:
+        module = importlib.import_module(DEEPSEEK_V4_FALLBACK_MODULE)
+    except ImportError:
+        module = None
+    if module is not None:
+        logger.warning(
+            "Using vLLM's vendored copy of the DeepSeek-V4 encoder: %s could not be fetched from %s. "
+            "It is a fork of the checkpoint's own encoder, not a mirror.",
+            DEEPSEEK_V4_ENCODER_FILE,
+            hf_model_id,
+        )
+        return DeepseekV4Formatter(module)
+
+    raise ChatFormatterUnavailable(
+        f"{hf_model_id} ships no chat template; its format is defined by "
+        f"{DEEPSEEK_V4_ENCODER_FILE}, which could not be fetched from the repo, and "
+        f"{DEEPSEEK_V4_FALLBACK_MODULE} is not importable either."
+    )
+
+
+# Architecture *prefixes* (as they appear in ``config.architectures``) whose chat format is
+# code rather than a template, mapped to the loader that builds it. Prefixes and families
+# rather than checkpoints, matching `facts.MANDATORY_KV_CACHE_DTYPES`: this follows from the
+# format the family was post-trained on, so every DeepSeek-V4 checkpoint shares it.
+CODE_CHAT_FORMATS: dict[str, Callable[..., ChatFormatter]] = {
+    "DeepseekV4": load_deepseek_v4_formatter,
+}
+
+
+_CACHE: dict[tuple[str, str, bool], ChatFormatter | None] = {}
+
+
+def resolve_chat_formatter(
+    architectures: Sequence[str] | None,
+    hf_model_id: str,
+    *,
+    trust_remote_code: bool = True,
+) -> ChatFormatter | None:
+    """The code formatter these architectures need, or ``None`` when a template is enough.
+
+    ``None`` is the answer for the great majority of models, and also the answer when a
+    formatter is needed but unreachable -- that case logs a warning naming the file. Loading a
+    model must not fail because one endpoint is unavailable, and the caller already has a
+    refusal for "this model cannot take chat input" (``NoChatTemplateError``) that says to send
+    raw text instead.
+
+    Takes the config's whole ``architectures`` list for the reason
+    :func:`facts.mandatory_kv_cache_dtype` does: it is the shape callers already hold, and a
+    composite config can name more than one.
+    """
+    for name in architectures or ():
+        for prefix, loader in CODE_CHAT_FORMATS.items():
+            if not str(name).startswith(prefix):
+                continue
+            key = (str(name), hf_model_id, bool(trust_remote_code))
+            if key not in _CACHE:
+                try:
+                    _CACHE[key] = loader(hf_model_id, trust_remote_code=trust_remote_code)
+                except ChatFormatterUnavailable as exc:
+                    logger.warning("No chat formatter for %s: %s", hf_model_id, exc)
+                    _CACHE[key] = None
+            return _CACHE[key]
+    return None

{interp_engine-1.2.2 → interp_engine-1.2.4}/interp_engine/model.py

@@ -38,6 +38,7 @@ from interp_engine import facts, moe_routing
 from interp_engine.address import Address, to_address
 from interp_engine.arch import ArchSpec, resolve_arch
 from interp_engine.autograd_support import GradSupport, eager_grad_support
+from interp_engine.chat_formatters import resolve_chat_formatter
 from interp_engine.facts import factored_projection, text_config
 from interp_engine.points import PointSpec, Scope, known_names, point_spec, points_for
 from interp_engine.protocol import Completion
@@ -310,6 +311,14 @@ class EagerModel:
             self.tokenizer,
             default_prepend_bos=default_prepend_bos,
             device=str(self.device),
+            # None for every family whose tokenizer carries its own chat template, which is
+            # nearly all of them. Never raises: a model whose code-defined format cannot be
+            # loaded still serves everything except chat input.
+            formatter=resolve_chat_formatter(
+                getattr(self.config, "architectures", None),
+                hf_model_id,
+                trust_remote_code=trust_remote,
+            ),
         )
 
         # `requires_grad=True` IS a gradient request, so this is its point of use and the same gate