renderers 0.1.8.dev41__tar.gz → 0.1.8.dev42__tar.gz

{renderers-0.1.8.dev41 → renderers-0.1.8.dev42}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: renderers
-Version: 0.1.8.dev41
+Version: 0.1.8.dev42
 Summary: Chat template renderers — deterministic message-to-token conversion for LLM training
 License-Expression: Apache-2.0
 License-File: LICENSE

{renderers-0.1.8.dev41 → renderers-0.1.8.dev42}/renderers/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.1.8.dev41'
-__version_tuple__ = version_tuple = (0, 1, 8, 'dev41')
+__version__ = version = '0.1.8.dev42'
+__version_tuple__ = version_tuple = (0, 1, 8, 'dev42')
 
 __commit_id__ = commit_id = None

{renderers-0.1.8.dev41 → renderers-0.1.8.dev42}/renderers/kimi_k25.py

@@ -42,7 +42,7 @@ from renderers.base import (
     trim_to_turn_close,
 )
 from renderers.configs import KimiK25RendererConfig
-from renderers.parsing import parse_kimi_k2_section
+from renderers.parsing import _reasoning_end_token_index, parse_kimi_k2_section
 from renderers.qwen3_vl import (
     _image_hash,
     _is_image_part,
@@ -452,6 +452,13 @@ def _parse_kimi_k2_response(
             ids = ids[:i]
             break
 
+    # Reasoning first: a tool-call section the model drafts *inside* its
+    # <think> trace must not be parsed as a real call (regression #78 — cf.
+    # parse_qwen3). K2.5 renders </think> as text, so locate the boundary by
+    # decoding; the section scan then starts past it. content_ids still begins
+    # at 0, so the </think> text-split below recovers reasoning unchanged.
+    reasoning_end = _reasoning_end_token_index(tokenizer, ids)
+
     # Token-ID path — produces spans. Only run if every relevant special
     # token resolved at init (i.e. is in the tokenizer's vocab).
     tool_calls: list[ParsedToolCall] = []
@@ -471,6 +478,7 @@ def _parse_kimi_k2_response(
             tool_call_begin_id=tool_call_begin_id,
             tool_call_argument_begin_id=tool_call_argument_begin_id,
             tool_call_end_id=tool_call_end_id,
+            scan_start=reasoning_end,
         )
         text = (
             tokenizer.decode(content_ids, skip_special_tokens=False)
@@ -481,9 +489,13 @@ def _parse_kimi_k2_response(
         text = tokenizer.decode(ids, skip_special_tokens=False) if ids else ""
 
     # Fallback path: model emitted literal-text section delimiters (singular
-    # variant) rather than special tokens. Spans unavailable here.
+    # variant) rather than special tokens. Spans unavailable here. Start the
+    # search past the first </think> so a literal section drafted inside the
+    # reasoning trace isn't matched as a real call (regression #78).
     if not tool_calls:
-        tc_match = _TOOL_CALLS_SECTION_RE.search(text)
+        think_close = text.find("</think>")
+        search_from = think_close + len("</think>") if think_close != -1 else 0
+        tc_match = _TOOL_CALLS_SECTION_RE.search(text, search_from)
         if tc_match:
             text = text[: tc_match.start()]
             tool_section = (

{renderers-0.1.8.dev41 → renderers-0.1.8.dev42}/renderers/parsing.py

@@ -133,6 +133,39 @@ def _decode(tokenizer, ids: list[int]) -> str:
     return tokenizer.decode(ids, skip_special_tokens=False)
 
 
+def _reasoning_end_token_index(
+    tokenizer, ids: list[int], marker: str = "</think>"
+) -> int:
+    """Token index immediately past the first ``</think>`` in ``ids``.
+
+    Returns 0 when ``ids`` has no closed reasoning region — callers treat
+    that as "scan from the start" (preserves pre-existing behavior for
+    non-thinking / truncated-reasoning completions).
+
+    Used by parsers whose ``</think>`` is *not* a single special token
+    (DeepSeek-V3, Kimi-K2.5) — where it tokenizes to several pieces and is
+    context-sensitive (the closing ``>`` merges differently depending on the
+    next char), so a token-id or fixed-subsequence search isn't reliable. We
+    instead locate the boundary in decoded text via binary search over prefix
+    decodes, which holds as long as ``decode(ids[:k])`` is prefix-stable in
+    ``k`` (true for the byte-level BPE tokenizers here; ``</think>`` is clean
+    ASCII that won't straddle a byte boundary). Single-token ``</think>``
+    parsers (Qwen3) anchor on the token id directly and don't need this.
+    """
+    if not ids or marker not in _decode(tokenizer, ids):
+        return 0
+    # Smallest prefix length (in tokens) whose decode already contains the
+    # full marker — i.e. the index just past where </think> completes.
+    lo, hi = 1, len(ids)
+    while lo < hi:
+        mid = (lo + hi) // 2
+        if marker in _decode(tokenizer, ids[:mid]):
+            hi = mid
+        else:
+            lo = mid + 1
+    return lo
+
+
 # ── Qwen3: <tool_call> JSON </tool_call> ────────────────────────────
 
 
@@ -143,11 +176,26 @@ def parse_qwen3(
     stop_ids: set[int],
     tool_call_id: int,
     tool_call_end_id: int,
+    reasoning_end_id: int | None = None,
 ) -> ParsedResponse:
     """Parse Qwen3 completion tokens. Hermes-style JSON tool calls."""
     ids = _strip_stop_tokens(token_ids, stop_ids)
 
-    tc_start = _find(ids, tool_call_id)
+    # Reasoning is resolved before tool calls. Thinking models (e.g.
+    # Qwen3-*-Thinking) routinely draft ``<tool_call>`` blocks *inside* their
+    # ``<think>...</think>`` trace while planning; those are reasoning, not
+    # real invocations. Anchoring the tool-call scan after the ``</think>``
+    # boundary keeps in-think drafts out of ``tool_calls`` (otherwise they
+    # surface as phantom/duplicate calls) and out of the reasoning/content
+    # split. Mirrors vLLM's DelegatingParser, which runs the reasoning parser
+    # first and tool-parses only the post-``</think>`` content.
+    # ``reasoning_end_id`` is the ``</think>`` token id; when it's absent
+    # (``None``) or the model never closed its reasoning, the scan falls back
+    # to the whole stream (prior behavior).
+    reasoning_end = _find(ids, reasoning_end_id) if reasoning_end_id is not None else -1
+    scan_start = reasoning_end + 1 if reasoning_end != -1 else 0
+
+    tc_start = _find(ids, tool_call_id, scan_start)
     tool_calls: list[ParsedToolCall] = []
     if tc_start != -1:
         content_ids = ids[:tc_start]
@@ -685,7 +733,15 @@ def parse_deepseek_v3(
     """
     ids = _strip_stop_tokens(token_ids, stop_ids)
 
-    tc_section_start = _find(ids, tool_calls_begin_id)
+    # Reasoning first: skip past </think> before looking for the tool-call
+    # section, so a section the model drafts *inside* its <think> trace isn't
+    # parsed as a real call (regression #78 — cf. parse_qwen3). content_ids
+    # still starts at 0, so the </think> text-split below recovers reasoning.
+    # DeepSeek-V3 renders </think> as multi-token text, hence the decode-based
+    # boundary finder rather than a token-id anchor.
+    reasoning_end = _reasoning_end_token_index(tokenizer, ids)
+
+    tc_section_start = _find(ids, tool_calls_begin_id, reasoning_end)
     tool_calls: list[ParsedToolCall] = []
     if tc_section_start != -1:
         content_ids = ids[:tc_section_start]
@@ -962,6 +1018,7 @@ def parse_kimi_k2_section(
     tool_call_begin_id: int,
     tool_call_argument_begin_id: int,
     tool_call_end_id: int,
+    scan_start: int = 0,
 ) -> tuple[list[int], list[ParsedToolCall]]:
     """Split ``ids`` into ``(content_before_section, tool_calls)`` by finding
     the Kimi-style tool-call section delimiters.
@@ -973,8 +1030,15 @@ def parse_kimi_k2_section(
     of the section and a list of ``ParsedToolCall`` covering every attempted
     block inside it; an unclosed section is still walked to whatever the model
     emitted before EOS. Returns ``(ids, [])`` when no section is present.
+
+    ``scan_start`` restricts the section search to ``ids[scan_start:]`` while
+    keeping ``content_ids = ids[:section_start]`` and all token spans relative
+    to the full ``ids``. Callers pass the post-``</think>`` index so a section
+    the model drafts inside its reasoning trace isn't parsed as a real call;
+    because ``content_ids`` still starts at 0, downstream text-based reasoning
+    extraction is unaffected (regression #78).
     """
-    section_start = _find_any(ids, tool_calls_section_begin_ids)
+    section_start = _find_any(ids, tool_calls_section_begin_ids, scan_start)
     if section_start == -1:
         return list(ids), []
     content_ids = ids[:section_start]

{renderers-0.1.8.dev41 → renderers-0.1.8.dev42}/renderers/qwen3.py

@@ -62,6 +62,7 @@ class Qwen3Renderer:
         self._tool_call_end = self._token_id("</tool_call>")
         self._tool_response = self._token_id("<tool_response>")
         self._tool_response_end = self._token_id("</tool_response>")
+        self._think_end = self._token_id("</think>")
 
     def _token_id(self, token: str) -> int:
         tid = self._tokenizer.convert_tokens_to_ids(token)
@@ -276,6 +277,7 @@ class Qwen3Renderer:
             stop_ids={self._im_end, self._endoftext},
             tool_call_id=self._tool_call,
             tool_call_end_id=self._tool_call_end,
+            reasoning_end_id=self._think_end,
         )
 
     def get_stop_token_ids(self) -> list[int]:

{renderers-0.1.8.dev41 → renderers-0.1.8.dev42}/renderers/qwen3_vl.py

@@ -325,6 +325,7 @@ class Qwen3VLRenderer:
         self._tool_call_end = self._token_id("</tool_call>")
         self._tool_response = self._token_id("<tool_response>")
         self._tool_response_end = self._token_id("</tool_response>")
+        self._think_end = self._token_id("</think>")
         self._vision_start = self._token_id("<|vision_start|>")
         self._vision_end = self._token_id("<|vision_end|>")
         self._image_pad = self._token_id("<|image_pad|>")
@@ -634,6 +635,7 @@ class Qwen3VLRenderer:
             stop_ids={self._im_end, self._endoftext},
             tool_call_id=self._tool_call,
             tool_call_end_id=self._tool_call_end,
+            reasoning_end_id=self._think_end,
         )
 
     def get_stop_token_ids(self) -> list[int]:

renderers-0.1.8.dev42/tests/test_parse_response.py

@@ -0,0 +1,276 @@
+"""Barrage test: renderer.parse_response() must correctly extract
+content, reasoning_content, and tool_calls from completion tokens.
+
+Runs against every (model, renderer) pair.
+"""
+
+from functools import lru_cache
+
+from renderers import create_renderer
+from renderers.base import ToolCallParseStatus, load_tokenizer
+
+
+@lru_cache
+def _qwen3_vl():
+    tokenizer = load_tokenizer("Qwen/Qwen3-VL-4B-Instruct")
+    renderer = create_renderer(tokenizer)
+    return tokenizer, renderer
+
+
+def test_parse_simple_content(model_name, tokenizer, renderer):
+    """Plain content, no thinking."""
+    text = "Hello there!"
+    ids = tokenizer.encode(text, add_special_tokens=False)
+    parsed = renderer.parse_response(ids)
+    assert "Hello" in parsed.content
+
+
+def test_parse_thinking_and_content(model_name, tokenizer, renderer):
+    """Content with <think>reasoning</think> block."""
+    text = "Let me think about this.\n</think>\n\nThe answer is 42."
+    ids = tokenizer.encode(text, add_special_tokens=False)
+    parsed = renderer.parse_response(ids)
+    # Should extract reasoning or at least not crash
+    assert (
+        "42" in parsed.content
+        or "think" in (parsed.reasoning_content or "").lower()
+        or parsed.content
+    )
+
+
+def test_parse_empty_completion(model_name, tokenizer, renderer):
+    """Empty completion should not crash."""
+    parsed = renderer.parse_response([])
+    assert parsed.content is not None
+
+
+def test_parse_response_returns_parsed_response(model_name, tokenizer, renderer):
+    """Return type must have content, reasoning_content, tool_calls."""
+    ids = tokenizer.encode("Hello!", add_special_tokens=False)
+    parsed = renderer.parse_response(ids)
+    assert hasattr(parsed, "content")
+    assert hasattr(parsed, "reasoning_content")
+    assert hasattr(parsed, "tool_calls")
+
+
+def test_qwen3_vl_parse_json_tool_call():
+    tokenizer, renderer = _qwen3_vl()
+    text = (
+        'Need a tool.\n<tool_call>\n{"name": "get_weather", '
+        '"arguments": {"city": "Paris"}}\n</tool_call>'
+    )
+    parsed = renderer.parse_response(tokenizer.encode(text, add_special_tokens=False))
+
+    assert parsed.content == "Need a tool."
+    assert len(parsed.tool_calls) == 1
+    tc = parsed.tool_calls[0]
+    assert tc.status == ToolCallParseStatus.OK
+    assert tc.name == "get_weather"
+    assert tc.arguments == {"city": "Paris"}
+
+
+def test_qwen3_vl_malformed_tool_call_surfaces_as_invalid_json():
+    """A malformed ``<tool_call>`` block lands as a non-OK ``ParsedToolCall``
+    rather than getting silently merged back into ``content``.
+
+    Before the per-call status redesign, the parser mirrored vLLM's
+    hermes parser and stuffed the raw block into ``content`` to avoid
+    downstream ``EmptyModelResponseError``. That hid the malformed signal
+    from verifiers — they couldn't tell "model wrote prose" from "model
+    tried a tool call and produced broken JSON." Now the failed attempt
+    is preserved with ``status=INVALID_JSON`` and ``raw`` text, which
+    also satisfies the EmptyModelResponseError prevention contract: the
+    response is non-empty (it has a tool-call attempt) without lying
+    about what kind of output the model produced.
+    """
+    tokenizer, renderer = _qwen3_vl()
+    # Note the trailing comma — malformed JSON
+    text = (
+        '<tool_call>\n{"name": "get_weather", '
+        '"arguments": {"city": "Paris",}}\n</tool_call>'
+    )
+    parsed = renderer.parse_response(tokenizer.encode(text, add_special_tokens=False))
+
+    assert len(parsed.tool_calls) == 1
+    tc = parsed.tool_calls[0]
+    assert tc.status == ToolCallParseStatus.INVALID_JSON
+    assert "get_weather" in tc.raw
+    assert tc.token_span is not None
+
+
+@lru_cache
+def _qwen3():
+    tokenizer = load_tokenizer("Qwen/Qwen3-0.6B")
+    renderer = create_renderer(tokenizer)
+    return tokenizer, renderer
+
+
+def test_qwen3_in_think_tool_call_is_not_a_real_call():
+    """A ``<tool_call>`` the model drafts *inside* its ``<think>`` trace must
+    stay reasoning — only the call emitted after ``</think>`` counts.
+
+    Regression for #78: Thinking models (e.g. Qwen3-*-Thinking-2507) draft
+    tool-call syntax while planning. Because ``<tool_call>`` is a real vocab
+    token, the parser used to scan the whole stream and emit the in-think
+    draft *and* the genuine post-``</think>`` call as two tool calls — a
+    phantom duplicate that made callers execute the same code twice. The scan
+    is now anchored after ``</think>``, mirroring vLLM's reasoning-then-tools
+    ordering.
+    """
+    tokenizer, renderer = _qwen3()
+    text = (
+        "<think>\nLet me draft the call:\n"
+        '<tool_call>\n{"name": "execute_code", "arguments": {"code": "print(1)"}}\n'
+        "</tool_call>\nYes, that looks right.\n</think>\n"
+        '<tool_call>\n{"name": "execute_code", "arguments": {"code": "print(1)"}}\n'
+        "</tool_call>"
+    )
+    parsed = renderer.parse_response(tokenizer.encode(text, add_special_tokens=False))
+
+    assert len(parsed.tool_calls) == 1
+    tc = parsed.tool_calls[0]
+    assert tc.status == ToolCallParseStatus.OK
+    assert tc.name == "execute_code"
+    assert tc.arguments == {"code": "print(1)"}
+    # The drafted call stays in the reasoning trace, not content.
+    assert parsed.reasoning_content is not None
+    assert "<tool_call>" in parsed.reasoning_content
+    assert parsed.content == ""
+
+
+def test_qwen3_distinct_parallel_calls_after_think_are_preserved():
+    """The fix must not over-correct: two *genuine* parallel calls emitted
+    after ``</think>`` are still both returned (no dedup), preserving the
+    faithful-transcription contract for real invocations.
+    """
+    tokenizer, renderer = _qwen3()
+    text = (
+        "<think>\nplan\n</think>\n"
+        '<tool_call>\n{"name": "execute_code", "arguments": {"code": "print(1)"}}\n'
+        "</tool_call>\n"
+        '<tool_call>\n{"name": "execute_code", "arguments": {"code": "print(2)"}}\n'
+        "</tool_call>"
+    )
+    parsed = renderer.parse_response(tokenizer.encode(text, add_special_tokens=False))
+
+    assert len(parsed.tool_calls) == 2
+    assert [tc.arguments for tc in parsed.tool_calls] == [
+        {"code": "print(1)"},
+        {"code": "print(2)"},
+    ]
+    assert parsed.reasoning_content == "plan"
+
+
+@lru_cache
+def _kimi_k25():
+    tokenizer = load_tokenizer("moonshotai/Kimi-K2.5")
+    renderer = create_renderer(tokenizer)
+    return tokenizer, renderer
+
+
+def test_kimi_k25_tool_call_carries_token_span():
+    """K2.5 was the lone parser without token spans before — its inline
+    text-walking implementation couldn't cheaply map regex hits back to
+    token offsets. We now walk token IDs via ``parse_kimi_k2_section`` for
+    the special-token path; spans must round-trip and point at a sensible
+    range within the original input token_ids.
+    """
+    tokenizer, renderer = _kimi_k25()
+    # K2.5 tool-call wire shape: section + per-call special tokens.
+    text = (
+        "<|tool_calls_section_begin|>"
+        "<|tool_call_begin|>functions.get_weather:0"
+        "<|tool_call_argument_begin|>"
+        '{"city": "Tokyo"}'
+        "<|tool_call_end|>"
+        "<|tool_calls_section_end|>"
+    )
+    token_ids = tokenizer.encode(text, add_special_tokens=False)
+    parsed = renderer.parse_response(token_ids)
+
+    assert len(parsed.tool_calls) == 1
+    tc = parsed.tool_calls[0]
+    assert tc.status == ToolCallParseStatus.OK
+    assert tc.name == "get_weather"
+    assert tc.arguments == {"city": "Tokyo"}
+    assert tc.token_span is not None
+    start, end = tc.token_span
+    assert 0 <= start < end <= len(token_ids), (
+        f"span {tc.token_span} out of range for {len(token_ids)} input tokens"
+    )
+
+
+def test_kimi_k25_in_think_section_is_not_a_real_call():
+    """A tool-call section the model drafts inside its ``<think>`` trace must
+    not be parsed — only the section after ``</think>`` counts.
+
+    Regression for #78. K2.5's failure mode differed from Qwen3's: the
+    in-think section tripped the "truncated reasoning" guard and the parser
+    *dropped every tool call* (returned zero), losing the genuine call. The
+    scan is now anchored past ``</think>``.
+    """
+    tokenizer, renderer = _kimi_k25()
+    section = (
+        "<|tool_calls_section_begin|>"
+        "<|tool_call_begin|>functions.execute_code:0"
+        '<|tool_call_argument_begin|>{"code": "print(1)"}'
+        "<|tool_call_end|><|tool_calls_section_end|>"
+    )
+    text = f"<think>\nLet me draft:\n{section}\nlooks right.\n</think>\nGo.\n{section}"
+    parsed = renderer.parse_response(tokenizer.encode(text, add_special_tokens=False))
+
+    assert len(parsed.tool_calls) == 1
+    tc = parsed.tool_calls[0]
+    assert tc.status == ToolCallParseStatus.OK
+    assert tc.name == "execute_code"
+    assert tc.arguments == {"code": "print(1)"}
+    # The drafted section stays in the reasoning trace.
+    assert parsed.reasoning_content is not None
+    assert "<|tool_calls_section_begin|>" in parsed.reasoning_content
+    assert parsed.content == "Go."
+
+
+@lru_cache
+def _deepseek_v3():
+    tokenizer = load_tokenizer("deepseek-ai/DeepSeek-V3")
+    renderer = create_renderer(tokenizer)
+    return tokenizer, renderer
+
+
+def test_deepseek_v3_in_think_section_is_not_a_real_call():
+    """A tool-call section drafted inside ``<think>`` must not be parsed —
+    only the section after ``</think>`` counts.
+
+    Regression for #78. DeepSeek-V3's failure mode: it returned the *wrong*
+    call (the in-think draft) and lost reasoning, because ``</think>`` is
+    multi-token text there and the scan wasn't anchored past it.
+    """
+    tokenizer, renderer = _deepseek_v3()
+
+    def section(name: str) -> str:
+        return (
+            "<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>"
+            f'{name}\n```json\n{{"code": "print(1)"}}\n```'
+            "<｜tool▁call▁end｜><｜tool▁calls▁end｜>"
+        )
+
+    text = (
+        f"<think>\nLet me draft:\n{section('draft_tool')}\nlooks right.\n</think>\n"
+        f"Go.\n{section('real_tool')}"
+    )
+    parsed = renderer.parse_response(tokenizer.encode(text, add_special_tokens=False))
+
+    assert len(parsed.tool_calls) == 1
+    tc = parsed.tool_calls[0]
+    # Assert the *post-``</think>``* section was chosen, not the in-think draft.
+    # (Use ``startswith`` rather than ``== "real_tool"``: under transformers
+    # 5.x the DeepSeek tokenizer's decode drops the ``\n`` between name and the
+    # ```json fence, so ``_parse_deepseek_tool_calls`` folds the fence into the
+    # name — a pre-existing, #78-unrelated quirk. What matters here is *which*
+    # section won.)
+    assert tc.name is not None and tc.name.startswith("real_tool")
+    assert "draft_tool" not in tc.name
+    # The drafted section stays in the reasoning trace, not content.
+    assert parsed.reasoning_content is not None
+    assert "draft_tool" in parsed.reasoning_content
+    assert parsed.content == "Go."

renderers-0.1.8.dev41/tests/test_parse_response.py

@@ -1,137 +0,0 @@
-"""Barrage test: renderer.parse_response() must correctly extract
-content, reasoning_content, and tool_calls from completion tokens.
-
-Runs against every (model, renderer) pair.
-"""
-
-from functools import lru_cache
-
-from renderers import create_renderer
-from renderers.base import ToolCallParseStatus, load_tokenizer
-
-
-@lru_cache
-def _qwen3_vl():
-    tokenizer = load_tokenizer("Qwen/Qwen3-VL-4B-Instruct")
-    renderer = create_renderer(tokenizer)
-    return tokenizer, renderer
-
-
-def test_parse_simple_content(model_name, tokenizer, renderer):
-    """Plain content, no thinking."""
-    text = "Hello there!"
-    ids = tokenizer.encode(text, add_special_tokens=False)
-    parsed = renderer.parse_response(ids)
-    assert "Hello" in parsed.content
-
-
-def test_parse_thinking_and_content(model_name, tokenizer, renderer):
-    """Content with <think>reasoning</think> block."""
-    text = "Let me think about this.\n</think>\n\nThe answer is 42."
-    ids = tokenizer.encode(text, add_special_tokens=False)
-    parsed = renderer.parse_response(ids)
-    # Should extract reasoning or at least not crash
-    assert (
-        "42" in parsed.content
-        or "think" in (parsed.reasoning_content or "").lower()
-        or parsed.content
-    )
-
-
-def test_parse_empty_completion(model_name, tokenizer, renderer):
-    """Empty completion should not crash."""
-    parsed = renderer.parse_response([])
-    assert parsed.content is not None
-
-
-def test_parse_response_returns_parsed_response(model_name, tokenizer, renderer):
-    """Return type must have content, reasoning_content, tool_calls."""
-    ids = tokenizer.encode("Hello!", add_special_tokens=False)
-    parsed = renderer.parse_response(ids)
-    assert hasattr(parsed, "content")
-    assert hasattr(parsed, "reasoning_content")
-    assert hasattr(parsed, "tool_calls")
-
-
-def test_qwen3_vl_parse_json_tool_call():
-    tokenizer, renderer = _qwen3_vl()
-    text = (
-        'Need a tool.\n<tool_call>\n{"name": "get_weather", '
-        '"arguments": {"city": "Paris"}}\n</tool_call>'
-    )
-    parsed = renderer.parse_response(tokenizer.encode(text, add_special_tokens=False))
-
-    assert parsed.content == "Need a tool."
-    assert len(parsed.tool_calls) == 1
-    tc = parsed.tool_calls[0]
-    assert tc.status == ToolCallParseStatus.OK
-    assert tc.name == "get_weather"
-    assert tc.arguments == {"city": "Paris"}
-
-
-def test_qwen3_vl_malformed_tool_call_surfaces_as_invalid_json():
-    """A malformed ``<tool_call>`` block lands as a non-OK ``ParsedToolCall``
-    rather than getting silently merged back into ``content``.
-
-    Before the per-call status redesign, the parser mirrored vLLM's
-    hermes parser and stuffed the raw block into ``content`` to avoid
-    downstream ``EmptyModelResponseError``. That hid the malformed signal
-    from verifiers — they couldn't tell "model wrote prose" from "model
-    tried a tool call and produced broken JSON." Now the failed attempt
-    is preserved with ``status=INVALID_JSON`` and ``raw`` text, which
-    also satisfies the EmptyModelResponseError prevention contract: the
-    response is non-empty (it has a tool-call attempt) without lying
-    about what kind of output the model produced.
-    """
-    tokenizer, renderer = _qwen3_vl()
-    # Note the trailing comma — malformed JSON
-    text = (
-        '<tool_call>\n{"name": "get_weather", '
-        '"arguments": {"city": "Paris",}}\n</tool_call>'
-    )
-    parsed = renderer.parse_response(tokenizer.encode(text, add_special_tokens=False))
-
-    assert len(parsed.tool_calls) == 1
-    tc = parsed.tool_calls[0]
-    assert tc.status == ToolCallParseStatus.INVALID_JSON
-    assert "get_weather" in tc.raw
-    assert tc.token_span is not None
-
-
-@lru_cache
-def _kimi_k25():
-    tokenizer = load_tokenizer("moonshotai/Kimi-K2.5")
-    renderer = create_renderer(tokenizer)
-    return tokenizer, renderer
-
-
-def test_kimi_k25_tool_call_carries_token_span():
-    """K2.5 was the lone parser without token spans before — its inline
-    text-walking implementation couldn't cheaply map regex hits back to
-    token offsets. We now walk token IDs via ``parse_kimi_k2_section`` for
-    the special-token path; spans must round-trip and point at a sensible
-    range within the original input token_ids.
-    """
-    tokenizer, renderer = _kimi_k25()
-    # K2.5 tool-call wire shape: section + per-call special tokens.
-    text = (
-        "<|tool_calls_section_begin|>"
-        "<|tool_call_begin|>functions.get_weather:0"
-        "<|tool_call_argument_begin|>"
-        '{"city": "Tokyo"}'
-        "<|tool_call_end|>"
-        "<|tool_calls_section_end|>"
-    )
-    token_ids = tokenizer.encode(text, add_special_tokens=False)
-    parsed = renderer.parse_response(token_ids)
-
-    assert len(parsed.tool_calls) == 1
-    tc = parsed.tool_calls[0]
-    assert tc.status == ToolCallParseStatus.OK
-    assert tc.name == "get_weather"
-    assert tc.arguments == {"city": "Tokyo"}
-    assert tc.token_span is not None
-    start, end = tc.token_span
-    assert 0 <= start < end <= len(token_ids), (
-        f"span {tc.token_span} out of range for {len(token_ids)} input tokens"
-    )