renderers 0.1.12.dev12__tar.gz → 0.1.12.dev13__tar.gz

{renderers-0.1.12.dev12 → renderers-0.1.12.dev13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: renderers
-Version: 0.1.12.dev12
+Version: 0.1.12.dev13
 Summary: Chat template renderers — deterministic message-to-token conversion for LLM training
 License-Expression: Apache-2.0
 License-File: LICENSE

{renderers-0.1.12.dev12 → renderers-0.1.12.dev13}/renderers/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.1.12.dev12'
-__version_tuple__ = version_tuple = (0, 1, 12, 'dev12')
+__version__ = version = '0.1.12.dev13'
+__version_tuple__ = version_tuple = (0, 1, 12, 'dev13')
 
 __commit_id__ = commit_id = None

{renderers-0.1.12.dev12 → renderers-0.1.12.dev13}/renderers/qwen35.py

@@ -23,6 +23,7 @@ path and remain byte-identical to ``apply_chat_template``.
 from __future__ import annotations
 
 import json
+from collections.abc import Iterable
 from typing import Any
 
 from renderers.reasoning import scan_reasoning, prompt_ends_in_reasoning
@@ -252,7 +253,11 @@ class Qwen35Renderer:
     # ------------------------------------------------------------------
 
     @staticmethod
-    def _render_content(content: Any) -> str:
+    def _iter_content_parts(content: list[Any]) -> Iterable[Any]:
+        return content
+
+    @classmethod
+    def _render_content(cls, content: Any) -> str:
         """Render message content to a text string (before tokenization).
 
         Handles string, list of text parts, and None. Image / video parts
@@ -268,7 +273,7 @@ class Qwen35Renderer:
             return content
         if isinstance(content, list):
             parts: list[str] = []
-            for item in content:
+            for item in cls._iter_content_parts(content):
                 if isinstance(item, str):
                     parts.append(item)
                 elif isinstance(item, dict):
@@ -489,7 +494,7 @@ class Qwen35Renderer:
                     emit_text_segments(buf_segments, msg_idx, is_sampled=False)
                     buf_segments.clear()
 
-            for item in content_list:
+            for item in self._iter_content_parts(content_list):
                 if isinstance(item, str):
                     if item:
                         buf_segments.append((item, True))
@@ -864,7 +869,7 @@ class Qwen35Renderer:
                     emit_text_segments(buf_segments, msg_idx)
                     buf_segments.clear()
 
-            for item in content_list:
+            for item in self._iter_content_parts(content_list):
                 if isinstance(item, str):
                     if item:
                         buf_segments.append((item, True))
@@ -1192,7 +1197,7 @@ class Qwen35Renderer:
                     emit_text_segments(buf_segments, msg_idx, is_sampled=False)
                     buf_segments.clear()
 
-            for item in raw_content:
+            for item in self._iter_content_parts(raw_content):
                 if isinstance(item, str):
                     if item:
                         buf_segments.append((item, True))

{renderers-0.1.12.dev12 → renderers-0.1.12.dev13}/renderers/qwen38.py

@@ -14,10 +14,13 @@ argument serialization, with three template changes:
 
 from __future__ import annotations
 
+from collections.abc import Iterable
+from typing import Any
+
 from renderers.base import Message
 from renderers.configs import Qwen38RendererConfig
-from renderers.qwen35 import Qwen35Renderer
 from renderers.qwen36 import Qwen36Renderer
+from renderers.qwen3_vl import _is_image_part, _is_video_part
 
 
 _REASONING_INSTRUCTIONS = {
@@ -48,12 +51,35 @@ class Qwen38Renderer(Qwen36Renderer):
     def _omit_empty_system_message(self) -> bool:
         return True
 
+    @staticmethod
+    def _iter_content_parts(content: list[Any]) -> Iterable[Any]:
+        """Keep media and string-valued text in every rendering path."""
+        for item in content:
+            if isinstance(item, str):
+                yield item
+            elif isinstance(item, dict) and (
+                _is_image_part(item)
+                or _is_video_part(item)
+                or isinstance(item.get("text"), str)
+            ):
+                yield item
+
+    @staticmethod
+    def _is_user_query_message(msg: Message) -> bool:
+        if msg.get("role") != "user":
+            return False
+        content = Qwen38Renderer._render_content(msg.get("content")).strip()
+        return not (
+            content.startswith("<tool_response>")
+            and content.endswith("</tool_response>")
+        )
+
     @staticmethod
     def _last_query_index(messages: list[Message]) -> int:
-        last_query_index = Qwen35Renderer._last_query_index(messages)
-        if last_query_index == len(messages):
-            raise ValueError("No user query found in messages.")
-        return last_query_index
+        for i in range(len(messages) - 1, -1, -1):
+            if Qwen38Renderer._is_user_query_message(messages[i]):
+                return i
+        raise ValueError("No user query found in messages.")
 
     @staticmethod
     def _extract_assistant_parts(msg: Message, content: str) -> tuple[str, str]:

{renderers-0.1.12.dev12 → renderers-0.1.12.dev13}/tests/test_qwen38.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 from functools import lru_cache
+from copy import deepcopy
 
 import pytest
 from pydantic import TypeAdapter
@@ -176,3 +177,76 @@ def test_qwen38_requires_a_real_user_query(qwen38_model):
 
     with pytest.raises(ValueError, match="No user query found"):
         renderer.render_ids([{"role": "system", "content": "System only."}])
+
+
+@pytest.mark.parametrize(
+    "role,with_image",
+    [
+        ("system", False),
+        ("user", False),
+        ("assistant", False),
+        ("tool", False),
+        ("user", True),
+        ("tool", True),
+    ],
+)
+def test_qwen38_structured_content_render_and_bridge(role, with_image):
+    _, renderer = _qwen38(QWEN38_MODELS[0])
+    parts = ["Hello", {"type": "text", "text": " world"}]
+    if with_image:
+        parts.append({"type": "image", "image": "unused", "text": 123})
+        parts.append({"type": "text", "text": "After image"})
+    noisy_parts = [
+        {"type": "unsupported"},
+        *parts,
+        {"type": "text", "text": None},
+        {"type": "text", "text": 123},
+        None,
+    ]
+    prior = [
+        {"role": "user", "content": "First question"},
+        {"role": "assistant", "content": "First answer"},
+    ]
+    clean = {"role": role, "content": parts}
+    noisy = {"role": role, "content": noisy_parts}
+    original = deepcopy(noisy)
+    messages = [noisy, *prior] if role == "system" else [*prior, noisy]
+    expected_messages = [clean, *prior] if role == "system" else [*prior, clean]
+    assert renderer.render(messages, process_multimodal=False) == renderer.render(
+        expected_messages, process_multimodal=False
+    )
+    if role in {"user", "tool"}:
+        previous = renderer.render_ids(prior)
+        actual = renderer.bridge_to_next_turn(
+            previous, [], [noisy], process_multimodal=False
+        )
+        assert actual is not None
+        assert actual == renderer.bridge_to_next_turn(
+            previous, [], [clean], process_multimodal=False
+        )
+    assert noisy == original
+
+
+def test_qwen38_structured_tool_response_keeps_current_reasoning():
+    tokenizer, _ = _qwen38(QWEN38_MODELS[0])
+    renderer = Qwen38Renderer(tokenizer, Qwen38RendererConfig(preserve_thinking=False))
+    prior = [
+        {"role": "user", "content": "First question"},
+        {"role": "assistant", "reasoning_content": "Keep me", "content": ""},
+    ]
+    content = [
+        {"type": "text", "text": "  <tool_response>"},
+        {"type": "unsupported"},
+        {"type": "text", "text": None},
+        "result",
+        {"type": "text", "text": "</tool_response>  "},
+    ]
+    messages = [*prior, {"role": "user", "content": content}]
+    expected = [
+        *prior,
+        {"role": "user", "content": "<tool_response>result</tool_response>"},
+    ]
+    assert renderer.render(messages) == renderer.render(expected)
+    assert "Keep me" in tokenizer.decode(renderer.render_ids(messages))
+    with pytest.raises(ValueError, match="No user query found"):
+        renderer.render([messages[-1]])