renderers 0.1.12.dev8__tar.gz → 0.1.12.dev9__tar.gz

{renderers-0.1.12.dev8 → renderers-0.1.12.dev9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: renderers
-Version: 0.1.12.dev8
+Version: 0.1.12.dev9
 Summary: Chat template renderers — deterministic message-to-token conversion for LLM training
 License-Expression: Apache-2.0
 License-File: LICENSE

{renderers-0.1.12.dev8 → renderers-0.1.12.dev9}/renderers/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.1.12.dev8'
-__version_tuple__ = version_tuple = (0, 1, 12, 'dev8')
+__version__ = version = '0.1.12.dev9'
+__version_tuple__ = version_tuple = (0, 1, 12, 'dev9')
 
 __commit_id__ = commit_id = None

{renderers-0.1.12.dev8 → renderers-0.1.12.dev9}/renderers/qwen3.py

@@ -93,9 +93,16 @@ class Qwen3Renderer:
         return self._tokenizer.encode(text, add_special_tokens=False)
 
     @staticmethod
-    def _query_boundary_text(content) -> str:
+    def _content_text(content) -> str:
+        """Return the text parts of a message in a renderer-safe form.
+
+        Qwen3's native template is text-only, but text-only messages can also
+        use OpenAI-style content lists: ``[{"type": "text", "text": ...}]``.
+        Treat both representations identically instead of silently dropping
+        the message body.
+        """
         if isinstance(content, str):
-            return content.strip()
+            return content
         if isinstance(content, list):
             parts: list[str] = []
             for item in content:
@@ -103,9 +110,13 @@ class Qwen3Renderer:
                     parts.append(item)
                 elif isinstance(item, dict) and isinstance(item.get("text"), str):
                     parts.append(item["text"])
-            return "".join(parts).strip()
+            return "".join(parts)
         return ""
 
+    @staticmethod
+    def _query_boundary_text(content) -> str:
+        return Qwen3Renderer._content_text(content).strip()
+
     @staticmethod
     def _is_user_query_message(msg: Message) -> bool:
         if msg.get("role") != "user":
@@ -186,7 +197,7 @@ class Qwen3Renderer:
             # JSON as message body.
             segments: list[tuple[str, bool]] = [("system\n", False)]
             if first_is_system:
-                sys_content = messages[0].get("content") or ""
+                sys_content = self._content_text(messages[0].get("content"))
                 if sys_content:
                     segments.append((sys_content, True))
                 segments.append(("\n\n", False))
@@ -199,7 +210,7 @@ class Qwen3Renderer:
             emit_text("\n", sys_idx, is_sampled=False, is_content=False)
         elif first_is_system:
             emit_special(self._im_start, 0, is_sampled=False, is_content=False)
-            sys_content = messages[0].get("content") or ""
+            sys_content = self._content_text(messages[0].get("content"))
             sys_segments: list[tuple[str, bool]] = [("system\n", False)]
             if sys_content:
                 sys_segments.append((sys_content, True))
@@ -214,7 +225,7 @@ class Qwen3Renderer:
         num_messages = len(messages)
         for i, msg in enumerate(messages):
             role = msg["role"]
-            content = msg.get("content") if isinstance(msg.get("content"), str) else ""
+            content = self._content_text(msg.get("content"))
 
             if role == "system":
                 if i == 0:
@@ -415,7 +426,7 @@ class Qwen3Renderer:
 
         for i, msg in enumerate(new_messages):
             role = msg.get("role")
-            content = msg.get("content") if isinstance(msg.get("content"), str) else ""
+            content = self._content_text(msg.get("content"))
             if role == "user":
                 emit_special(self._im_start, i)
                 user_segments: list[tuple[str, bool]] = [("user\n", False)]

{renderers-0.1.12.dev8 → renderers-0.1.12.dev9}/tests/test_bridge.py

@@ -205,6 +205,24 @@ def test_bridge_extension_includes_new_message_text(
     )
 
 
+def test_qwen3_renders_openai_text_content_parts(br_renderer, br_tokenizer, br_model):
+    """Text content lists must not become an empty user turn."""
+    if br_model not in {"Qwen/Qwen3-8B", "Qwen/Qwen3-4B"}:
+        pytest.skip(f"{br_model}: not a Qwen3 text renderer")
+
+    text = "CONTENT_LIST_SENTINEL_XYZ"
+    string_ids = br_renderer.render_ids(
+        [{"role": "user", "content": text}], add_generation_prompt=True
+    )
+    list_ids = br_renderer.render_ids(
+        [{"role": "user", "content": [{"type": "text", "text": text}]}],
+        add_generation_prompt=True,
+    )
+
+    assert list_ids == string_ids
+    assert text in br_tokenizer.decode(list_ids, skip_special_tokens=False)
+
+
 def test_bridge_declines_across_user_query_when_template_drops_thinking():
     """Qwen3's template drops a past block's thinking once a new user turn
     arrives. The resolved ``tool_cycle`` bridge policy therefore treats a