renderers 0.1.11.dev5__tar.gz → 0.1.12.dev2__tar.gz

{renderers-0.1.11.dev5 → renderers-0.1.12.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: renderers
-Version: 0.1.11.dev5
+Version: 0.1.12.dev2
 Summary: Chat template renderers — deterministic message-to-token conversion for LLM training
 License-Expression: Apache-2.0
 License-File: LICENSE

{renderers-0.1.11.dev5 → renderers-0.1.12.dev2}/renderers/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.1.11.dev5'
-__version_tuple__ = version_tuple = (0, 1, 11, 'dev5')
+__version__ = version = '0.1.12.dev2'
+__version_tuple__ = version_tuple = (0, 1, 12, 'dev2')
 
 __commit_id__ = commit_id = None

{renderers-0.1.11.dev5 → renderers-0.1.12.dev2}/renderers/base.py

@@ -2095,7 +2095,7 @@ def build_trajectory_step(
         "completion_mask": [True] * len(completion_ids),
         "completion_logprobs": [0.0] * len(completion_ids),
         "routed_experts": None,
-        "kept_tokens": None,
+        "sampling_mask": None,
     }
     if (
         full_rendered.multi_modal_data is not None

{renderers-0.1.11.dev5 → renderers-0.1.12.dev2}/renderers/client.py

@@ -28,7 +28,6 @@ from renderers.base import (
 
 _request_logger = logging.getLogger("renderers.client")
 ROUTED_EXPERTS_DATA_PREFIX = b'"routed_experts":{"data":"'
-KEPT_TOKENS_IDS_PREFIX = b'"kept_tokens":{"ids":"'
 # vLLM uses this value both when sampled-token evidence is missing and as a
 # lower-bound clamp, so receiving it cannot prove the real logprob was returned.
 VLLM_LOGPROB_SENTINEL = -9999.0
@@ -129,12 +128,9 @@ def _strip_base64_field(raw: bytes, prefix: bytes) -> tuple[bytes, memoryview |
 
 def parse_generate_response(raw: bytes) -> dict[str, Any]:
     stripped, routed_data = _strip_base64_field(raw, ROUTED_EXPERTS_DATA_PREFIX)
-    stripped, kept_ids_data = _strip_base64_field(stripped, KEPT_TOKENS_IDS_PREFIX)
     payload: dict[str, Any] = json.loads(stripped)
     if routed_data is not None:
         payload["choices"][0]["routed_experts"]["data"] = routed_data
-    if kept_ids_data is not None:
-        payload["choices"][0]["kept_tokens"]["ids"] = kept_ids_data
     return payload
 
 
@@ -207,6 +203,7 @@ async def generate(
     priority: int | None = None,
     extra_headers: dict[str, str] | None = None,
     max_prompt_len: int | None = None,
+    process_multimodal: bool = True,
 ) -> dict[str, Any]:
     """Tokenize messages, call vLLM /inference/v1/generate, parse the response.
 
@@ -226,6 +223,9 @@ async def generate(
     sidecar, then serializes it to vLLM's ``features`` schema (mm_hashes,
     mm_placeholders, kwargs_data) before POSTing. The serializer imports
     ``vllm.*`` lazily so text-only consumers never pay for the import.
+    With ``process_multimodal=False``, rendering skips image processing and
+    the request carries ``content_parts`` instead; vLLM must return the
+    expanded prompt as ``prompt_token_ids``.
 
     ``max_prompt_len`` controls the pre-flight overflow check. When the
     rendered prompt is strictly longer than the cap, the request is never
@@ -236,10 +236,15 @@ async def generate(
     cache a ``None`` cap and the pre-flight silently disables. Engine 4xx
     that still slip through propagate raw — converting them into a domain
     error is the calling client's job (its error shape is engine-specific).
+    Calls with ``process_multimodal=False`` skip this pre-flight because only
+    vLLM knows the expanded prompt length.
 
-    Returns a dict with: request_id, prompt_ids, completion_ids,
-    completion_logprobs, content, reasoning_content, tool_calls,
-    finish_reason, routed_experts, multi_modal_data, prompt_attribution.
+    Returns a dict with: request_id, prompt_ids, renderer_prompt_ids,
+    mm_placeholders, completion_ids, completion_logprobs, content,
+    reasoning_content, tool_calls, finish_reason, routed_experts,
+    multi_modal_data, prompt_attribution. ``renderer_prompt_ids`` is the
+    unexpanded logical prompt when ``process_multimodal=False`` and ``None``
+    otherwise.
 
     ``prompt_attribution`` is the renderer's :class:`RenderedTokens` for
     the prompt — either the one this call computed via
@@ -258,6 +263,12 @@ async def generate(
             f"{type(renderer).__name__} does not support tools. "
             "Choose a model-specific renderer instead of the default fallback."
         )
+    if not process_multimodal and not getattr(
+        renderer, "supports_process_multimodal", False
+    ):
+        raise NotImplementedError(
+            f"{type(renderer).__name__} does not support process_multimodal=False"
+        )
 
     def _prepare():
         if prompt_ids is not None:
@@ -270,7 +281,15 @@ async def generate(
                 multi_modal_data,
                 prompt_attribution,
             )
-        rendered = renderer.render(messages, tools=tools, add_generation_prompt=True)
+        render_kwargs: dict[str, Any] = {}
+        if not process_multimodal:
+            render_kwargs["process_multimodal"] = False
+        rendered = renderer.render(
+            messages,
+            tools=tools,
+            add_generation_prompt=True,
+            **render_kwargs,
+        )
         return (
             rendered.token_ids,
             renderer.get_stop_token_ids(),
@@ -280,12 +299,13 @@ async def generate(
 
     prompt_ids, stop_token_ids, mm_data, prompt_attr = _prepare()
 
-    if max_prompt_len is None:
-        max_prompt_len = await _resolve_max_prompt_len(client, model)
-    if max_prompt_len is not None and len(prompt_ids) > max_prompt_len:
-        raise OverlongPromptError(
-            prompt_len=len(prompt_ids), max_prompt_len=max_prompt_len
-        )
+    if process_multimodal:
+        if max_prompt_len is None:
+            max_prompt_len = await _resolve_max_prompt_len(client, model)
+        if max_prompt_len is not None and len(prompt_ids) > max_prompt_len:
+            raise OverlongPromptError(
+                prompt_len=len(prompt_ids), max_prompt_len=max_prompt_len
+            )
 
     sp: dict[str, Any] = dict(sampling_params or {})
     sp["stop_token_ids"] = stop_token_ids
@@ -297,11 +317,14 @@ async def generate(
         "token_ids": prompt_ids,
         "sampling_params": sp,
     }
+    content_parts = _content_parts(messages) if not process_multimodal else None
     features = (
         _build_mm_features(renderer, mm_data)
-        if mm_data and not mm_data.is_empty()
+        if process_multimodal and mm_data and not mm_data.is_empty()
         else None
     )
+    if content_parts:
+        body["content_parts"] = content_parts
     if features is not None:
         body["features"] = features
     if cache_salt is not None:
@@ -331,13 +354,25 @@ async def generate(
 
     choice = (data.get("choices") or [{}])[0]
     completion_ids = choice.get("token_ids") or []
+    effective_prompt_ids = data.get("prompt_token_ids")
+    if content_parts and not isinstance(effective_prompt_ids, list):
+        raise MalformedGenerateResponseError(
+            "Engine response must include prompt_token_ids when process_multimodal=False."
+        )
+    mm_placeholders = data.get("mm_placeholders")
+    if content_parts and not isinstance(mm_placeholders, dict):
+        raise MalformedGenerateResponseError(
+            "Engine response must include mm_placeholders when process_multimodal=False."
+        )
 
     completion_logprobs = _parse_completion_logprobs(choice, completion_ids)
 
     parsed = renderer.parse_response(completion_ids, tools=tools)
 
     routed_experts = choice.get("routed_experts")
-    kept_tokens = choice.get("kept_tokens")
+    # vLLM's native kept-set sampling masks (``--return-sampling-mask``):
+    # one list of surviving vocab ids per completion token.
+    sampling_mask = choice.get("sampling_mask")
 
     # /inference/v1/generate returns finish_reason in {"stop","length",...} —
     # never "tool_calls" (a chat-completions concept). Promote stop→tool_calls
@@ -356,7 +391,9 @@ async def generate(
 
     return {
         "request_id": data.get("request_id") or "",
-        "prompt_ids": list(prompt_ids),
+        "prompt_ids": list(effective_prompt_ids or prompt_ids),
+        "renderer_prompt_ids": list(prompt_ids) if content_parts else None,
+        "mm_placeholders": mm_placeholders,
         "completion_ids": list(completion_ids),
         "completion_logprobs": completion_logprobs,
         "content": parsed.content,
@@ -364,7 +401,7 @@ async def generate(
         "tool_calls": parsed.tool_calls,
         "finish_reason": finish_reason,
         "routed_experts": routed_experts,
-        "kept_tokens": kept_tokens,
+        "sampling_mask": sampling_mask,
         # The mm sidecar consumed on the request side, surfaced back so
         # callers can persist it on the trajectory step for downstream
         # multi-turn bridging and training-sample construction.
@@ -380,6 +417,43 @@ async def generate(
     }
 
 
+_MEDIA_URL_TYPES = {
+    "image": "image_url",
+    "image_url": "image_url",
+    "audio": "audio_url",
+    "audio_url": "audio_url",
+    "video": "video_url",
+    "video_url": "video_url",
+}
+
+
+def _content_parts(messages: list[Message]) -> list[dict[str, Any]]:
+    """Flatten raw media in prompt order for vLLM's token generate endpoint."""
+    parts: list[dict[str, Any]] = []
+    for message in messages:
+        content = message.get("content")
+        if not isinstance(content, list):
+            continue
+        for part in content:
+            if not isinstance(part, Mapping):
+                continue
+            part_type = part.get("type")
+            if part_type is None:
+                part_type = next(
+                    (key for key in _MEDIA_URL_TYPES if part.get(key)), None
+                )
+            if not isinstance(part_type, str) or part_type not in _MEDIA_URL_TYPES:
+                continue
+            source = part.get(part_type)
+            if source is None:
+                source = part.get(_MEDIA_URL_TYPES[part_type]) or part.get("url")
+            url = source.get("url") if isinstance(source, Mapping) else source
+            if not isinstance(url, str) or not url:
+                raise ValueError(f"{part_type} content part is missing a URL")
+            parts.append({"type": _MEDIA_URL_TYPES[part_type], "url": url})
+    return parts
+
+
 def _build_mm_features(
     renderer: Renderer,
     mm_data: MultiModalData,

{renderers-0.1.11.dev5 → renderers-0.1.12.dev2}/renderers/kimi_k25.py

@@ -591,6 +591,8 @@ class KimiK25Renderer:
     The tokenizer should be ``moonshotai/Kimi-K2-Instruct`` (same as K2).
     """
 
+    supports_process_multimodal = True
+
     def __init__(
         self,
         tokenizer: Tokenizer,
@@ -734,6 +736,7 @@ class KimiK25Renderer:
         *,
         tools: list[ToolSpec] | None = None,
         add_generation_prompt: bool = False,
+        process_multimodal: bool = True,
     ) -> RenderedTokens:
         """Render messages to tokens, matching the K2.5 chat template.
 
@@ -821,7 +824,10 @@ class KimiK25Renderer:
             ``<|media_content|>``, ``<|media_end|>``, the trailing
             ``\\n``) are template-injected scaffold.
             """
-            _, out, _num_patches, h = self._process_image(part)
+            if process_multimodal:
+                _, out, _num_patches, h = self._process_image(part)
+            else:
+                out = h = None
             emit_special(
                 self._media_begin, msg_idx, is_sampled=is_sampled, is_content=False
             )
@@ -840,6 +846,9 @@ class KimiK25Renderer:
                 self._media_end, msg_idx, is_sampled=is_sampled, is_content=False
             )
             emit_text("\n", msg_idx, is_sampled=is_sampled, is_content=False)
+            if not process_multimodal:
+                return
+            assert out is not None and h is not None
             mm_hashes.setdefault("image", []).append(h)
             mm_placeholders.setdefault("image", []).append(
                 PlaceholderRange(offset=offset, length=1)
@@ -1027,6 +1036,7 @@ class KimiK25Renderer:
         *,
         tools: list[ToolSpec] | None = None,
         previous_multi_modal_data: MultiModalData | None = None,
+        process_multimodal: bool = True,
     ) -> "RenderedTokens | None":
         if (
             not previous_prompt_ids
@@ -1115,7 +1125,10 @@ class KimiK25Renderer:
             is_sampled: bool = False,
             is_content: bool = False,
         ) -> None:
-            _, out, _num_patches, h = self._process_image(part)
+            if process_multimodal:
+                _, out, _num_patches, h = self._process_image(part)
+            else:
+                out = h = None
             emit_special(self._media_begin, msg_idx)
             emit_text("image", msg_idx)
             emit_special(self._media_content, msg_idx)
@@ -1125,6 +1138,9 @@ class KimiK25Renderer:
             emit_special(self._media_pad, msg_idx, is_content=is_content)
             emit_special(self._media_end, msg_idx)
             emit_text("\n", msg_idx)
+            if not process_multimodal:
+                return
+            assert out is not None and h is not None
             new_hashes.setdefault("image", []).append(h)
             new_placeholders.setdefault("image", []).append(
                 PlaceholderRange(offset=offset, length=1)
@@ -1188,17 +1204,17 @@ class KimiK25Renderer:
         # below never mutates the caller's previous_multi_modal_data.
         merged_hashes: dict[str, list[str]] = (
             {k: list(v) for k, v in previous_multi_modal_data.mm_hashes.items()}
-            if previous_multi_modal_data
+            if process_multimodal and previous_multi_modal_data
             else {}
         )
         merged_placeholders: dict[str, list[PlaceholderRange]] = (
             {k: list(v) for k, v in previous_multi_modal_data.mm_placeholders.items()}
-            if previous_multi_modal_data
+            if process_multimodal and previous_multi_modal_data
             else {}
         )
         merged_items: dict[str, list[dict[str, Any]]] = (
             {k: list(v) for k, v in previous_multi_modal_data.mm_items.items()}
-            if previous_multi_modal_data
+            if process_multimodal and previous_multi_modal_data
             else {}
         )
         for modality, vals in new_hashes.items():

{renderers-0.1.11.dev5 → renderers-0.1.12.dev2}/renderers/qwen35.py

@@ -123,6 +123,7 @@ def _default_enable_thinking(tokenizer) -> bool:
 class Qwen35Renderer:
     """Deterministic message → token renderer for Qwen3.5 models."""
 
+    supports_process_multimodal = True
     _config_cls: type = Qwen35RendererConfig
 
     def __init__(
@@ -353,6 +354,7 @@ class Qwen35Renderer:
         *,
         tools: list[ToolSpec] | None = None,
         add_generation_prompt: bool = False,
+        process_multimodal: bool = True,
     ) -> RenderedTokens:
         if not messages:
             raise ValueError("No messages provided.")
@@ -424,7 +426,11 @@ class Qwen35Renderer:
             # image data, so they ARE body content (is_content=True);
             # the surrounding ``<|vision_start|>`` / ``<|vision_end|>``
             # specials are template scaffold.
-            _, out, n, h = self._process_image(part)
+            if process_multimodal:
+                _, out, n, h = self._process_image(part)
+            else:
+                out = h = None
+                n = 1
             vision_counts["image"] += 1
             if self.config.add_vision_id:
                 emit_text(
@@ -442,16 +448,18 @@ class Qwen35Renderer:
                     self._image_pad, msg_idx, is_sampled=False, is_content=True
                 )
             emit_special(self._vision_end, msg_idx, is_sampled=False, is_content=False)
-            mm_hashes.setdefault("image", []).append(h)
-            mm_placeholders.setdefault("image", []).append(
-                PlaceholderRange(offset=offset, length=n)
-            )
-            mm_items.setdefault("image", []).append(
-                {
-                    "pixel_values": out["pixel_values"],
-                    "image_grid_thw": out["image_grid_thw"],
-                }
-            )
+            if process_multimodal:
+                assert out is not None and h is not None
+                mm_hashes.setdefault("image", []).append(h)
+                mm_placeholders.setdefault("image", []).append(
+                    PlaceholderRange(offset=offset, length=n)
+                )
+                mm_items.setdefault("image", []).append(
+                    {
+                        "pixel_values": out["pixel_values"],
+                        "image_grid_thw": out["image_grid_thw"],
+                    }
+                )
 
         def emit_user_with_media(content_list: list[Any], msg_idx: int) -> None:
             """Emit a user message whose content list contains image parts.
@@ -685,6 +693,7 @@ class Qwen35Renderer:
         *,
         tools: list[ToolSpec] | None = None,
         previous_multi_modal_data: MultiModalData | None = None,
+        process_multimodal: bool = True,
     ) -> "RenderedTokens | None":
         if (
             not previous_prompt_ids
@@ -721,6 +730,7 @@ class Qwen35Renderer:
         # scratch correctly.
         if (
             self.config.add_vision_id
+            and process_multimodal
             and previous_multi_modal_data is None
             and self._vision_start in previous_ids
         ):
@@ -751,9 +761,11 @@ class Qwen35Renderer:
         # ``add_vision_id`` parity across turns.
         prev_image_count = 0
         prev_video_count = 0
-        if previous_multi_modal_data is not None:
+        if process_multimodal and previous_multi_modal_data is not None:
             prev_image_count = len(previous_multi_modal_data.mm_items.get("image", []))
             prev_video_count = len(previous_multi_modal_data.mm_items.get("video", []))
+        elif not process_multimodal:
+            prev_image_count = previous_ids.count(self._image_pad)
         vision_counts = {"image": prev_image_count, "video": prev_video_count}
 
         def emit_special(
@@ -796,7 +808,11 @@ class Qwen35Renderer:
                 content_mask.append(is_content)
 
         def emit_image(part: dict[str, Any], msg_idx: int = -1) -> None:
-            _, out, n, h = self._process_image(part)
+            if process_multimodal:
+                _, out, n, h = self._process_image(part)
+            else:
+                out = h = None
+                n = 1
             vision_counts["image"] += 1
             if self.config.add_vision_id:
                 emit_text(f"Picture {vision_counts['image']}: ", msg_idx)
@@ -805,16 +821,18 @@ class Qwen35Renderer:
             for _ in range(n):
                 emit_special(self._image_pad, msg_idx, is_content=True)
             emit_special(self._vision_end, msg_idx)
-            new_hashes.setdefault("image", []).append(h)
-            new_placeholders.setdefault("image", []).append(
-                PlaceholderRange(offset=offset, length=n)
-            )
-            new_items.setdefault("image", []).append(
-                {
-                    "pixel_values": out["pixel_values"],
-                    "image_grid_thw": out["image_grid_thw"],
-                }
-            )
+            if process_multimodal:
+                assert out is not None and h is not None
+                new_hashes.setdefault("image", []).append(h)
+                new_placeholders.setdefault("image", []).append(
+                    PlaceholderRange(offset=offset, length=n)
+                )
+                new_items.setdefault("image", []).append(
+                    {
+                        "pixel_values": out["pixel_values"],
+                        "image_grid_thw": out["image_grid_thw"],
+                    }
+                )
 
         def emit_user_with_media(content_list: list[Any], msg_idx: int) -> None:
             emit_special(self._im_start, msg_idx)
@@ -905,17 +923,17 @@ class Qwen35Renderer:
         # below never mutates the caller's previous_multi_modal_data.
         merged_hashes: dict[str, list[str]] = (
             {k: list(v) for k, v in previous_multi_modal_data.mm_hashes.items()}
-            if previous_multi_modal_data
+            if process_multimodal and previous_multi_modal_data
             else {}
         )
         merged_placeholders: dict[str, list[PlaceholderRange]] = (
             {k: list(v) for k, v in previous_multi_modal_data.mm_placeholders.items()}
-            if previous_multi_modal_data
+            if process_multimodal and previous_multi_modal_data
             else {}
         )
         merged_items: dict[str, list[dict[str, Any]]] = (
             {k: list(v) for k, v in previous_multi_modal_data.mm_items.items()}
-            if previous_multi_modal_data
+            if process_multimodal and previous_multi_modal_data
             else {}
         )
         for modality, vals in new_hashes.items():

{renderers-0.1.11.dev5 → renderers-0.1.12.dev2}/renderers/qwen3_vl.py

@@ -310,6 +310,8 @@ class Qwen3VLRenderer:
     ``thinking_retention`` still controls whether the bridge is attempted.
     """
 
+    supports_process_multimodal = True
+
     def __init__(
         self,
         tokenizer: Tokenizer,
@@ -457,6 +459,7 @@ class Qwen3VLRenderer:
         *,
         tools: list[ToolSpec] | None = None,
         add_generation_prompt: bool = False,
+        process_multimodal: bool = True,
     ) -> RenderedTokens:
         if not messages:
             raise ValueError("No messages provided.")
@@ -480,7 +483,11 @@ class Qwen3VLRenderer:
             # image data, so they ARE body content (is_content=True);
             # the surrounding ``<|vision_start|>`` / ``<|vision_end|>``
             # markers are renderer-emitted scaffold.
-            _, out, n, h = self._process_image(part)
+            if process_multimodal:
+                _, out, n, h = self._process_image(part)
+            else:
+                out = h = None
+                n = 1
             vision_counts["image"] += 1
             if self.config.add_vision_id:
                 em.text(
@@ -493,16 +500,18 @@ class Qwen3VLRenderer:
             for _ in range(n):
                 em.special(self._image_pad, is_sampled=False, is_content=True)
             em.special(self._vision_end, is_sampled=False, is_content=False)
-            mm_hashes.setdefault("image", []).append(h)
-            mm_placeholders.setdefault("image", []).append(
-                PlaceholderRange(offset=offset, length=n)
-            )
-            mm_items.setdefault("image", []).append(
-                {
-                    "pixel_values": out["pixel_values"],
-                    "image_grid_thw": out["image_grid_thw"],
-                }
-            )
+            if process_multimodal:
+                assert out is not None and h is not None
+                mm_hashes.setdefault("image", []).append(h)
+                mm_placeholders.setdefault("image", []).append(
+                    PlaceholderRange(offset=offset, length=n)
+                )
+                mm_items.setdefault("image", []).append(
+                    {
+                        "pixel_values": out["pixel_values"],
+                        "image_grid_thw": out["image_grid_thw"],
+                    }
+                )
 
         def render_media_content(content: Any) -> None:
             """Emit a user/tool content list with media handled inline.
@@ -665,6 +674,7 @@ class Qwen3VLRenderer:
         *,
         tools: list[ToolSpec] | None = None,
         previous_multi_modal_data: MultiModalData | None = None,
+        process_multimodal: bool = True,
     ) -> "RenderedTokens | None":
         """Extend the prior turn with ``new_messages``.
 
@@ -709,6 +719,7 @@ class Qwen3VLRenderer:
         # correctly.
         if (
             self.config.add_vision_id
+            and process_multimodal
             and previous_multi_modal_data is None
             and self._vision_start in previous_ids
         ):
@@ -746,13 +757,19 @@ class Qwen3VLRenderer:
         # ``add_vision_id`` parity across turns.
         prev_image_count = 0
         prev_video_count = 0
-        if previous_multi_modal_data is not None:
+        if process_multimodal and previous_multi_modal_data is not None:
             prev_image_count = len(previous_multi_modal_data.mm_items.get("image", []))
             prev_video_count = len(previous_multi_modal_data.mm_items.get("video", []))
+        elif not process_multimodal:
+            prev_image_count = previous_ids.count(self._image_pad)
         vision_counts = {"image": prev_image_count, "video": prev_video_count}
 
         def emit_image(part: dict[str, Any]) -> None:
-            _, out, n, h = self._process_image(part)
+            if process_multimodal:
+                _, out, n, h = self._process_image(part)
+            else:
+                out = h = None
+                n = 1
             vision_counts["image"] += 1
             if self.config.add_vision_id:
                 em.text(
@@ -765,16 +782,18 @@ class Qwen3VLRenderer:
             for _ in range(n):
                 em.special(self._image_pad, is_sampled=False, is_content=True)
             em.special(self._vision_end, is_sampled=False, is_content=False)
-            new_hashes.setdefault("image", []).append(h)
-            new_placeholders.setdefault("image", []).append(
-                PlaceholderRange(offset=offset, length=n)
-            )
-            new_items.setdefault("image", []).append(
-                {
-                    "pixel_values": out["pixel_values"],
-                    "image_grid_thw": out["image_grid_thw"],
-                }
-            )
+            if process_multimodal:
+                assert out is not None and h is not None
+                new_hashes.setdefault("image", []).append(h)
+                new_placeholders.setdefault("image", []).append(
+                    PlaceholderRange(offset=offset, length=n)
+                )
+                new_items.setdefault("image", []).append(
+                    {
+                        "pixel_values": out["pixel_values"],
+                        "image_grid_thw": out["image_grid_thw"],
+                    }
+                )
 
         def render_media_content(content: Any) -> None:
             if isinstance(content, str):
@@ -833,17 +852,17 @@ class Qwen3VLRenderer:
         # caller's previous_multi_modal_data.
         merged_hashes = (
             {k: list(v) for k, v in previous_multi_modal_data.mm_hashes.items()}
-            if previous_multi_modal_data
+            if process_multimodal and previous_multi_modal_data
             else {}
         )
         merged_placeholders = (
             {k: list(v) for k, v in previous_multi_modal_data.mm_placeholders.items()}
-            if previous_multi_modal_data
+            if process_multimodal and previous_multi_modal_data
             else {}
         )
         merged_items = (
             {k: list(v) for k, v in previous_multi_modal_data.mm_items.items()}
-            if previous_multi_modal_data
+            if process_multimodal and previous_multi_modal_data
             else {}
         )
         for modality, vals in new_hashes.items():

{renderers-0.1.11.dev5 → renderers-0.1.12.dev2}/tests/test_client.py

@@ -93,6 +93,9 @@ class _FakeClient:
             "request_id": "gen-test",
             "choices": [self.choice],
         }
+        if body and body.get("content_parts"):
+            payload["prompt_token_ids"] = [1, 2, 2, 3]
+            payload["mm_placeholders"] = {"image": [{"offset": 1, "length": 2}]}
         return httpx.Response(
             200,
             content=json.dumps(payload, separators=(",", ":")).encode("utf-8"),
@@ -182,6 +185,49 @@ def test_generate_builds_request_body_and_parses_response():
     assert tc.status == ToolCallParseStatus.OK
 
 
+def test_generate_process_multimodal_false_sends_content_parts():
+    class DeferredMultimodalRenderer(_FakeRenderer):
+        supports_process_multimodal = True
+
+        def render(
+            self,
+            messages,
+            *,
+            tools=None,
+            add_generation_prompt=False,
+            process_multimodal=True,
+        ):
+            assert process_multimodal is False
+            return RenderedTokens(token_ids=[1, 2, 3])
+
+    client = _FakeClient()
+    image_url = "data:image/png;base64,aW1hZ2U="
+    result = asyncio.run(
+        generate(
+            client=client,
+            renderer=DeferredMultimodalRenderer(),
+            messages=[
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": "look"},
+                        {"type": "image_url", "image_url": {"url": image_url}},
+                    ],
+                }
+            ],
+            model="test-model",
+            process_multimodal=False,
+        )
+    )
+
+    body = client.calls[0]["body"]
+    assert body["content_parts"] == [{"type": "image_url", "url": image_url}]
+    assert "features" not in body
+    assert result["renderer_prompt_ids"] == [1, 2, 3]
+    assert result["prompt_ids"] == [1, 2, 2, 3]
+    assert result["mm_placeholders"] == {"image": [{"offset": 1, "length": 2}]}
+
+
 def test_generate_rejects_missing_completion_logprobs_before_parsing():
     client = _FakeClient()
     client.choice.pop("logprobs")