renderers 0.1.12.dev6__tar.gz → 0.1.12.dev8__tar.gz

{renderers-0.1.12.dev6 → renderers-0.1.12.dev8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: renderers
-Version: 0.1.12.dev6
+Version: 0.1.12.dev8
 Summary: Chat template renderers — deterministic message-to-token conversion for LLM training
 License-Expression: Apache-2.0
 License-File: LICENSE

{renderers-0.1.12.dev6 → renderers-0.1.12.dev8}/renderers/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.1.12.dev6'
-__version_tuple__ = version_tuple = (0, 1, 12, 'dev6')
+__version__ = version = '0.1.12.dev8'
+__version_tuple__ = version_tuple = (0, 1, 12, 'dev8')
 
 __commit_id__ = commit_id = None

{renderers-0.1.12.dev6 → renderers-0.1.12.dev8}/renderers/base.py

@@ -1017,6 +1017,7 @@ MODEL_RENDERER_MAP: dict[str, str] = {
     # Nemotron 3.5 (Lightning). Its template is the Ultra variant's minus the
     # effort kwarg (``nemotron-3.5``).
     "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16": "nemotron-3.5",
+    "nvidia/NVIDIA-Nemotron-3.5-Super-EA-09112026": "nemotron-3.5",
     # Llama 3.2 (Instruct). Tested against the gated meta-llama repos and
     # the unrestricted unsloth/... mirror, which ships a byte-identical
     # chat template. ``Llama3Renderer`` defaults ``date_string`` to
@@ -1057,6 +1058,7 @@ MODEL_RENDERER_MAP: dict[str, str] = {
 # Modality values: ``"image"``, ``"video"``, ``"audio"``. Text is implicit
 # (every model supports it), so it doesn't appear in the set.
 MULTIMODAL_MODELS: dict[str, set[str]] = {
+    "nvidia/NVIDIA-Nemotron-3.5-Super-EA-09112026": {"image"},
     "Qwen/Qwen3-VL-4B-Instruct": {"image"},
     "Qwen/Qwen3-VL-8B-Instruct": {"image"},
     "Qwen/Qwen3-VL-30B-A3B-Instruct": {"image"},

{renderers-0.1.12.dev6 → renderers-0.1.12.dev8}/renderers/client.py

@@ -486,6 +486,7 @@ def _build_mm_features(
     to change. Don't pre-build the abstraction with one engine in tree.
     """
     from renderers.gemma4 import Gemma4Renderer
+    from renderers.nemotron3 import Nemotron35Renderer
     from renderers.qwen3_vl import Qwen3VLRenderer
     from renderers.qwen35 import Qwen35Renderer
 
@@ -498,6 +499,8 @@ def _build_mm_features(
         return _build_qwen_vl_features(mm_data, spatial_merge_size=2)
     if issubclass(renderer_cls, Gemma4Renderer):
         return _build_gemma4_features(mm_data)
+    if issubclass(renderer_cls, Nemotron35Renderer):
+        return _build_nemotron35_features(mm_data)
 
     raise NotImplementedError(
         f"Multimodal serialization not implemented for {renderer_cls.__name__}. "
@@ -505,6 +508,79 @@ def _build_mm_features(
     )
 
 
+def _build_nemotron35_features(mm_data: MultiModalData) -> dict[str, Any]:
+    """Encode dynamic-resolution Nemotron image inputs for vLLM."""
+    _require_transformers("Encoding Nemotron 3.5 multimodal features for vLLM")
+    try:
+        import torch
+        from transformers.feature_extraction_utils import BatchFeature
+        from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import (
+            encode_mm_kwargs_item,
+        )
+        from vllm.multimodal.inputs import (
+            MultiModalFieldConfig,
+            MultiModalKwargsItems,
+        )
+    except ImportError as exc:
+        raise RuntimeError(
+            "Nemotron 3.5 multimodal generate requires vLLM and torch."
+        ) from exc
+
+    out: dict[str, Any] = {
+        "mm_hashes": {},
+        "mm_placeholders": {},
+        "kwargs_data": {},
+    }
+    image_items = mm_data.mm_items.get("image") or []
+    if image_items:
+        pixel_values_flat = []
+        imgs_sizes = []
+        num_tokens_per_image = []
+        for item in image_items:
+            pixel_values = torch.as_tensor(item["pixel_values"])
+            if pixel_values.ndim != 4 or pixel_values.shape[0] != 1:
+                raise ValueError(
+                    "Each Nemotron pixel_values item must have shape (1, C, H, W)."
+                )
+            sizes = item["imgs_sizes"]
+            token_counts = item["num_tokens"]
+            if not (len(sizes) == len(token_counts) == 1):
+                raise ValueError(
+                    "Each Nemotron image item must contain one size and token count."
+                )
+            pixel_values_flat.append(pixel_values[0])
+            imgs_sizes.append(tuple(int(value) for value in sizes[0]))
+            num_tokens_per_image.append(int(token_counts[0]))
+
+        hf_inputs = BatchFeature(
+            data={
+                "pixel_values_flat": pixel_values_flat,
+                "imgs_sizes": imgs_sizes,
+                "num_tokens_per_image": num_tokens_per_image,
+            }
+        )
+        field_config = {
+            "pixel_values_flat": MultiModalFieldConfig.batched("image"),
+            "num_tokens_per_image": MultiModalFieldConfig.batched(
+                "image", keep_on_cpu=True
+            ),
+            "imgs_sizes": MultiModalFieldConfig.batched("image", keep_on_cpu=True),
+        }
+        kwargs_items = MultiModalKwargsItems.from_hf_inputs(hf_inputs, field_config)
+        out["kwargs_data"]["image"] = [
+            encode_mm_kwargs_item(item) for item in kwargs_items["image"]
+        ]
+        out["mm_hashes"]["image"] = list(mm_data.mm_hashes.get("image") or [])
+        out["mm_placeholders"]["image"] = [
+            {"offset": placeholder.offset, "length": placeholder.length}
+            for placeholder in mm_data.mm_placeholders.get("image") or []
+        ]
+
+    if not any(out["kwargs_data"].values()):
+        out["kwargs_data"] = None
+    return out
+
+
 def _build_gemma4_features(mm_data: MultiModalData) -> dict[str, Any]:
     """vLLM features payload for Gemma 4 image inputs.
 

{renderers-0.1.12.dev6 → renderers-0.1.12.dev8}/renderers/nemotron3.py

@@ -14,18 +14,23 @@ Nemotron 3 uses the same <|im_start|>/<|im_end|> format as Qwen3.5 but differs i
 
 from __future__ import annotations
 
+from dataclasses import replace
 import json
+from pathlib import Path
 from typing import Any
 
 from renderers.reasoning import scan_reasoning, prompt_ends_in_reasoning
 
 from renderers.base import (
     Message,
+    MultiModalData,
     ParsedResponse,
+    PlaceholderRange,
     RenderedTokens,
     ToolSpec,
     Tokenizer,
     _content_mask_or_empty,
+    _require_transformers,
     attribute_text_segments,
     extract_message_tool_names,
     reject_assistant_in_extension,
@@ -39,6 +44,7 @@ from renderers.configs import (
     Nemotron35RendererConfig,
 )
 from renderers.parsing import parse_qwen35
+from renderers.qwen3_vl import _image_hash, _is_image_part, _load_pil_image
 
 # ---------------------------------------------------------------------------
 # Tool system prompt constants
@@ -894,3 +900,249 @@ class Nemotron35Renderer(Nemotron3Renderer):
 
     _config_cls = Nemotron35RendererConfig
     _ultra = True
+    supports_process_multimodal = True
+    supports_multimodal_bridge = False
+
+    _processor_model_prefix = "nvidia/NVIDIA-Nemotron-3.5"
+
+    def __init__(
+        self,
+        tokenizer: Tokenizer,
+        config: Nemotron35RendererConfig | None = None,
+        *,
+        processor: Any = None,
+    ):
+        super().__init__(tokenizer, config)
+        self._processor = processor
+        self._image = self._token_id("<image>", optional=True)
+
+    @property
+    def mm_token_type_id_map(self) -> dict[int, int]:
+        return {self._image: 1} if self._image is not None else {}
+
+    def _image_token_id(self) -> int:
+        if self._image is None:
+            raise RuntimeError("This Nemotron 3.5 tokenizer has no <image> token.")
+        return self._image
+
+    def _get_processor(self):
+        if self._processor is not None:
+            return self._processor
+        name = getattr(self._tokenizer, "name_or_path", None)
+        if not name:
+            raise RuntimeError(
+                "Nemotron35Renderer needs a processor for image content."
+            )
+        if Path(name).expanduser().is_dir():
+            raise RuntimeError(
+                "Local Nemotron remote-code processor paths require an explicitly "
+                "injected reviewed processor."
+            )
+        if not name.startswith(self._processor_model_prefix):
+            raise RuntimeError(
+                f"Nemotron processor {name!r} is not approved for remote-code "
+                "loading. Inject a reviewed processor."
+            )
+        transformers = _require_transformers("Auto-loading a Nemotron 3.5 processor")
+        self._processor = transformers.AutoProcessor.from_pretrained(
+            name, trust_remote_code=True
+        )
+        return self._processor
+
+    @staticmethod
+    def _is_image_part(item: Any) -> bool:
+        return _is_image_part(item) or (
+            isinstance(item, dict) and item.get("type") == "input_image"
+        )
+
+    @staticmethod
+    def _image_source(part: dict[str, Any]) -> dict[str, Any]:
+        if "input_image" not in part:
+            return part
+        return {**part, "image": part["input_image"]}
+
+    @classmethod
+    def _logical_multimodal_content(
+        cls, content: list[Any]
+    ) -> tuple[str, list[dict[str, Any]]]:
+        text = "".join(
+            item if isinstance(item, str) else str(item.get("text") or "")
+            for item in content
+            if isinstance(item, str)
+            or (isinstance(item, dict) and item.get("type") in ("text", "input_text"))
+        )
+        image_parts = [
+            item
+            for item in content
+            if isinstance(item, dict) and cls._is_image_part(item)
+        ]
+        emit_structured_images = "<image>" not in text
+        rendered: list[str] = []
+        image_index = 0
+        for item in content:
+            if isinstance(item, str):
+                rendered.append(item)
+            elif not isinstance(item, dict):
+                raise ValueError(f"Unexpected content item: {item}")
+            elif cls._is_image_part(item):
+                image_index += 1
+                if emit_structured_images:
+                    if len(image_parts) > 1:
+                        rendered.append(f"<image {image_index}><image>")
+                    else:
+                        rendered.append("<image>")
+            elif item.get("type") in ("video", "video_url", "input_video"):
+                raise NotImplementedError(
+                    "Video parts are not yet supported by Nemotron35Renderer."
+                )
+            elif item.get("type") in ("text", "input_text") or "text" in item:
+                rendered.append(str(item.get("text") or ""))
+        return "".join(rendered), image_parts
+
+    @staticmethod
+    def _load_image(part: dict[str, Any]):
+        source = Nemotron35Renderer._image_source(part)
+        if "image" in source:
+            raw = source["image"]
+        elif "image_url" in source:
+            image_url = source.get("image_url")
+            raw = image_url.get("url") if isinstance(image_url, dict) else image_url
+        else:
+            raw = source.get("url") or source.get("path")
+        if isinstance(raw, str) and not raw.startswith("data:image/"):
+            raise ValueError(
+                "Nemotron 3.5 accepts in-memory images or data:image URIs; "
+                "network and filesystem image sources are disabled."
+            )
+        return _load_pil_image(source)
+
+    def _process_image(self, part: dict[str, Any]) -> tuple[dict[str, Any], int, str]:
+        self._image_token_id()
+        image = self._load_image(part)
+        output = self._get_processor().image_processor(
+            images=[image], return_tensors="np"
+        )
+        return output, int(output["num_tokens"][0]), _image_hash(image)
+
+    @staticmethod
+    def _expand_image_tokens(content: str, token_counts: list[int]) -> str:
+        if content.count("<image>") != len(token_counts):
+            raise ValueError(
+                "Nemotron image content must contain one <image> placeholder per image."
+            )
+        segments = content.split("<image>")
+        expanded = [segments[0]]
+        for count, suffix in zip(token_counts, segments[1:]):
+            expanded.append("<img>" + "<image>" * count + "</img>" + suffix)
+        return "".join(expanded)
+
+    def _transform_multimodal_messages(
+        self,
+        messages: list[Message],
+        *,
+        process_multimodal: bool,
+    ) -> tuple[list[Message], list[tuple[dict[str, Any], int, str]]]:
+        transformed: list[Message] = []
+        processed_images: list[tuple[dict[str, Any], int, str]] = []
+        for message in messages:
+            copied = dict(message)
+            content = message.get("content")
+            if isinstance(content, list) and any(
+                self._is_image_part(item) for item in content
+            ):
+                logical_content, image_parts = self._logical_multimodal_content(content)
+                if process_multimodal:
+                    current_images = [self._process_image(part) for part in image_parts]
+                    logical_content = self._expand_image_tokens(
+                        logical_content, [item[1] for item in current_images]
+                    )
+                    processed_images.extend(current_images)
+                copied["content"] = logical_content
+            transformed.append(copied)
+        return transformed, processed_images
+
+    def render(
+        self,
+        messages: list[Message],
+        *,
+        tools: list[ToolSpec] | None = None,
+        add_generation_prompt: bool = False,
+        process_multimodal: bool = True,
+    ) -> RenderedTokens:
+        transformed, processed_images = self._transform_multimodal_messages(
+            messages, process_multimodal=process_multimodal
+        )
+
+        rendered = super().render(
+            transformed,
+            tools=tools,
+            add_generation_prompt=add_generation_prompt,
+        )
+        if not processed_images:
+            return rendered
+
+        image_token_id = self._image_token_id()
+        placeholder_runs: list[PlaceholderRange] = []
+        cursor = 0
+        for _, length, _ in processed_images:
+            while (
+                cursor < len(rendered.token_ids)
+                and rendered.token_ids[cursor] != image_token_id
+            ):
+                cursor += 1
+            if (
+                rendered.token_ids[cursor : cursor + length]
+                != [image_token_id] * length
+            ):
+                raise ValueError(
+                    "Rendered Nemotron image placeholders are not contiguous."
+                )
+            placeholder_runs.append(PlaceholderRange(offset=cursor, length=length))
+            cursor += length
+
+        mm_data = MultiModalData(
+            mm_hashes={"image": [item[2] for item in processed_images]},
+            mm_placeholders={"image": placeholder_runs},
+            mm_items={
+                "image": [
+                    {
+                        key: processor_output[key]
+                        for key in (
+                            "pixel_values",
+                            "imgs_sizes",
+                            "num_tokens",
+                            "num_patches",
+                        )
+                    }
+                    for processor_output, _, _ in processed_images
+                ],
+            },
+        )
+        return replace(rendered, multi_modal_data=mm_data)
+
+    def bridge_to_next_turn(
+        self,
+        previous_prompt_ids: list[int],
+        previous_completion_ids: list[int],
+        new_messages: list[Message],
+        *,
+        tools: list[ToolSpec] | None = None,
+        previous_multi_modal_data: MultiModalData | None = None,
+        process_multimodal: bool = True,
+    ) -> RenderedTokens | None:
+        has_new_images = any(
+            isinstance(message.get("content"), list)
+            and any(self._is_image_part(item) for item in message["content"])
+            for message in new_messages
+        )
+        if has_new_images or (
+            previous_multi_modal_data is not None
+            and not previous_multi_modal_data.is_empty()
+        ):
+            return None
+        return super().bridge_to_next_turn(
+            previous_prompt_ids,
+            previous_completion_ids,
+            new_messages,
+            tools=tools,
+        )

{renderers-0.1.12.dev6 → renderers-0.1.12.dev8}/tests/test_client.py

@@ -595,6 +595,105 @@ def test_generate_serializes_multimodal_features_for_gemma4():
     )
 
 
+def test_generate_serializes_nemotron_dynamic_image_features():
+    pytest.importorskip("torch")
+    pytest.importorskip("vllm", reason="vllm needed for features serialization")
+
+    from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import (
+        decode_mm_kwargs_item,
+    )
+    from vllm.model_executor.models.nano_nemotron_vl import NemotronH_Nano_VL_V2
+    from vllm.multimodal.inputs import MultiModalKwargsItems
+
+    from renderers.base import MultiModalData, PlaceholderRange
+    from renderers.nemotron3 import Nemotron35Renderer
+
+    class _FakeNemotronRenderer(Nemotron35Renderer):
+        def get_stop_token_ids(self):
+            return [99]
+
+        def parse_response(self, completion_ids, *, tools=None, prompt_ids=None):
+            return ParsedResponse(content="done")
+
+    renderer = object.__new__(_FakeNemotronRenderer)
+    mm_data = MultiModalData(
+        mm_hashes={"image": ["wide", "tall"]},
+        mm_placeholders={
+            "image": [
+                PlaceholderRange(offset=2, length=6),
+                PlaceholderRange(offset=10, length=6),
+            ]
+        },
+        mm_items={
+            "image": [
+                {
+                    "pixel_values": np.zeros((1, 3, 32, 48), dtype=np.float32),
+                    "imgs_sizes": np.array([[32, 48]], dtype=np.int64),
+                    "num_tokens": [6],
+                    "num_patches": [1],
+                },
+                {
+                    "pixel_values": np.ones((1, 3, 48, 32), dtype=np.float32),
+                    "imgs_sizes": np.array([[48, 32]], dtype=np.int64),
+                    "num_tokens": [6],
+                    "num_patches": [1],
+                },
+            ]
+        },
+    )
+    client = _FakeClient()
+
+    asyncio.run(
+        generate(
+            client=client,
+            renderer=renderer,
+            messages=[],
+            model="nemotron35-super",
+            prompt_ids=list(range(20)),
+            multi_modal_data=mm_data,
+            sampling_params={"max_tokens": 4},
+        )
+    )
+
+    features = client.calls[0]["body"]["features"]
+    assert features["mm_hashes"] == {"image": ["wide", "tall"]}
+    assert features["mm_placeholders"] == {
+        "image": [{"offset": 2, "length": 6}, {"offset": 10, "length": 6}]
+    }
+    decoded_items = [
+        decode_mm_kwargs_item(item) for item in features["kwargs_data"]["image"]
+    ]
+    decoded = [item.get_data() for item in decoded_items]
+    assert [tuple(item["pixel_values_flat"].shape) for item in decoded] == [
+        (3, 32, 48),
+        (3, 48, 32),
+    ]
+    assert [tuple(item["imgs_sizes"]) for item in decoded] == [
+        (32, 48),
+        (48, 32),
+    ]
+    assert [int(item["num_tokens_per_image"]) for item in decoded] == [6, 6]
+    assert all("image_num_patches" not in item for item in decoded)
+
+    kwargs = MultiModalKwargsItems({"image": decoded_items}).get_data()
+    model = object.__new__(NemotronH_Nano_VL_V2)
+    model.dynamic_resolution = True
+    model.patch_size = 16
+    image_input = model._parse_and_validate_image_input(**kwargs)
+
+    assert image_input is not None
+    assert tuple(image_input["pixel_values_flat"].shape) == (
+        1,
+        12,
+        3 * 16 * 16,
+    )
+    assert [tuple(size) for size in image_input["imgs_sizes"]] == [
+        (32, 48),
+        (48, 32),
+    ]
+    assert image_input["num_tokens_per_image"] == [6, 6]
+
+
 # ---------------------------------------------------------------------------
 # Prompt overflow handling.
 # ---------------------------------------------------------------------------

{renderers-0.1.12.dev6 → renderers-0.1.12.dev8}/tests/test_multimodal.py

@@ -105,8 +105,8 @@ _loaded: dict[str, tuple] = {}
 
 
 # Models whose processors need ``trust_remote_code=True`` (custom Python
-# in the repo) AND a pinned revision for security. Mirrors the
-# ``TRUSTED_REVISIONS`` policy in ``renderers.base`` for tokenizers.
+# in the repo) and still use a pinned revision. Nemotron 3.5 is handled
+# separately below so its selected model revision is not overridden.
 _PROCESSOR_TRUSTED_REVISIONS: dict[str, str] = {
     "moonshotai/Kimi-K2.5": "4d01dfe0332d63057c186e0b262165819efb6611",
     "moonshotai/Kimi-K2.6": "2755962d07cb42aa2d988a35bcb65cd4a9c2de82",
@@ -119,7 +119,12 @@ def _load_processor_and_renderer(model_name: str):
 
         tokenizer = load_tokenizer(model_name)
         revision = _PROCESSOR_TRUSTED_REVISIONS.get(model_name)
-        if revision is not None:
+        if model_name.startswith("nvidia/NVIDIA-Nemotron-3.5"):
+            processor = AutoProcessor.from_pretrained(
+                model_name,
+                trust_remote_code=True,
+            )
+        elif revision is not None:
             processor = AutoProcessor.from_pretrained(
                 model_name,
                 trust_remote_code=True,
@@ -173,6 +178,8 @@ def _detect_family(model_name: str) -> str:
     - ``gemma4``: canonical Gemma turn grammar plus dynamic
       ``<|image>`` + N x ``<|image|>`` + ``<image|>`` expansion.
     """
+    if model_name.startswith("nvidia/NVIDIA-Nemotron-3.5"):
+        return "nemotron35"
     if model_name.startswith("moonshotai/Kimi-K2.5") or model_name.startswith(
         "moonshotai/Kimi-K2.6"
     ):
@@ -332,6 +339,12 @@ def _modality_kit(modality: str, model_name: str):
                 "placeholder_token": "<|unused_200054|>",
                 "processor_input_ids": _qwen_vl_processor_input_ids,
             }
+        if family == "nemotron35":
+            return {
+                "make_part": _image_content_part,
+                "placeholder_token": "<image>",
+                "processor_input_ids": _qwen_vl_processor_input_ids,
+            }
         # Default: Qwen-VL family (Qwen3-VL, Qwen3.5, Qwen3.6).
         return {
             "make_part": _image_content_part,
@@ -594,9 +607,13 @@ def _supports_tool_message_images(renderer) -> bool:
     asserting against them."""
     from renderers.gemma4 import Gemma4Renderer
     from renderers.kimi_k25 import KimiK25Renderer
+    from renderers.nemotron3 import Nemotron35Renderer
     from renderers.qwen35 import Qwen35Renderer
 
-    return isinstance(renderer, (Qwen35Renderer, KimiK25Renderer, Gemma4Renderer))
+    return isinstance(
+        renderer,
+        (Qwen35Renderer, KimiK25Renderer, Gemma4Renderer, Nemotron35Renderer),
+    )
 
 
 @pytest.mark.parametrize(
@@ -715,6 +732,8 @@ def test_multimodal_bridge_extends_and_carries_mm_data(
         tokenizer,
         _config_for_model(mm_model_name, thinking_retention="all"),
     )
+    if not getattr(renderer, "supports_multimodal_bridge", True):
+        pytest.skip(f"{type(renderer).__name__} uses full multimodal re-rendering")
     if hasattr(renderer, "_processor") and renderer._processor is None:
         renderer._processor = processor
     sample = _sample_for(modality, tiny_image)

renderers-0.1.12.dev8/tests/test_nemotron35_vlm.py

@@ -0,0 +1,322 @@
+"""Focused multimodal tests for the private Nemotron 3.5 Super checkpoint."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from types import SimpleNamespace
+from typing import Any
+
+import pytest
+
+from renderers.base import MODEL_RENDERER_MAP, MULTIMODAL_MODELS, PlaceholderRange
+from renderers.nemotron3 import Nemotron35Renderer
+
+
+MODEL = "nvidia/NVIDIA-Nemotron-3.5-Super-EA-09112026"
+
+
+class _Tokenizer:
+    name_or_path = MODEL
+    unk_token_id = -1
+    eos_token_id = 2
+
+    _specials = {
+        "<|im_start|>": 1,
+        "<|im_end|>": 2,
+        "<|endoftext|>": 3,
+        "<think>": 4,
+        "</think>": 5,
+        "<tool_call>": 6,
+        "</tool_call>": 7,
+        "<tool_response>": 8,
+        "</tool_response>": 9,
+        "<img>": 10,
+        "</img>": 11,
+        "<image>": 18,
+    }
+
+    def convert_tokens_to_ids(self, token: str) -> int:
+        return self._specials.get(token, self.unk_token_id)
+
+    def encode(self, text: str, *, add_special_tokens: bool = False) -> list[int]:
+        assert not add_special_tokens
+        token_ids: list[int] = []
+        index = 0
+        specials = sorted(self._specials, key=len, reverse=True)
+        while index < len(text):
+            special = next(
+                (value for value in specials if text.startswith(value, index)), None
+            )
+            if special is not None:
+                token_ids.append(self._specials[special])
+                index += len(special)
+            else:
+                token_ids.append(1000 + ord(text[index]))
+                index += 1
+        return token_ids
+
+    def decode(self, token_ids: list[int], **_: Any) -> str:
+        reverse_specials = {value: key for key, value in self._specials.items()}
+        return "".join(
+            reverse_specials[token_id]
+            if token_id in reverse_specials
+            else chr(token_id - 1000)
+            for token_id in token_ids
+        )
+
+
+class _ImageProcessor:
+    def __init__(self) -> None:
+        self.calls = 0
+
+    def __call__(self, *, images: list[Any], return_tensors: str):
+        assert len(images) == 1
+        assert return_tensors == "np"
+        self.calls += 1
+        return {
+            "pixel_values": [[[[0.0]]]],
+            "num_tokens": [3],
+            "num_patches": [1],
+            "imgs_sizes": [(1, 1)],
+        }
+
+
+def test_private_super_is_registered_as_image_model():
+    assert MODEL_RENDERER_MAP[MODEL] == "nemotron-3.5"
+    assert MULTIMODAL_MODELS[MODEL] == {"image"}
+
+
+def test_image_content_expands_and_attaches_only_model_inputs():
+    Image = pytest.importorskip("PIL.Image")
+    image = Image.new("RGB", (4, 3), color=(10, 20, 30))
+    image_processor = _ImageProcessor()
+    processor = SimpleNamespace(image_processor=image_processor)
+    renderer = Nemotron35Renderer(_Tokenizer(), processor=processor)
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "What is shown? "},
+                {"type": "image", "image": image},
+                {"type": "text", "text": " Be concise."},
+            ],
+        }
+    ]
+
+    rendered = renderer.render(messages, add_generation_prompt=True)
+
+    assert messages[0]["content"][1]["image"] is image
+    assert image_processor.calls == 1
+    assert renderer.mm_token_type_id_map == {18: 1}
+    assert rendered.multi_modal_data is not None
+    assert rendered.multi_modal_data.mm_hashes.keys() == {"image"}
+    assert len(rendered.multi_modal_data.mm_hashes["image"]) == 1
+    placeholder = rendered.multi_modal_data.mm_placeholders["image"]
+    assert placeholder == [PlaceholderRange(offset=placeholder[0].offset, length=3)]
+    assert rendered.token_ids[placeholder[0].offset : placeholder[0].offset + 3] == [
+        18,
+        18,
+        18,
+    ]
+    assert rendered.multi_modal_data.mm_items == {
+        "image": [
+            {
+                "pixel_values": [[[[0.0]]]],
+                "num_tokens": [3],
+                "num_patches": [1],
+                "imgs_sizes": [(1, 1)],
+            }
+        ]
+    }
+
+
+def test_deferred_multimodal_processing_keeps_one_logical_image_token():
+    Image = pytest.importorskip("PIL.Image")
+    image = Image.new("RGB", (2, 2))
+    renderer = Nemotron35Renderer(
+        _Tokenizer(), processor=SimpleNamespace(image_processor=_ImageProcessor())
+    )
+
+    rendered = renderer.render(
+        [{"role": "user", "content": [{"type": "image", "image": image}]}],
+        process_multimodal=False,
+    )
+
+    assert rendered.token_ids.count(18) == 1
+    assert rendered.multi_modal_data is None
+
+
+def test_two_images_produce_two_distinct_placeholder_ranges():
+    Image = pytest.importorskip("PIL.Image")
+    renderer = Nemotron35Renderer(
+        _Tokenizer(), processor=SimpleNamespace(image_processor=_ImageProcessor())
+    )
+
+    rendered = renderer.render(
+        [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "image", "image": Image.new("RGB", (2, 2))},
+                    {"type": "text", "text": "compare"},
+                    {"type": "image", "image": Image.new("RGB", (3, 2))},
+                ],
+            }
+        ]
+    )
+
+    assert rendered.multi_modal_data is not None
+    placeholders = rendered.multi_modal_data.mm_placeholders["image"]
+    assert [placeholder.length for placeholder in placeholders] == [3, 3]
+    assert placeholders[0].offset + placeholders[0].length < placeholders[1].offset
+    assert len(rendered.multi_modal_data.mm_items["image"]) == 2
+
+
+def test_multimodal_bridge_uses_full_rerender_fallback():
+    renderer = Nemotron35Renderer(_Tokenizer())
+
+    assert not renderer.supports_multimodal_bridge
+    assert (
+        renderer.bridge_to_next_turn(
+            [1],
+            [2],
+            [{"role": "user", "content": [{"type": "image", "image": object()}]}],
+        )
+        is None
+    )
+
+
+def test_text_only_nemotron35_tokenizer_does_not_need_image_token():
+    class _TextTokenizer(_Tokenizer):
+        _specials = {
+            key: value
+            for key, value in _Tokenizer._specials.items()
+            if key != "<image>"
+        }
+
+    renderer = Nemotron35Renderer(_TextTokenizer())
+
+    assert renderer.mm_token_type_id_map == {}
+    assert renderer.render_ids([{"role": "user", "content": "Hello"}])
+
+
+@pytest.mark.parametrize(
+    "source",
+    [
+        "https://example.test/image.png",
+        "file:///tmp/image.png",
+        "/tmp/image.png",
+    ],
+)
+def test_nemotron35_rejects_network_and_filesystem_image_sources(source):
+    renderer = Nemotron35Renderer(
+        _Tokenizer(), processor=SimpleNamespace(image_processor=_ImageProcessor())
+    )
+
+    with pytest.raises(ValueError, match="in-memory images or data:image"):
+        renderer.render(
+            [
+                {
+                    "role": "user",
+                    "content": [{"type": "image_url", "image_url": {"url": source}}],
+                }
+            ]
+        )
+
+
+def test_nemotron35_rejects_unpinned_remote_processor(monkeypatch):
+    class _RemoteTokenizer(_Tokenizer):
+        name_or_path = "example/unreviewed-model"
+
+    monkeypatch.setattr(
+        "renderers.nemotron3._require_transformers",
+        lambda _: pytest.fail(
+            "unknown remote model must fail before transformers load"
+        ),
+    )
+    renderer = Nemotron35Renderer(_RemoteTokenizer())
+
+    with pytest.raises(RuntimeError, match="not approved for remote-code loading"):
+        renderer._get_processor()
+
+
+def test_nemotron35_rejects_unapproved_local_processor(tmp_path, monkeypatch):
+    class _LocalTokenizer(_Tokenizer):
+        name_or_path = str(tmp_path)
+
+    monkeypatch.setattr(
+        "renderers.nemotron3._require_transformers",
+        lambda _: pytest.fail("unapproved local model must fail before load"),
+    )
+
+    with pytest.raises(RuntimeError, match="explicitly injected reviewed processor"):
+        Nemotron35Renderer(_LocalTokenizer())._get_processor()
+
+
+def test_nemotron35_loads_approved_remote_processor_without_revision(monkeypatch):
+    calls = []
+    expected = object()
+    transformers = SimpleNamespace(
+        AutoProcessor=SimpleNamespace(
+            from_pretrained=lambda name, **kwargs: (
+                calls.append((name, kwargs)) or expected
+            )
+        )
+    )
+    monkeypatch.setattr(
+        "renderers.nemotron3._require_transformers", lambda _: transformers
+    )
+
+    assert Nemotron35Renderer(_Tokenizer())._get_processor() is expected
+    assert calls == [
+        (
+            MODEL,
+            {"trust_remote_code": True},
+        )
+    ]
+
+
+def test_real_processor_token_parity_when_metadata_is_available():
+    metadata = os.environ.get("NEMOTRON35_VLM_METADATA")
+    if metadata is None:
+        pytest.skip("set NEMOTRON35_VLM_METADATA to the downloaded model metadata")
+    metadata_path = Path(metadata)
+    if not metadata_path.is_dir():
+        pytest.skip(f"Nemotron metadata directory does not exist: {metadata_path}")
+
+    Image = pytest.importorskip("PIL.Image")
+    transformers = pytest.importorskip("transformers")
+    tokenizer = transformers.AutoTokenizer.from_pretrained(metadata_path)
+    processor = transformers.AutoProcessor.from_pretrained(
+        metadata_path, trust_remote_code=True
+    )
+    renderer = Nemotron35Renderer(tokenizer, processor=processor)
+    images = [
+        Image.new("RGB", (2048, 256), color=(10, 20, 30)),
+        Image.new("RGB", (256, 2048), color=(30, 20, 10)),
+    ]
+    for selected_images in (images[:1], images):
+        content = [{"type": "text", "text": "Describe: "}]
+        content.extend({"type": "image", "image": image} for image in selected_images)
+        messages = [{"role": "user", "content": content}]
+
+        rendered = renderer.render(messages, add_generation_prompt=True)
+        prompt = processor.apply_chat_template(
+            messages, tokenize=False, add_generation_prompt=True
+        )
+        expected = processor(
+            images=selected_images,
+            text=prompt,
+            return_tensors=None,
+        )["input_ids"][0]
+
+        assert rendered.token_ids == expected
+        assert rendered.multi_modal_data is not None
+        assert len(rendered.multi_modal_data.mm_items["image"]) == len(selected_images)
+        if len(selected_images) == 2:
+            shapes = [
+                tuple(item["pixel_values"].shape)
+                for item in rendered.multi_modal_data.mm_items["image"]
+            ]
+            assert shapes[0] != shapes[1]

{renderers-0.1.12.dev6 → renderers-0.1.12.dev8}/uv.lock

@@ -33,16 +33,16 @@ wheels = [
 
 [[package]]
 name = "anyio"
-version = "4.13.0"
+version = "4.14.2"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "exceptiongroup", marker = "python_full_version < '3.11'" },
     { name = "idna" },
     { name = "typing-extensions", marker = "python_full_version < '3.13'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/19/14/2c5dd9f512b66549ae92767a9c7b330ae88e1932ca57876909410251fe13/anyio-4.13.0.tar.gz", hash = "sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc", size = 231622, upload-time = "2026-03-24T12:59:09.671Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353, upload-time = "2026-03-24T12:59:08.246Z" },
+    { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" },
 ]
 
 [[package]]