renderers 0.1.12.dev7__tar.gz → 0.1.12.dev9__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/PKG-INFO +1 -1
  2. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/renderers/_version.py +2 -2
  3. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/renderers/base.py +2 -0
  4. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/renderers/client.py +76 -0
  5. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/renderers/nemotron3.py +252 -0
  6. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/renderers/qwen3.py +18 -7
  7. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/test_bridge.py +18 -0
  8. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/test_client.py +99 -0
  9. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/test_multimodal.py +23 -4
  10. renderers-0.1.12.dev9/tests/test_nemotron35_vlm.py +322 -0
  11. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/.github/workflows/publish-dev.yml +0 -0
  12. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/.github/workflows/publish.yml +0 -0
  13. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/.github/workflows/style.yml +0 -0
  14. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/.github/workflows/test.yml +0 -0
  15. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/.gitignore +0 -0
  16. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/.pre-commit-config.yaml +0 -0
  17. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/LICENSE +0 -0
  18. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/README.md +0 -0
  19. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/docs/renderer-config.md +0 -0
  20. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/examples/README.md +0 -0
  21. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/examples/sglang/multiturn_generate_sglang.py +0 -0
  22. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/examples/sglang/online_multiturn_sglang.py +0 -0
  23. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/examples/tinker/multiturn_generate_tinker.py +0 -0
  24. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/examples/transformers/multiturn_generate_transformers.py +0 -0
  25. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/examples/vllm/multiturn_generate_vllm.py +0 -0
  26. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/pyproject.toml +0 -0
  27. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/renderers/__init__.py +0 -0
  28. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/renderers/configs.py +0 -0
  29. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/renderers/deepseek_r1.py +0 -0
  30. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/renderers/deepseek_v3.py +0 -0
  31. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/renderers/deepseek_v4.py +0 -0
  32. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/renderers/default.py +0 -0
  33. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/renderers/gemma4.py +0 -0
  34. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/renderers/glm45.py +0 -0
  35. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/renderers/glm5.py +0 -0
  36. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/renderers/gpt_oss.py +0 -0
  37. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/renderers/hy3.py +0 -0
  38. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/renderers/inkling.py +0 -0
  39. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/renderers/kimi_k2.py +0 -0
  40. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/renderers/kimi_k25.py +0 -0
  41. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/renderers/laguna_s21.py +0 -0
  42. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/renderers/laguna_xs2.py +0 -0
  43. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/renderers/llama_3.py +0 -0
  44. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/renderers/minimax_m2.py +0 -0
  45. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/renderers/parsers.py +0 -0
  46. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/renderers/parsing.py +0 -0
  47. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/renderers/prime_qwen3.py +0 -0
  48. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/renderers/qwen35.py +0 -0
  49. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/renderers/qwen36.py +0 -0
  50. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/renderers/qwen38.py +0 -0
  51. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/renderers/qwen3_vl.py +0 -0
  52. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/renderers/reasoning.py +0 -0
  53. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/conftest.py +0 -0
  54. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/parity.py +0 -0
  55. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/reference_rendering.py +0 -0
  56. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/test_build_helpers.py +0 -0
  57. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/test_deepseek_r1.py +0 -0
  58. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/test_deepseek_v4.py +0 -0
  59. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/test_disabled_thinking_stability.py +0 -0
  60. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/test_gemma4.py +0 -0
  61. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/test_glm_tool_name_validation.py +0 -0
  62. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/test_hy3.py +0 -0
  63. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/test_incremental.py +0 -0
  64. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/test_inkling.py +0 -0
  65. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/test_is_content.py +0 -0
  66. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/test_kimi_k25_tool_schema.py +0 -0
  67. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/test_laguna_m1.py +0 -0
  68. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/test_laguna_xs21.py +0 -0
  69. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/test_llama_3.py +0 -0
  70. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/test_load_tokenizer.py +0 -0
  71. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/test_message_indices.py +0 -0
  72. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/test_message_tool_names.py +0 -0
  73. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/test_nemotron3_ultra.py +0 -0
  74. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/test_offsetless_tokenizers.py +0 -0
  75. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/test_parity.py +0 -0
  76. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/test_parse_response.py +0 -0
  77. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/test_parse_response_robustness.py +0 -0
  78. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/test_parsers.py +0 -0
  79. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/test_preserve_thinking.py +0 -0
  80. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/test_qwen35_size_coverage.py +0 -0
  81. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/test_qwen38.py +0 -0
  82. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/test_reasoning_boundaries.py +0 -0
  83. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/test_renderer_config.py +0 -0
  84. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/test_roundtrip.py +0 -0
  85. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/test_sampled_mask.py +0 -0
  86. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/test_tokens_per_message.py +0 -0
  87. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/tests/test_tool_arg_type_preservation.py +0 -0
  88. {renderers-0.1.12.dev7 → renderers-0.1.12.dev9}/uv.lock +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: renderers
3
- Version: 0.1.12.dev7
3
+ Version: 0.1.12.dev9
4
4
  Summary: Chat template renderers — deterministic message-to-token conversion for LLM training
5
5
  License-Expression: Apache-2.0
6
6
  License-File: LICENSE
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
18
18
  commit_id: str | None
19
19
  __commit_id__: str | None
20
20
 
21
- __version__ = version = '0.1.12.dev7'
22
- __version_tuple__ = version_tuple = (0, 1, 12, 'dev7')
21
+ __version__ = version = '0.1.12.dev9'
22
+ __version_tuple__ = version_tuple = (0, 1, 12, 'dev9')
23
23
 
24
24
  __commit_id__ = commit_id = None
@@ -1017,6 +1017,7 @@ MODEL_RENDERER_MAP: dict[str, str] = {
1017
1017
  # Nemotron 3.5 (Lightning). Its template is the Ultra variant's minus the
1018
1018
  # effort kwarg (``nemotron-3.5``).
1019
1019
  "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16": "nemotron-3.5",
1020
+ "nvidia/NVIDIA-Nemotron-3.5-Super-EA-09112026": "nemotron-3.5",
1020
1021
  # Llama 3.2 (Instruct). Tested against the gated meta-llama repos and
1021
1022
  # the unrestricted unsloth/... mirror, which ships a byte-identical
1022
1023
  # chat template. ``Llama3Renderer`` defaults ``date_string`` to
@@ -1057,6 +1058,7 @@ MODEL_RENDERER_MAP: dict[str, str] = {
1057
1058
  # Modality values: ``"image"``, ``"video"``, ``"audio"``. Text is implicit
1058
1059
  # (every model supports it), so it doesn't appear in the set.
1059
1060
  MULTIMODAL_MODELS: dict[str, set[str]] = {
1061
+ "nvidia/NVIDIA-Nemotron-3.5-Super-EA-09112026": {"image"},
1060
1062
  "Qwen/Qwen3-VL-4B-Instruct": {"image"},
1061
1063
  "Qwen/Qwen3-VL-8B-Instruct": {"image"},
1062
1064
  "Qwen/Qwen3-VL-30B-A3B-Instruct": {"image"},
@@ -486,6 +486,7 @@ def _build_mm_features(
486
486
  to change. Don't pre-build the abstraction with one engine in tree.
487
487
  """
488
488
  from renderers.gemma4 import Gemma4Renderer
489
+ from renderers.nemotron3 import Nemotron35Renderer
489
490
  from renderers.qwen3_vl import Qwen3VLRenderer
490
491
  from renderers.qwen35 import Qwen35Renderer
491
492
 
@@ -498,6 +499,8 @@ def _build_mm_features(
498
499
  return _build_qwen_vl_features(mm_data, spatial_merge_size=2)
499
500
  if issubclass(renderer_cls, Gemma4Renderer):
500
501
  return _build_gemma4_features(mm_data)
502
+ if issubclass(renderer_cls, Nemotron35Renderer):
503
+ return _build_nemotron35_features(mm_data)
501
504
 
502
505
  raise NotImplementedError(
503
506
  f"Multimodal serialization not implemented for {renderer_cls.__name__}. "
@@ -505,6 +508,79 @@ def _build_mm_features(
505
508
  )
506
509
 
507
510
 
511
+ def _build_nemotron35_features(mm_data: MultiModalData) -> dict[str, Any]:
512
+ """Encode dynamic-resolution Nemotron image inputs for vLLM."""
513
+ _require_transformers("Encoding Nemotron 3.5 multimodal features for vLLM")
514
+ try:
515
+ import torch
516
+ from transformers.feature_extraction_utils import BatchFeature
517
+ from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import (
518
+ encode_mm_kwargs_item,
519
+ )
520
+ from vllm.multimodal.inputs import (
521
+ MultiModalFieldConfig,
522
+ MultiModalKwargsItems,
523
+ )
524
+ except ImportError as exc:
525
+ raise RuntimeError(
526
+ "Nemotron 3.5 multimodal generate requires vLLM and torch."
527
+ ) from exc
528
+
529
+ out: dict[str, Any] = {
530
+ "mm_hashes": {},
531
+ "mm_placeholders": {},
532
+ "kwargs_data": {},
533
+ }
534
+ image_items = mm_data.mm_items.get("image") or []
535
+ if image_items:
536
+ pixel_values_flat = []
537
+ imgs_sizes = []
538
+ num_tokens_per_image = []
539
+ for item in image_items:
540
+ pixel_values = torch.as_tensor(item["pixel_values"])
541
+ if pixel_values.ndim != 4 or pixel_values.shape[0] != 1:
542
+ raise ValueError(
543
+ "Each Nemotron pixel_values item must have shape (1, C, H, W)."
544
+ )
545
+ sizes = item["imgs_sizes"]
546
+ token_counts = item["num_tokens"]
547
+ if not (len(sizes) == len(token_counts) == 1):
548
+ raise ValueError(
549
+ "Each Nemotron image item must contain one size and token count."
550
+ )
551
+ pixel_values_flat.append(pixel_values[0])
552
+ imgs_sizes.append(tuple(int(value) for value in sizes[0]))
553
+ num_tokens_per_image.append(int(token_counts[0]))
554
+
555
+ hf_inputs = BatchFeature(
556
+ data={
557
+ "pixel_values_flat": pixel_values_flat,
558
+ "imgs_sizes": imgs_sizes,
559
+ "num_tokens_per_image": num_tokens_per_image,
560
+ }
561
+ )
562
+ field_config = {
563
+ "pixel_values_flat": MultiModalFieldConfig.batched("image"),
564
+ "num_tokens_per_image": MultiModalFieldConfig.batched(
565
+ "image", keep_on_cpu=True
566
+ ),
567
+ "imgs_sizes": MultiModalFieldConfig.batched("image", keep_on_cpu=True),
568
+ }
569
+ kwargs_items = MultiModalKwargsItems.from_hf_inputs(hf_inputs, field_config)
570
+ out["kwargs_data"]["image"] = [
571
+ encode_mm_kwargs_item(item) for item in kwargs_items["image"]
572
+ ]
573
+ out["mm_hashes"]["image"] = list(mm_data.mm_hashes.get("image") or [])
574
+ out["mm_placeholders"]["image"] = [
575
+ {"offset": placeholder.offset, "length": placeholder.length}
576
+ for placeholder in mm_data.mm_placeholders.get("image") or []
577
+ ]
578
+
579
+ if not any(out["kwargs_data"].values()):
580
+ out["kwargs_data"] = None
581
+ return out
582
+
583
+
508
584
  def _build_gemma4_features(mm_data: MultiModalData) -> dict[str, Any]:
509
585
  """vLLM features payload for Gemma 4 image inputs.
510
586
 
@@ -14,18 +14,23 @@ Nemotron 3 uses the same <|im_start|>/<|im_end|> format as Qwen3.5 but differs i
14
14
 
15
15
  from __future__ import annotations
16
16
 
17
+ from dataclasses import replace
17
18
  import json
19
+ from pathlib import Path
18
20
  from typing import Any
19
21
 
20
22
  from renderers.reasoning import scan_reasoning, prompt_ends_in_reasoning
21
23
 
22
24
  from renderers.base import (
23
25
  Message,
26
+ MultiModalData,
24
27
  ParsedResponse,
28
+ PlaceholderRange,
25
29
  RenderedTokens,
26
30
  ToolSpec,
27
31
  Tokenizer,
28
32
  _content_mask_or_empty,
33
+ _require_transformers,
29
34
  attribute_text_segments,
30
35
  extract_message_tool_names,
31
36
  reject_assistant_in_extension,
@@ -39,6 +44,7 @@ from renderers.configs import (
39
44
  Nemotron35RendererConfig,
40
45
  )
41
46
  from renderers.parsing import parse_qwen35
47
+ from renderers.qwen3_vl import _image_hash, _is_image_part, _load_pil_image
42
48
 
43
49
  # ---------------------------------------------------------------------------
44
50
  # Tool system prompt constants
@@ -894,3 +900,249 @@ class Nemotron35Renderer(Nemotron3Renderer):
894
900
 
895
901
  _config_cls = Nemotron35RendererConfig
896
902
  _ultra = True
903
+ supports_process_multimodal = True
904
+ supports_multimodal_bridge = False
905
+
906
+ _processor_model_prefix = "nvidia/NVIDIA-Nemotron-3.5"
907
+
908
+ def __init__(
909
+ self,
910
+ tokenizer: Tokenizer,
911
+ config: Nemotron35RendererConfig | None = None,
912
+ *,
913
+ processor: Any = None,
914
+ ):
915
+ super().__init__(tokenizer, config)
916
+ self._processor = processor
917
+ self._image = self._token_id("<image>", optional=True)
918
+
919
+ @property
920
+ def mm_token_type_id_map(self) -> dict[int, int]:
921
+ return {self._image: 1} if self._image is not None else {}
922
+
923
+ def _image_token_id(self) -> int:
924
+ if self._image is None:
925
+ raise RuntimeError("This Nemotron 3.5 tokenizer has no <image> token.")
926
+ return self._image
927
+
928
+ def _get_processor(self):
929
+ if self._processor is not None:
930
+ return self._processor
931
+ name = getattr(self._tokenizer, "name_or_path", None)
932
+ if not name:
933
+ raise RuntimeError(
934
+ "Nemotron35Renderer needs a processor for image content."
935
+ )
936
+ if Path(name).expanduser().is_dir():
937
+ raise RuntimeError(
938
+ "Local Nemotron remote-code processor paths require an explicitly "
939
+ "injected reviewed processor."
940
+ )
941
+ if not name.startswith(self._processor_model_prefix):
942
+ raise RuntimeError(
943
+ f"Nemotron processor {name!r} is not approved for remote-code "
944
+ "loading. Inject a reviewed processor."
945
+ )
946
+ transformers = _require_transformers("Auto-loading a Nemotron 3.5 processor")
947
+ self._processor = transformers.AutoProcessor.from_pretrained(
948
+ name, trust_remote_code=True
949
+ )
950
+ return self._processor
951
+
952
+ @staticmethod
953
+ def _is_image_part(item: Any) -> bool:
954
+ return _is_image_part(item) or (
955
+ isinstance(item, dict) and item.get("type") == "input_image"
956
+ )
957
+
958
+ @staticmethod
959
+ def _image_source(part: dict[str, Any]) -> dict[str, Any]:
960
+ if "input_image" not in part:
961
+ return part
962
+ return {**part, "image": part["input_image"]}
963
+
964
+ @classmethod
965
+ def _logical_multimodal_content(
966
+ cls, content: list[Any]
967
+ ) -> tuple[str, list[dict[str, Any]]]:
968
+ text = "".join(
969
+ item if isinstance(item, str) else str(item.get("text") or "")
970
+ for item in content
971
+ if isinstance(item, str)
972
+ or (isinstance(item, dict) and item.get("type") in ("text", "input_text"))
973
+ )
974
+ image_parts = [
975
+ item
976
+ for item in content
977
+ if isinstance(item, dict) and cls._is_image_part(item)
978
+ ]
979
+ emit_structured_images = "<image>" not in text
980
+ rendered: list[str] = []
981
+ image_index = 0
982
+ for item in content:
983
+ if isinstance(item, str):
984
+ rendered.append(item)
985
+ elif not isinstance(item, dict):
986
+ raise ValueError(f"Unexpected content item: {item}")
987
+ elif cls._is_image_part(item):
988
+ image_index += 1
989
+ if emit_structured_images:
990
+ if len(image_parts) > 1:
991
+ rendered.append(f"<image {image_index}><image>")
992
+ else:
993
+ rendered.append("<image>")
994
+ elif item.get("type") in ("video", "video_url", "input_video"):
995
+ raise NotImplementedError(
996
+ "Video parts are not yet supported by Nemotron35Renderer."
997
+ )
998
+ elif item.get("type") in ("text", "input_text") or "text" in item:
999
+ rendered.append(str(item.get("text") or ""))
1000
+ return "".join(rendered), image_parts
1001
+
1002
+ @staticmethod
1003
+ def _load_image(part: dict[str, Any]):
1004
+ source = Nemotron35Renderer._image_source(part)
1005
+ if "image" in source:
1006
+ raw = source["image"]
1007
+ elif "image_url" in source:
1008
+ image_url = source.get("image_url")
1009
+ raw = image_url.get("url") if isinstance(image_url, dict) else image_url
1010
+ else:
1011
+ raw = source.get("url") or source.get("path")
1012
+ if isinstance(raw, str) and not raw.startswith("data:image/"):
1013
+ raise ValueError(
1014
+ "Nemotron 3.5 accepts in-memory images or data:image URIs; "
1015
+ "network and filesystem image sources are disabled."
1016
+ )
1017
+ return _load_pil_image(source)
1018
+
1019
+ def _process_image(self, part: dict[str, Any]) -> tuple[dict[str, Any], int, str]:
1020
+ self._image_token_id()
1021
+ image = self._load_image(part)
1022
+ output = self._get_processor().image_processor(
1023
+ images=[image], return_tensors="np"
1024
+ )
1025
+ return output, int(output["num_tokens"][0]), _image_hash(image)
1026
+
1027
+ @staticmethod
1028
+ def _expand_image_tokens(content: str, token_counts: list[int]) -> str:
1029
+ if content.count("<image>") != len(token_counts):
1030
+ raise ValueError(
1031
+ "Nemotron image content must contain one <image> placeholder per image."
1032
+ )
1033
+ segments = content.split("<image>")
1034
+ expanded = [segments[0]]
1035
+ for count, suffix in zip(token_counts, segments[1:]):
1036
+ expanded.append("<img>" + "<image>" * count + "</img>" + suffix)
1037
+ return "".join(expanded)
1038
+
1039
+ def _transform_multimodal_messages(
1040
+ self,
1041
+ messages: list[Message],
1042
+ *,
1043
+ process_multimodal: bool,
1044
+ ) -> tuple[list[Message], list[tuple[dict[str, Any], int, str]]]:
1045
+ transformed: list[Message] = []
1046
+ processed_images: list[tuple[dict[str, Any], int, str]] = []
1047
+ for message in messages:
1048
+ copied = dict(message)
1049
+ content = message.get("content")
1050
+ if isinstance(content, list) and any(
1051
+ self._is_image_part(item) for item in content
1052
+ ):
1053
+ logical_content, image_parts = self._logical_multimodal_content(content)
1054
+ if process_multimodal:
1055
+ current_images = [self._process_image(part) for part in image_parts]
1056
+ logical_content = self._expand_image_tokens(
1057
+ logical_content, [item[1] for item in current_images]
1058
+ )
1059
+ processed_images.extend(current_images)
1060
+ copied["content"] = logical_content
1061
+ transformed.append(copied)
1062
+ return transformed, processed_images
1063
+
1064
+ def render(
1065
+ self,
1066
+ messages: list[Message],
1067
+ *,
1068
+ tools: list[ToolSpec] | None = None,
1069
+ add_generation_prompt: bool = False,
1070
+ process_multimodal: bool = True,
1071
+ ) -> RenderedTokens:
1072
+ transformed, processed_images = self._transform_multimodal_messages(
1073
+ messages, process_multimodal=process_multimodal
1074
+ )
1075
+
1076
+ rendered = super().render(
1077
+ transformed,
1078
+ tools=tools,
1079
+ add_generation_prompt=add_generation_prompt,
1080
+ )
1081
+ if not processed_images:
1082
+ return rendered
1083
+
1084
+ image_token_id = self._image_token_id()
1085
+ placeholder_runs: list[PlaceholderRange] = []
1086
+ cursor = 0
1087
+ for _, length, _ in processed_images:
1088
+ while (
1089
+ cursor < len(rendered.token_ids)
1090
+ and rendered.token_ids[cursor] != image_token_id
1091
+ ):
1092
+ cursor += 1
1093
+ if (
1094
+ rendered.token_ids[cursor : cursor + length]
1095
+ != [image_token_id] * length
1096
+ ):
1097
+ raise ValueError(
1098
+ "Rendered Nemotron image placeholders are not contiguous."
1099
+ )
1100
+ placeholder_runs.append(PlaceholderRange(offset=cursor, length=length))
1101
+ cursor += length
1102
+
1103
+ mm_data = MultiModalData(
1104
+ mm_hashes={"image": [item[2] for item in processed_images]},
1105
+ mm_placeholders={"image": placeholder_runs},
1106
+ mm_items={
1107
+ "image": [
1108
+ {
1109
+ key: processor_output[key]
1110
+ for key in (
1111
+ "pixel_values",
1112
+ "imgs_sizes",
1113
+ "num_tokens",
1114
+ "num_patches",
1115
+ )
1116
+ }
1117
+ for processor_output, _, _ in processed_images
1118
+ ],
1119
+ },
1120
+ )
1121
+ return replace(rendered, multi_modal_data=mm_data)
1122
+
1123
+ def bridge_to_next_turn(
1124
+ self,
1125
+ previous_prompt_ids: list[int],
1126
+ previous_completion_ids: list[int],
1127
+ new_messages: list[Message],
1128
+ *,
1129
+ tools: list[ToolSpec] | None = None,
1130
+ previous_multi_modal_data: MultiModalData | None = None,
1131
+ process_multimodal: bool = True,
1132
+ ) -> RenderedTokens | None:
1133
+ has_new_images = any(
1134
+ isinstance(message.get("content"), list)
1135
+ and any(self._is_image_part(item) for item in message["content"])
1136
+ for message in new_messages
1137
+ )
1138
+ if has_new_images or (
1139
+ previous_multi_modal_data is not None
1140
+ and not previous_multi_modal_data.is_empty()
1141
+ ):
1142
+ return None
1143
+ return super().bridge_to_next_turn(
1144
+ previous_prompt_ids,
1145
+ previous_completion_ids,
1146
+ new_messages,
1147
+ tools=tools,
1148
+ )
@@ -93,9 +93,16 @@ class Qwen3Renderer:
93
93
  return self._tokenizer.encode(text, add_special_tokens=False)
94
94
 
95
95
  @staticmethod
96
- def _query_boundary_text(content) -> str:
96
+ def _content_text(content) -> str:
97
+ """Return the text parts of a message in a renderer-safe form.
98
+
99
+ Qwen3's native template is text-only, but text-only messages can also
100
+ use OpenAI-style content lists: ``[{"type": "text", "text": ...}]``.
101
+ Treat both representations identically instead of silently dropping
102
+ the message body.
103
+ """
97
104
  if isinstance(content, str):
98
- return content.strip()
105
+ return content
99
106
  if isinstance(content, list):
100
107
  parts: list[str] = []
101
108
  for item in content:
@@ -103,9 +110,13 @@ class Qwen3Renderer:
103
110
  parts.append(item)
104
111
  elif isinstance(item, dict) and isinstance(item.get("text"), str):
105
112
  parts.append(item["text"])
106
- return "".join(parts).strip()
113
+ return "".join(parts)
107
114
  return ""
108
115
 
116
+ @staticmethod
117
+ def _query_boundary_text(content) -> str:
118
+ return Qwen3Renderer._content_text(content).strip()
119
+
109
120
  @staticmethod
110
121
  def _is_user_query_message(msg: Message) -> bool:
111
122
  if msg.get("role") != "user":
@@ -186,7 +197,7 @@ class Qwen3Renderer:
186
197
  # JSON as message body.
187
198
  segments: list[tuple[str, bool]] = [("system\n", False)]
188
199
  if first_is_system:
189
- sys_content = messages[0].get("content") or ""
200
+ sys_content = self._content_text(messages[0].get("content"))
190
201
  if sys_content:
191
202
  segments.append((sys_content, True))
192
203
  segments.append(("\n\n", False))
@@ -199,7 +210,7 @@ class Qwen3Renderer:
199
210
  emit_text("\n", sys_idx, is_sampled=False, is_content=False)
200
211
  elif first_is_system:
201
212
  emit_special(self._im_start, 0, is_sampled=False, is_content=False)
202
- sys_content = messages[0].get("content") or ""
213
+ sys_content = self._content_text(messages[0].get("content"))
203
214
  sys_segments: list[tuple[str, bool]] = [("system\n", False)]
204
215
  if sys_content:
205
216
  sys_segments.append((sys_content, True))
@@ -214,7 +225,7 @@ class Qwen3Renderer:
214
225
  num_messages = len(messages)
215
226
  for i, msg in enumerate(messages):
216
227
  role = msg["role"]
217
- content = msg.get("content") if isinstance(msg.get("content"), str) else ""
228
+ content = self._content_text(msg.get("content"))
218
229
 
219
230
  if role == "system":
220
231
  if i == 0:
@@ -415,7 +426,7 @@ class Qwen3Renderer:
415
426
 
416
427
  for i, msg in enumerate(new_messages):
417
428
  role = msg.get("role")
418
- content = msg.get("content") if isinstance(msg.get("content"), str) else ""
429
+ content = self._content_text(msg.get("content"))
419
430
  if role == "user":
420
431
  emit_special(self._im_start, i)
421
432
  user_segments: list[tuple[str, bool]] = [("user\n", False)]
@@ -205,6 +205,24 @@ def test_bridge_extension_includes_new_message_text(
205
205
  )
206
206
 
207
207
 
208
+ def test_qwen3_renders_openai_text_content_parts(br_renderer, br_tokenizer, br_model):
209
+ """Text content lists must not become an empty user turn."""
210
+ if br_model not in {"Qwen/Qwen3-8B", "Qwen/Qwen3-4B"}:
211
+ pytest.skip(f"{br_model}: not a Qwen3 text renderer")
212
+
213
+ text = "CONTENT_LIST_SENTINEL_XYZ"
214
+ string_ids = br_renderer.render_ids(
215
+ [{"role": "user", "content": text}], add_generation_prompt=True
216
+ )
217
+ list_ids = br_renderer.render_ids(
218
+ [{"role": "user", "content": [{"type": "text", "text": text}]}],
219
+ add_generation_prompt=True,
220
+ )
221
+
222
+ assert list_ids == string_ids
223
+ assert text in br_tokenizer.decode(list_ids, skip_special_tokens=False)
224
+
225
+
208
226
  def test_bridge_declines_across_user_query_when_template_drops_thinking():
209
227
  """Qwen3's template drops a past block's thinking once a new user turn
210
228
  arrives. The resolved ``tool_cycle`` bridge policy therefore treats a
@@ -595,6 +595,105 @@ def test_generate_serializes_multimodal_features_for_gemma4():
595
595
  )
596
596
 
597
597
 
598
+ def test_generate_serializes_nemotron_dynamic_image_features():
599
+ pytest.importorskip("torch")
600
+ pytest.importorskip("vllm", reason="vllm needed for features serialization")
601
+
602
+ from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import (
603
+ decode_mm_kwargs_item,
604
+ )
605
+ from vllm.model_executor.models.nano_nemotron_vl import NemotronH_Nano_VL_V2
606
+ from vllm.multimodal.inputs import MultiModalKwargsItems
607
+
608
+ from renderers.base import MultiModalData, PlaceholderRange
609
+ from renderers.nemotron3 import Nemotron35Renderer
610
+
611
+ class _FakeNemotronRenderer(Nemotron35Renderer):
612
+ def get_stop_token_ids(self):
613
+ return [99]
614
+
615
+ def parse_response(self, completion_ids, *, tools=None, prompt_ids=None):
616
+ return ParsedResponse(content="done")
617
+
618
+ renderer = object.__new__(_FakeNemotronRenderer)
619
+ mm_data = MultiModalData(
620
+ mm_hashes={"image": ["wide", "tall"]},
621
+ mm_placeholders={
622
+ "image": [
623
+ PlaceholderRange(offset=2, length=6),
624
+ PlaceholderRange(offset=10, length=6),
625
+ ]
626
+ },
627
+ mm_items={
628
+ "image": [
629
+ {
630
+ "pixel_values": np.zeros((1, 3, 32, 48), dtype=np.float32),
631
+ "imgs_sizes": np.array([[32, 48]], dtype=np.int64),
632
+ "num_tokens": [6],
633
+ "num_patches": [1],
634
+ },
635
+ {
636
+ "pixel_values": np.ones((1, 3, 48, 32), dtype=np.float32),
637
+ "imgs_sizes": np.array([[48, 32]], dtype=np.int64),
638
+ "num_tokens": [6],
639
+ "num_patches": [1],
640
+ },
641
+ ]
642
+ },
643
+ )
644
+ client = _FakeClient()
645
+
646
+ asyncio.run(
647
+ generate(
648
+ client=client,
649
+ renderer=renderer,
650
+ messages=[],
651
+ model="nemotron35-super",
652
+ prompt_ids=list(range(20)),
653
+ multi_modal_data=mm_data,
654
+ sampling_params={"max_tokens": 4},
655
+ )
656
+ )
657
+
658
+ features = client.calls[0]["body"]["features"]
659
+ assert features["mm_hashes"] == {"image": ["wide", "tall"]}
660
+ assert features["mm_placeholders"] == {
661
+ "image": [{"offset": 2, "length": 6}, {"offset": 10, "length": 6}]
662
+ }
663
+ decoded_items = [
664
+ decode_mm_kwargs_item(item) for item in features["kwargs_data"]["image"]
665
+ ]
666
+ decoded = [item.get_data() for item in decoded_items]
667
+ assert [tuple(item["pixel_values_flat"].shape) for item in decoded] == [
668
+ (3, 32, 48),
669
+ (3, 48, 32),
670
+ ]
671
+ assert [tuple(item["imgs_sizes"]) for item in decoded] == [
672
+ (32, 48),
673
+ (48, 32),
674
+ ]
675
+ assert [int(item["num_tokens_per_image"]) for item in decoded] == [6, 6]
676
+ assert all("image_num_patches" not in item for item in decoded)
677
+
678
+ kwargs = MultiModalKwargsItems({"image": decoded_items}).get_data()
679
+ model = object.__new__(NemotronH_Nano_VL_V2)
680
+ model.dynamic_resolution = True
681
+ model.patch_size = 16
682
+ image_input = model._parse_and_validate_image_input(**kwargs)
683
+
684
+ assert image_input is not None
685
+ assert tuple(image_input["pixel_values_flat"].shape) == (
686
+ 1,
687
+ 12,
688
+ 3 * 16 * 16,
689
+ )
690
+ assert [tuple(size) for size in image_input["imgs_sizes"]] == [
691
+ (32, 48),
692
+ (48, 32),
693
+ ]
694
+ assert image_input["num_tokens_per_image"] == [6, 6]
695
+
696
+
598
697
  # ---------------------------------------------------------------------------
599
698
  # Prompt overflow handling.
600
699
  # ---------------------------------------------------------------------------
@@ -105,8 +105,8 @@ _loaded: dict[str, tuple] = {}
105
105
 
106
106
 
107
107
  # Models whose processors need ``trust_remote_code=True`` (custom Python
108
- # in the repo) AND a pinned revision for security. Mirrors the
109
- # ``TRUSTED_REVISIONS`` policy in ``renderers.base`` for tokenizers.
108
+ # in the repo) and still use a pinned revision. Nemotron 3.5 is handled
109
+ # separately below so its selected model revision is not overridden.
110
110
  _PROCESSOR_TRUSTED_REVISIONS: dict[str, str] = {
111
111
  "moonshotai/Kimi-K2.5": "4d01dfe0332d63057c186e0b262165819efb6611",
112
112
  "moonshotai/Kimi-K2.6": "2755962d07cb42aa2d988a35bcb65cd4a9c2de82",
@@ -119,7 +119,12 @@ def _load_processor_and_renderer(model_name: str):
119
119
 
120
120
  tokenizer = load_tokenizer(model_name)
121
121
  revision = _PROCESSOR_TRUSTED_REVISIONS.get(model_name)
122
- if revision is not None:
122
+ if model_name.startswith("nvidia/NVIDIA-Nemotron-3.5"):
123
+ processor = AutoProcessor.from_pretrained(
124
+ model_name,
125
+ trust_remote_code=True,
126
+ )
127
+ elif revision is not None:
123
128
  processor = AutoProcessor.from_pretrained(
124
129
  model_name,
125
130
  trust_remote_code=True,
@@ -173,6 +178,8 @@ def _detect_family(model_name: str) -> str:
173
178
  - ``gemma4``: canonical Gemma turn grammar plus dynamic
174
179
  ``<|image>`` + N x ``<|image|>`` + ``<image|>`` expansion.
175
180
  """
181
+ if model_name.startswith("nvidia/NVIDIA-Nemotron-3.5"):
182
+ return "nemotron35"
176
183
  if model_name.startswith("moonshotai/Kimi-K2.5") or model_name.startswith(
177
184
  "moonshotai/Kimi-K2.6"
178
185
  ):
@@ -332,6 +339,12 @@ def _modality_kit(modality: str, model_name: str):
332
339
  "placeholder_token": "<|unused_200054|>",
333
340
  "processor_input_ids": _qwen_vl_processor_input_ids,
334
341
  }
342
+ if family == "nemotron35":
343
+ return {
344
+ "make_part": _image_content_part,
345
+ "placeholder_token": "<image>",
346
+ "processor_input_ids": _qwen_vl_processor_input_ids,
347
+ }
335
348
  # Default: Qwen-VL family (Qwen3-VL, Qwen3.5, Qwen3.6).
336
349
  return {
337
350
  "make_part": _image_content_part,
@@ -594,9 +607,13 @@ def _supports_tool_message_images(renderer) -> bool:
594
607
  asserting against them."""
595
608
  from renderers.gemma4 import Gemma4Renderer
596
609
  from renderers.kimi_k25 import KimiK25Renderer
610
+ from renderers.nemotron3 import Nemotron35Renderer
597
611
  from renderers.qwen35 import Qwen35Renderer
598
612
 
599
- return isinstance(renderer, (Qwen35Renderer, KimiK25Renderer, Gemma4Renderer))
613
+ return isinstance(
614
+ renderer,
615
+ (Qwen35Renderer, KimiK25Renderer, Gemma4Renderer, Nemotron35Renderer),
616
+ )
600
617
 
601
618
 
602
619
  @pytest.mark.parametrize(
@@ -715,6 +732,8 @@ def test_multimodal_bridge_extends_and_carries_mm_data(
715
732
  tokenizer,
716
733
  _config_for_model(mm_model_name, thinking_retention="all"),
717
734
  )
735
+ if not getattr(renderer, "supports_multimodal_bridge", True):
736
+ pytest.skip(f"{type(renderer).__name__} uses full multimodal re-rendering")
718
737
  if hasattr(renderer, "_processor") and renderer._processor is None:
719
738
  renderer._processor = processor
720
739
  sample = _sample_for(modality, tiny_image)