renderers 0.1.8.dev35__tar.gz → 0.1.8.dev37__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/PKG-INFO +1 -1
  2. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/renderers/__init__.py +2 -0
  3. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/renderers/_version.py +2 -2
  4. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/renderers/base.py +96 -5
  5. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/renderers/deepseek_v3.py +3 -0
  6. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/renderers/default.py +2 -0
  7. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/renderers/glm45.py +3 -0
  8. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/renderers/glm5.py +3 -0
  9. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/renderers/gpt_oss.py +3 -0
  10. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/renderers/kimi_k2.py +3 -0
  11. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/renderers/kimi_k25.py +5 -0
  12. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/renderers/laguna_xs2.py +3 -0
  13. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/renderers/minimax_m2.py +3 -0
  14. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/renderers/nemotron3.py +3 -0
  15. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/renderers/qwen3.py +3 -0
  16. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/renderers/qwen35.py +5 -0
  17. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/renderers/qwen3_vl.py +3 -0
  18. renderers-0.1.8.dev37/tests/test_message_tool_names.py +97 -0
  19. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/.github/workflows/publish-dev.yml +0 -0
  20. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/.github/workflows/publish.yml +0 -0
  21. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/.github/workflows/style.yml +0 -0
  22. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/.github/workflows/test.yml +0 -0
  23. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/.gitignore +0 -0
  24. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/.pre-commit-config.yaml +0 -0
  25. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/LICENSE +0 -0
  26. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/README.md +0 -0
  27. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/docs/renderer-config.md +0 -0
  28. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/examples/README.md +0 -0
  29. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/examples/sglang/multiturn_generate_sglang.py +0 -0
  30. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/examples/sglang/online_multiturn_sglang.py +0 -0
  31. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/examples/tinker/multiturn_generate_tinker.py +0 -0
  32. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/examples/transformers/multiturn_generate_transformers.py +0 -0
  33. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/examples/vllm/multiturn_generate_vllm.py +0 -0
  34. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/pyproject.toml +0 -0
  35. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/renderers/client.py +0 -0
  36. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/renderers/configs.py +0 -0
  37. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/renderers/parsers.py +0 -0
  38. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/renderers/parsing.py +0 -0
  39. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/renderers/qwen36.py +0 -0
  40. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/tests/conftest.py +0 -0
  41. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/tests/test_bridge.py +0 -0
  42. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/tests/test_build_helpers.py +0 -0
  43. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/tests/test_client.py +0 -0
  44. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/tests/test_gpt_oss_harmony_parity.py +0 -0
  45. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/tests/test_incremental.py +0 -0
  46. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/tests/test_is_content.py +0 -0
  47. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/tests/test_kimi_k25_tool_schema.py +0 -0
  48. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/tests/test_load_tokenizer.py +0 -0
  49. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/tests/test_load_tokenizer_fastokens.py +0 -0
  50. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/tests/test_message_indices.py +0 -0
  51. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/tests/test_multimodal.py +0 -0
  52. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/tests/test_parse_response.py +0 -0
  53. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/tests/test_parse_response_robustness.py +0 -0
  54. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/tests/test_parsers.py +0 -0
  55. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/tests/test_preserve_thinking.py +0 -0
  56. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/tests/test_qwen35_size_coverage.py +0 -0
  57. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/tests/test_render_ids.py +0 -0
  58. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/tests/test_renderer_config.py +0 -0
  59. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/tests/test_renderer_config_parity.py +0 -0
  60. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/tests/test_roundtrip.py +0 -0
  61. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/tests/test_sampled_mask.py +0 -0
  62. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/tests/test_tokens_per_message.py +0 -0
  63. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/tests/test_tool_arg_type_preservation.py +0 -0
  64. {renderers-0.1.8.dev35 → renderers-0.1.8.dev37}/uv.lock +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: renderers
3
- Version: 0.1.8.dev35
3
+ Version: 0.1.8.dev37
4
4
  Summary: Chat template renderers — deterministic message-to-token conversion for LLM training
5
5
  License-Expression: Apache-2.0
6
6
  License-File: LICENSE
@@ -33,6 +33,7 @@ from renderers.base import (
33
33
  build_trajectory_step,
34
34
  create_renderer,
35
35
  create_renderer_pool,
36
+ extract_message_tool_names,
36
37
  is_multimodal,
37
38
  reject_assistant_in_extension,
38
39
  trim_to_turn_close,
@@ -168,6 +169,7 @@ __all__ = [
168
169
  "config_from_name",
169
170
  "create_renderer",
170
171
  "create_renderer_pool",
172
+ "extract_message_tool_names",
171
173
  "is_multimodal",
172
174
  "reject_assistant_in_extension",
173
175
  "trim_to_turn_close",
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
18
18
  commit_id: str | None
19
19
  __commit_id__: str | None
20
20
 
21
- __version__ = version = '0.1.8.dev35'
22
- __version_tuple__ = version_tuple = (0, 1, 8, 'dev35')
21
+ __version__ = version = '0.1.8.dev37'
22
+ __version_tuple__ = version_tuple = (0, 1, 8, 'dev37')
23
23
 
24
24
  __commit_id__ = commit_id = None
@@ -6,6 +6,7 @@ import io
6
6
  import logging
7
7
  import queue
8
8
  import threading
9
+ from collections.abc import Mapping
9
10
  from contextlib import contextmanager
10
11
  from dataclasses import dataclass, field
11
12
  from typing import (
@@ -117,6 +118,68 @@ class Message(TypedDict, total=False):
117
118
  reasoning_content: str
118
119
 
119
120
 
121
+ def extract_message_tool_names(messages: list[Message]) -> list[str | None]:
122
+ """Per-message tool function names parallel to ``message_roles``.
123
+
124
+ Returns one entry per message: the function name for ``role="tool"``
125
+ messages, ``None`` for every other message. Length matches the
126
+ input list.
127
+
128
+ For tool messages the name is taken from ``msg["name"]`` when set
129
+ (caller-provided), otherwise recovered by joining
130
+ ``msg["tool_call_id"]`` against any prior assistant's
131
+ ``tool_calls[i].function.name`` in the same list. Tool messages
132
+ whose issuing assistant lives outside the provided list (e.g. on
133
+ a :meth:`Renderer.bridge_to_next_turn` call where ``new_messages``
134
+ covers only the new turn) resolve to ``None``.
135
+
136
+ Pure metadata: this never mutates the caller's messages and has
137
+ no effect on the rendered token stream. It runs independently of
138
+ the render path so the renderer can populate the field on
139
+ :class:`RenderedTokens` without breaking HF byte parity for tool
140
+ messages that carry no ``name``. Callers who *also* want the
141
+ function name to appear in the rendered scaffold (e.g. GPT-OSS
142
+ Harmony's ``functions.{name}`` prefix) must attach ``name`` to
143
+ their tool messages before calling :meth:`Renderer.render`
144
+ themselves — renderers don't synthesize ``name`` into the input,
145
+ only into this metadata field.
146
+
147
+ Trainers join this list with :attr:`RenderedTokens.message_indices`
148
+ to recover per-token tool attribution — the canonical use case is
149
+ SFT on tool response bodies while RL acts only on assistant tokens
150
+ (tool body tokens get a constant positive advantage so the model
151
+ learns to anticipate tool outputs without learning to emit
152
+ ``<|tool_response>`` itself).
153
+
154
+ Per-message rather than per-token because the data is naturally
155
+ per-message — storing it per-token would duplicate the same
156
+ string across every body token of the same tool message.
157
+ """
158
+ lookup: dict[str, str] = {}
159
+ for m in messages:
160
+ if not isinstance(m, Mapping) or m.get("role") != "assistant":
161
+ continue
162
+ for tc in m.get("tool_calls") or []:
163
+ if not isinstance(tc, Mapping):
164
+ continue
165
+ tc_id = tc.get("id")
166
+ fn = tc.get("function")
167
+ tc_name = fn.get("name") if isinstance(fn, Mapping) else None
168
+ if isinstance(tc_id, str) and isinstance(tc_name, str):
169
+ lookup[tc_id] = tc_name
170
+ out: list[str | None] = []
171
+ for m in messages:
172
+ if not isinstance(m, Mapping) or m.get("role") != "tool":
173
+ out.append(None)
174
+ continue
175
+ name = m.get("name")
176
+ if not (isinstance(name, str) and name):
177
+ tc_id = m.get("tool_call_id")
178
+ name = lookup.get(tc_id) if isinstance(tc_id, str) else None
179
+ out.append(name if isinstance(name, str) and name else None)
180
+ return out
181
+
182
+
120
183
  # ---------------------------------------------------------------------------
121
184
  # Renderer data types
122
185
  # ---------------------------------------------------------------------------
@@ -208,6 +271,32 @@ class RenderedTokens:
208
271
  renderer doesn't provide the signal. ``DefaultRenderer`` leaves it
209
272
  empty for the same reason.
210
273
 
274
+ ``message_tool_names`` is the per-message tool function name list,
275
+ parallel to ``message_roles`` (same length). For tool-role
276
+ messages it carries the function name — either taken from
277
+ ``msg["name"]`` (caller-provided) or recovered by joining
278
+ ``msg["tool_call_id"]`` against a prior assistant's
279
+ ``tool_calls[i].function.name`` in the rendered slice. Every
280
+ other message is ``None``, as are tool messages whose issuing
281
+ assistant lives outside the rendered slice (e.g. on a
282
+ :meth:`Renderer.bridge_to_next_turn` call where ``new_messages``
283
+ covers only the new turn).
284
+
285
+ This is pure metadata, computed by :func:`extract_message_tool_names`
286
+ independently of the render path: populating it never touches the
287
+ rendered token stream, so HF chat-template byte parity is
288
+ preserved for tool messages carrying no ``name``. Callers who
289
+ *also* want the function name to appear in the rendered scaffold
290
+ (e.g. GPT-OSS Harmony's ``functions.{name}`` prefix) must attach
291
+ ``name`` to their tool messages before calling
292
+ :meth:`Renderer.render` themselves.
293
+
294
+ Trainers join this with ``message_indices`` to build per-tool
295
+ selective loss masks (SFT on tool response bodies of a specific
296
+ tool while RL acts on assistant tokens). Empty
297
+ ``message_tool_names`` (``[]``) means the renderer doesn't
298
+ provide the signal.
299
+
211
300
  ``multi_modal_data`` is populated by multimodal renderers (e.g.
212
301
  ``Qwen3VLRenderer``) when image / video content parts are present;
213
302
  text-only renderers leave it as ``None``.
@@ -218,6 +307,7 @@ class RenderedTokens:
218
307
  sampled_mask: list[bool] = field(default_factory=list)
219
308
  is_content: list[bool] = field(default_factory=list)
220
309
  message_roles: list[str] = field(default_factory=list)
310
+ message_tool_names: list[str | None] = field(default_factory=list)
221
311
  multi_modal_data: "MultiModalData | None" = None
222
312
 
223
313
  def tokens_per_message(
@@ -1587,7 +1677,6 @@ def _get_offset_tokenizer(tokenizer):
1587
1677
  cached = _offset_tokenizers.get(name_or_path)
1588
1678
  if cached is not None:
1589
1679
  return cached
1590
- from transformers import AutoTokenizer
1591
1680
 
1592
1681
  kwargs: dict[str, Any] = {}
1593
1682
  revision = TRUSTED_REVISIONS.get(name_or_path)
@@ -1597,10 +1686,12 @@ def _get_offset_tokenizer(tokenizer):
1597
1686
  kwargs = {"trust_remote_code": False}
1598
1687
  # Explicitly vanilla — we want HF's Rust tokenizer with offset
1599
1688
  # tracking, not the fastokens shim. ``load_tokenizer`` would
1600
- # patch fastokens in by default; calling
1601
- # ``AutoTokenizer.from_pretrained`` directly here keeps the
1602
- # fastokens patch out of this code path entirely.
1603
- offset_tok = AutoTokenizer.from_pretrained(name_or_path, **kwargs)
1689
+ # patch fastokens in by default; routing through
1690
+ # ``_load_tokenizer_via_auto`` keeps the fastokens patch out
1691
+ # of this code path while still applying the config-build
1692
+ # fallback (RoPE-validation failures on nested
1693
+ # ``rope_parameters``, etc.).
1694
+ offset_tok = _load_tokenizer_via_auto(name_or_path, **kwargs)
1604
1695
  if not getattr(offset_tok, "is_fast", False):
1605
1696
  raise RuntimeError(
1606
1697
  f"Vanilla tokenizer for {name_or_path!r} is not a fast "
@@ -22,6 +22,7 @@ from renderers.base import (
22
22
  RenderedTokens,
23
23
  ToolSpec,
24
24
  attribute_text_segments,
25
+ extract_message_tool_names,
25
26
  reject_assistant_in_extension,
26
27
  trim_to_turn_close,
27
28
  )
@@ -247,6 +248,7 @@ class DeepSeekV3Renderer:
247
248
  sampled_mask=sampled,
248
249
  is_content=content_mask,
249
250
  message_roles=[m.get("role") or "" for m in messages],
251
+ message_tool_names=extract_message_tool_names(messages),
250
252
  )
251
253
 
252
254
  def render_ids(
@@ -390,6 +392,7 @@ class DeepSeekV3Renderer:
390
392
  sampled_mask=[False] * total_len,
391
393
  is_content=[False] * len(previous_ids) + ext_content,
392
394
  message_roles=[m.get("role") or "" for m in new_messages],
395
+ message_tool_names=extract_message_tool_names(new_messages),
393
396
  )
394
397
 
395
398
  # ------------------------------------------------------------------
@@ -18,6 +18,7 @@ from renderers.base import (
18
18
  ParsedResponse,
19
19
  RenderedTokens,
20
20
  ToolSpec,
21
+ extract_message_tool_names,
21
22
  )
22
23
  from renderers.configs import DefaultRendererConfig
23
24
  from renderers.parsers import (
@@ -141,6 +142,7 @@ class DefaultRenderer:
141
142
  token_ids=token_ids,
142
143
  message_indices=message_indices,
143
144
  message_roles=message_roles,
145
+ message_tool_names=extract_message_tool_names(messages),
144
146
  )
145
147
 
146
148
  def _apply(self, messages, *, tools=None, add_generation_prompt=False) -> list[int]:
@@ -21,6 +21,7 @@ from renderers.base import (
21
21
  RenderedTokens,
22
22
  ToolSpec,
23
23
  attribute_text_segments,
24
+ extract_message_tool_names,
24
25
  reject_assistant_in_extension,
25
26
  should_preserve_past_thinking,
26
27
  )
@@ -265,6 +266,7 @@ class GLM45Renderer:
265
266
  sampled_mask=sampled,
266
267
  is_content=content_mask,
267
268
  message_roles=[m.get("role") or "" for m in messages],
269
+ message_tool_names=extract_message_tool_names(messages),
268
270
  )
269
271
 
270
272
  def render_ids(
@@ -445,6 +447,7 @@ class GLM45Renderer:
445
447
  sampled_mask=[False] * total_len,
446
448
  is_content=[False] * len(previous_ids) + ext_content,
447
449
  message_roles=[m.get("role") or "" for m in new_messages],
450
+ message_tool_names=extract_message_tool_names(new_messages),
448
451
  )
449
452
 
450
453
  def _render_assistant(
@@ -22,6 +22,7 @@ from renderers.base import (
22
22
  RenderedTokens,
23
23
  ToolSpec,
24
24
  attribute_text_segments,
25
+ extract_message_tool_names,
25
26
  reject_assistant_in_extension,
26
27
  should_preserve_past_thinking,
27
28
  )
@@ -281,6 +282,7 @@ class GLM5Renderer:
281
282
  sampled_mask=sampled,
282
283
  is_content=content_mask,
283
284
  message_roles=[m.get("role") or "" for m in messages],
285
+ message_tool_names=extract_message_tool_names(messages),
284
286
  )
285
287
 
286
288
  def render_ids(
@@ -456,6 +458,7 @@ class GLM5Renderer:
456
458
  sampled_mask=[False] * total_len,
457
459
  is_content=[False] * len(previous_ids) + ext_content,
458
460
  message_roles=[m.get("role") or "" for m in new_messages],
461
+ message_tool_names=extract_message_tool_names(new_messages),
459
462
  )
460
463
 
461
464
  def _render_assistant(
@@ -56,6 +56,7 @@ from renderers.base import (
56
56
  ParsedResponse,
57
57
  RenderedTokens,
58
58
  ToolSpec,
59
+ extract_message_tool_names,
59
60
  reject_assistant_in_extension,
60
61
  should_preserve_past_thinking,
61
62
  trim_to_turn_close,
@@ -465,6 +466,7 @@ class GptOssRenderer:
465
466
  sampled_mask=sampled,
466
467
  is_content=content_mask,
467
468
  message_roles=[m.get("role") or "" for m in messages],
469
+ message_tool_names=extract_message_tool_names(messages),
468
470
  )
469
471
 
470
472
  def render_ids(
@@ -594,6 +596,7 @@ class GptOssRenderer:
594
596
  sampled_mask=[False] * total_len,
595
597
  is_content=[False] * len(previous_ids) + ext_content,
596
598
  message_roles=[m.get("role") or "" for m in new_messages],
599
+ message_tool_names=extract_message_tool_names(new_messages),
597
600
  )
598
601
 
599
602
  # ── message conversion ───────────────────────────────────────────────────
@@ -23,6 +23,7 @@ from renderers.base import (
23
23
  ParsedResponse,
24
24
  RenderedTokens,
25
25
  ToolSpec,
26
+ extract_message_tool_names,
26
27
  reject_assistant_in_extension,
27
28
  trim_to_turn_close,
28
29
  )
@@ -305,6 +306,7 @@ class KimiK2Renderer:
305
306
  sampled_mask=sampled,
306
307
  is_content=content_mask,
307
308
  message_roles=[m.get("role") or "" for m in caller_messages],
309
+ message_tool_names=extract_message_tool_names(caller_messages),
308
310
  )
309
311
 
310
312
  def render_ids(
@@ -454,6 +456,7 @@ class KimiK2Renderer:
454
456
  sampled_mask=[False] * total_len,
455
457
  is_content=[False] * len(previous_ids) + ext_content,
456
458
  message_roles=[m.get("role") or "" for m in new_messages],
459
+ message_tool_names=extract_message_tool_names(new_messages),
457
460
  )
458
461
 
459
462
  def _render_assistant(
@@ -36,6 +36,7 @@ from renderers.base import (
36
36
  RenderedTokens,
37
37
  ToolCallParseStatus,
38
38
  ToolSpec,
39
+ extract_message_tool_names,
39
40
  reject_assistant_in_extension,
40
41
  should_preserve_past_thinking,
41
42
  trim_to_turn_close,
@@ -946,6 +947,7 @@ class KimiK25Renderer:
946
947
  sampled_mask=sampled,
947
948
  is_content=content_mask,
948
949
  message_roles=[m.get("role") or "" for m in messages],
950
+ message_tool_names=extract_message_tool_names(messages),
949
951
  multi_modal_data=mm_data,
950
952
  )
951
953
 
@@ -1188,6 +1190,7 @@ class KimiK25Renderer:
1188
1190
  merged_items.setdefault(modality, []).extend(vals)
1189
1191
 
1190
1192
  bridge_roles = [m.get("role") or "" for m in new_messages]
1193
+ bridge_tool_names = extract_message_tool_names(new_messages)
1191
1194
  if not (merged_hashes or merged_placeholders or merged_items):
1192
1195
  return RenderedTokens(
1193
1196
  token_ids=tokens,
@@ -1195,6 +1198,7 @@ class KimiK25Renderer:
1195
1198
  sampled_mask=sampled,
1196
1199
  is_content=content_mask,
1197
1200
  message_roles=bridge_roles,
1201
+ message_tool_names=bridge_tool_names,
1198
1202
  )
1199
1203
 
1200
1204
  mm_data = MultiModalData(
@@ -1208,6 +1212,7 @@ class KimiK25Renderer:
1208
1212
  sampled_mask=sampled,
1209
1213
  is_content=content_mask,
1210
1214
  message_roles=bridge_roles,
1215
+ message_tool_names=bridge_tool_names,
1211
1216
  multi_modal_data=mm_data,
1212
1217
  )
1213
1218
 
@@ -36,6 +36,7 @@ from renderers.base import (
36
36
  RenderedTokens,
37
37
  ToolSpec,
38
38
  attribute_text_segments,
39
+ extract_message_tool_names,
39
40
  reject_assistant_in_extension,
40
41
  )
41
42
  from renderers.configs import LagunaXS2RendererConfig
@@ -275,6 +276,7 @@ class LagunaXS2Renderer:
275
276
  sampled_mask=sampled,
276
277
  is_content=content_mask,
277
278
  message_roles=[m.get("role") or "" for m in messages],
279
+ message_tool_names=extract_message_tool_names(messages),
278
280
  )
279
281
 
280
282
  def render_ids(
@@ -426,6 +428,7 @@ class LagunaXS2Renderer:
426
428
  sampled_mask=[False] * total_len,
427
429
  is_content=[False] * len(previous_ids) + ext_content,
428
430
  message_roles=[m.get("role") or "" for m in new_messages],
431
+ message_tool_names=extract_message_tool_names(new_messages),
429
432
  )
430
433
 
431
434
  def _render_assistant(
@@ -22,6 +22,7 @@ from renderers.base import (
22
22
  RenderedTokens,
23
23
  ToolSpec,
24
24
  attribute_text_segments,
25
+ extract_message_tool_names,
25
26
  reject_assistant_in_extension,
26
27
  should_preserve_past_thinking,
27
28
  trim_to_turn_close,
@@ -278,6 +279,7 @@ class MiniMaxM2Renderer:
278
279
  sampled_mask=sampled,
279
280
  is_content=content_mask,
280
281
  message_roles=[m.get("role") or "" for m in messages],
282
+ message_tool_names=extract_message_tool_names(messages),
281
283
  )
282
284
 
283
285
  def render_ids(
@@ -459,6 +461,7 @@ class MiniMaxM2Renderer:
459
461
  sampled_mask=[False] * total_len,
460
462
  is_content=[False] * len(previous_ids) + ext_content,
461
463
  message_roles=[m.get("role") or "" for m in new_messages],
464
+ message_tool_names=extract_message_tool_names(new_messages),
462
465
  )
463
466
 
464
467
  def _render_assistant(
@@ -25,6 +25,7 @@ from renderers.base import (
25
25
  RenderedTokens,
26
26
  ToolSpec,
27
27
  attribute_text_segments,
28
+ extract_message_tool_names,
28
29
  reject_assistant_in_extension,
29
30
  should_preserve_past_thinking,
30
31
  trim_to_turn_close,
@@ -411,6 +412,7 @@ class Nemotron3Renderer:
411
412
  sampled_mask=sampled,
412
413
  is_content=content_mask,
413
414
  message_roles=[m.get("role") or "" for m in original_messages],
415
+ message_tool_names=extract_message_tool_names(original_messages),
414
416
  )
415
417
 
416
418
  def render_ids(
@@ -581,6 +583,7 @@ class Nemotron3Renderer:
581
583
  sampled_mask=[False] * total_len,
582
584
  is_content=[False] * len(previous_ids) + ext_content,
583
585
  message_roles=[m.get("role") or "" for m in new_messages],
586
+ message_tool_names=extract_message_tool_names(new_messages),
584
587
  )
585
588
 
586
589
  # ------------------------------------------------------------------
@@ -19,6 +19,7 @@ from renderers.base import (
19
19
  RenderedTokens,
20
20
  ToolSpec,
21
21
  attribute_text_segments,
22
+ extract_message_tool_names,
22
23
  reject_assistant_in_extension,
23
24
  should_preserve_past_thinking,
24
25
  trim_to_turn_close,
@@ -247,6 +248,7 @@ class Qwen3Renderer:
247
248
  sampled_mask=sampled,
248
249
  is_content=content_mask,
249
250
  message_roles=[m.get("role") or "" for m in messages],
251
+ message_tool_names=extract_message_tool_names(messages),
250
252
  )
251
253
 
252
254
  def render_ids(
@@ -403,6 +405,7 @@ class Qwen3Renderer:
403
405
  sampled_mask=[False] * total_len,
404
406
  is_content=[False] * len(previous_ids) + ext_content,
405
407
  message_roles=[m.get("role") or "" for m in new_messages],
408
+ message_tool_names=extract_message_tool_names(new_messages),
406
409
  )
407
410
 
408
411
  def _render_assistant(
@@ -27,6 +27,7 @@ from renderers.base import (
27
27
  RenderedTokens,
28
28
  ToolSpec,
29
29
  attribute_text_segments,
30
+ extract_message_tool_names,
30
31
  reject_assistant_in_extension,
31
32
  should_preserve_past_thinking,
32
33
  trim_to_turn_close,
@@ -565,6 +566,7 @@ class Qwen35Renderer:
565
566
  sampled_mask=sampled,
566
567
  is_content=content_mask,
567
568
  message_roles=[m.get("role") or "" for m in messages],
569
+ message_tool_names=extract_message_tool_names(messages),
568
570
  multi_modal_data=mm_data,
569
571
  )
570
572
 
@@ -841,6 +843,7 @@ class Qwen35Renderer:
841
843
  merged_items.setdefault(modality, []).extend(vals)
842
844
 
843
845
  bridge_roles = [m.get("role") or "" for m in new_messages]
846
+ bridge_tool_names = extract_message_tool_names(new_messages)
844
847
  if not (merged_hashes or merged_placeholders or merged_items):
845
848
  return RenderedTokens(
846
849
  token_ids=tokens,
@@ -848,6 +851,7 @@ class Qwen35Renderer:
848
851
  sampled_mask=sampled,
849
852
  is_content=content_mask,
850
853
  message_roles=bridge_roles,
854
+ message_tool_names=bridge_tool_names,
851
855
  )
852
856
 
853
857
  mm_data = MultiModalData(
@@ -861,6 +865,7 @@ class Qwen35Renderer:
861
865
  sampled_mask=sampled,
862
866
  is_content=content_mask,
863
867
  message_roles=bridge_roles,
868
+ message_tool_names=bridge_tool_names,
864
869
  multi_modal_data=mm_data,
865
870
  )
866
871
 
@@ -43,6 +43,7 @@ from renderers.base import (
43
43
  RenderedTokens,
44
44
  ToolSpec,
45
45
  attribute_text_segments,
46
+ extract_message_tool_names,
46
47
  reject_assistant_in_extension,
47
48
  trim_to_turn_close,
48
49
  )
@@ -604,6 +605,7 @@ class Qwen3VLRenderer:
604
605
  sampled_mask=em.sampled,
605
606
  is_content=em.is_content,
606
607
  message_roles=[m.get("role") or "" for m in messages],
608
+ message_tool_names=extract_message_tool_names(messages),
607
609
  multi_modal_data=mm_data,
608
610
  )
609
611
 
@@ -839,6 +841,7 @@ class Qwen3VLRenderer:
839
841
  sampled_mask=em.sampled,
840
842
  is_content=em.is_content,
841
843
  message_roles=[m.get("role") or "" for m in new_messages],
844
+ message_tool_names=extract_message_tool_names(new_messages),
842
845
  multi_modal_data=mm_data,
843
846
  )
844
847
 
@@ -0,0 +1,97 @@
1
+ """Tests for ``RenderedTokens.message_tool_names`` and its populating helper.
2
+
3
+ ``message_tool_names`` is a per-message sidecar parallel to
4
+ ``message_roles``: for each tool-role message in the rendered slice
5
+ it carries the tool function name, ``None`` everywhere else. The name
6
+ comes from ``msg["name"]`` when set, otherwise from a
7
+ ``tool_call_id`` join against any prior assistant's ``tool_calls`` in
8
+ the same slice. Pure metadata — does not affect the rendered token
9
+ stream, does not mutate the caller's messages.
10
+
11
+ Unit tests below cover the join's case matrix without a tokenizer.
12
+ The single integration test runs every renderer in the conftest
13
+ matrix to catch any of the ~25 ``RenderedTokens(...)`` construction
14
+ sites that might fail to wire the field through.
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ from renderers.base import extract_message_tool_names
20
+
21
+
22
+ def test_extract_empty():
23
+ assert extract_message_tool_names([]) == []
24
+
25
+
26
+ def test_extract_caller_provided_name_wins():
27
+ """``msg['name']`` set by the caller is used verbatim — no join attempted."""
28
+ messages = [
29
+ {"role": "tool", "tool_call_id": "c1", "name": "caller_set", "content": "x"},
30
+ ]
31
+ assert extract_message_tool_names(messages) == ["caller_set"]
32
+
33
+
34
+ def test_extract_resolves_from_prior_assistant():
35
+ """Tool message without ``name``: recovered via tool_call_id → assistant.tool_calls."""
36
+ messages = [
37
+ {"role": "user", "content": "go"},
38
+ {
39
+ "role": "assistant",
40
+ "tool_calls": [{"id": "c1", "function": {"name": "screenshot"}}],
41
+ },
42
+ {"role": "tool", "tool_call_id": "c1", "content": "ok"},
43
+ ]
44
+ assert extract_message_tool_names(messages) == [None, None, "screenshot"]
45
+
46
+
47
+ def test_extract_orphan_tool_message_is_none():
48
+ """``tool_call_id`` matching no in-slice assistant resolves to ``None``
49
+ (bridge case: the issuing assistant lives in the prior portion that
50
+ ``new_messages`` doesn't cover).
51
+ """
52
+ messages = [{"role": "tool", "tool_call_id": "orphan", "content": "x"}]
53
+ assert extract_message_tool_names(messages) == [None]
54
+
55
+
56
+ def test_extract_does_not_mutate_caller():
57
+ """Caller's tool message must not gain a ``name`` field after extraction —
58
+ the helper produces a sidecar list, not a mutated view of the input.
59
+ """
60
+ messages = [
61
+ {
62
+ "role": "assistant",
63
+ "tool_calls": [{"id": "c1", "function": {"name": "f"}}],
64
+ },
65
+ {"role": "tool", "tool_call_id": "c1", "content": "x"},
66
+ ]
67
+ extract_message_tool_names(messages)
68
+ assert "name" not in messages[1]
69
+
70
+
71
+ def test_renderer_populates_message_tool_names(model_name, renderer):
72
+ """Every renderer wires ``message_tool_names`` through ``RenderedTokens``.
73
+
74
+ Catches missed wire-up at any of the ~25 ``RenderedTokens(...)``
75
+ construction sites across concrete renderers. The input is
76
+ spec-conformant (tool message carries ``tool_call_id`` but no
77
+ ``name``) so the resolution path exercises the internal join.
78
+ """
79
+ messages = [
80
+ {"role": "user", "content": "go"},
81
+ {
82
+ "role": "assistant",
83
+ "content": "",
84
+ "tool_calls": [
85
+ {
86
+ "id": "c1",
87
+ "type": "function",
88
+ "function": {"name": "screenshot", "arguments": {}},
89
+ }
90
+ ],
91
+ },
92
+ {"role": "tool", "tool_call_id": "c1", "content": "ok"},
93
+ ]
94
+ rt = renderer.render(messages)
95
+ assert rt.message_tool_names == [None, None, "screenshot"], (
96
+ f"{model_name}: got {rt.message_tool_names}"
97
+ )
File without changes
File without changes