renderers 0.1.11.dev2__tar.gz → 0.1.11.dev3__tar.gz

{renderers-0.1.11.dev2 → renderers-0.1.11.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: renderers
-Version: 0.1.11.dev2
+Version: 0.1.11.dev3
 Summary: Chat template renderers — deterministic message-to-token conversion for LLM training
 License-Expression: Apache-2.0
 License-File: LICENSE
@@ -39,11 +39,13 @@ uv add 'renderers[transformers]'
 uv add 'renderers[multimodal]'
 ```
 
-A BYO tokenizer must expose `encode`, `decode`, `convert_tokens_to_ids`, token
-IDs such as `eos_token_id`, and `return_offsets_mapping=True` through its call
-interface. `DefaultRenderer` additionally requires `apply_chat_template`.
-This includes text-only Inkling training: `InklingRenderer` loads its
-Transformers processor only when image or audio content is actually rendered.
+A BYO tokenizer must expose `encode`, `decode`, `convert_tokens_to_ids`, and
+token IDs such as `eos_token_id`. Character offsets are optional: tokenizers
+supporting `return_offsets_mapping=True` also receive precise per-token
+`is_content` attribution; without offsets, renderers return `is_content=[]`.
+`DefaultRenderer` additionally requires `apply_chat_template`. This includes
+text-only Inkling training: `InklingRenderer` loads its Transformers processor
+only when image or audio content is actually rendered.
 
 ## At a glance
 

{renderers-0.1.11.dev2 → renderers-0.1.11.dev3}/README.md

@@ -19,11 +19,13 @@ uv add 'renderers[transformers]'
 uv add 'renderers[multimodal]'
 ```
 
-A BYO tokenizer must expose `encode`, `decode`, `convert_tokens_to_ids`, token
-IDs such as `eos_token_id`, and `return_offsets_mapping=True` through its call
-interface. `DefaultRenderer` additionally requires `apply_chat_template`.
-This includes text-only Inkling training: `InklingRenderer` loads its
-Transformers processor only when image or audio content is actually rendered.
+A BYO tokenizer must expose `encode`, `decode`, `convert_tokens_to_ids`, and
+token IDs such as `eos_token_id`. Character offsets are optional: tokenizers
+supporting `return_offsets_mapping=True` also receive precise per-token
+`is_content` attribution; without offsets, renderers return `is_content=[]`.
+`DefaultRenderer` additionally requires `apply_chat_template`. This includes
+text-only Inkling training: `InklingRenderer` loads its Transformers processor
+only when image or audio content is actually rendered.
 
 ## At a glance
 

{renderers-0.1.11.dev2 → renderers-0.1.11.dev3}/pyproject.toml

@@ -39,8 +39,8 @@ dependencies = [
 
 [project.optional-dependencies]
 # Tokenizer loading uses Hugging Face. Text-only renderers can instead be
-# constructed with an offset-capable BYO tokenizer and do not import this
-# dependency.
+# constructed with a compatible BYO tokenizer and do not import this
+# dependency. Character offsets are optional.
 transformers = [
     # Keep this floor compatible with prime-rl's transformers pin. Inkling's
     # tokenizer and text-only renderer work on older releases; image/audio

{renderers-0.1.11.dev2 → renderers-0.1.11.dev3}/renderers/__init__.py

@@ -15,6 +15,7 @@ from renderers.base import (
     Message,
     MultiModalData,
     MultimodalRenderer,
+    OffsetTokenizer,
     ParsedResponse,
     ParsedToolCall,
     PlaceholderRange,
@@ -181,6 +182,7 @@ __all__ = [
     "Nemotron3RendererConfig",
     "Nemotron3UltraRenderer",
     "Nemotron3UltraRendererConfig",
+    "OffsetTokenizer",
     "OverlongPromptError",
     "ParsedResponse",
     "ParsedToolCall",

{renderers-0.1.11.dev2 → renderers-0.1.11.dev3}/renderers/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.1.11.dev2'
-__version_tuple__ = version_tuple = (0, 1, 11, 'dev2')
+__version__ = version = '0.1.11.dev3'
+__version_tuple__ = version_tuple = (0, 1, 11, 'dev3')
 
 __commit_id__ = commit_id = None

{renderers-0.1.11.dev2 → renderers-0.1.11.dev3}/renderers/base.py

@@ -11,6 +11,7 @@ from typing import (
     Literal,
     Protocol,
     TypedDict,
+    cast,
     runtime_checkable,
 )
 
@@ -243,7 +244,9 @@ class RenderedTokens:
     Empty ``sampled_mask`` (``[]``) means the renderer doesn't provide
     this signal — consumers should fall back to attribution-only
     masking. ``DefaultRenderer`` leaves it empty because the Jinja
-    template is opaque; hand-coded renderers populate it.
+    template is opaque. Hand-coded renderers normally populate it; a
+    renderer whose sampled/scaffold boundary depends on character
+    attribution may leave it empty for an offsetless tokenizer.
 
     ``is_content`` is a per-token signal generalizing the "scaffold vs
     body" distinction across all roles: ``True`` iff the token was
@@ -269,7 +272,8 @@ class RenderedTokens:
 
     Empty ``is_content`` (``[]``) — like ``sampled_mask`` — means the
     renderer doesn't provide the signal. ``DefaultRenderer`` leaves it
-    empty for the same reason.
+    empty because its Jinja template is opaque; all renderers leave it
+    empty when the supplied tokenizer cannot return character offsets.
 
     ``message_tool_names`` is the per-message tool function name list,
     parallel to ``message_roles`` (same length). For tool-role
@@ -476,7 +480,7 @@ class RenderedTokens:
 
         Returns an empty dict when :attr:`is_content` or
         :attr:`message_roles` is empty (renderer didn't populate the
-        signal — e.g. ``DefaultRenderer``).
+        signal — e.g. ``DefaultRenderer`` or an offsetless tokenizer).
 
         Intended for selective loss masking: SFT on tool response
         bodies while RL acts only on assistant turns is the canonical
@@ -667,8 +671,8 @@ class Tokenizer(Protocol):
     Hugging Face tokenizers satisfy this protocol, as can lightweight BYO
     adapters around ``tokenizers.Tokenizer`` or another tokenizer backend.
     Keeping the renderer-facing contract here makes ``transformers`` optional
-    for text rendering. Offset-capable ``__call__`` behavior is required by
-    :func:`attribute_text_segments` to preserve BPE boundary attribution.
+    for text rendering. Character offsets are a separate optional capability;
+    see :class:`OffsetTokenizer`.
     """
 
     name_or_path: str
@@ -681,6 +685,17 @@ class Tokenizer(Protocol):
 
     def convert_tokens_to_ids(self, tokens: Any) -> Any: ...
 
+
+@runtime_checkable
+class OffsetTokenizer(Tokenizer, Protocol):
+    """Tokenizer that can return character offsets alongside token IDs.
+
+    Hand-coded renderers use this optional capability to distinguish caller
+    content from adjacent template scaffold without changing the underlying
+    BPE pass. A basic :class:`Tokenizer` remains sufficient for rendering token
+    IDs; when offsets are unavailable, renderers leave ``is_content`` empty.
+    """
+
     def __call__(self, *args: Any, **kwargs: Any) -> Any: ...
 
 
@@ -1064,7 +1079,7 @@ _TRANSFORMERS_INSTALL_HINT = (
     "Install the optional dependency with "
     "`pip install 'renderers[transformers]'` (or "
     "`uv add 'renderers[transformers]'`). Text-only renderers work without "
-    "it when constructed with an offset-capable tokenizer object."
+    "it when constructed with a compatible tokenizer object."
 )
 
 
@@ -1759,36 +1774,122 @@ def trim_to_turn_close(
     return previous_ids
 
 
-def _get_offset_tokenizer(tokenizer):
-    """Assert ``tokenizer`` supports ``return_offsets_mapping=True``.
+class AttributedTextSegments(list[tuple[int, bool]]):
+    """Token/content pairs with an explicit attribution-availability flag."""
+
+    def __init__(
+        self,
+        values=(),
+        *,
+        has_content_attribution: bool,
+    ) -> None:
+        super().__init__(values)
+        self.has_content_attribution = has_content_attribution
+
+
+def _get_offset_tokenizer(tokenizer: Tokenizer) -> OffsetTokenizer | None:
+    """Return ``tokenizer`` when it supports character offsets, else ``None``.
 
     Hand-coded renderers concatenate scaffold + body in one BPE pass to
     preserve cross-boundary merges, then attribute each resulting token
     back to its source segment via the fast tokenizer's
-    ``offset_mapping`` (see :func:`attribute_text_segments`). The
-    contract: every BYO tokenizer must be a fast tokenizer with offset
-    support. Tokenizers loaded via :func:`load_tokenizer` are
-    ``PreTrainedTokenizerFast`` instances that satisfy this trivially.
+    ``offset_mapping`` (see :func:`attribute_text_segments`). Tokenizers
+    loaded via :func:`load_tokenizer` are ``PreTrainedTokenizerFast``
+    instances that satisfy this capability, but BYO tokenizers need not.
     """
+    call = getattr(tokenizer, "__call__", None)
+    if not callable(call):
+        return None
     try:
-        tokenizer("a", add_special_tokens=False, return_offsets_mapping=True)
-    except (NotImplementedError, ValueError, TypeError) as exc:
-        raise RuntimeError(
-            "Hand-coded renderers require a fast tokenizer with "
-            "``return_offsets_mapping=True`` support for body/scaffold "
-            "attribution. Pass a tokenizer loaded via "
-            "``renderers.base.load_tokenizer``, or any "
-            "``transformers.PreTrainedTokenizerFast`` instance."
-        ) from exc
-    return tokenizer
+        encoding = call("a", add_special_tokens=False, return_offsets_mapping=True)
+        encoding["input_ids"]
+        encoding["offset_mapping"]
+    except (KeyError, NotImplementedError, TypeError, ValueError):
+        return None
+    return cast(OffsetTokenizer, tokenizer)
+
+
+def _infer_offsets_from_decode(
+    tokenizer: Tokenizer,
+    token_ids: list[int],
+    text: str,
+) -> list[tuple[int, int]] | None:
+    """Recover token character spans from an exact decoder round-trip.
+
+    This is a narrow fallback for metadata that does not require exposing
+    content attribution. Some renderers join text from multiple messages in a
+    single BPE pass, so they still need to associate the resulting tokens with
+    the right message when a BYO tokenizer has no native offset mapping.
+
+    Decoding individual tokens is linear and exact for the common BPE/SentencePiece
+    backends. Byte-fallback tokenizers can require multiple tokens before text
+    becomes valid, so a validated cumulative-prefix pass handles that case.
+    If either strategy cannot reconstruct ``text`` exactly, callers must use a
+    conservative renderer-specific message-index fallback. This helper never
+    upgrades the tokenizer's content-attribution capability: ``is_content``
+    remains unavailable without native offsets.
+    """
+
+    def decode(ids: list[int]) -> str | None:
+        variants = (
+            {"skip_special_tokens": False, "clean_up_tokenization_spaces": False},
+            {"skip_special_tokens": False},
+            {},
+        )
+        for kwargs in variants:
+            try:
+                decoded = tokenizer.decode(ids, **kwargs)
+            except TypeError:
+                continue
+            except (KeyError, NotImplementedError, UnicodeError, ValueError):
+                return None
+            return decoded if isinstance(decoded, str) else None
+        return None
+
+    pieces: list[str] = []
+    for token_id in token_ids:
+        piece = decode([token_id])
+        if piece is None:
+            break
+        pieces.append(piece)
+    if len(pieces) == len(token_ids) and "".join(pieces) == text:
+        offsets: list[tuple[int, int]] = []
+        position = 0
+        for piece in pieces:
+            end = position + len(piece)
+            offsets.append((position, end))
+            position = end
+        return offsets
+
+    offsets = []
+    previous_end = 0
+    for end_index in range(1, len(token_ids) + 1):
+        prefix = decode(token_ids[:end_index])
+        if prefix is None or len(prefix) < previous_end or not text.startswith(prefix):
+            return None
+        current_end = len(prefix)
+        offsets.append((previous_end, current_end))
+        previous_end = current_end
+    if previous_end != len(text):
+        return None
+    return offsets
+
+
+def _content_mask_or_empty(
+    tokenizer: Tokenizer, content_mask: list[bool]
+) -> list[bool]:
+    """Return exact content attribution, or the empty-list unavailable sentinel."""
+    if _get_offset_tokenizer(tokenizer) is None:
+        return []
+    return content_mask
 
 
 def attribute_text_segments(
-    tokenizer,
+    tokenizer: Tokenizer,
     segments: "list[tuple[str, bool]]",
     *,
     overlap_is_content: bool = False,
-) -> "list[tuple[int, bool]]":
+) -> AttributedTextSegments:
     """Tokenize concatenated segments as a single BPE pass and return
     ``(token_id, is_content)`` pairs.
 
@@ -1816,23 +1917,29 @@ def attribute_text_segments(
     every body byte inside the ``is_content=True`` run at the cost of a
     few adjacent wrap bytes.
 
-    Requires a HuggingFace fast tokenizer with offset tracking. Every
-    model in ``MODEL_RENDERER_MAP`` ships one, so the offset lookup
-    always succeeds for tokenizers obtained via :func:`load_tokenizer`.
-    BYO tokenizers must be a ``PreTrainedTokenizerFast`` (or anything
-    else exposing ``return_offsets_mapping=True``); slow tokenizers
-    aren't supported — BPE drift at the wrap/body boundary would
-    defeat the whole point.
+    When ``tokenizer`` implements :class:`OffsetTokenizer`, the result's
+    ``has_content_attribution`` flag is true and each bool is exact. For a
+    basic :class:`Tokenizer`, the joined text is still encoded in one pass so
+    token IDs remain identical, but the bools are placeholders and
+    ``has_content_attribution`` is false. Renderers propagate that state as an
+    empty ``RenderedTokens.is_content`` list rather than exposing a partial or
+    inaccurate mask.
 
     Empty input or empty joined text returns an empty list.
     """
     if not segments:
-        return []
+        return AttributedTextSegments([], has_content_attribution=True)
     full_text = "".join(text for text, _ in segments)
     if not full_text:
-        return []
+        return AttributedTextSegments([], has_content_attribution=True)
 
     offset_tokenizer = _get_offset_tokenizer(tokenizer)
+    if offset_tokenizer is None:
+        token_ids = tokenizer.encode(full_text, add_special_tokens=False)
+        return AttributedTextSegments(
+            ((token_id, False) for token_id in token_ids),
+            has_content_attribution=False,
+        )
     encoding = offset_tokenizer(
         full_text,
         add_special_tokens=False,
@@ -1887,7 +1994,7 @@ def attribute_text_segments(
             # the last non-empty segment's bit.
             pass
         out.append((tok_id, is_content))
-    return out
+    return AttributedTextSegments(out, has_content_attribution=True)
 
 
 def reject_assistant_in_extension(new_messages: list[Message]) -> bool:

{renderers-0.1.11.dev2 → renderers-0.1.11.dev3}/renderers/deepseek_v3.py

@@ -20,6 +20,7 @@ from renderers.base import (
     RenderedTokens,
     ToolSpec,
     Tokenizer,
+    _content_mask_or_empty,
     attribute_text_segments,
     extract_message_tool_names,
     reject_assistant_in_extension,
@@ -259,7 +260,7 @@ class DeepSeekV3Renderer:
             token_ids=tokens,
             message_indices=indices,
             sampled_mask=sampled,
-            is_content=content_mask,
+            is_content=_content_mask_or_empty(self._tokenizer, content_mask),
             message_roles=[m.get("role") or "" for m in messages],
             message_tool_names=extract_message_tool_names(messages),
         )
@@ -408,7 +409,9 @@ class DeepSeekV3Renderer:
             token_ids=previous_ids + ext,
             message_indices=[-1] * len(previous_ids) + ext_indices,
             sampled_mask=[False] * total_len,
-            is_content=[False] * len(previous_ids) + ext_content,
+            is_content=_content_mask_or_empty(
+                self._tokenizer, [False] * len(previous_ids) + ext_content
+            ),
             message_roles=[m.get("role") or "" for m in new_messages],
             message_tool_names=extract_message_tool_names(new_messages),
         )

{renderers-0.1.11.dev2 → renderers-0.1.11.dev3}/renderers/gemma4.py

@@ -34,6 +34,7 @@ from renderers.base import (
     ToolCallParseStatus,
     ToolSpec,
     Tokenizer,
+    _content_mask_or_empty,
     _require_transformers,
     attribute_text_segments,
     extract_message_tool_names,
@@ -1025,7 +1026,7 @@ class Gemma4Renderer:
             token_ids=em.token_ids,
             message_indices=em.message_indices,
             sampled_mask=em.sampled,
-            is_content=em.is_content,
+            is_content=_content_mask_or_empty(self._tokenizer, em.is_content),
             message_roles=[m.get("role") or "" for m in messages],
             message_tool_names=extract_message_tool_names(messages),
             multi_modal_data=multi_modal_data,
@@ -1356,7 +1357,7 @@ class Gemma4Renderer:
             token_ids=em.token_ids,
             message_indices=em.message_indices,
             sampled_mask=em.sampled,
-            is_content=em.is_content,
+            is_content=_content_mask_or_empty(self._tokenizer, em.is_content),
             message_roles=[m.get("role") or "" for m in new_messages],
             message_tool_names=extract_message_tool_names(new_messages),
             multi_modal_data=self._merge_multi_modal_data(

{renderers-0.1.11.dev2 → renderers-0.1.11.dev3}/renderers/glm45.py

@@ -19,6 +19,7 @@ from renderers.base import (
     RenderedTokens,
     ToolSpec,
     Tokenizer,
+    _content_mask_or_empty,
     attribute_text_segments,
     extract_message_tool_names,
     reject_assistant_in_extension,
@@ -261,7 +262,7 @@ class GLM45Renderer:
             token_ids=tokens,
             message_indices=indices,
             sampled_mask=sampled,
-            is_content=content_mask,
+            is_content=_content_mask_or_empty(self._tokenizer, content_mask),
             message_roles=[m.get("role") or "" for m in messages],
             message_tool_names=extract_message_tool_names(messages),
         )
@@ -448,7 +449,9 @@ class GLM45Renderer:
             token_ids=previous_ids + ext,
             message_indices=[-1] * len(previous_ids) + ext_indices,
             sampled_mask=[False] * total_len,
-            is_content=[False] * len(previous_ids) + ext_content,
+            is_content=_content_mask_or_empty(
+                self._tokenizer, [False] * len(previous_ids) + ext_content
+            ),
             message_roles=[m.get("role") or "" for m in new_messages],
             message_tool_names=extract_message_tool_names(new_messages),
         )

{renderers-0.1.11.dev2 → renderers-0.1.11.dev3}/renderers/glm5.py

@@ -20,6 +20,7 @@ from renderers.base import (
     RenderedTokens,
     ToolSpec,
     Tokenizer,
+    _content_mask_or_empty,
     attribute_text_segments,
     extract_message_tool_names,
     reject_assistant_in_extension,
@@ -283,7 +284,7 @@ class GLM5Renderer:
             token_ids=tokens,
             message_indices=indices,
             sampled_mask=sampled,
-            is_content=content_mask,
+            is_content=_content_mask_or_empty(self._tokenizer, content_mask),
             message_roles=[m.get("role") or "" for m in messages],
             message_tool_names=extract_message_tool_names(messages),
         )
@@ -465,7 +466,9 @@ class GLM5Renderer:
             token_ids=previous_ids + ext,
             message_indices=[-1] * len(previous_ids) + ext_indices,
             sampled_mask=[False] * total_len,
-            is_content=[False] * len(previous_ids) + ext_content,
+            is_content=_content_mask_or_empty(
+                self._tokenizer, [False] * len(previous_ids) + ext_content
+            ),
             message_roles=[m.get("role") or "" for m in new_messages],
             message_tool_names=extract_message_tool_names(new_messages),
         )

{renderers-0.1.11.dev2 → renderers-0.1.11.dev3}/renderers/gpt_oss.py

@@ -55,6 +55,7 @@ from renderers.base import (
     RenderedTokens,
     ToolSpec,
     Tokenizer,
+    _content_mask_or_empty,
     extract_message_tool_names,
     reject_assistant_in_extension,
     resolve_thinking_retention,
@@ -461,7 +462,7 @@ class GptOssRenderer:
             token_ids=tokens,
             message_indices=indices,
             sampled_mask=sampled,
-            is_content=content_mask,
+            is_content=_content_mask_or_empty(self._tokenizer, content_mask),
             message_roles=[m.get("role") or "" for m in messages],
             message_tool_names=extract_message_tool_names(messages),
         )
@@ -597,7 +598,9 @@ class GptOssRenderer:
             token_ids=previous_ids + ext,
             message_indices=[-1] * len(previous_ids) + ext_indices,
             sampled_mask=[False] * total_len,
-            is_content=[False] * len(previous_ids) + ext_content,
+            is_content=_content_mask_or_empty(
+                self._tokenizer, [False] * len(previous_ids) + ext_content
+            ),
             message_roles=[m.get("role") or "" for m in new_messages],
             message_tool_names=extract_message_tool_names(new_messages),
         )

{renderers-0.1.11.dev2 → renderers-0.1.11.dev3}/renderers/hy3.py

@@ -33,6 +33,9 @@ from renderers.base import (
     RenderedTokens,
     ToolSpec,
     Tokenizer,
+    _content_mask_or_empty,
+    _get_offset_tokenizer,
+    _infer_offsets_from_decode,
     attribute_text_segments,
     extract_message_tool_names,
     reject_assistant_in_extension,
@@ -238,13 +241,34 @@ class Hy3Renderer:
         if not segments:
             return []
         full_text = "".join(text for text, _, _ in segments)
-        encoding = self._tokenizer(
-            full_text,
-            add_special_tokens=False,
-            return_offsets_mapping=True,
-        )
-        token_ids = list(encoding["input_ids"])
-        offsets = list(encoding["offset_mapping"])
+        offset_tokenizer = _get_offset_tokenizer(self._tokenizer)
+        if offset_tokenizer is None:
+            token_ids = self._encode(full_text)
+            offsets = _infer_offsets_from_decode(
+                self._tokenizer,
+                token_ids,
+                full_text,
+            )
+            if offsets is None:
+                # Token IDs remain exact even when a lossy decoder prevents
+                # reconstructing boundaries. Associate the opaque joined run
+                # with a contributing caller system message rather than
+                # silently classifying its body as global scaffold.
+                fallback_idx = next(
+                    (msg_idx for text, _, msg_idx in segments if text and msg_idx >= 0),
+                    -1,
+                )
+                return [(token_id, False, fallback_idx) for token_id in token_ids]
+            has_content_attribution = False
+        else:
+            encoding = offset_tokenizer(
+                full_text,
+                add_special_tokens=False,
+                return_offsets_mapping=True,
+            )
+            token_ids = list(encoding["input_ids"])
+            offsets = list(encoding["offset_mapping"])
+            has_content_attribution = True
 
         spans: list[tuple[int, int, bool, int]] = []
         pos = 0
@@ -254,13 +278,19 @@ class Hy3Renderer:
         total_len = pos
 
         out: list[tuple[int, bool, int]] = []
-        last = (spans[-1][2], spans[-1][3])
+        last = (
+            spans[-1][2] if has_content_attribution else False,
+            spans[-1][3],
+        )
         for tok_id, (start, _end) in zip(token_ids, offsets):
             attr = last
             if start < total_len:
                 for seg_start, seg_end, seg_is_content, seg_idx in spans:
                     if seg_start <= start < seg_end:
-                        attr = (seg_is_content, seg_idx)
+                        attr = (
+                            seg_is_content if has_content_attribution else False,
+                            seg_idx,
+                        )
                         break
             out.append((tok_id, attr[0], attr[1]))
         return out
@@ -431,7 +461,7 @@ class Hy3Renderer:
             token_ids=tokens,
             message_indices=indices,
             sampled_mask=sampled,
-            is_content=content_mask,
+            is_content=_content_mask_or_empty(self._tokenizer, content_mask),
             message_roles=[m.get("role") or "" for m in messages],
             message_tool_names=extract_message_tool_names(messages),
         )
@@ -699,7 +729,9 @@ class Hy3Renderer:
             token_ids=previous_ids + ext,
             message_indices=[-1] * len(previous_ids) + ext_indices,
             sampled_mask=[False] * total_len,
-            is_content=[False] * len(previous_ids) + ext_content,
+            is_content=_content_mask_or_empty(
+                self._tokenizer, [False] * len(previous_ids) + ext_content
+            ),
             message_roles=[m.get("role") or "" for m in new_messages],
             message_tool_names=extract_message_tool_names(new_messages),
         )

{renderers-0.1.11.dev2 → renderers-0.1.11.dev3}/renderers/inkling.py

@@ -48,6 +48,7 @@ from renderers.base import (
     RenderedTokens,
     ToolSpec,
     Tokenizer,
+    _content_mask_or_empty,
     _require_transformers,
     extract_message_tool_names,
     reject_assistant_in_extension,
@@ -492,7 +493,7 @@ class InklingRenderer:
             token_ids=tokens,
             message_indices=indices,
             sampled_mask=sampled,
-            is_content=content_mask,
+            is_content=_content_mask_or_empty(self._tokenizer, content_mask),
             message_roles=[m.get("role") or "" for m in messages],
             message_tool_names=tool_names,
             multi_modal_data=mm_data,
@@ -989,7 +990,7 @@ class InklingRenderer:
             token_ids=tokens,
             message_indices=indices,
             sampled_mask=sampled,
-            is_content=content_mask,
+            is_content=_content_mask_or_empty(self._tokenizer, content_mask),
             message_roles=[m.get("role") or "" for m in new_messages],
             message_tool_names=tool_names,
             multi_modal_data=mm_data,

{renderers-0.1.11.dev2 → renderers-0.1.11.dev3}/renderers/kimi_k2.py

@@ -22,6 +22,7 @@ from renderers.base import (
     RenderedTokens,
     ToolSpec,
     Tokenizer,
+    _content_mask_or_empty,
     extract_message_tool_names,
     reject_assistant_in_extension,
     resolve_thinking_retention,
@@ -309,7 +310,7 @@ class KimiK2Renderer:
             token_ids=token_ids,
             message_indices=indices,
             sampled_mask=sampled,
-            is_content=content_mask,
+            is_content=_content_mask_or_empty(self._tokenizer, content_mask),
             message_roles=[m.get("role") or "" for m in caller_messages],
             message_tool_names=extract_message_tool_names(caller_messages),
         )
@@ -464,7 +465,9 @@ class KimiK2Renderer:
             token_ids=previous_ids + ext,
             message_indices=[-1] * len(previous_ids) + ext_indices,
             sampled_mask=[False] * total_len,
-            is_content=[False] * len(previous_ids) + ext_content,
+            is_content=_content_mask_or_empty(
+                self._tokenizer, [False] * len(previous_ids) + ext_content
+            ),
             message_roles=[m.get("role") or "" for m in new_messages],
             message_tool_names=extract_message_tool_names(new_messages),
         )