renderers 0.1.12.dev19__tar.gz → 0.1.12.dev20__tar.gz

{renderers-0.1.12.dev19 → renderers-0.1.12.dev20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: renderers
-Version: 0.1.12.dev19
+Version: 0.1.12.dev20
 Summary: Chat template renderers — deterministic message-to-token conversion for LLM training
 License-Expression: Apache-2.0
 License-File: LICENSE

{renderers-0.1.12.dev19 → renderers-0.1.12.dev20}/renderers/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.1.12.dev19'
-__version_tuple__ = version_tuple = (0, 1, 12, 'dev19')
+__version__ = version = '0.1.12.dev20'
+__version_tuple__ = version_tuple = (0, 1, 12, 'dev20')
 
 __commit_id__ = commit_id = None

{renderers-0.1.12.dev19 → renderers-0.1.12.dev20}/renderers/base.py

@@ -782,8 +782,9 @@ class Renderer(Protocol):
         Pass the exact ``prompt_ids`` used for sampling whenever available.
         This identifies reasoning prefilled by the prompt, including Gemma's
         post-tool thought channel and partial assistant prefixes. Omitting it,
-        passing None, and passing [] all parse a self-contained completion;
-        generation settings do not imply an initial reasoning channel.
+        passing None, and passing [] are equivalent: the format's completion
+        grammar applies without prompt context. GLM and DeepSeek V4 also use
+        the configured generation mode to determine the initial channel.
 
         ``tools`` is the same list passed to ``render`` for this turn.
         XML-style formats (Qwen3.5, GLM, MiniMax, Laguna) render argument

{renderers-0.1.12.dev19 → renderers-0.1.12.dev20}/renderers/deepseek_v4.py

@@ -22,8 +22,6 @@ import json
 from dataclasses import dataclass, field
 from typing import Any, Mapping
 
-from renderers.reasoning import scan_reasoning, prompt_ends_in_reasoning
-
 from renderers.base import (
     Message,
     ParsedResponse,
@@ -41,7 +39,7 @@ from renderers.base import (
 )
 from renderers.configs import DeepSeekV4RendererConfig
 from renderers.parsing import parse_deepseek_v4
-
+from renderers.reasoning import prompt_ends_in_reasoning
 
 _BOS = "<｜begin▁of▁sentence｜>"
 _EOS = "<｜end▁of▁sentence｜>"
@@ -798,15 +796,18 @@ class DeepSeekV4Renderer:
             self._tokenizer,
             token_ids,
             stop_ids={self._eos},
-            thinking_enabled=prompt_ends_in_reasoning(
-                self._tokenizer,
-                prompt_ids,
-                stop_ids=set(self.get_stop_token_ids()),
-                assistant_prefix="<｜Assistant｜>",
+            thinking_enabled=(
+                prompt_ends_in_reasoning(
+                    self._tokenizer,
+                    prompt_ids,
+                    stop_ids=set(self.get_stop_token_ids()),
+                    assistant_prefix="<｜Assistant｜>",
+                )
+                if prompt_ids
+                else self.config.enable_thinking
             ),
             think_end_id=self._think_end,
             think_start_id=self._think_start,
-            dsml_id=self._dsml,
         )
 
     def get_stop_token_ids(self) -> list[int]:
@@ -827,16 +828,10 @@ class DeepSeekV4Renderer:
         ):
             return None
 
-        boundary = scan_reasoning(
-            self._tokenizer,
-            previous_completion_ids,
-            prompt_ids=previous_prompt_ids,
-            stop_ids=set(self.get_stop_token_ids()),
-            tool_start_id=self._dsml,
-            tool_start_closes_reasoning=True,
-            assistant_prefix="<｜Assistant｜>",
+        boundary = self.parse_response(
+            previous_completion_ids, prompt_ids=previous_prompt_ids, tools=tools
         )
-        if boundary.is_open:
+        if boundary.reasoning_complete is False:
             if any(t in self.get_stop_token_ids() for t in previous_completion_ids):
                 return None
             previous_completion_ids = [*previous_completion_ids, self._think_end]

{renderers-0.1.12.dev19 → renderers-0.1.12.dev20}/renderers/glm45.py

@@ -13,14 +13,12 @@ from __future__ import annotations
 import json
 from typing import Any
 
-from renderers.reasoning import scan_reasoning, prompt_ends_in_reasoning
-
 from renderers.base import (
     Message,
     ParsedResponse,
     RenderedTokens,
-    ToolSpec,
     Tokenizer,
+    ToolSpec,
     _content_mask_or_empty,
     attribute_text_segments,
     extract_message_tool_names,
@@ -296,12 +294,7 @@ class GLM45Renderer:
             arg_value_id=self._arg_value,
             arg_value_end_id=self._arg_value_end,
             tools=tools,
-            prefilled_thinking=prompt_ends_in_reasoning(
-                self._tokenizer,
-                prompt_ids,
-                stop_ids=set(self.get_stop_token_ids()),
-                assistant_prefix="<|assistant|>",
-            ),
+            thinking_enabled=self.config.enable_thinking,
         )
 
     def get_stop_token_ids(self) -> list[int]:
@@ -322,16 +315,10 @@ class GLM45Renderer:
         ):
             return None
 
-        boundary = scan_reasoning(
-            self._tokenizer,
-            previous_completion_ids,
-            prompt_ids=previous_prompt_ids,
-            stop_ids=set(self.get_stop_token_ids()),
-            tool_start_id=self._tool_call_tok,
-            tool_start_closes_reasoning=True,
-            assistant_prefix="<|assistant|>",
+        boundary = self.parse_response(
+            previous_completion_ids, prompt_ids=previous_prompt_ids, tools=tools
         )
-        if boundary.is_open:
+        if boundary.reasoning_complete is False:
             if any(t in self.get_stop_token_ids() for t in previous_completion_ids):
                 return None
             previous_completion_ids = [*previous_completion_ids, self._think_end]

{renderers-0.1.12.dev19 → renderers-0.1.12.dev20}/renderers/glm5.py

@@ -14,8 +14,6 @@ from __future__ import annotations
 import json
 from typing import Any
 
-from renderers.reasoning import scan_reasoning, prompt_ends_in_reasoning
-
 from renderers.base import (
     Message,
     ParsedResponse,
@@ -351,12 +349,7 @@ class GLM5Renderer:
             arg_value_id=self._arg_value,
             arg_value_end_id=self._arg_value_end,
             tools=tools,
-            prefilled_thinking=prompt_ends_in_reasoning(
-                self._tokenizer,
-                prompt_ids,
-                stop_ids=set(self.get_stop_token_ids()),
-                assistant_prefix="<|assistant|>",
-            ),
+            thinking_enabled=getattr(self.config, "enable_thinking", True),
         )
 
     def get_stop_token_ids(self) -> list[int]:
@@ -377,16 +370,10 @@ class GLM5Renderer:
         ):
             return None
 
-        boundary = scan_reasoning(
-            self._tokenizer,
-            previous_completion_ids,
-            prompt_ids=previous_prompt_ids,
-            stop_ids=set(self.get_stop_token_ids()),
-            tool_start_id=self._tool_call_tok,
-            tool_start_closes_reasoning=True,
-            assistant_prefix="<|assistant|>",
+        boundary = self.parse_response(
+            previous_completion_ids, prompt_ids=previous_prompt_ids, tools=tools
         )
-        if boundary.is_open:
+        if boundary.reasoning_complete is False:
             if any(t in self.get_stop_token_ids() for t in previous_completion_ids):
                 return None
             previous_completion_ids = [*previous_completion_ids, self._think_end]