verifiers 0.1.15.dev401__tar.gz → 0.1.15.dev403__tar.gz

{verifiers-0.1.15.dev401 → verifiers-0.1.15.dev403}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev401
+Version: 0.1.15.dev403
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev401 → verifiers-0.1.15.dev403}/tests/test_renderer_client.py

@@ -83,15 +83,13 @@ def test_renderer_client_uses_renderer_model_name_override():
 
 
 def test_renderer_client_threads_chat_template_kwargs_into_pool():
-    """``sampling_args.extra_body.chat_template_kwargs`` land on the
-    concrete RendererConfig (resolving Auto/None against
-    ``MODEL_RENDERER_MAP`` when needed) and are stripped from the params
-    forwarded to /generate. Covers explicit / Auto / None bases."""
-    from renderers import AutoRendererConfig, Qwen3RendererConfig
+    """``sampling_args.extra_body.chat_template_kwargs`` are routed to
+    renderer pool construction and stripped from /generate sampling params.
+    Renderers owns concrete config resolution/validation."""
+    from renderers import Qwen3RendererConfig
 
     bases = [
         Qwen3RendererConfig(enable_thinking=True),
-        AutoRendererConfig(preserve_all_thinking=True),
         None,
     ]
     for base in bases:
@@ -134,29 +132,16 @@ def test_renderer_client_threads_chat_template_kwargs_into_pool():
                 )
             )
 
-        expected_preserve_all = (
-            base.preserve_all_thinking
-            if isinstance(base, AutoRendererConfig)
-            else False
-        )
         create_pool_mock.assert_called_once_with(
             "Qwen/Qwen3-8B",
-            Qwen3RendererConfig(
-                enable_thinking=False,
-                preserve_all_thinking=expected_preserve_all,
-            ),
+            base,
             size=1,
+            chat_template_kwargs={"enable_thinking": False},
         )
         assert captured["sampling_params"] == {"top_k": 20}
 
 
-def test_renderer_client_auto_resolves_against_renderer_model_name_override():
-    """When ``ClientConfig.renderer_model_name`` overrides the API request
-    model, auto-resolution looks up the OVERRIDE in ``MODEL_RENDERER_MAP``
-    (it's what loads the tokenizer the renderer holds) — not the request
-    model, which may not be in the map at all."""
-    from renderers import Qwen3RendererConfig
-
+def test_renderer_client_forwards_renderer_model_name_with_chat_template_kwargs():
     RendererClient._shared_pools.clear()
 
     client = object.__new__(RendererClient)
@@ -183,7 +168,7 @@ def test_renderer_client_auto_resolves_against_renderer_model_name_override():
         asyncio.run(
             client.get_native_response(
                 prompt=[{"role": "user", "content": "hi"}],
-                model="r8-smoke",  # not in MODEL_RENDERER_MAP
+                model="r8-smoke",
                 sampling_args={
                     "extra_body": {"chat_template_kwargs": {"enable_thinking": False}}
                 },
@@ -191,46 +176,14 @@ def test_renderer_client_auto_resolves_against_renderer_model_name_override():
             )
         )
 
-    # Resolves against renderer_model_name (Qwen/Qwen3-8B → "qwen3") rather
-    # than the request "r8-smoke" (which would fall through to default).
     create_pool_mock.assert_called_once_with(
         "Qwen/Qwen3-8B",
-        Qwen3RendererConfig(enable_thinking=False),
+        None,
         size=1,
+        chat_template_kwargs={"enable_thinking": False},
     )
 
 
-def test_renderer_client_rejects_invalid_chat_template_kwargs():
-    """Unknown / mistyped chat_template_kwargs surface as a pydantic
-    ``ValidationError`` (``extra="forbid"`` on the typed RendererConfig)."""
-    from pydantic import ValidationError
-    from renderers import Qwen3RendererConfig
-
-    RendererClient._shared_pools.clear()
-
-    client = object.__new__(RendererClient)
-    client._renderer = None
-    client._pool_size = 1
-    client._config = vf.ClientConfig(
-        client_type="renderer", renderer_config=Qwen3RendererConfig()
-    )
-    client._client = object()  # type: ignore[attr-defined]
-
-    with pytest.raises(ValidationError, match="enable_thinkng"):
-        asyncio.run(
-            client.get_native_response(
-                prompt=[{"role": "user", "content": "hi"}],
-                model="Qwen/Qwen3-8B",
-                sampling_args={
-                    "extra_body": {
-                        "chat_template_kwargs": {"enable_thinkng": False},  # typo
-                    }
-                },
-                tools=None,
-            )
-        )
-
-
 # Provenance: Eli's review on PR #1068, comment 3150580768.
 #   "RendererClient parses the GPT-OSS assistant tool call into ToolCall(name=...),
 #   but ToolEnv returns ToolMessage with only content/tool_call_id, and
@@ -675,11 +628,26 @@ _TRUNCATED_ANCHOR_MODELS = [
 
 
 @lru_cache(maxsize=None)
-def _load_tokenizer_and_renderer(model_name: str, renderer_name: str):
+def _load_tokenizer_and_renderer(
+    model_name: str, renderer_name: str, thinking_retention: str | None = None
+):
+    from renderers import AutoRendererConfig
     from transformers import AutoTokenizer
 
     tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
-    renderer = create_renderer(tokenizer, config_from_name(renderer_name))
+    config = config_from_name(renderer_name)
+    if thinking_retention is not None:
+        if config is None and "thinking_retention" in AutoRendererConfig.model_fields:
+            config = AutoRendererConfig(thinking_retention=thinking_retention)
+        elif config is not None and "thinking_retention" in type(config).model_fields:
+            updates = {"thinking_retention": thinking_retention}
+            if (
+                thinking_retention == "all"
+                and "auto_drop_analysis" in type(config).model_fields
+            ):
+                updates["auto_drop_analysis"] = False
+            config = type(config).model_validate({**config.model_dump(), **updates})
+    renderer = create_renderer(tokenizer, config)
     return tokenizer, renderer
 
 
@@ -729,11 +697,14 @@ def _build_truncated_state(tokenizer, renderer):
 async def test_get_incremental_prompt_ids_bridges_over_truncated_step(
     model_id, renderer_id
 ):
-    """The bridge anchors on the truncated step and returns new prompt_ids
-    that start with prev_prompt + prev_completion byte-identically (the
-    extension invariant). This is what keeps interleave_rollout from
-    fragmenting the rollout into two samples."""
-    tokenizer, renderer = _load_tokenizer_and_renderer(model_id, renderer_id)
+    """With full thinking retention, the bridge anchors on the truncated
+    step and returns new prompt_ids that start with prev_prompt +
+    prev_completion byte-identically (the extension invariant)."""
+    tokenizer, renderer = _load_tokenizer_and_renderer(
+        model_id,
+        renderer_id,
+        "all",
+    )
     prev_prompt_ids, prev_completion_ids, state, next_turn_prompt = (
         _build_truncated_state(tokenizer, renderer)
     )

{verifiers-0.1.15.dev401 → verifiers-0.1.15.dev403}/verifiers/clients/renderer_client.py

@@ -18,7 +18,6 @@ from openai import AsyncOpenAI
 from renderers import Message as RendererMessage
 from renderers import OverlongPromptError as RendererOverlongPromptError
 from renderers import (
-    AutoRendererConfig,
     MultimodalRenderer,
     ParsedToolCall,
     RenderedTokens,
@@ -26,13 +25,11 @@ from renderers import (
     RendererConfig,
     RendererPool,
     ToolSpec,
-    config_from_name,
     create_renderer_pool,
     is_multimodal,
 )
 from renderers import ToolCall as RendererToolCall
 from renderers import ToolCallFunction
-from renderers.base import MODEL_RENDERER_MAP
 from renderers.client import _maybe_offload, generate
 
 from verifiers.clients.client import Client
@@ -100,6 +97,14 @@ def _get_value(obj: Any, key: str, default: Any = None) -> Any:
     return getattr(obj, key, default)
 
 
+def _chat_template_kwargs_key(
+    chat_template_kwargs: Mapping[str, Any] | None,
+) -> str | None:
+    if not chat_template_kwargs:
+        return None
+    return json.dumps(chat_template_kwargs, sort_keys=True, separators=(",", ":"))
+
+
 def _normalize_for_comparison(value: Any, _key: str | None = None) -> Any:
     # tool_call.arguments is serialized as a string on one side (our trajectory
     # uses json.dumps with default separators) and often comes back from
@@ -380,53 +385,6 @@ async def _get_incremental_prompt_ids(
     return None
 
 
-def _resolve_renderer_config(
-    base: RendererConfig | None,
-    chat_template_kwargs: Mapping[str, Any] | None,
-    *,
-    renderer_model: str,
-) -> RendererConfig | None:
-    """Merge ``chat_template_kwargs`` into a typed ``RendererConfig``.
-
-    When ``base`` is ``None`` or ``AutoRendererConfig`` (would auto-resolve
-    inside ``renderers.create_renderer``), we pull resolution forward via
-    ``MODEL_RENDERER_MAP`` so kwargs land on the concrete config variant
-    and pydantic validates them against the actual renderer's schema —
-    ``AutoRendererConfig`` intentionally carries only ``preserve_*`` and
-    would reject template kwargs like ``enable_thinking``. ``renderer_model``
-    must match what the pool will tokenize with (i.e.
-    ``ClientConfig.renderer_model_name`` when set, else the request model),
-    so resolution agrees with the tokenizer the renderer will hold.
-
-    Kwargs override fields with the same name on the (resolved) base.
-    Typed configs (``extra="forbid"``) reject unknown keys with a
-    field-path error; ``DefaultRendererConfig`` (``extra="allow"``) keeps
-    the escape hatch for arbitrary jinja kwargs.
-    """
-    if not chat_template_kwargs:
-        return base
-
-    # Resolve auto → concrete (mirrors ``renderers._resolve_auto``) so
-    # ``enable_thinking`` etc. validate against the right schema instead of
-    # ``AutoRendererConfig``'s minimal one. Carries ``preserve_*`` across.
-    if base is None or isinstance(base, AutoRendererConfig):
-        renderer_name = MODEL_RENDERER_MAP.get(renderer_model, "default")
-        # ``config_from_name`` returns ``None`` only for ``"auto"``, which
-        # ``MODEL_RENDERER_MAP.get(..., "default")`` excludes — assert for ty.
-        concrete = config_from_name(renderer_name)
-        assert concrete is not None
-        if isinstance(base, AutoRendererConfig):
-            concrete = concrete.model_copy(
-                update={
-                    "preserve_all_thinking": base.preserve_all_thinking,
-                    "preserve_thinking_between_tool_calls": base.preserve_thinking_between_tool_calls,
-                }
-            )
-        base = cast(RendererConfig, concrete)
-
-    return type(base).model_validate({**base.model_dump(), **chat_template_kwargs})
-
-
 class RendererClient(
     Client[AsyncOpenAI, list[RendererMessage], dict[str, Any], ToolSpec]
 ):
@@ -439,13 +397,12 @@ class RendererClient(
     so that concurrent rollouts tokenize in parallel threads.
     """
 
-    # Cache key is ``(renderer_model_name, pool_size, renderer_config_json)``.
-    # ``renderer_config`` is a frozen pydantic model so it's hashable directly,
-    # but we serialize it via ``model_dump_json()`` for a stable, deterministic
-    # key shape that's safe across pydantic version bumps.
+    # Cache key is ``(renderer_model_name, pool_size, renderer_config_json,
+    # chat_template_kwargs_json)``. Renderers owns the concrete config
+    # resolution; verifiers only separates pools whose construction inputs differ.
     _shared_pools: ClassVar[
         dict[
-            tuple[str, int, str | None],
+            tuple[str, int, str | None, str | None],
             RendererPool,
         ]
     ] = {}
@@ -477,6 +434,7 @@ class RendererClient(
         model: str,
         *,
         renderer_config: RendererConfig | None = None,
+        chat_template_kwargs: Mapping[str, Any] | None = None,
     ) -> Renderer | RendererPool:
         if self._renderer is not None:
             return self._renderer
@@ -493,14 +451,18 @@ class RendererClient(
         cfg_key = (
             renderer_config.model_dump_json() if renderer_config is not None else None
         )
-        cache_key = (renderer_model, self._pool_size, cfg_key)
+        kwargs_key = _chat_template_kwargs_key(chat_template_kwargs)
+        cache_key = (renderer_model, self._pool_size, cfg_key, kwargs_key)
 
         with self._shared_pools_lock:
             if cache_key not in self._shared_pools:
+                pool_kwargs: dict[str, Any] = {"size": self._pool_size}
+                if chat_template_kwargs:
+                    pool_kwargs["chat_template_kwargs"] = chat_template_kwargs
                 self._shared_pools[cache_key] = create_renderer_pool(
                     renderer_model,
                     renderer_config,
-                    size=self._pool_size,
+                    **pool_kwargs,
                 )
 
         return self._shared_pools[cache_key]
@@ -543,25 +505,14 @@ class RendererClient(
         }
         sampling_params: dict[str, Any] = dict(args.pop("extra_body", None) or {})
 
-        # ``chat_template_kwargs`` belong to the renderer, not the engine —
-        # peel them off the per-request sampling and fold them into the
-        # typed RendererConfig. Pool cache key already includes the
-        # effective config so identical kwargs reuse the same renderer.
+        # ``chat_template_kwargs`` belong to the renderer, not the engine.
+        # Renderers owns resolving them against the concrete chat template;
+        # verifiers only routes them into pool construction.
         chat_template_kwargs = sampling_params.pop("chat_template_kwargs", None)
-        # Auto-resolution must agree with the model the pool will tokenize
-        # against — ``renderer_model_name`` overrides the request ``model``
-        # when set (same precedence ``_get_renderer_or_pool`` uses below).
-        renderer_model = (
-            self._config.renderer_model_name
-            if self._config is not None and self._config.renderer_model_name is not None
-            else model
-        )
-        effective_cfg = _resolve_renderer_config(
-            self._config.renderer_config if self._config is not None else None,
-            chat_template_kwargs,
-            renderer_model=renderer_model,
+        renderer = self._get_renderer_or_pool(
+            model,
+            chat_template_kwargs=chat_template_kwargs,
         )
-        renderer = self._get_renderer_or_pool(model, renderer_config=effective_cfg)
 
         for key in (
             "temperature",

{verifiers-0.1.15.dev401 → verifiers-0.1.15.dev403}/verifiers/envs/experimental/composable/tasksets/__init__.py

@@ -17,10 +17,6 @@ from verifiers.envs.experimental.composable.tasksets.swe.swe_tasksets import (
     make_swesmith_taskset,
     make_swesmith_ts_taskset,
 )
-from verifiers.envs.experimental.composable.tasksets.lean.lean_task import (
-    LEAN_SYSTEM_PROMPT,
-    LeanTaskSet,
-)
 from verifiers.envs.experimental.composable.tasksets.math.math_task import MathTaskSet
 from verifiers.envs.experimental.composable.tasksets.cp.cp_task import (
     CPRubric,
@@ -59,8 +55,6 @@ __all__ = [
     "make_swesmith_rs_taskset",
     "make_swesmith_cpp_taskset",
     "make_swesmith_php_taskset",
-    "LeanTaskSet",
-    "LEAN_SYSTEM_PROMPT",
     "MathTaskSet",
     "CPTaskSet",
     "CPRubric",

{verifiers-0.1.15.dev401 → verifiers-0.1.15.dev403}/verifiers/v1/clients/train.py

@@ -193,12 +193,23 @@ class TrainClient(Client):
         self.renderer_model_name = renderer_model_name
         self._pool = None
 
-    def _renderer_pool(self, model: str):
+    def _renderer_pool(
+        self,
+        model: str,
+        *,
+        chat_template_kwargs: Mapping[str, Any] | None = None,
+    ):
+        renderer_model = self.renderer_model_name or model
         if self._pool is None:
             from renderers import create_renderer_pool
 
+            pool_kwargs: dict[str, Any] = {"size": self.pool_size}
+            if chat_template_kwargs:
+                pool_kwargs["chat_template_kwargs"] = chat_template_kwargs
             self._pool = create_renderer_pool(
-                self.renderer_model_name or model, self.config, size=self.pool_size
+                renderer_model,
+                self.config,
+                **pool_kwargs,
             )
         return self._pool
 
@@ -231,7 +242,6 @@ class TrainClient(Client):
             tools = parse_tools(body.get("tools"))
         else:
             prompt, tools = dialect.parse_request(body)
-        renderer = self._renderer_pool(model)
         from renderers.client import _maybe_offload, generate
 
         wire_tools = [tool_to_wire(t) for t in tools] if tools else None
@@ -241,7 +251,16 @@ class TrainClient(Client):
         prompt_ids: list[int] | None = None
         multi_modal_data = None
         prompt_attribution: RenderedTokens | None = None
-        sampling_params = sampling_args.model_dump(exclude_none=True)
+        raw_sampling = sampling_args.model_dump(exclude_none=True)
+        sampling_params: dict[str, Any] = dict(
+            raw_sampling.pop("extra_body", None) or {}
+        )
+        chat_template_kwargs = sampling_params.pop("chat_template_kwargs", None)
+        sampling_params.update(raw_sampling)
+        renderer = self._renderer_pool(
+            model,
+            chat_template_kwargs=chat_template_kwargs,
+        )
         bridged_turn: PendingTurn | None = None
 
         # Only build the (O(context)) previous-turn token ids once the cheap guards pass — a

{verifiers-0.1.15.dev401 → verifiers-0.1.15.dev403}/verifiers/v1/tasksets/__init__.py

@@ -5,5 +5,18 @@ here — it imports the optional `textarena` dependency at module load — but s
 `verifiers.v1.tasksets.textarena`."""
 
 from verifiers.v1.tasksets.harbor import HarborConfig, HarborTaskset
+from verifiers.v1.tasksets.lean import (
+    LeanConfig,
+    LeanDatasetConfig,
+    LeanTask,
+    LeanTaskset,
+)
 
-__all__ = ["HarborConfig", "HarborTaskset"]
+__all__ = [
+    "HarborConfig",
+    "HarborTaskset",
+    "LeanConfig",
+    "LeanDatasetConfig",
+    "LeanTask",
+    "LeanTaskset",
+]

verifiers-0.1.15.dev403/verifiers/v1/tasksets/lean/__init__.py

@@ -0,0 +1,31 @@
+from verifiers.v1.tasksets.lean.scoring import (
+    build_starter_file,
+    expected_protected_signature,
+    parse_compile_output,
+    protected_signature_substring_present,
+    strip_lean_comments,
+)
+from verifiers.v1.tasksets.lean.taskset import (
+    DEFAULT_DOCKER_IMAGE,
+    LEAN_PROJECT_PATH,
+    PROOF_FILE_PATH,
+    LeanConfig,
+    LeanDatasetConfig,
+    LeanTask,
+    LeanTaskset,
+)
+
+__all__ = [
+    "DEFAULT_DOCKER_IMAGE",
+    "LEAN_PROJECT_PATH",
+    "PROOF_FILE_PATH",
+    "LeanConfig",
+    "LeanDatasetConfig",
+    "LeanTask",
+    "LeanTaskset",
+    "build_starter_file",
+    "expected_protected_signature",
+    "parse_compile_output",
+    "protected_signature_substring_present",
+    "strip_lean_comments",
+]