verifiers 0.1.15.dev180__tar.gz → 0.1.15.dev186__tar.gz

{verifiers-0.1.15.dev180 → verifiers-0.1.15.dev186}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev180
+Version: 0.1.15.dev186
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

verifiers-0.1.15.dev186/tests/test_sandbox_debug_env.py

@@ -0,0 +1,44 @@
+import pytest
+
+import verifiers as vf
+from verifiers.envs.experimental.composable import (
+    SandboxDebugEnv,
+    SandboxDebugRubric,
+    SandboxSpec,
+    SandboxTaskSet,
+    SWEDebugEnv,
+)
+
+
+class DummySandboxTaskSet(SandboxTaskSet):
+    def __init__(self):
+        from datasets import Dataset
+
+        super().__init__(
+            dataset=Dataset.from_list([{"question": "noop", "info": {}, "answer": ""}])
+        )
+
+    def get_instruction(self, info: dict) -> str:
+        return "Run the debug step."
+
+    def get_rubric(self) -> vf.Rubric:
+        return vf.Rubric()
+
+    def get_sandbox_spec(self, info: dict) -> SandboxSpec:
+        return SandboxSpec()
+
+
+def test_swe_debug_env_is_deprecated_subclass_not_alias():
+    assert SWEDebugEnv is not SandboxDebugEnv
+    assert issubclass(SWEDebugEnv, SandboxDebugEnv)
+    assert SandboxDebugRubric.__name__ == "SandboxDebugRubric"
+
+    with pytest.warns(DeprecationWarning, match="use SandboxDebugEnv"):
+        env = SWEDebugEnv(
+            DummySandboxTaskSet(),
+            debug_step="command",
+            debug_command="true",
+        )
+
+    assert isinstance(env, SandboxDebugEnv)
+    assert env.labels == ["sandbox-debug"]

{verifiers-0.1.15.dev180 → verifiers-0.1.15.dev186}/verifiers/envs/experimental/__init__.py

@@ -8,6 +8,8 @@ __all__ = [
     "TaskSet",
     "Harness",
     "ComposableEnv",
+    "SandboxDebugEnv",
+    "SandboxDebugRubric",
     "SWEDebugEnv",
 ]
 
@@ -20,6 +22,8 @@ def __getattr__(name: str):
         "TaskSet": "verifiers.envs.experimental.composable:TaskSet",
         "Harness": "verifiers.envs.experimental.composable:Harness",
         "ComposableEnv": "verifiers.envs.experimental.composable:ComposableEnv",
+        "SandboxDebugEnv": "verifiers.envs.experimental.composable:SandboxDebugEnv",
+        "SandboxDebugRubric": "verifiers.envs.experimental.composable:SandboxDebugRubric",
         "SWEDebugEnv": "verifiers.envs.experimental.composable:SWEDebugEnv",
     }
     if name in _lazy:

{verifiers-0.1.15.dev180 → verifiers-0.1.15.dev186}/verifiers/envs/experimental/composable/__init__.py

@@ -7,6 +7,10 @@ from verifiers.envs.experimental.composable.task import (
 )
 from verifiers.envs.experimental.composable.harness import Harness
 from verifiers.envs.experimental.composable.composable_env import ComposableEnv
+from verifiers.envs.experimental.composable.sandbox_debug_env import (
+    SandboxDebugEnv,
+    SandboxDebugRubric,
+)
 from verifiers.envs.experimental.composable.swe_debug_env import SWEDebugEnv
 
 __all__ = [
@@ -16,6 +20,8 @@ __all__ = [
     "SandboxTaskSet",
     "Harness",
     "ComposableEnv",
+    "SandboxDebugEnv",
+    "SandboxDebugRubric",
     "SWEDebugEnv",
     "discover_sibling_dir",
 ]

verifiers-0.1.15.dev180/verifiers/envs/experimental/composable/swe_debug_env.py → verifiers-0.1.15.dev186/verifiers/envs/experimental/composable/sandbox_debug_env.py

@@ -1,4 +1,4 @@
-"""No-agent debugger for SWE-style SandboxTaskSet instances."""
+"""No-agent debugger for sandbox-backed TaskSet instances."""
 
 import shlex
 import time
@@ -18,8 +18,8 @@ from .task import SandboxTaskSet
 DebugStep = Literal["none", "gold_patch", "command", "script"]
 
 
-class SWEDebugRubric(SandboxMonitorRubric):
-    """Reads the reward set by SWEDebugEnv during setup."""
+class SandboxDebugRubric(SandboxMonitorRubric):
+    """Reads the reward set by SandboxDebugEnv during setup."""
 
     def __init__(self, **kwargs: Any):
         super().__init__(**kwargs)
@@ -29,7 +29,7 @@ class SWEDebugRubric(SandboxMonitorRubric):
         return float(state.get("reward") or 0.0)
 
 
-class SWEDebugEnv(SandboxMixin, vf.MultiTurnEnv):
+class SandboxDebugEnv(SandboxMixin, vf.MultiTurnEnv):
     """Create a task sandbox, optionally mutate it, optionally run tests.
 
     Pipeline:
@@ -80,13 +80,13 @@ class SWEDebugEnv(SandboxMixin, vf.MultiTurnEnv):
         self._cpu_cores = cpu_cores
         self._memory_gb = memory_gb
         self._disk_size_gb = disk_size_gb
-        self.labels = labels or ["swe-debug"]
+        self.labels = labels or ["sandbox-debug"]
         self.timeout_seconds = timeout_seconds
         self.output_tail_chars = output_tail_chars
 
         super().__init__(
             dataset=dataset or taskset.get_dataset,
-            rubric=SWEDebugRubric(),
+            rubric=SandboxDebugRubric(),
             timeout_seconds=timeout_seconds,
         )
         self.init_sandbox_client(**sandbox_kwargs)
@@ -94,7 +94,7 @@ class SWEDebugEnv(SandboxMixin, vf.MultiTurnEnv):
     async def env_response(
         self, messages: Messages, state: State, **kwargs: Any
     ) -> Messages:
-        raise NotImplementedError("SWEDebugEnv does not use multi-turn interaction")
+        raise NotImplementedError("SandboxDebugEnv does not use multi-turn interaction")
 
     async def setup_state(self, state: State) -> None:
         await super().setup_state(state)
@@ -111,7 +111,7 @@ class SWEDebugEnv(SandboxMixin, vf.MultiTurnEnv):
             valid = await self._run_debug_pipeline(state)
         except Exception as e:  # noqa: BLE001
             exc = e
-            state["error"] = vf.SandboxError(f"SWE debug failed: {repr(e)}")
+            state["error"] = vf.SandboxError(f"Sandbox debug failed: {repr(e)}")
             state["reward"] = 0.0
         finally:
             state["elapsed_s"] = time.perf_counter() - t0
@@ -130,7 +130,7 @@ class SWEDebugEnv(SandboxMixin, vf.MultiTurnEnv):
             else self.compute_sandbox_timeout_minutes()
         )
         request = CreateSandboxRequest(
-            name=f"swe-debug-{state.get('example_id', 'unknown')}",
+            name=f"sandbox-debug-{state.get('example_id', 'unknown')}",
             docker_image=spec.image,
             cpu_cores=spec.cpu_cores if self._cpu_cores is None else self._cpu_cores,
             memory_gb=spec.memory_gb if self._memory_gb is None else self._memory_gb,
@@ -170,7 +170,7 @@ class SWEDebugEnv(SandboxMixin, vf.MultiTurnEnv):
                 return False
         elif self.debug_step == "script":
             sandbox_id = state["sandbox_id"]
-            remote_path = "/tmp/swe_debug_script.sh"
+            remote_path = "/tmp/sandbox_debug_script.sh"
             if self.debug_script_path:
                 await self.upload_file(sandbox_id, remote_path, self.debug_script_path)
             else:

verifiers-0.1.15.dev186/verifiers/envs/experimental/composable/swe_debug_env.py

@@ -0,0 +1,39 @@
+"""Deprecated SWE-specific debug environment wrappers."""
+
+from typing import Any
+from warnings import warn
+
+from .sandbox_debug_env import DebugStep, SandboxDebugEnv, SandboxDebugRubric
+
+
+class SWEDebugRubric(SandboxDebugRubric):
+    """Deprecated wrapper for SandboxDebugRubric."""
+
+    def __init__(self, **kwargs: Any):
+        warn(
+            "SWEDebugRubric is deprecated; use SandboxDebugRubric.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        super().__init__(**kwargs)
+
+
+class SWEDebugEnv(SandboxDebugEnv):
+    """Deprecated wrapper for SandboxDebugEnv."""
+
+    def __init__(self, *args: Any, **kwargs: Any):
+        warn(
+            "SWEDebugEnv is deprecated; use SandboxDebugEnv.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        super().__init__(*args, **kwargs)
+
+
+__all__ = [
+    "DebugStep",
+    "SandboxDebugEnv",
+    "SandboxDebugRubric",
+    "SWEDebugEnv",
+    "SWEDebugRubric",
+]

{verifiers-0.1.15.dev180 → verifiers-0.1.15.dev186}/verifiers/envs/experimental/composable/tasksets/swe/README.md

@@ -6,7 +6,7 @@
   they are not known to be available there yet.
 - Validation: ✅ means the taskset has a linked `prime-data` PR and was
   validated with
-  [`SWEDebugEnv`](../../../../../../docs/environments.md#integrations-and-experimental-environments),
+  [`SandboxDebugEnv`](../../../../../../docs/environments.md#integrations-and-experimental-environments),
   — not yet complete.
 
 ## Progress
@@ -306,6 +306,6 @@
    filtered Prime dataset through `prime-data` when validation identifies rows
    to exclude.
 4. Validate with
-   [`SWEDebugEnv`](../../swe_debug_env.py): no-op runs should fail real tasks,
+   [`SandboxDebugEnv`](../../sandbox_debug_env.py): no-op runs should fail real tasks,
    gold-patch runs should pass, and repeated passes should separate task
    quality issues from sandbox or infrastructure failures.

{verifiers-0.1.15.dev180 → verifiers-0.1.15.dev186}/verifiers/v1/harness.py

@@ -220,6 +220,18 @@ class Harness(RuntimeOwnerMixin[ConfigT], Generic[ConfigT]):
     def load_system_prompt(self, config: ConfigT) -> SystemPrompt:
         return config.system_prompt
 
+    def prepare_prompt(self, prompt: Messages, state: State) -> Messages:
+        """Hook to prepare the prompt before each model request.
+
+        Default identity — vf applies no transform of its own. Override in a
+        subclass to return a modified prompt. Applied at the single point every
+        model request passes through (``Runtime.submit_model_request``), so it
+        covers base-program, fn-mode, and host-loop rollouts uniformly. May be
+        sync or async. The returned messages are what the host tokenizes, sends
+        to the model, and records as the trajectory step.
+        """
+        return prompt
+
     def load_program_config(self, config: ConfigT) -> ProgramConfig:
         return config.program.resolve()
 
@@ -521,6 +533,20 @@ class Harness(RuntimeOwnerMixin[ConfigT], Generic[ConfigT]):
                 tool_defs=self.runtime.tool_defs(state),
             )
             turn += 1
+            # Align the local host-loop thread with the exact prompt sent to the model
+            # (which may have been modified by the harness's prepare_prompt hook).
+            trajectory = state.get("trajectory")
+            if isinstance(trajectory, list) and trajectory:
+                last_step = trajectory[-1]
+                if isinstance(last_step, dict):
+                    used_prompt = last_step.get("prompt")
+                    if isinstance(used_prompt, list):
+                        messages = normalize_messages(
+                            cast(Messages, used_prompt), field_name="used_prompt"
+                        )
+                        prompt_messages = [
+                            m.model_dump(exclude_none=True) for m in messages
+                        ]
             messages.extend(await parse_response_message(response))
             rendered_messages = sync_completion()
             tool_calls = list(response.message.tool_calls or [])

{verifiers-0.1.15.dev180 → verifiers-0.1.15.dev186}/verifiers/v1/runtime.py

@@ -883,6 +883,7 @@ class Runtime:
             return self._completed_model_response(state)
         released = False
         try:
+            prompt = await self._prepare_prompt(prompt, state)
             client = self.model_client(state)
             request_start = time.time()
             response = await client.get_response(
@@ -925,6 +926,21 @@ class Runtime:
             if not released:
                 self._release_model_request(state, context)
 
+    async def _prepare_prompt(self, prompt: Messages, state: State) -> Messages:
+        """Apply the harness ``prepare_prompt`` hook (default identity).
+
+        The single point every model request passes through, so a subclass's
+        prompt preparation applies uniformly across base-program, fn-mode, and
+        host-loop rollouts. Tolerates sync or async hooks.
+        """
+        harness = self.harness
+        if harness is None:
+            return prompt
+        result = harness.prepare_prompt(prompt, state)
+        if inspect.isawaitable(result):
+            result = await cast(Awaitable[Messages], result)
+        return result if result is not None else prompt
+
     async def _reserve_model_request(
         self, task: Task, state: State, context: ModelRequestContext
     ) -> bool: