verifiers 0.3.2.dev31__py3-none-any.whl → 0.3.2.dev33__py3-none-any.whl

verifiers/v1/agent.py

@@ -362,21 +362,26 @@ class Agent:
         runtime: Runtime | None = None,
         tools: Mapping[str, SharedToolServer] | None = None,
         on_trace: Callable[[Trace], None] | None = None,
+        collect_artifacts: bool = False,
     ) -> Trace:
         """Run this agent on `task` once and return the trace: one segment — the
         program runs on the task's prompt until it exits (a multi-turn exchange
         is `interaction()`). `runtime` places it into a live borrowed box instead of
         provisioning one; `tools` are live servers borrowed from their
         owner, counted in the pairing check; `on_trace` observes the trace the
-        moment it's minted, before any I/O. Retries whole while the trace ends
-        with a retryable error (`config.retries`) — never into a borrowed box;
-        the final trace keeps earlier attempts' errors."""
+        moment it's minted, before any I/O. `collect_artifacts` captures the task's
+        declared artifacts after its finalizer while its container runtime is still
+        alive. Retries whole while the trace ends with a retryable error
+        (`config.retries`) — never into a borrowed box; the final trace keeps earlier
+        attempts' errors."""
         if self._closed:
             raise RuntimeError("Agent is closed; create a new agent")
         retry = self.config.retries
         history: list = []
         for attempt in range(retry.max_retries + 1):
-            trace = await self._run_once(task, runtime, tools, on_trace)
+            trace = await self._run_once(
+                task, runtime, tools, on_trace, collect_artifacts
+            )
             if attempt == retry.max_retries or not trace_should_retry(trace, retry):
                 break
             if runtime is not None:
@@ -407,9 +412,20 @@ class Agent:
         runtime: Runtime | None,
         shared_tools: Mapping[str, SharedToolServer] | None,
         on_trace: Callable[[Trace], None] | None,
+        collect_artifacts: bool,
     ) -> Trace:
         params = self._rollout_params(task, runtime, dict(shared_tools or {}))
-        run = Rollout(task=task, on_trace=on_trace, **params)
+        if collect_artifacts and isinstance(params["runtime_config"], SubprocessConfig):
+            raise TypeError(
+                "artifact collection requires a container runtime; subprocess "
+                "artifacts live in a host-only temporary working directory"
+            )
+        run = Rollout(
+            task=task,
+            on_trace=on_trace,
+            collect_artifacts=collect_artifacts,
+            **params,
+        )
         try:
             if await run.open():
                 await run.step()
@@ -626,6 +642,7 @@ class _EpisodeAgent(Agent):
         runtime: Runtime | None = None,
         tools: Mapping[str, SharedToolServer] | None = None,
         on_trace: Callable[[Trace], None] | None = None,
+        collect_artifacts: bool = False,
     ) -> Trace:
         async with self._gate or nullcontext():
             trace = await super().run(
@@ -633,6 +650,7 @@ class _EpisodeAgent(Agent):
                 runtime=runtime,
                 tools=tools if tools is not None else self._shared_for(task),
                 on_trace=self._watch(on_trace),
+                collect_artifacts=collect_artifacts,
             )
         self._completed.append(trace)
         return trace

verifiers/v1/envs/agentic_judge/env.py

@@ -342,7 +342,7 @@ class IsolatedAgenticJudgeEnv(AgenticJudgeEnv):
     """Judge only collected artifacts in a fresh box with the solver's policy."""
 
     async def run(self, task: vf.Task, agents: vf.Agents) -> None:
-        solution = await agents.solver.run(task)
+        solution = await agents.solver.run(task, collect_artifacts=True)
         if not solution.ok:
             raise RuntimeError("the solver's rollout failed, so the judge never ran")
         await agents.judge.run(

verifiers/v1/graph.py

@@ -121,6 +121,12 @@ class MessageNode(BaseModel):
     reference_logprobs: list[float] | None = None
     """Reference-model logprobs over the sampled tokens, in the same compact layout as
     `logprobs`. None means no reference model scored this node."""
+    trainer_logprobs: list[float] | None = None
+    """Trainer-recomputed logprobs over the sampled tokens, in the same compact layout as
+    `logprobs`. None means no trainer forward annotated this node."""
+    entropies: list[float] | None = None
+    """Trainer policy entropies over the sampled tokens, in the same compact layout as
+    `logprobs`. None means no trainer forward annotated this node."""
     loss_weights: dict[str, list[float]] | None = None
     """Named loss-weight streams aligned to `token_ids`, consumer-stamped."""
     multi_modal_data: SkipJsonSchema[MultiModalData | None] = None

verifiers/v1/rollout.py

@@ -32,6 +32,7 @@ from verifiers.v1.state import state_cls
 from verifiers.v1.task import Task
 from verifiers.v1.trace import AgentInfo, Trace, TraceTask
 from verifiers.v1.types import Messages, Request, Response, SystemMessage, UserMessage
+from verifiers.v1.utils.artifacts import collect
 from verifiers.v1.utils.decorators import discover_decorated, invoke
 
 logger = logging.getLogger(__name__)
@@ -69,6 +70,7 @@ class Rollout:
         interception: Interception | None = None,
         runtime: Runtime | None = None,
         on_trace: Callable[[Trace], None] | None = None,
+        collect_artifacts: bool = False,
     ) -> None:
         self.task = task
         self.harness = harness
@@ -81,6 +83,7 @@ class Rollout:
         self._interception = interception
         self.runtime = runtime
         self._borrowed_runtime = runtime
+        self._collect_artifacts = collect_artifacts
         self.trace: Trace = Trace(
             task=TraceTask(
                 type=type(task).__name__,
@@ -466,12 +469,14 @@ class Rollout:
             if not self._failed and self._opened:
                 trace.timing.finalize.start = time.time()
                 async with boundary(TaskError, "task finalize"):
-                    await asyncio.wait_for(
-                        invoke(
+                    async with asyncio.timeout(self._timeouts.finalize):
+                        await invoke(
                             self.task.finalize, {"trace": trace, "runtime": runtime}
-                        ),
-                        self._timeouts.finalize,
-                    )
+                        )
+                        if self._collect_artifacts and not trace.state.artifacts:
+                            trace.state.artifacts = await collect(
+                                runtime, self.task.data.artifacts
+                            )
                 now = time.time()
                 trace.timing.finalize.end = now
                 trace.timing.scoring.start = now

verifiers/v1/trace.py

@@ -271,6 +271,20 @@ class Branch(BaseModel):
             return None
         return self.spread(lambda node: node.reference_logprobs)
 
+    @property
+    def trainer_logprobs(self) -> list[float] | None:
+        """Trainer-recomputed logprobs aligned to `token_ids`, or None when unannotated."""
+        if all(node.trainer_logprobs is None for node in self.nodes):
+            return None
+        return self.spread(lambda node: node.trainer_logprobs)
+
+    @property
+    def entropies(self) -> list[float] | None:
+        """Trainer policy entropies aligned to `token_ids`, or None when unannotated."""
+        if all(node.entropies is None for node in self.nodes):
+            return None
+        return self.spread(lambda node: node.entropies)
+
     def loss_weights(self, name: str) -> list[float] | None:
         """One named loss-weight stream aligned to `token_ids`, or None when absent."""
         if all(

{verifiers-0.3.2.dev31.dist-info → verifiers-0.3.2.dev33.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev31
+Version: 0.3.2.dev33
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.2.dev31.dist-info → verifiers-0.3.2.dev33.dist-info}/RECORD

@@ -1,19 +1,19 @@
 verifiers/__init__.py,sha256=yDYsckVQbUkwoh9mU_hBeCLGvS_aWgAxuoNm5LxQpDI,517
 verifiers/v1/__init__.py,sha256=AYjjpOBfpSlWZRN51V1KaxMYMBWSZhBEIOzvNk23pxg,8348
-verifiers/v1/agent.py,sha256=AfjyYmcGpVaOlZkElIz1JZeNRQuZsE_KMZ-LV3uLvzA,31644
+verifiers/v1/agent.py,sha256=QvP7jJMR1NQEzAdcqSxcPRgYYTZYt6gxst3vxmXymOE,32384
 verifiers/v1/env.py,sha256=CEhzjojhg5jqbOYruUnc5U7TtgDHDSx8NiiFA0qfbt8,18234
 verifiers/v1/episode.py,sha256=uStvI4WqgJT50hmHIB55m5B-ASqqLz-fZ-ynBz9uzEY,5474
 verifiers/v1/errors.py,sha256=uQySmCUrjDYFkBIii1vBFPiEcPFPQZ9EGpas_6HvcQI,5279
-verifiers/v1/graph.py,sha256=09uysIuX7kHCt9XRndpACkQtWocgVbtiLewcEmnzmD4,35787
+verifiers/v1/graph.py,sha256=y8B9mW-8BUEs0B7vg7YEU1WblfsgagIHmDCK0MWujSQ,36193
 verifiers/v1/harness.py,sha256=ziECqEBWV34UWbLlxrkhY6MwkbVYkoXBOoaY7mZxwTs,14753
 verifiers/v1/judge.py,sha256=Pvr0C41ah1qkNSZ0WXdkDvBMfv5YSfY1REHncwlJ8cg,9346
-verifiers/v1/rollout.py,sha256=EnF3RuBfMyKNjJIs0LZaImkSvfe9LYLXc8dYP-1a2Ew,23628
+verifiers/v1/rollout.py,sha256=hoKGn1CbShbyjEM9PgYq8pdKgoC91VebTkBSgUHLtiE,23978
 verifiers/v1/semantic.py,sha256=wssvBSEpY_tgzpFUADG9-N4HkDXgCZvNQ1pdf6qAsS0,4680
 verifiers/v1/session.py,sha256=xM7JJJVJACk0JfH9RAN1Ts8gJJKr2tGGRiIgQTrdieY,18692
 verifiers/v1/state.py,sha256=EckF2bWp-vV4b1jYJ9sLI5xrfGuI5spIgYYwW926toI,595
 verifiers/v1/task.py,sha256=vaVkBu00riiNU1Ke4NeeY2NQTXxOguz9d6NLe55hO_Q,11468
 verifiers/v1/taskset.py,sha256=fp2E0IEhL_Ybj9cegZwljfTmW27p_30zTWHFKw_kXE4,4374
-verifiers/v1/trace.py,sha256=ABAr9hxpLiAwVOb4h6djAom9DJbOy4BuDqvx0wJDUDY,28439
+verifiers/v1/trace.py,sha256=Adj9iR7ilLE0UscNMiiQTt3NMPPFPx1wpYP17UVr1Bc,29051
 verifiers/v1/types.py,sha256=a1wqL8T7KQg9WxkaDIZpmOyU6xT2VeSr1K4SWiW8ef4,9660
 verifiers/v1/acp/__init__.py,sha256=wUPZVYBFjJuOnIIoYONBFuA04A1DJf_dOyKY3QwK37Q,12722
 verifiers/v1/acp/runner.py,sha256=gMXFmsVg2qD0Zhrx7kvC6Me2Z-wsfBamRIO0i45UW8g,11097
@@ -65,7 +65,7 @@ verifiers/v1/dialects/chat.py,sha256=_pMsNpbffiXiK0_0tboAKtuj7-smUnkS9H0LPK4j1mU
 verifiers/v1/dialects/responses.py,sha256=KRuS9TKw2uoNz4NsG7E-H6eyhm_EJQwVtZ5_XW9iB3c,29090
 verifiers/v1/envs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 verifiers/v1/envs/agentic_judge/__init__.py,sha256=LfEcRHLG_siKzhzWmODDXWSLpLgfum3UFdQDK9XG0gk,325
-verifiers/v1/envs/agentic_judge/env.py,sha256=yVMdq0zBOX7vkEkhnhHTUDM-KKcoOIHUt7yEggOs4ng,14615
+verifiers/v1/envs/agentic_judge/env.py,sha256=IZ8G8OgMI4P5JKJhXM0rrsBloWKgqUSzYiKHpqhEPDY,14639
 verifiers/v1/envs/best_of_n/__init__.py,sha256=mubASiwXMNIhnQFDccNAc7yZ0OFsOVK8hH8WIx9b2A4,119
 verifiers/v1/envs/best_of_n/env.py,sha256=KEUJBCVqblnXP_lEwm3mjGdaikBMkecz8UjOFPBhxc0,1987
 verifiers/v1/envs/shared_agentic_judge/__init__.py,sha256=9T4vyYAuqgZSQjqTIceghxnpauvqeW6XGhZSQDCttMA,168
@@ -185,8 +185,8 @@ verifiers/v1/utils/prime.py,sha256=UTYRjp9cbjNb6CVmBHda-1wWZAIIfxNOmTyNuT7_wL4,9
 verifiers/v1/utils/retries.py,sha256=Y2ZgrAjn-qNkRKZP_RVNL_7EK0iaRcZeCa404lpGYi4,5417
 verifiers/v1/utils/score.py,sha256=493yJVMw8teCu9JxapxMFPFzI0hNUqdo0Y2nGW4kckk,6200
 verifiers/v1/utils/version.py,sha256=-obEo_-l9-D8FLef4hYxncOe-uJpxrM1g2Hig_37Sgs,1607
-verifiers-0.3.2.dev31.dist-info/METADATA,sha256=0h0tNHbRYVLoDa0whEvfRNXDH9SkdX41bAn-Ct2aC2A,4144
-verifiers-0.3.2.dev31.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
-verifiers-0.3.2.dev31.dist-info/entry_points.txt,sha256=uqQje0TMsr7k6nXlWxlPs13Si0evb3pVCEDF6c-YL80,241
-verifiers-0.3.2.dev31.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
-verifiers-0.3.2.dev31.dist-info/RECORD,,
+verifiers-0.3.2.dev33.dist-info/METADATA,sha256=kK1FyrtG26OzrKqPEZXVv-Us3NU2OcvSGNbGN4Ji5c8,4144
+verifiers-0.3.2.dev33.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+verifiers-0.3.2.dev33.dist-info/entry_points.txt,sha256=uqQje0TMsr7k6nXlWxlPs13Si0evb3pVCEDF6c-YL80,241
+verifiers-0.3.2.dev33.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
+verifiers-0.3.2.dev33.dist-info/RECORD,,