verifiers 0.3.2.dev171__py3-none-any.whl → 0.3.2.dev185__py3-none-any.whl

verifiers/v1/__init__.py

@@ -66,6 +66,8 @@ from verifiers.v1.harness import Harness, HarnessSession
 from verifiers.v1.judge import Judge, JudgeResponse, JudgeView
 from verifiers.v1.judges import (
     Criterion,
+    DecisionJudge,
+    DecisionJudgeConfig,
     ReferenceJudge,
     ReferenceJudgeConfig,
     RubricJudge,
@@ -358,6 +360,8 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "Judges",
     "JudgeResponse",
     "JudgeView",
+    "DecisionJudge",
+    "DecisionJudgeConfig",
     "ReferenceJudge",
     "ReferenceJudgeConfig",
     "RubricJudge",

verifiers/v1/acp/__init__.py

@@ -231,6 +231,10 @@ class ACPHarnessSession(HarnessSession):
         self._stderr_tail = bytearray()
         self._stderr_task: asyncio.Task[None] | None = None
         self._lock = asyncio.Lock()
+        self._write_lock = asyncio.Lock()
+        self._responses: dict[int, asyncio.Future] = {}
+        self._request_id = 0
+        self._response_task: asyncio.Task | None = None
 
     async def _start(self) -> None:
         self._stderr_tail.clear()
@@ -242,8 +246,57 @@ class ACPHarnessSession(HarnessSession):
         process = await self.runtime.open_process(program, self.config.env)
         self._process = process
         self._reader = _PacketReader(process.stdout)
+        self._response_task = asyncio.create_task(self._read_responses(self._reader))
         self._stderr_task = asyncio.create_task(self._drain_stderr(process.stderr))
 
+    async def _read_responses(self, reader: _PacketReader) -> None:
+        try:
+            while True:
+                response = await reader.read()
+                future = self._responses.get(response.get("id"))
+                if future is not None and not future.done():
+                    future.set_result(response)
+        except BaseException as error:  # noqa: BLE001 - settle callers on disconnect or shutdown
+            for future in self._responses.values():
+                if not future.done():
+                    future.set_exception(
+                        RuntimeError(f"ACP connection closed: {error}")
+                    )
+
+    async def _request(self, operation: str, **payload: Any) -> JsonObject:
+        process = self._process
+        if process is None or self._response_task is None or self._response_task.done():
+            raise RuntimeError("ACP process is not running")
+        self._request_id += 1
+        request_id = self._request_id
+        future = asyncio.get_running_loop().create_future()
+        self._responses[request_id] = future
+        try:
+            async with self._write_lock:
+                await process.write(
+                    _packet({"id": request_id, "operation": operation, **payload})
+                )
+            return await future
+        finally:
+            self._responses.pop(request_id, None)
+            if not future.done():
+                future.cancel()
+
+    async def steer(self, message: str, *, message_id: str | None = None) -> dict:
+        if self._closed:
+            raise RuntimeError("ACP session is closed")
+        if not isinstance(message, str) or not message.strip():
+            raise ValueError("steering requires a nonempty user message")
+        if self._process is None:
+            return {"outcome": "promptRequired", "reason": "noRunningTurn"}
+        response = await self._request("steer", message=message, message_id=message_id)
+        if not response.get("ok"):
+            raise RuntimeError(response.get("error") or "ACP steering failed")
+        result = response.get("result")
+        if not isinstance(result, dict):
+            raise TypeError("invalid ACP steering receipt")
+        return result
+
     async def _drain_stderr(self, stream: AsyncIterator[bytes]) -> None:
         async for chunk in stream:
             self._stderr_tail.extend(chunk)
@@ -295,10 +348,7 @@ class ACPHarnessSession(HarnessSession):
             assert self._reader is not None
             calls_before = len(self.trace.calls)
             try:
-                await self._process.write(
-                    _packet({"operation": "prompt", "config": config})
-                )
-                response = await self._reader.read()
+                response = await self._request("prompt", config=config)
             except BaseException:
                 await run_shielded(self._stop(graceful=False))
                 raise
@@ -323,17 +373,18 @@ class ACPHarnessSession(HarnessSession):
         return result
 
     async def _stop(self, *, graceful: bool) -> dict[str, Any]:
-        process, self._process = self._process, None
-        reader, self._reader = self._reader, None
+        process = self._process
+        response_task = self._response_task
         stderr_task, self._stderr_task = self._stderr_task, None
         if process is None:
             return {}
         response_metadata: dict[str, Any] = {}
         try:
-            if graceful and reader is not None:
+            if graceful:
                 with contextlib.suppress(BaseException):
-                    await process.write(_packet({"operation": "shutdown"}))
-                    response = await asyncio.wait_for(reader.read(), timeout=10)
+                    response = await asyncio.wait_for(
+                        self._request("shutdown"), timeout=10
+                    )
                     result = response.get("result")
                     if response.get("ok") and isinstance(result, dict):
                         metadata = result.get("response_metadata")
@@ -353,6 +404,13 @@ class ACPHarnessSession(HarnessSession):
                 except TimeoutError:
                     continue
         finally:
+            self._process = None
+            self._reader = None
+            self._response_task = None
+            if response_task is not None:
+                response_task.cancel()
+                with contextlib.suppress(BaseException):
+                    await response_task
             if stderr_task is not None:
                 if not stderr_task.done():
                     stderr_task.cancel()

verifiers/v1/acp/runner.py

@@ -240,6 +240,7 @@ class ACPSession:
         self.stack = AsyncExitStack()
         self.connection: Any = None
         self.capabilities: Any = None
+        self.supports_steering = False
         self.session_id: str | None = None
         self.is_new = True
 
@@ -263,6 +264,9 @@ class ACPSession:
                 ),
             )
             self.capabilities = initialized.agent_capabilities
+            self.supports_steering = (initialized.field_meta or {}).get(
+                "steering", {}
+            ).get("supported") is True
             session = await self.connection.new_session(
                 cwd=os.getcwd(),
                 mcp_servers=mcp_servers(config),
@@ -294,6 +298,20 @@ class ACPSession:
         self.is_new = False
         return result
 
+    async def steer(self, message: str, message_id: str | None = None) -> dict:
+        if self.connection is None or self.session_id is None:
+            return {"outcome": "promptRequired", "reason": "noRunningTurn"}
+        if not self.supports_steering:
+            raise NotImplementedError("ACP agent does not advertise steering support")
+        params = {
+            "sessionId": self.session_id,
+            "prompt": [{"type": "text", "text": message}],
+            "_meta": {"steering": {"idleBehavior": "promptRequired"}},
+        }
+        if message_id is not None:
+            params["messageId"] = message_id
+        return await self.connection.ext_method("session/steering", params)
+
     async def close(self) -> dict[str, Any]:
         response_metadata: dict[str, Any] = {}
         try:
@@ -350,39 +368,50 @@ async def serve_stream() -> None:
     await asyncio.get_running_loop().connect_read_pipe(
         lambda: protocol, sys.stdin.buffer
     )
+    tasks: set[asyncio.Task] = set()
+    prompt_lock = asyncio.Lock()
+
+    async def handle(request: dict) -> None:
+        operation = request.get("operation")
+        try:
+            if operation == "prompt":
+                async with prompt_lock:
+                    result = asdict(await session.run(request["config"]))
+            elif operation == "steer":
+                result = await session.steer(
+                    request["message"], request.get("message_id")
+                )
+            elif operation == "shutdown":
+                result = {"response_metadata": await session.close()}
+            else:
+                raise ValueError(f"unknown ACP session operation: {operation!r}")
+            response = {"ok": True, "result": result}
+        except Exception as error:  # noqa: BLE001 - serialize protocol failures
+            traceback.print_exc()
+            response = {"ok": False, "error": f"{type(error).__name__}: {error}"}
+            if isinstance(error, RequestError) and isinstance(error.data, dict):
+                response["error"] = error.data.get("details") or str(error)
+                response["error_data"] = error.data
+            if operation == "prompt":
+                response["result"] = asdict(session.client.turn_result())
+        response["id"] = request.get("id")
+        write_packet(sys.stdout.buffer, response)
+
     try:
         while request := await read_packet(reader):
-            stop = False
-            try:
-                operation = request.get("operation")
-                if operation == "prompt":
-                    response = {
-                        "ok": True,
-                        "result": asdict(await session.run(request["config"])),
-                    }
-                elif operation == "shutdown":
-                    stop = True
-                    response = {
-                        "ok": True,
-                        "result": {"response_metadata": await session.close()},
-                    }
-                else:
-                    raise ValueError(f"unknown ACP session operation: {operation!r}")
-            except Exception as error:  # noqa: BLE001 - serialize protocol failures
-                traceback.print_exc()
-                response = {
-                    "ok": False,
-                    "error": f"{type(error).__name__}: {error}",
-                }
-                if isinstance(error, RequestError) and isinstance(error.data, dict):
-                    response["error"] = error.data.get("details") or str(error)
-                    response["error_data"] = error.data
-                if operation == "prompt":
-                    response["result"] = asdict(session.client.turn_result())
-            write_packet(sys.stdout.buffer, response)
-            if stop:
+            if request.get("operation") == "shutdown":
+                if tasks:
+                    await asyncio.gather(*tasks)
+                await handle(request)
                 break
+            task = asyncio.create_task(handle(request))
+            tasks.add(task)
+            task.add_done_callback(tasks.discard)
     finally:
+        for task in tasks:
+            task.cancel()
+        if tasks:
+            await asyncio.gather(*tasks, return_exceptions=True)
         await session.close()
 
 

verifiers/v1/agent.py

@@ -11,8 +11,9 @@ import asyncio
 import logging
 from collections.abc import AsyncIterator, Callable, Mapping
 from contextlib import asynccontextmanager, nullcontext
-from dataclasses import dataclass, replace
-from typing import Generic, Self, cast
+from dataclasses import dataclass, field, replace
+from typing import Generic, Literal, Self, cast
+from weakref import WeakValueDictionary
 
 from typing_extensions import TypeVar
 
@@ -162,6 +163,15 @@ class Segment:
         return ""
 
 
+@dataclass
+class _PendingMessage:
+    text: str
+    message_id: str | None
+    receipt: dict | None = None
+    error: BaseException | None = None
+    done: asyncio.Event = field(default_factory=asyncio.Event)
+
+
 class Interaction:
     """An agent's rollout, held open turn-by-turn: the caller IS the run's user.
 
@@ -170,8 +180,8 @@ class Interaction:
     program, resumed onto the conversation, until it yields — returning the
     resulting `Segment`. A prompt-less (or masked) task is opened by the first
     `turn(message)`; a prompted task speaks first — a bare `turn()` takes its
-    opening reply. One consumer at a time — `turn()` is a strict
-    request/response alternation, not a mailbox. `interaction.trace` is live from
+    opening reply. Turns are serialized; `steer(..., wake=True)` can run a
+    background turn when idle. `interaction.trace` is live from
     the moment the interaction exists: watch tokens and turns mid-exchange, read
     rewards after close. Leaving the `interaction()` context closes the exchange
     as `user_closed` and finishes the rollout — hooks and scoring included."""
@@ -182,11 +192,263 @@ class Interaction:
         self._over = False  # a terminated segment was already delivered
         self._started = False  # a segment has run (the exchange is under way)
         self._lock = asyncio.Lock()
+        self._steering_locks: WeakValueDictionary[str, asyncio.Lock] = (
+            WeakValueDictionary()
+        )
+        self._steering_receipts: dict[str, tuple[str, dict]] = {}
+        self._steering_tasks: dict[object, tuple[str, asyncio.Task[dict]]] = {}
+        self._pending_messages: list[_PendingMessage] = []
+        self._pending_by_id: dict[str, _PendingMessage] = {}
+        self._message_drain: asyncio.Task[None] | None = None
+        self._closing = False
 
     @property
     def trace(self) -> Trace:
         return self._run.trace
 
+    async def send(
+        self,
+        message: str,
+        *,
+        mode: Literal["steer", "queue"] = "steer",
+        message_id: str | None = None,
+    ) -> dict:
+        """Deliver a user message to this open interaction.
+
+        ``steer`` delivers at the harness's next safe point during active work.
+        ``queue`` waits for the current turn to yield, then delivers all pending
+        messages together in arrival order in one new turn. It does not wake a
+        wait tool inside the active turn. Messages arriving after the batch is
+        drained wait for the following turn.
+        Both resume an idle session. Queue mode also works without ACP steering.
+        Unsupported steering raises; it never silently changes to queue mode.
+        Sends return on acceptance, not turn completion. For queued sends and
+        idle wake-ups, the first model request acknowledges the resumed input.
+        Cancellation withdraws a queued message that has not started. After a batch starts, cancelling
+        a sender does not cancel the shared turn. Callers must await or schedule
+        this coroutine to submit input; closing the interaction cancels any
+        outstanding delivery tasks.
+        The receipt confirms delivery, not that the interaction is finished;
+        keep its context open
+        while more messages may arrive. A prompted task must start its opening
+        turn first. Stable IDs deduplicate successful deliveries.
+        """
+        if mode not in ("steer", "queue"):
+            raise ValueError("message mode must be 'steer' or 'queue'")
+        return await self._send(message, message_id=message_id, mode=mode, wake=True)
+
+    async def steer(
+        self, message: str, *, message_id: str | None = None, wake: bool = False
+    ) -> dict:
+        """Inject a user message, optionally resuming an idle interaction.
+
+        With ``wake=True``, an idle session runs another turn on this message.
+        The interaction must remain open, and a prompted task must have started
+        its opening turn. Wake-up uses the normal turn budget and lifecycle.
+        Successful deliveries with a stable ID are deduplicated in this interaction.
+        """
+        return await self._send(message, message_id=message_id, mode="steer", wake=wake)
+
+    async def _send(
+        self,
+        message: str,
+        *,
+        message_id: str | None,
+        mode: Literal["steer", "queue"],
+        wake: bool,
+    ) -> dict:
+        if not isinstance(message, str) or not message.strip():
+            raise ValueError("delivery requires a nonempty user message")
+        # Only retries of the same message share a lock. A wake-up can run a
+        # full turn; unrelated messages must still reach that turn through ACP.
+        lock = (
+            self._steering_locks.setdefault(message_id, asyncio.Lock())
+            if message_id is not None
+            else nullcontext()
+        )
+        async with lock:
+            if self._closing or self._run.closed:
+                raise RuntimeError("this interaction is closed")
+            if message_id is not None and message_id in self._steering_receipts:
+                previous, receipt = self._steering_receipts[message_id]
+                if previous != message:
+                    raise ValueError(
+                        "steering message ID reused with different content"
+                    )
+                return dict(receipt)
+            if message_id is not None and message_id in self._pending_by_id:
+                pending = self._pending_by_id[message_id]
+                if pending.text != message:
+                    raise ValueError(
+                        "steering message ID reused with different content"
+                    )
+                await pending.done.wait()
+                if pending.error is not None:
+                    raise pending.error
+                assert pending.receipt is not None
+                return dict(pending.receipt)
+            if message_id in self._steering_tasks:
+                receipt = await self._deliver_steer(message, message_id)
+            elif self._over or not self._run.ok:
+                return {"outcome": "promptRequired", "reason": "interactionStopped"}
+            elif mode == "steer":
+                receipt = await self._deliver_steer(message, message_id)
+            else:
+                receipt = {"outcome": "promptRequired", "reason": "noRunningTurn"}
+            if wake and receipt.get("outcome") == "promptRequired":
+                receipt = await self._resume_messages(message, message_id)
+            if receipt.get("outcome") == "injected" and message_id is not None:
+                self._steering_receipts[message_id] = (message, dict(receipt))
+            return receipt
+
+    async def _deliver_steer(self, message: str, message_id: str | None) -> dict:
+        key = message_id if message_id is not None else object()
+        if key in self._steering_tasks:
+            previous, task = self._steering_tasks[key]
+            if previous != message:
+                raise ValueError("steering message ID reused with different content")
+        else:
+
+            async def deliver() -> dict:
+                try:
+                    receipt = await self._run.steer(message, message_id=message_id)
+                    if receipt.get("outcome") == "injected" and message_id is not None:
+                        self._steering_receipts[message_id] = (message, dict(receipt))
+                    return receipt
+                finally:
+                    self._steering_tasks.pop(key, None)
+
+            # The harness may already have accepted input when the caller is
+            # cancelled. Finish and cache that delivery before allowing retries.
+            task = asyncio.create_task(deliver())
+            self._steering_tasks[key] = (message, task)
+            task.add_done_callback(
+                lambda done: None if done.cancelled() else done.exception()
+            )
+        return await asyncio.shield(task)
+
+    async def _cancel_deliveries(self) -> None:
+        self._closing = True
+        tasks = [task for _, task in self._steering_tasks.values()]
+        if self._message_drain is not None:
+            tasks.append(self._message_drain)
+        for task in tasks:
+            task.cancel()
+        await asyncio.gather(*tasks, return_exceptions=True)
+        self._steering_tasks.clear()
+        # A task cancelled before it starts never enters its finally block.
+        self._finish_pending(asyncio.CancelledError())
+
+    def _finish_pending(self, error: BaseException) -> None:
+        for item in self._pending_messages:
+            item.error = error
+            item.done.set()
+            if item.message_id is not None:
+                self._pending_by_id.pop(item.message_id, None)
+        self._pending_messages.clear()
+
+    async def _resume_messages(self, message: str, message_id: str | None) -> dict:
+        if self._closing:
+            raise RuntimeError("this interaction is closed")
+        pending = _PendingMessage(message, message_id)
+        self._pending_messages.append(pending)
+        if message_id is not None:
+            self._pending_by_id[message_id] = pending
+        if self._message_drain is None or self._message_drain.done():
+            self._message_drain = asyncio.create_task(self._drain_messages())
+            self._message_drain.add_done_callback(
+                lambda done: None if done.cancelled() else done.exception()
+            )
+        try:
+            await pending.done.wait()
+            if pending.error is not None:
+                raise pending.error
+            assert pending.receipt is not None
+            return dict(pending.receipt)
+        finally:
+            # Cancellation withdraws only input the worker has not drained.
+            if message_id is not None and any(
+                item is pending for item in self._pending_messages
+            ):
+                self._pending_by_id.pop(message_id, None)
+            self._pending_messages = [
+                item for item in self._pending_messages if item is not pending
+            ]
+
+    async def _drain_messages(self) -> None:
+        try:
+            while self._pending_messages:
+                async with self._lock, self._gate or nullcontext():
+                    batch, self._pending_messages = self._pending_messages, []
+                    if not batch:
+                        continue
+                    try:
+                        if self._closing or self._run.closed:
+                            raise RuntimeError("this interaction is closed")
+                        if self._over or not self._run.ok:
+                            receipt = {
+                                "outcome": "promptRequired",
+                                "reason": "interactionStopped",
+                            }
+                        elif (
+                            not self._started
+                            and self.trace.task.data.prompt is not None
+                        ):
+                            receipt = {
+                                "outcome": "promptRequired",
+                                "reason": "noRunningTurn",
+                            }
+                        else:
+
+                            def accepted(batch: list[_PendingMessage] = batch) -> None:
+                                for item in batch:
+                                    item.receipt = {"outcome": "injected"}
+                                    if item.message_id is not None:
+                                        self._steering_receipts[item.message_id] = (
+                                            item.text,
+                                            dict(item.receipt),
+                                        )
+                                        self._pending_by_id.pop(item.message_id, None)
+                                    item.done.set()
+
+                            segment = await self._turn(
+                                [UserMessage(content=item.text) for item in batch],
+                                on_input=accepted,
+                            )
+                            receipt = (
+                                {
+                                    "outcome": "promptRequired",
+                                    "reason": "interactionStopped",
+                                }
+                                if segment.terminated
+                                else {"outcome": "injected"}
+                            )
+                        for item in batch:
+                            if item.done.is_set():
+                                continue
+                            item.receipt = receipt
+                            if (
+                                receipt.get("outcome") == "injected"
+                                and item.message_id is not None
+                            ):
+                                self._steering_receipts[item.message_id] = (
+                                    item.text,
+                                    dict(receipt),
+                                )
+                    except BaseException as error:
+                        for item in batch:
+                            if not item.done.is_set():
+                                item.error = error
+                        raise
+                    finally:
+                        for item in batch:
+                            if item.message_id is not None:
+                                self._pending_by_id.pop(item.message_id, None)
+                            item.done.set()
+        except BaseException as error:
+            self._finish_pending(error)
+            raise
+
     async def turn(self, message: str | Messages | None = None) -> Segment:
         """Send one user turn (a string, or full `Messages` for multimodal /
         multi-message turns); run one segment; return its `Segment`. A
@@ -196,8 +458,13 @@ class Interaction:
         async with self._lock, self._gate or nullcontext():
             return await self._turn(message)
 
-    async def _turn(self, message: str | Messages | None) -> Segment:
-        if self._run.closed:
+    async def _turn(
+        self,
+        message: str | Messages | None,
+        *,
+        on_input: Callable[[], None] | None = None,
+    ) -> Segment:
+        if self._closing or self._run.closed:
             raise RuntimeError("this interaction is closed")
         if self._over:
             raise RuntimeError(
@@ -225,7 +492,7 @@ class Interaction:
         self._started = True
         turns_before = self.trace.num_turns
         nodes_before = len(self.trace.nodes)
-        await self._run.step(messages)
+        await self._run.step(messages, on_input=on_input)
         if self.trace.num_turns > turns_before:
             # The segment answered — even if a limit or @stop then ended the
             # exchange, that surfaces as the NEXT turn's terminated segment.
@@ -247,6 +514,7 @@ class Interaction:
     async def close(self) -> Trace:
         """End the exchange and finish the rollout (idempotent): scoring and hooks
         run, then the finished trace returns (also on `interaction.trace`)."""
+        await self._cancel_deliveries()
         async with self._lock, self._gate or nullcontext():
             if not self._run.closed and self._run.ok:
                 self.trace.stop("user_closed")
@@ -525,6 +793,7 @@ class Agent:
             run.fail(e)
             raise
         except BaseException:
+            await interaction._cancel_deliveries()
             await run.abort()
             raise
         finally:

verifiers/v1/configs/client.py

@@ -60,7 +60,10 @@ class BaseClientConfig(BaseConfig):
             or prime_config.get("inference_url")
             or DEFAULT_PRIME_INFERENCE_URL
         )
-        if "base_url" not in self.model_fields_set:
+        if (
+            "base_url" not in self.model_fields_set
+            and self.base_url == DEFAULT_PRIME_INFERENCE_URL
+        ):
             self.base_url = prime_base_url
         host = urlparse(self.base_url).hostname or ""
         if host != PRIME_INFERENCE_HOST and not host.endswith(

verifiers/v1/configs/judge.py

@@ -40,7 +40,7 @@ def resolve_judges(entries: Sequence[Any]) -> list[JudgeConfig]:
         if not raw.get("id"):
             raise ValueError(
                 "each `judges` entry needs an `id` (a judge plugin: `reference`, "
-                "`rubric`, or an installed package)"
+                "`rubric`, `decision`, or an installed package)"
             )
         resolved.append(judge_config_type(raw["id"]).model_validate(raw))
     return resolved
@@ -51,7 +51,7 @@ def check_judges(entries: Sequence[JudgeConfig]) -> None:
         if not entry.id:
             raise ValueError(
                 "each `judges` entry needs an `id` (a judge plugin: `reference`, "
-                "`rubric`, or an installed package)"
+                "`rubric`, `decision`, or an installed package)"
             )
     keys = [judge_key(entry) for entry in entries]
     if duplicates := {key for key in keys if keys.count(key) > 1}:

verifiers/v1/harness.py

@@ -347,6 +347,12 @@ class HarnessSession:
         self.tool_interception_url = tool_interception_url
         self._closed = False
 
+    async def steer(self, message: str, *, message_id: str | None = None) -> dict:
+        """Deliver a user message during a running turn, if supported by the harness."""
+        raise NotImplementedError(
+            f"harness {self.harness.config.id!r} does not support steering"
+        )
+
     async def turn(self, messages: Messages | None = None) -> None:
         """Run one harness segment while retaining session state for the next."""
         if self._closed:

verifiers/v1/interception/server.py

@@ -752,6 +752,9 @@ class InterceptionServer(Interception):
         # The tail is what the harness added since the last turn (tool results, user
         # turns): live watchers see it now rather than with the model's reply.
         session.trace.preview(turn, turn.tail)
+        if session.on_turn_input is not None:
+            on_input, session.on_turn_input = session.on_turn_input, None
+            on_input()
 
         def serve(response: Response, events: bytes | None) -> web.Response:
             if streaming:

verifiers/v1/judges/__init__.py

@@ -1,8 +1,11 @@
+from verifiers.v1.judges.decision import DecisionJudge, DecisionJudgeConfig
 from verifiers.v1.judges.reference import ReferenceJudge, ReferenceJudgeConfig
 from verifiers.v1.judges.rubric import Criterion, RubricJudge, RubricJudgeConfig
 
 __all__ = [
     "Criterion",
+    "DecisionJudge",
+    "DecisionJudgeConfig",
     "ReferenceJudge",
     "ReferenceJudgeConfig",
     "RubricJudge",

verifiers/v1/judges/decision.py

@@ -0,0 +1,105 @@
+"""Decision-model judge using Prime's evaluations API by default."""
+
+import json
+from typing import Any, cast
+
+from pydantic import Field
+
+from verifiers.v1.clients.base import build_async_openai
+from verifiers.v1.configs.judge import JudgeConfig
+from verifiers.v1.judge import (
+    Judge,
+    JudgeResponse,
+    JudgeView,
+    judge_question,
+    judge_response,
+)
+from verifiers.v1.task import TaskData
+from verifiers.v1.trace import Trace
+from verifiers.v1.types import ID, Usage
+
+
+class DecisionJudgeConfig(JudgeConfig):
+    id: ID = "decision"
+    model: str = "typesafe-ai/jev"
+    base_url: str = "https://api.pinference.ai/api/v1/evaluations"
+    """Full URL to POST evaluation requests to."""
+    levels: list[str] = Field(
+        default_factory=lambda: ["Incorrect", "Correct"], min_length=2, max_length=10
+    )
+    """Grading levels, worst to best. The model's probability-weighted level is
+    normalized to [0, 1], so a two-level grade is the probability of correctness."""
+    question_field: str = ""
+    """Task field containing the question; empty uses the task's prompt text."""
+    answer_field: str = ""
+    """Optional task field containing a reference answer to include in the state."""
+    view: JudgeView = "last_reply"
+
+
+class DecisionJudge(Judge[float, DecisionJudgeConfig]):
+    prompt = (
+        "Grade how well the response satisfies the task in question. "
+        "Use the reference answer when provided; equivalent answers are acceptable."
+    )
+
+    async def evaluate(
+        self, *, trace: Trace | None = None, **fields: Any
+    ) -> JudgeResponse[float]:
+        request = {
+            "model": self.config.model,
+            "state": fields,
+            "questions": {
+                "grade": {
+                    "type": "score",
+                    "instructions": self.build_messages(**fields),
+                    "criteria": self.config.levels,
+                }
+            },
+        }
+        async with build_async_openai(self.config) as client:
+            raw = await client.post(
+                self.config.base_url, cast_to=dict[str, Any], body=request
+            )
+        result = raw.get("result", raw)  # Workers AI wraps its response in `result`.
+        response = JudgeResponse[float](text=json.dumps(raw))
+        try:
+            if usage := result.get("usage"):
+                response.usage = Usage(
+                    prompt_tokens=usage.get("inputTokens", usage.get("input_tokens")),
+                    completion_tokens=usage.get(
+                        "outputTokens", usage.get("output_tokens")
+                    ),
+                    cost=usage.get("cost"),
+                )
+            score = result["answers"]["grade"]["score"] / (len(self.config.levels) - 1)
+            if not 0 <= score <= 1:
+                raise ValueError(f"decision judge score {score!r} is outside [0, 1]")
+            response.parsed = score
+            return response
+        finally:
+            # The call was billed even if the score could not be parsed.
+            if trace is not None:
+                trace.record_judge_call(
+                    name=self.reward_name, request=request, response=response
+                )
+
+    async def score(self, task: TaskData, trace: Trace) -> float:
+        response = judge_response(trace, self.config.view)
+        if not response.strip():
+            return 0.0
+        fields = {
+            "question": judge_question(task, self.config.question_field),
+            "response": response,
+        }
+        if self.config.answer_field:
+            answer = getattr(task, self.config.answer_field, None)
+            if answer is None:
+                raise ValueError(
+                    f"decision judge found no {self.config.answer_field!r} field on the task"
+                )
+            fields["answer"] = answer
+        result = await self.evaluate(trace=trace, **fields)
+        return cast(float, result.parsed)
+
+
+__all__ = ["DecisionJudge", "DecisionJudgeConfig"]

verifiers/v1/rollout.py

@@ -405,7 +405,19 @@ class Rollout:
         self.trace.notify()
         return not self._session.stopped
 
-    async def step(self, messages: Messages | None = None) -> bool:
+    async def steer(self, message: str, *, message_id: str | None = None) -> dict:
+        if self._closed or not self.ok:
+            raise RuntimeError("this rollout is closed or stopped")
+        if self._harness_session is None:
+            return {"outcome": "promptRequired", "reason": "noRunningTurn"}
+        return await self._harness_session.steer(message, message_id=message_id)
+
+    async def step(
+        self,
+        messages: Messages | None = None,
+        *,
+        on_input: Callable[[], None] | None = None,
+    ) -> bool:
         """Run ONE segment: the harness program to its exit. With `messages`, the
         segment resumes the exchange with the user's turn(s) (`Harness.resume` —
         for an exchange the user opens, this is also the first segment, on an
@@ -438,6 +450,7 @@ class Rollout:
                     self.trace.request_rewrites.extend(rewrites)
                     if self._session.stopped:
                         return False
+                self._session.on_turn_input = on_input
                 await self._harness_session.turn(messages)
         except TimeoutError as e:
             # An expired rollout deadline is the agent breaking its time budget: a
@@ -459,6 +472,7 @@ class Rollout:
                 self.fail(e)
             return False
         finally:
+            self._session.on_turn_input = None
             if trace.num_turns == turns_before:
                 trace.root_reply = root_reply_before
             if self._agent_time_remaining is not None:

verifiers/v1/session.py

@@ -148,6 +148,8 @@ class RolloutSession:
     response_interceptors: list[Callable] = field(default_factory=list)
     request_stops: list[Callable] = field(default_factory=list)
     response_stops: list[Callable] = field(default_factory=list)
+    on_turn_input: Callable[[], None] | None = None
+    """One-shot acknowledgement when a resumed harness reaches model inference."""
     gates_tools: bool = False
     """Whether the harness asks `/tool` before executing each call (see
     `Harness.SUPPORTS_TOOL_INTERCEPTION`). Without that gate a pre-execution rewrite

{verifiers-0.3.2.dev171.dist-info → verifiers-0.3.2.dev185.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev171
+Version: 0.3.2.dev185
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.2.dev171.dist-info → verifiers-0.3.2.dev185.dist-info}/RECORD

@@ -1,22 +1,22 @@
 verifiers/__init__.py,sha256=F5IVMw6mhGaPBMCw7QKk0la_r5mMwydP2hdTTdQucik,352
-verifiers/v1/__init__.py,sha256=ybCeLKoVNDU9nwla4BG8HrcZ2oXOuXquRBhaorGevxI,8792
-verifiers/v1/agent.py,sha256=746LE0HE4DPyxVcgEkAAs8z9qjsR7CQncfKhT1h4jek,32433
+verifiers/v1/__init__.py,sha256=xvS_VfN7QwTethhEJsZCi7lHicKvQdQKh2nhB2h9Lu8,8884
+verifiers/v1/agent.py,sha256=zBqbHDeqdPghg_QOkcyFplIDY3f34qCOtwcxJnl5N-U,45088
 verifiers/v1/env.py,sha256=fxXBjVyi0milYViHt6O0vBK3JdM2TYSOoQpXd0P0rFE,18110
 verifiers/v1/episode.py,sha256=UPtA02aGWA-fY8cJKEuYCMtQVnPN1C0BSN13LggNT8U,5986
 verifiers/v1/errors.py,sha256=nNabzi55J5aHmAmW9UHZVpI7kcBfy4Xcyt4cQthZbM8,5875
 verifiers/v1/graph.py,sha256=LW0pJRxCCOXZa2KwB0edHIIqmRrzLFBDNyEeOFVskl8,41502
-verifiers/v1/harness.py,sha256=AcsCmNHB1UQ75nD7RUawsb65q63NdkcMykD0E6oRDOA,15957
+verifiers/v1/harness.py,sha256=4hnAh7tjiXfXoEYAzn8XhHy8sJ3PhZ0Tti5VqwxUQDs,16251
 verifiers/v1/judge.py,sha256=uqn-djkK2_7Q3kOjkDo6ipOvV_HOvuLa9y6AoDjRux8,9618
-verifiers/v1/rollout.py,sha256=sx9ilprIu-5zrYH9G4yBWZdqzDdb_ctwLJVItLdTo88,26960
+verifiers/v1/rollout.py,sha256=fmDU1qlodpS1A7f-rlSMixMeGwf69hg9O3bLYxIR-ec,27539
 verifiers/v1/semantic.py,sha256=tX8FYEceuMYauuB3Bc8jLDMS2eR06JjLjTo66R190Fg,4053
-verifiers/v1/session.py,sha256=qVMgKmqkItemR3uYUDAFuDz7huP593kHQbge8FlRba8,28420
+verifiers/v1/session.py,sha256=p3Rm0MU-6fPsJLjQcIMg-NOIV-8_tsqcXNTftJ94brY,28555
 verifiers/v1/state.py,sha256=EckF2bWp-vV4b1jYJ9sLI5xrfGuI5spIgYYwW926toI,595
 verifiers/v1/task.py,sha256=YN_P3912NYbK4VPTPHu6kyYCpeyoyhDVxBbhpjg8AA0,13185
 verifiers/v1/taskset.py,sha256=LE_3ogL7itU2Amk-N20cHkZ7Ev6MznJ0HpC8B1xb4w0,9482
 verifiers/v1/trace.py,sha256=Npc-TrBh-B3H0OvtfA5X31VAlOhfgLDKGdXohqC2uIk,31156
 verifiers/v1/types.py,sha256=-In5plgB3mQLqtRKHC3QDCPDvuruAqVd8WIjKrsGB_I,10578
-verifiers/v1/acp/__init__.py,sha256=Omle2zo7aly_7CCg1N9yQjQcJe4UpqTw9xVBP8_tuT8,13891
-verifiers/v1/acp/runner.py,sha256=OrJ1VSWTnNosV8OklqM2YdERlbF7olT9PHEdQfXvgGQ,14137
+verifiers/v1/acp/__init__.py,sha256=gb2tkeq27oLHMPrCFOAGQi4iS2g3nzhpbC1kM_YCS8g,16479
+verifiers/v1/acp/runner.py,sha256=sFsg0UgWeDjpjihejUidIV2yTapVj1Ap0cDTaEktjkI,15484
 verifiers/v1/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 verifiers/v1/cli/debug.py,sha256=OgzXCW9HaTNruZil4f70WP5tR6toKliKaVvV98C5qzE,11665
 verifiers/v1/cli/gepa.py,sha256=IyS3YZCCHVydb-OyPeGdtCYsz6OYqXXddXqYq8RLucM,4374
@@ -43,10 +43,10 @@ verifiers/v1/clients/eval.py,sha256=x_5E8XnhFu4v-k4yI8WWHd39rDkvB7mSEPCVAWL4AiE,
 verifiers/v1/clients/train.py,sha256=lSJTwHjwOET3D_CI_eYZSoXSQcv2H0K5sYJ7kD_kdw4,19489
 verifiers/v1/configs/__init__.py,sha256=X7u6X7B3ieD1HZl79Tv1kg-yPHbT0QLvAebaGqxVtvA,317
 verifiers/v1/configs/agent.py,sha256=Vut73R2EW8QuL5c0iTnNFC_g-Obvl8YUKsjlvkABuL0,5311
-verifiers/v1/configs/client.py,sha256=vl46nrayZa84zrPBuEVpJIjDjbmYUIrDIuR0vZvvugw,5212
+verifiers/v1/configs/client.py,sha256=i45QOop2Wb-LQRpNgUZGTCT1_HpNmdcG41eQrGMgJKQ,5297
 verifiers/v1/configs/env.py,sha256=k47XBYYWLOf-8P1dbek3x_s0ucNb726xxoryG69DIr4,7256
 verifiers/v1/configs/harness.py,sha256=ZrR-Egzfft95oMoqQ78bAq9TqP882Q68M_47Ajw9lE8,2918
-verifiers/v1/configs/judge.py,sha256=WEwA8D4X5JSaIP3c-E2wPnvgByWeT6JRAn1d8gfiQ40,2142
+verifiers/v1/configs/judge.py,sha256=3ZvxcTf-3clj8bDLyaqsMSKCRclTQCvbyb2qp3QSRD8,2166
 verifiers/v1/configs/retries.py,sha256=BgpGbdMBmjyjcIGnvb5VTJzKMDA6TdKil-PBQaRwGZM,1770
 verifiers/v1/configs/runtime.py,sha256=3LDAnKy-BotE6MqyT1l07Y_bAq9Wye1OU8JZ7lyUVvY,5474
 verifiers/v1/configs/select.py,sha256=_HdcepQaJ1sDCYcs7Vqx72npDwHGOgUjsLDZmcgnonU,4743
@@ -126,12 +126,13 @@ verifiers/v1/harnesses/utils/mcp.py,sha256=oNhKfLjAW-wScJcDIsW_HfuvKY-JkaGBBZrRL
 verifiers/v1/interception/__init__.py,sha256=5Kvo7tcBUVaa65s9A1NiOH3CPpzw-P5btXwxB4Nbbvs,4251
 verifiers/v1/interception/base.py,sha256=Mg16Y104lyZilF9shdivdqYoqz4OYp978i_myyrulu8,2529
 verifiers/v1/interception/pool.py,sha256=sk_zcdTV8EVRuzqlYMXxon_42tSDAlFvwcyOO2EF7hw,6473
-verifiers/v1/interception/server.py,sha256=DnPvQcZZX0JYvxrTLOQuJn4QFX5Z44edGjpvfdxZmRw,45773
+verifiers/v1/interception/server.py,sha256=8cvco2PnW8nW7rVYZAndsmzU0I9uRowCoYziM1CqLeI,45916
 verifiers/v1/interception/tunnel/__init__.py,sha256=eVNZJszj6myrKhx9jmJz7VRd9qlfj0AUvUZwI2bXEj8,893
 verifiers/v1/interception/tunnel/base.py,sha256=DZB6uPLwM4Qy7n7m0vKeyg-Px87MhsmpYTpe2vpPEng,1998
 verifiers/v1/interception/tunnel/custom.py,sha256=yL4UbGf4bAuMsxk42mEP-t1wITQCT8BE9FHQIz0qRMA,1708
 verifiers/v1/interception/tunnel/prime.py,sha256=jVk_6xyrxXiHtsVFErcRNd4MoENdFbqzfC0i20nlizU,2803
-verifiers/v1/judges/__init__.py,sha256=MUIBWcx6c70BykrTDHhB0ITIyJPxXe6xDBou4VM7jDQ,286
+verifiers/v1/judges/__init__.py,sha256=z-16awBqEZJrlVYIb9HD-rk12RaABh6jodJgwX4blnA,410
+verifiers/v1/judges/decision.py,sha256=U8BKuzJVMWfX76fa6jaG_rmcOhzMkZoXSnkUuHKpoV4,3991
 verifiers/v1/judges/reference.py,sha256=iEVHw-iJ6dbwLOOqrvWrEMJOcUEYJ0YspT04rZz7Sd4,3868
 verifiers/v1/judges/reference.txt,sha256=Ej35kGXiT2uJ0LcHIeez49rCS_9uHmCQKoVdn93V6A8,353
 verifiers/v1/judges/rubric.py,sha256=HGQYI3xdq8I84Ce0_sDeNgk5JM597es8rOVPipjCyxU,12025
@@ -194,8 +195,8 @@ verifiers/v1/utils/scope.py,sha256=bzUEiWDOVdbj7dXOwGOlzsQ_-Xu0j9NR5TalQB-8TS8,8
 verifiers/v1/utils/score.py,sha256=493yJVMw8teCu9JxapxMFPFzI0hNUqdo0Y2nGW4kckk,6200
 verifiers/v1/utils/trace_store.py,sha256=RsCDonGR-bs-tvEstcppiJm3jFmAdIiG83fA5KtCbAw,2801
 verifiers/v1/utils/version.py,sha256=-obEo_-l9-D8FLef4hYxncOe-uJpxrM1g2Hig_37Sgs,1607
-verifiers-0.3.2.dev171.dist-info/METADATA,sha256=FiDjnhp-MuIN3vAbb1f9oBgKh5ZnPrX_9h-6B0YSe0E,4227
-verifiers-0.3.2.dev171.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-verifiers-0.3.2.dev171.dist-info/entry_points.txt,sha256=iugElcdWPKbQM7uFF0lZ8iUpHsNr17-BwEAAjJWxV3U,259
-verifiers-0.3.2.dev171.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
-verifiers-0.3.2.dev171.dist-info/RECORD,,
+verifiers-0.3.2.dev185.dist-info/METADATA,sha256=DF9K7iZDqE_b9Sf8WSNbMwPW1mZlDdTxqrUZtxUQNrY,4227
+verifiers-0.3.2.dev185.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+verifiers-0.3.2.dev185.dist-info/entry_points.txt,sha256=iugElcdWPKbQM7uFF0lZ8iUpHsNr17-BwEAAjJWxV3U,259
+verifiers-0.3.2.dev185.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
+verifiers-0.3.2.dev185.dist-info/RECORD,,