verifiers 0.3.2.dev171__tar.gz → 0.3.2.dev185__tar.gz

{verifiers-0.3.2.dev171 → verifiers-0.3.2.dev185}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev171
+Version: 0.3.2.dev185
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

verifiers-0.3.2.dev185/tests/v1/fixtures/delivery_behaviors.py

@@ -0,0 +1,196 @@
+"""Scripted inference and event log for the real-harness delivery examples."""
+
+import asyncio
+import json
+import time
+from dataclasses import dataclass, field
+
+from starlette.responses import JSONResponse, StreamingResponse
+
+LABELS = ("MESSAGE_ALPHA", "MESSAGE_BETA")
+
+
+def labels(value):
+    text = json.dumps(value)
+    return sorted((label for label in LABELS if label in text), key=text.index)
+
+
+@dataclass
+class Scenario:
+    harness: str
+    state: str
+    events: list = field(default_factory=list)
+    calls: list = field(default_factory=list)
+    model_entered: asyncio.Event = field(default_factory=asyncio.Event)
+    model_release: asyncio.Event = field(default_factory=asyncio.Event)
+    tool_entered: asyncio.Event = field(default_factory=asyncio.Event)
+    tool_release: asyncio.Event = field(default_factory=asyncio.Event)
+    response_sent: asyncio.Event = field(default_factory=asyncio.Event)
+    endpoint: str = ""
+    hold_call: int = 1
+
+    def record(self, event, **data):
+        self.events.append({"index": len(self.events), "event": event, **data})
+
+    async def barrier(self, request):
+        self.record("tool_started")
+        self.tool_entered.set()
+        await self.tool_release.wait()
+        self.record("tool_finished")
+        return JSONResponse({"result": "work finished"})
+
+    async def respond(self, request):
+        data = await request.json()
+        number = len(self.calls) + 1
+        visible = labels(data.get("messages", data.get("input", [])))
+        self.calls.append({"number": number, "messages": visible})
+        self.record("model_request", call=number, messages=visible)
+        if number == self.hold_call:
+            self.model_entered.set()
+        text = "Observed: " + (", ".join(visible) or "opening prompt") + "."
+        tool = None
+        if number == 1 and self.state in ("tool", "wait"):
+            if self.state == "wait":
+                tool = ("wait", {"timeout": 2})
+            elif self.harness == "rlm":
+                tool = (
+                    "ipython",
+                    {
+                        "code": f"import urllib.request\nprint(urllib.request.urlopen('{self.endpoint}/barrier').read().decode())"
+                    },
+                )
+            else:
+                tool = (
+                    "Bash",
+                    {
+                        "command": f"curl -fsS {self.endpoint}/barrier",
+                        "description": "Wait for the demonstration work to finish",
+                    },
+                )
+        if self.harness == "rlm":
+            if number == self.hold_call and self.state in (
+                "model",
+                "idle",
+                "idle-burst",
+            ):
+                await self.model_release.wait()
+            message = {"role": "assistant", "content": text}
+            if tool:
+                message = {
+                    "role": "assistant",
+                    "content": None,
+                    "tool_calls": [
+                        {
+                            "id": f"tool_{number}",
+                            "type": "function",
+                            "function": {
+                                "name": tool[0],
+                                "arguments": json.dumps(tool[1]),
+                            },
+                        }
+                    ],
+                }
+            self.record("model_response", call=number, tool=tool[0] if tool else None)
+            if number == 1:
+                self.response_sent.set()
+            return JSONResponse(
+                {
+                    "id": f"chatcmpl-{number}",
+                    "object": "chat.completion",
+                    "created": int(time.time()),
+                    "model": data["model"],
+                    "choices": [
+                        {
+                            "index": 0,
+                            "message": message,
+                            "finish_reason": "tool_calls" if tool else "stop",
+                        }
+                    ],
+                    "usage": {
+                        "prompt_tokens": 100,
+                        "completion_tokens": 10,
+                        "total_tokens": 110,
+                    },
+                }
+            )
+
+        async def stream():
+            def event(kind, **fields):
+                return (
+                    "event: "
+                    + kind
+                    + "\ndata: "
+                    + json.dumps({"type": kind, **fields})
+                    + "\n\n"
+                )
+
+            yield event(
+                "message_start",
+                message={
+                    "id": f"msg_{number}",
+                    "type": "message",
+                    "role": "assistant",
+                    "model": data["model"],
+                    "content": [],
+                    "stop_reason": None,
+                    "stop_sequence": None,
+                    "usage": {"input_tokens": 100, "output_tokens": 0},
+                },
+            )
+            try:
+                if number == self.hold_call and self.state in (
+                    "model",
+                    "idle",
+                    "idle-burst",
+                ):
+                    await self.model_release.wait()
+                if tool:
+                    yield event(
+                        "content_block_start",
+                        index=0,
+                        content_block={
+                            "type": "tool_use",
+                            "id": f"tool_{number}",
+                            "name": tool[0],
+                            "input": {},
+                        },
+                    )
+                    yield event(
+                        "content_block_delta",
+                        index=0,
+                        delta={
+                            "type": "input_json_delta",
+                            "partial_json": json.dumps(tool[1]),
+                        },
+                    )
+                else:
+                    yield event(
+                        "content_block_start",
+                        index=0,
+                        content_block={"type": "text", "text": ""},
+                    )
+                    yield event(
+                        "content_block_delta",
+                        index=0,
+                        delta={"type": "text_delta", "text": text},
+                    )
+                yield event("content_block_stop", index=0)
+                yield event(
+                    "message_delta",
+                    delta={
+                        "stop_reason": "tool_use" if tool else "end_turn",
+                        "stop_sequence": None,
+                    },
+                    usage={"output_tokens": 10},
+                )
+                yield event("message_stop")
+                self.record(
+                    "model_response", call=number, tool=tool[0] if tool else None
+                )
+                if number == 1:
+                    self.response_sent.set()
+            except asyncio.CancelledError:
+                self.record("model_stream_cancelled", call=number)
+                raise
+
+        return StreamingResponse(stream(), media_type="text/event-stream")

verifiers-0.3.2.dev185/tests/v1/test_delivery_behaviors.py

@@ -0,0 +1,288 @@
+"""Executable delivery examples using real harness processes and scripted inference.
+
+Put `rlm`, `claude-agent-acp`, `claude`, and `node` on PATH, then run:
+    VF_DELIVERY_BEHAVIORS=1 uv run pytest tests/v1/test_delivery_behaviors.py -v -s
+
+For nano-rlm alone, add `-k rlm`; neither subset needs PRIME_API_KEY.
+VF_DELIVERY_OUTPUT optionally selects a directory for timeline.json and normal
+Verifiers traces.jsonl files (one run per case). No model API key is required.
+`idle` sends A after the opening turn finishes, then B during A's resumed work.
+`idle-burst` submits A and B together after the opening turn finishes.
+Other cases submit A and B during the same held model/tool operation. `queue`
+means an ordinary user message deferred until the current turn yields.
+"""
+
+import asyncio
+import json
+import os
+import shutil
+import socket
+from pathlib import Path
+
+import pytest
+import uvicorn
+from delivery_behaviors import LABELS, Scenario, labels
+from starlette.applications import Starlette
+from starlette.routing import Route
+
+import verifiers.v1 as vf
+from verifiers.v1.acp import ACPHarness
+from verifiers.v1.episode import Episode, EvalRunInfo
+from verifiers.v1.harnesses.claude_code.harness import (
+    ClaudeCodeHarness,
+    ClaudeCodeHarnessConfig,
+)
+from verifiers.v1.harnesses.rlm.harness import RLMHarness, RLMHarnessConfig
+
+pytestmark = [
+    pytest.mark.asyncio,
+    pytest.mark.skipif(
+        os.environ.get("VF_DELIVERY_BEHAVIORS") != "1",
+        reason="opt-in real-harness behavioral examples",
+    ),
+]
+
+
+@pytest.mark.parametrize("kind", ["rlm", "claude-code"])
+@pytest.mark.parametrize("state", ["idle-burst", "idle", "model", "tool", "wait"])
+@pytest.mark.parametrize(
+    "modes",
+    [("steer", "steer"), ("steer", "queue"), ("queue", "steer"), ("queue", "queue")],
+    ids=["steer-steer", "steer-message", "message-steer", "message-message"],
+)
+async def test_delivery_behavior(kind, state, modes, tmp_path, monkeypatch):
+    if state == "wait" and kind != "rlm":
+        pytest.skip("wait is a nano-rlm native tool")
+    commands = ["rlm"] if kind == "rlm" else ["node", "claude-agent-acp", "claude"]
+    bins = {command: shutil.which(command) for command in commands}
+    if not all(bins.values()):
+        pytest.fail(f"install these harness executables first: {commands}")
+    scenario = Scenario(kind, state, hold_call=2 if state.startswith("idle") else 1)
+    sock = socket.socket()
+    sock.bind(("127.0.0.1", 0))
+    scenario.endpoint = f"http://127.0.0.1:{sock.getsockname()[1]}"
+    server = uvicorn.Server(
+        uvicorn.Config(
+            Starlette(
+                routes=[
+                    Route("/v1/chat/completions", scenario.respond, methods=["POST"]),
+                    Route("/v1/messages", scenario.respond, methods=["POST"]),
+                    Route("/barrier", scenario.barrier),
+                ]
+            ),
+            log_level="error",
+        )
+    )
+    serving = asyncio.create_task(server.serve(sockets=[sock]))
+    monkeypatch.setenv("DELIVERY_SCRIPTED_KEY", "local-scripted")
+    base = RLMHarness if kind == "rlm" else ClaudeCodeHarness
+
+    class LocalHarness(base):
+        NEEDS_CONTAINER = False
+
+        async def setup(self, runtime):
+            await ACPHarness.setup(self, runtime)
+
+        async def prepare_acp(self, *args, **kwargs):
+            config = await super().prepare_acp(*args, **kwargs)
+            if kind == "rlm":
+                config.command = [bins["rlm"], "--acp"]
+            else:
+                config.command = [bins["node"], bins["claude-agent-acp"]]
+                config.env["CLAUDE_CODE_EXECUTABLE"] = bins["claude"]
+            return config
+
+    config = (
+        RLMHarnessConfig(id="rlm", builtin_skills=[], max_depth=0, compaction=False)
+        if kind == "rlm"
+        else ClaudeCodeHarnessConfig(id="claude-code")
+    )
+    agent = vf.make_agent(
+        vf.AgentConfig(
+            harness=config,
+            model="scripted-delivery" if kind == "rlm" else "claude-sonnet-4-6",
+            client={
+                "type": "eval",
+                "base_url": scenario.endpoint + "/v1",
+                "api_key_var": "DELIVERY_SCRIPTED_KEY",
+            },
+            runtime={"type": "subprocess"},
+            max_turns=8,
+            timeout={"rollout": 45},
+        )
+    )
+    agent.harness = LocalHarness(config)
+    name = f"delivery-{kind}-{state}-{'-'.join(modes)}"
+    task = vf.Task(
+        vf.TaskData(
+            idx=0,
+            name=name,
+            prompt="Report which incoming messages you have received.",
+            system_prompt="Behavioral demonstration with scripted inference. Each reply states which messages are visible in that model request.",
+        )
+    )
+    sends = []
+    opening = None
+    try:
+        async with asyncio.timeout(50), agent, agent.interaction(task) as interaction:
+            real_step = interaction._run.step
+
+            async def step(messages=None, **kwargs):
+                scenario.record(
+                    "turn_started",
+                    messages=labels(
+                        [m.model_dump() for m in messages] if messages else []
+                    ),
+                )
+                result = await real_step(messages, **kwargs)
+                scenario.record("turn_yielded")
+                return result
+
+            interaction._run.step = step
+
+            async def send(index):
+                scenario.record("submitted", message=LABELS[index], mode=modes[index])
+                receipt = await interaction.send(
+                    LABELS[index], mode=modes[index], message_id=LABELS[index]
+                )
+                scenario.record("receipt", message=LABELS[index], **receipt)
+                return receipt
+
+            opening = asyncio.create_task(interaction.turn())
+            if state == "idle-burst":
+                await opening
+                scenario.record("agent_idle")
+                sends.extend(asyncio.create_task(send(i)) for i in range(2))
+                await scenario.model_entered.wait()
+            elif state == "idle":
+                await opening
+                scenario.record("agent_idle")
+                sends.append(asyncio.create_task(send(0)))
+                # A sequential caller can await the wake-up receipt and still
+                # steer the next message into the held, resumed turn.
+                receipt = await sends[0]
+                assert (
+                    await interaction.send(
+                        LABELS[0], mode=modes[0], message_id=LABELS[0]
+                    )
+                    == receipt
+                )
+                await scenario.model_entered.wait()
+                assert not scenario.model_release.is_set()
+                assert not any(
+                    e["event"] == "turn_yielded"
+                    for e in scenario.events[
+                        next(
+                            e["index"]
+                            for e in scenario.events
+                            if e["event"] == "agent_idle"
+                        )
+                        + 1 :
+                    ]
+                )
+                sends.append(asyncio.create_task(send(1)))
+            else:
+                if state == "model":
+                    await scenario.model_entered.wait()
+                elif state == "tool":
+                    await scenario.tool_entered.wait()
+                else:
+                    await scenario.response_sent.wait()
+                    # Let the harness enter its native wait after reading the response.
+                    await asyncio.sleep(0.2)
+                sends.extend(asyncio.create_task(send(i)) for i in range(2))
+            # Keep the operation held while both ACP requests traverse the real processes.
+            await asyncio.sleep(0.25)
+            scenario.record("release_operation")
+            scenario.model_release.set()
+            scenario.tool_release.set()
+            await opening
+            receipts = await asyncio.gather(*sends)
+            # Delivery receipts no longer wait for completion. Keep the session
+            # open until its owned turns finish so the trace includes the replies.
+            if interaction._message_drain is not None:
+                await interaction._message_drain
+            assert all(r["outcome"] == "injected" for r in receipts)
+            assert set(scenario.calls[-1]["messages"]) == set(LABELS)
+            for label, mode in zip(LABELS, modes, strict=True):
+                submitted = next(
+                    e["index"]
+                    for e in scenario.events
+                    if e["event"] == "submitted" and e["message"] == label
+                )
+                seen = next(
+                    e["index"]
+                    for e in scenario.events
+                    if e["event"] == "model_request" and label in e["messages"]
+                )
+                assert seen > submitted
+                if mode == "steer" and (
+                    not state.startswith("idle")
+                    or (state == "idle" and label == LABELS[1])
+                ):
+                    assert not any(
+                        e["event"] == "turn_yielded"
+                        for e in scenario.events[submitted:seen]
+                    )
+                if (
+                    mode == "queue"
+                    and state != "idle-burst"
+                    and not (state == "idle" and label == LABELS[0])
+                ):
+                    assert any(
+                        e["event"] == "turn_yielded"
+                        for e in scenario.events[submitted:seen]
+                    )
+            if (
+                kind == "rlm"
+                and not state.startswith("idle")
+                and modes == ("steer", "steer")
+            ):
+                assert scenario.calls[1]["messages"] == list(LABELS)
+            if not state.startswith("idle") and modes == ("queue", "queue"):
+                assert next(c for c in scenario.calls if c["messages"])[
+                    "messages"
+                ] == list(LABELS)
+            interaction.trace.info.update(
+                scenario=name,
+                scripted_inference=True,
+                executables=bins,
+                events=scenario.events,
+                model_inputs=scenario.calls,
+            )
+        assert interaction.trace.ok, interaction.trace.errors
+    finally:
+        scenario.model_release.set()
+        scenario.tool_release.set()
+        for pending in [opening, *sends]:
+            if pending is not None and not pending.done():
+                pending.cancel()
+        await asyncio.gather(
+            *(p for p in [opening, *sends] if p is not None), return_exceptions=True
+        )
+        server.should_exit = True
+        await serving
+        output = Path(os.environ.get("VF_DELIVERY_OUTPUT", tmp_path)) / name
+        output.mkdir(parents=True, exist_ok=True)
+        (output / "timeline.json").write_text(
+            json.dumps(
+                {
+                    "scenario": name,
+                    "events": scenario.events,
+                    "model_inputs": scenario.calls,
+                },
+                indent=2,
+            )
+        )
+        if "interaction" in locals():
+            trace = interaction.trace
+            episode = Episode(
+                task=trace.task,
+                traces=[trace],
+                ok=trace.ok,
+                run=EvalRunInfo(id=name, name=name),
+            )
+            (output / "traces.jsonl").write_text(episode.model_dump_json() + "\n")
+        print(
+            "\n" + name + ": " + " → ".join(str(c["messages"]) for c in scenario.calls)
+        )

{verifiers-0.3.2.dev171 → verifiers-0.3.2.dev185}/verifiers/v1/__init__.py

@@ -66,6 +66,8 @@ from verifiers.v1.harness import Harness, HarnessSession
 from verifiers.v1.judge import Judge, JudgeResponse, JudgeView
 from verifiers.v1.judges import (
     Criterion,
+    DecisionJudge,
+    DecisionJudgeConfig,
     ReferenceJudge,
     ReferenceJudgeConfig,
     RubricJudge,
@@ -358,6 +360,8 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "Judges",
     "JudgeResponse",
     "JudgeView",
+    "DecisionJudge",
+    "DecisionJudgeConfig",
     "ReferenceJudge",
     "ReferenceJudgeConfig",
     "RubricJudge",

{verifiers-0.3.2.dev171 → verifiers-0.3.2.dev185}/verifiers/v1/acp/__init__.py

@@ -231,6 +231,10 @@ class ACPHarnessSession(HarnessSession):
         self._stderr_tail = bytearray()
         self._stderr_task: asyncio.Task[None] | None = None
         self._lock = asyncio.Lock()
+        self._write_lock = asyncio.Lock()
+        self._responses: dict[int, asyncio.Future] = {}
+        self._request_id = 0
+        self._response_task: asyncio.Task | None = None
 
     async def _start(self) -> None:
         self._stderr_tail.clear()
@@ -242,8 +246,57 @@ class ACPHarnessSession(HarnessSession):
         process = await self.runtime.open_process(program, self.config.env)
         self._process = process
         self._reader = _PacketReader(process.stdout)
+        self._response_task = asyncio.create_task(self._read_responses(self._reader))
         self._stderr_task = asyncio.create_task(self._drain_stderr(process.stderr))
 
+    async def _read_responses(self, reader: _PacketReader) -> None:
+        try:
+            while True:
+                response = await reader.read()
+                future = self._responses.get(response.get("id"))
+                if future is not None and not future.done():
+                    future.set_result(response)
+        except BaseException as error:  # noqa: BLE001 - settle callers on disconnect or shutdown
+            for future in self._responses.values():
+                if not future.done():
+                    future.set_exception(
+                        RuntimeError(f"ACP connection closed: {error}")
+                    )
+
+    async def _request(self, operation: str, **payload: Any) -> JsonObject:
+        process = self._process
+        if process is None or self._response_task is None or self._response_task.done():
+            raise RuntimeError("ACP process is not running")
+        self._request_id += 1
+        request_id = self._request_id
+        future = asyncio.get_running_loop().create_future()
+        self._responses[request_id] = future
+        try:
+            async with self._write_lock:
+                await process.write(
+                    _packet({"id": request_id, "operation": operation, **payload})
+                )
+            return await future
+        finally:
+            self._responses.pop(request_id, None)
+            if not future.done():
+                future.cancel()
+
+    async def steer(self, message: str, *, message_id: str | None = None) -> dict:
+        if self._closed:
+            raise RuntimeError("ACP session is closed")
+        if not isinstance(message, str) or not message.strip():
+            raise ValueError("steering requires a nonempty user message")
+        if self._process is None:
+            return {"outcome": "promptRequired", "reason": "noRunningTurn"}
+        response = await self._request("steer", message=message, message_id=message_id)
+        if not response.get("ok"):
+            raise RuntimeError(response.get("error") or "ACP steering failed")
+        result = response.get("result")
+        if not isinstance(result, dict):
+            raise TypeError("invalid ACP steering receipt")
+        return result
+
     async def _drain_stderr(self, stream: AsyncIterator[bytes]) -> None:
         async for chunk in stream:
             self._stderr_tail.extend(chunk)
@@ -295,10 +348,7 @@ class ACPHarnessSession(HarnessSession):
             assert self._reader is not None
             calls_before = len(self.trace.calls)
             try:
-                await self._process.write(
-                    _packet({"operation": "prompt", "config": config})
-                )
-                response = await self._reader.read()
+                response = await self._request("prompt", config=config)
             except BaseException:
                 await run_shielded(self._stop(graceful=False))
                 raise
@@ -323,17 +373,18 @@ class ACPHarnessSession(HarnessSession):
         return result
 
     async def _stop(self, *, graceful: bool) -> dict[str, Any]:
-        process, self._process = self._process, None
-        reader, self._reader = self._reader, None
+        process = self._process
+        response_task = self._response_task
         stderr_task, self._stderr_task = self._stderr_task, None
         if process is None:
             return {}
         response_metadata: dict[str, Any] = {}
         try:
-            if graceful and reader is not None:
+            if graceful:
                 with contextlib.suppress(BaseException):
-                    await process.write(_packet({"operation": "shutdown"}))
-                    response = await asyncio.wait_for(reader.read(), timeout=10)
+                    response = await asyncio.wait_for(
+                        self._request("shutdown"), timeout=10
+                    )
                     result = response.get("result")
                     if response.get("ok") and isinstance(result, dict):
                         metadata = result.get("response_metadata")
@@ -353,6 +404,13 @@ class ACPHarnessSession(HarnessSession):
                 except TimeoutError:
                     continue
         finally:
+            self._process = None
+            self._reader = None
+            self._response_task = None
+            if response_task is not None:
+                response_task.cancel()
+                with contextlib.suppress(BaseException):
+                    await response_task
             if stderr_task is not None:
                 if not stderr_task.done():
                     stderr_task.cancel()