verifiers 0.1.15.dev415__tar.gz → 0.1.15.dev417__tar.gz

{verifiers-0.1.15.dev415 → verifiers-0.1.15.dev417}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev415
+Version: 0.1.15.dev417
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev415 → verifiers-0.1.15.dev417}/tests/v1/test_e2e.py

@@ -9,8 +9,8 @@ carries a pytest mark, so subsets select with `-m` (see `conftest.py`).
 `test_user` and `test_tool` fan a server's own runtime against the harness runtime (the full
 reachability matrix); `test_single_turn`/`test_agentic` fan the harness against the harness runtime.
 `test_shared_tool_isolation` runs two concurrent rollouts against one SHARED writable tool server
-(fork off/on) across the full `harness_runtime` x (shared server's own) `tool_runtime` matrix —
-incl. mixed-locality combos — asserting each keeps its own state.
+across the full `harness_runtime` x (shared server's own) `tool_runtime` matrix — incl. mixed-locality
+combos — asserting each keeps its own `self.state`.
 """
 
 import pytest
@@ -126,23 +126,16 @@ async def test_tool_state(run_v1, harness_runtime, tool_runtime, tmp_path):
 
 
 @pytest.mark.e2e
-@pytest.mark.parametrize(
-    "fork",
-    [pytest.param(False, id="fork-off"), pytest.param(True, id="fork-on")],
-)
 async def test_shared_tool_isolation(
-    run_v1_server, harness_runtime, tool_runtime, fork, tmp_path
+    run_v1_server, harness_runtime, tool_runtime, tmp_path
 ):
-    """A SHARED, writable tool server keeps each rollout's state isolated across concurrent rollouts,
-    over the FULL `harness_runtime` x (shared server's own) `tool_runtime` x fork matrix — including
+    """A SHARED, writable tool server keeps each rollout's `self.state` isolated across concurrent
+    rollouts, over the FULL `harness_runtime` x (shared server's own) `tool_runtime` matrix — including
     the mixed-locality combos (e.g. a local harness with a remote shared tool), which is exactly where
-    the rollout's `/state` + `/task` channel must be bridged to the tool's runtime. `scratchpad-v1`
-    gives each task a unique word and rewards 1.0 iff the rollout reads back its OWN word, so two
-    concurrent rollouts (two distinct words on the ONE shared instance) both scoring 1.0 proves no
-    cross-rollout corruption. Each fork mode exercises a different isolation path: fork=off keeps
-    `isolate=True` (the per-rollout `self.state` channel); fork=on sets `isolate=False` (the server
-    writes a process-global slot), so ONLY the forked-child process isolation can keep it correct.
-    `isolate` is a config field (not an env var), so it reaches the shared server in any runtime.
+    the rollout's `/state` channel must be bridged to the tool's runtime. `scratchpad-v1` gives each
+    task a unique word and rewards 1.0 iff the rollout reads back its OWN word, so two concurrent
+    rollouts (two distinct words on the ONE shared instance) both scoring 1.0 proves the per-rollout
+    `self.state` channel keeps them isolated with no cross-rollout corruption.
 
     Placement is fixed to `shared`, so only the own-runtime cases of `tool_runtime` apply (the
     colocated/shared params have no distinct runtime to fan — skipped). Runs through the env-server
@@ -158,13 +151,6 @@ async def test_shared_tool_isolation(
         pytest.skip(
             "tool server in a prime sandbox needs prime port exposure (unreachable from host here)"
         )
-    # A prime harness relays every model call through the sandbox, so two concurrent rollouts driving
-    # the EXPERIMENTAL fork-per-rollout server can't finish within the rollout cap (reward stays 0).
-    # The fork-isolation path is covered by the subprocess/docker harness cases.
-    if harness_runtime == "prime" and fork:
-        pytest.skip(
-            "prime harness + experimental fork-per-rollout is too slow under concurrency to finish in the rollout cap"
-        )
     traces = await run_v1_server(
         "scratchpad-v1",
         harness="null",
@@ -176,8 +162,6 @@ async def test_shared_tool_isolation(
         taskset_overrides={
             "tools": {
                 "shared": True,
-                "fork": fork,
-                "isolate": not fork,  # fork=on: write a global slot so ONLY fork can isolate it
                 **tool_runtime,  # the shared tool's own runtime ({"runtime": {"type": ...}})
             }
         },

verifiers-0.1.15.dev417/tests/v1/test_scoring.py

@@ -0,0 +1,33 @@
+import verifiers.v1 as vf
+
+
+def test_compare_stdout_results_accepts_token_equal_text() -> None:
+    assert vf.compare_stdout_results("hello   world\n", "hello world\n")
+
+
+def test_compare_stdout_results_keeps_numeric_tolerance() -> None:
+    assert vf.compare_stdout_results("1.0001 2.0\n", "1.0002 2.0009\n")
+
+
+def test_parse_pytest_outcomes_strips_xfail_xpass_reasons() -> None:
+    output = "\n".join(
+        [
+            "XFAIL tests/test_mod.py::test_xfail - known bug - still tracked",
+            "XPASS tests/test_mod.py::test_xpass - always xfail - unexpectedly passed",
+            "FAILED tests/test_mod.py::test_param[a - b] - assert left - right",
+            "PASSED tests/test_mod.py::test_ok",
+        ]
+    )
+
+    assert vf.parse_pytest_outcomes(output) == {
+        "tests/test_mod.py::test_xfail": "XFAIL",
+        "tests/test_mod.py::test_xpass": "XPASS",
+        "tests/test_mod.py::test_param[a - b]": "FAILED",
+        "tests/test_mod.py::test_ok": "PASSED",
+    }
+
+
+def test_parse_judge_choice_uses_first_choice_after_verdict_marker() -> None:
+    response = "Final Judgment: B because it is a better answer"
+
+    assert vf.parse_judge_choice(response, choices=("A", "B")) == "B"

{verifiers-0.1.15.dev415 → verifiers-0.1.15.dev417}/verifiers/v1/GUIDE.md

@@ -511,19 +511,17 @@ the default is the cheapest correct thing; the rest trade setup cost for isolati
 | **own runtime** *(default)* | *(nothing; `runtime = {type = "docker"\|"prime"}` for a sandbox)* | own runtime per rollout — `subprocess` on the host (default), or a `docker`/`prime` sandbox over a tunnel | full per-rollout isolation; a sandbox also isolates untrusted code / deps / network | pays `setup` every rollout (a sandbox adds spin-up + env install) |
 | **colocated** | `colocated = true` | inside the harness's runtime, one per rollout (no tunnel) | no extra runtime/tunnel; can touch the harness's filesystem | couples to the harness; `setup` per rollout |
 | **shared** | `shared = true` | one instance for the whole eval | `setup` once; writable per-rollout if state lives in `self.state` | state outside `self.state` corrupts across rollouts; `setup_task` skipped |
-| **shared + fork** | `shared = true, fork = true` | warm parent + forked child per rollout (copy-on-write) | `setup` once **and** isolates arbitrary in-process/on-disk state; runs `setup_task` per child | a process per concurrent rollout; Linux only |
 | **remote** *(tools only)* | `url = "https://…"` | connects to an already-running MCP endpoint | zero hosting; use a public/third-party server | no isolation, state, or lifecycle control |
 
-`shared` (and `fork`) work on any runtime — the framework makes the rollout's `/state` channel
-reachable from the shared server (localhost, or a host tunnel when remote). Keep big shared data
-off the Python heap (numpy / mmap / an on-disk index) so fork's copy-on-write actually saves
-memory.
+`shared` works on any runtime — the framework makes the rollout's `/state` channel reachable from the
+shared server (localhost, or a host tunnel when remote). Keep big shared data off the Python heap
+(numpy / mmap / an on-disk index) so one instance scales across rollouts.
 
 **Choosing per-rollout state:** read-only resource → `shared`; state that fits a `State` model →
 `shared` + `self.state` (no extra process, the scalable default); state that can't (module globals,
-on-disk scratch, a stateful C library) → `shared + fork` or a per-rollout placement that pays
-`setup` each time; cheap `setup` → just use the default. **User simulators** support only the
-per-rollout placements (own runtime or `colocated`); `shared` / `fork` / `url` are tools-only.
+on-disk scratch, a stateful C library) → a per-rollout placement (own runtime or `colocated`) that
+pays `setup` each time; cheap `setup` → just use the default. **User simulators** support only the
+per-rollout placements (own runtime or `colocated`); `shared` / `url` are tools-only.
 
 ## Learn from the examples
 

{verifiers-0.1.15.dev415 → verifiers-0.1.15.dev417}/verifiers/v1/__init__.py

@@ -49,6 +49,14 @@ from verifiers.v1.loaders import (
     task_type,
     taskset_config_type,
 )
+from verifiers.v1.scoring import (
+    compare_stdout_results as compare_stdout_results,
+    extract_boxed_answer as extract_boxed_answer,
+    parse_judge_choice as parse_judge_choice,
+    parse_pytest_outcomes as parse_pytest_outcomes,
+    read_answer_file_or_last_reply as read_answer_file_or_last_reply,
+    verify_boxed_math_answer as verify_boxed_math_answer,
+)
 from verifiers.v1.retries import RetryConfig, RolloutRetryConfig
 from verifiers.v1.rollout import Rollout
 from verifiers.v1.runtimes import (
@@ -195,6 +203,13 @@ __all__ = [
     "JudgeConfig",
     "JudgeSamplingConfig",
     "JudgeResponse",
+    # scoring
+    "compare_stdout_results",
+    "extract_boxed_answer",
+    "parse_judge_choice",
+    "parse_pytest_outcomes",
+    "read_answer_file_or_last_reply",
+    "verify_boxed_math_answer",
     # mcp
     "Toolset",
     "ToolsetConfig",

{verifiers-0.1.15.dev415 → verifiers-0.1.15.dev417}/verifiers/v1/interception/server.py

@@ -216,8 +216,8 @@ class InterceptionServer:
         # GET/PUT their `self.state` here, keyed by the same bearer secret as the model routes.
         app.router.add_get("/state", self.handle_state_get)
         app.router.add_put("/state", self.handle_state_put)
-        # A forked shared server (see `verifiers.v1.mcp.multiplex`) fetches its rollout's task here
-        # to run `setup_task` per child — a shared server gets no task via env, keyed by the secret.
+        # A launched tool/user server fetches its rollout's task here to run `setup_task` — the task
+        # is never passed via env, only over this channel, keyed by the same bearer secret.
         app.router.add_get("/task", self.handle_task_get)
         self.runner = web.AppRunner(app)
         await self.runner.setup()
@@ -585,7 +585,7 @@ class InterceptionServer:
         )
 
     async def handle_task_get(self, request: web.Request) -> web.Response:
-        """Hand a forked shared server the rollout's task (class ref + JSON) so its child can run
+        """Hand a launched tool/user server the rollout's task (class ref + JSON) so it can run
         `setup_task` for this rollout — keyed by the same bearer secret as the state channel."""
         session = self._session_for(request)
         if session is None:

{verifiers-0.1.15.dev415 → verifiers-0.1.15.dev417}/verifiers/v1/mcp/launch.py

@@ -287,23 +287,11 @@ async def serve(
             "is built once for the whole eval and must be task-agnostic — it receives no task. "
             "Drop `shared` to run it per-rollout (with the task), or make its `setup` task-independent."
         )
-    if (
-        shared
-        and not getattr(cfg, "fork", False)
-        and type(server).setup_task is not ServerBase.setup_task
-    ):
+    if shared and type(server).setup_task is not ServerBase.setup_task:
         logger.warning(
             "shared server %r overrides `setup_task`, but `setup_task` is NEVER called for a shared "
             "server (it's built once, task-agnostic) — its per-task logic will not run. Move "
-            "task-agnostic work into `setup`, drop `shared` to run it per-rollout, or set `fork=True` "
-            "(each rollout gets a forked child that runs `setup_task`).",
-            server.server_name,
-        )
-    if shared and getattr(cfg, "fork", False):
-        # `serve_tools` makes the rollout's /state + /task channel reachable from the shared server's
-        # runtime (a host tunnel when it's remote), so any harness/runtime combo works.
-        logger.warning(
-            "shared server %r uses fork-per-rollout isolation, which is EXPERIMENTAL",
+            "task-agnostic work into `setup`, or drop `shared` to run it per-rollout.",
             server.server_name,
         )
     async with contextlib.AsyncExitStack() as stack:
@@ -383,11 +371,11 @@ async def serve_shared(toolsets: list[Toolset], harness_is_local: bool = True):
 
 def _shared_url_for_rollout(url: str, state_base: str | None, state_secret: str) -> str:
     """Tag a `shared` server's eval-level URL with this rollout's state-channel coordinates, so the
-    one shared process serves each rollout its OWN `self.state` (and a forked child fetches its task).
-    `state_base` is the interception server's reachable base for THIS rollout, chosen by the caller to
-    be reachable from the shared server's runtime (localhost, or a host tunnel for a remote runtime).
-    The secret is the bearer the harness already holds, so it's no new exposure — but it must not be
-    logged (callers log the untagged base)."""
+    one shared process serves each rollout its OWN `self.state`. `state_base` is the interception
+    server's reachable base for THIS rollout, chosen by the caller to be reachable from the shared
+    server's runtime (localhost, or a host tunnel for a remote runtime). The secret is the bearer the
+    harness already holds, so it's no new exposure — but it must not be logged (callers log the
+    untagged base)."""
     if not state_base:
         return url
     parts = urlsplit(url)
@@ -397,19 +385,6 @@ def _shared_url_for_rollout(url: str, state_base: str | None, state_secret: str)
     return urlunsplit(parts._replace(query=urlencode(query)))
 
 
-async def reap_forked_child(shared_url: str, secret: str) -> None:
-    """Best-effort: POST `/vf/close` to reap this rollout's forked child on teardown (a `fork` shared
-    server, see `multiplex`); the multiplexer otherwise only reaps a child when it exits."""
-    close_url = (
-        f"{shared_url.rsplit('/mcp', 1)[0]}/vf/close?{STATE_SECRET_PARAM}={secret}"
-    )
-    with contextlib.suppress(Exception):
-        import httpx
-
-        async with httpx.AsyncClient(timeout=10.0) as client:
-            await client.post(close_url)
-
-
 @contextlib.asynccontextmanager
 async def serve_tools(
     toolsets: list[Toolset],
@@ -456,11 +431,6 @@ async def serve_tools(
                 # log the untagged base, NOT urls[name] — the per-rollout tag carries the rollout's
                 # bearer secret (`vf_state_secret`), which must not reach a log sink
                 logger.info("tool server '%s' (shared): %s", name, shared_urls[name])
-                if getattr(cfg, "fork", False) and state_secret:
-                    # reap this rollout's forked child when the rollout's tool-serving scope exits
-                    stack.push_async_callback(
-                        reap_forked_child, shared_urls[name], state_secret
-                    )
             else:
                 urls[name] = await stack.enter_async_context(
                     serve(

{verifiers-0.1.15.dev415 → verifiers-0.1.15.dev417}/verifiers/v1/mcp/server.py

@@ -118,8 +118,7 @@ def _request_query(name: str) -> str | None:
 def _die_with_parent() -> None:
     """Ask the kernel to SIGKILL this process when its parent (the launcher) dies, so a server is
     never orphaned if its launcher is torn down without stopping it (a backstop to the runtime's
-    own cleanup). Linux-only; a no-op elsewhere. Cleared across `fork`, so a forked child must call
-    it again."""
+    own cleanup). Linux-only; a no-op elsewhere."""
     import ctypes
     import signal
 
@@ -241,9 +240,8 @@ class ServerBase(Generic[ConfigT, StateT]):
         self, state_url: str | None, secret: str
     ) -> None:
         """Fetch this rollout's task over the state channel and run `setup_task` for it — a no-op
-        without a channel or task (a shared, task-agnostic server). The single place task setup
-        happens: `_serve` calls it with the env channel; a forked child calls it with its per-request
-        channel (see `verifiers.v1.mcp.multiplex`)."""
+        without a channel or task (a shared, task-agnostic server). Called by `_serve` with the
+        per-rollout env channel a launched server carries."""
         task = await self._fetch_task(state_url, secret)
         if task is not None:
             await self.setup_task(task)
@@ -332,8 +330,8 @@ class ServerBase(Generic[ConfigT, StateT]):
         async def _setup() -> None:
             await self.setup()
             # Per-rollout servers carry the rollout's state channel in their env; fetch this rollout's
-            # task over it and run `setup_task`. A shared server has no env channel (its forked
-            # children fetch /task per-request instead — see multiplex), so this is a no-op for it.
+            # task over it and run `setup_task`. A shared server has no env channel, so this is a
+            # no-op for it (it's task-agnostic — one instance for the whole eval).
             await self._setup_task_from_channel(*self._state_channel())
 
         asyncio.run(_setup())
@@ -369,13 +367,6 @@ class ServerBase(Generic[ConfigT, StateT]):
                 self._state_client = None
 
         app.router.lifespan_context = serving_lifespan
-        if getattr(self.config, "fork", False):
-            # `setup` ran once above (warm); fork a child per rollout that inherits it and runs
-            # `setup_task` for the rollout's task (see multiplex).
-            from verifiers.v1.mcp.multiplex import serve_forked
-
-            serve_forked(app, sock, self)
-            return
         server = uvicorn.Server(uvicorn.Config(app, log_level="critical"))
         asyncio.run(server.serve(sockets=[sock]))
 

{verifiers-0.1.15.dev415 → verifiers-0.1.15.dev417}/verifiers/v1/mcp/toolset.py

@@ -32,8 +32,6 @@ class ToolsetConfig(BaseConfig):
         costs a per-rollout install.
       - shared: one instance for the whole eval, in its own `runtime` (pays `setup` once; per-rollout
         writable state stays isolated via `self.state`).
-      - shared + fork: like shared, but a forked child per rollout (copy-on-write) also isolates
-        state that can't live in `self.state`.
     See the placement/isolation section of `verifiers/v1/GUIDE.md` for the trade-offs of each.
     Subclass to add the server's own knobs (the data its `@tool` methods read). The server name is
     the class's `TOOL_PREFIX` ClassVar, not a field here — it's an identity (the model sees
@@ -51,15 +49,6 @@ class ToolsetConfig(BaseConfig):
     server's URL), so concurrent rollouts don't corrupt each other — provided the shared server runs
     on a local runtime (the default), which can reach the host's interception server. Mutually
     exclusive with `colocated`."""
-    fork: bool = False
-    """For a `shared` server: fork a child process per rollout (copy-on-write memory + a private
-    working dir), so per-rollout state that can't live in `self.state` — module globals, a mutated
-    in-memory object, relative-path on-disk writes — is isolated per rollout automatically. The
-    expensive `setup` runs once in the parent; each child inherits it warm and runs `setup_task` for
-    its rollout's task (see `verifiers.v1.mcp.multiplex`) — so a stateful per-rollout server pays its
-    `setup` once yet stays isolated. Requires `shared`; works on any runtime (a remote one reaches the
-    rollout's state/task channel over a host tunnel). Linux/fork only; not for CUDA/GPU state or
-    background threads in the server."""
     runtime: RuntimeConfig = SubprocessConfig()
     """The server's own runtime, used unless `colocated` (host/subprocess by default — always
     reachable from any harness runtime; set docker/prime to isolate it in its own sandbox)."""
@@ -72,10 +61,6 @@ class ToolsetConfig(BaseConfig):
     def _validate_placement(self) -> "ToolsetConfig":
         if self.colocated and self.shared:
             raise ValueError("colocated and shared are mutually exclusive")
-        if self.fork and not self.shared:
-            raise ValueError(
-                "fork requires shared — a per-rollout server already runs in its own process"
-            )
         return self
 
 

verifiers-0.1.15.dev417/verifiers/v1/scoring.py

@@ -0,0 +1,166 @@
+from __future__ import annotations
+
+import re
+from collections.abc import Sequence
+from decimal import Decimal, InvalidOperation
+from typing import TYPE_CHECKING
+
+from math_verify import parse, verify
+from math_verify.errors import TimeoutException as MathVerifyTimeout
+
+from verifiers.v1.errors import SandboxError
+
+if TYPE_CHECKING:
+    from verifiers.v1.runtimes import Runtime
+    from verifiers.v1.trace import Trace
+
+BOXED_START = "\\boxed{"
+ANSI_RE = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]")
+PYTEST_OUTCOMES = ("PASSED", "FAILED", "ERROR", "SKIPPED", "XFAIL", "XPASS")
+
+
+def extract_boxed_answer(text: str, strict: bool = False) -> str:
+    start = text.rfind(BOXED_START)
+    if start == -1:
+        return "" if strict else text
+
+    # Regex is the wrong tool for nested braces, so walk the final box by hand.
+    answer_start = start + len(BOXED_START)
+    depth = 1
+    for index, char in enumerate(text[answer_start:], start=answer_start):
+        if char == "{":
+            depth += 1
+        elif char == "}":
+            depth -= 1
+        if depth == 0:
+            return text[answer_start:index]
+
+    return "" if strict else text
+
+
+async def read_answer_file_or_last_reply(
+    runtime: "Runtime", path: str, trace: "Trace"
+) -> str:
+    try:
+        answer = (await runtime.read(path)).decode(errors="replace").strip()
+    except (FileNotFoundError, OSError, SandboxError):
+        answer = ""
+    return answer or trace.last_reply
+
+
+def parse_judge_choice(
+    content: str | None, choices: Sequence[str] = ("A", "B", "C")
+) -> str | None:
+    if not content or ("<think>" in content and "</think>" not in content):
+        return None
+
+    text = content.rsplit("</think>", 1)[-1].strip()
+    text = extract_boxed_answer(text, strict=True).strip() or text
+
+    text_upper = text.upper()
+    choices_by_upper = {choice.upper(): choice for choice in choices}
+    allowed = "|".join(re.escape(choice) for choice in choices_by_upper)
+    choice_re = rf"(?<!\w)({allowed})(?!\w)"
+    verdict_re = (
+        r"(?:^|\n)\s*(?:FINAL\s+JUDGMENT|FINAL\s+ANSWER|FINAL\s+VERDICT|"
+        r"JUDGMENT|VERDICT|ANSWER)\s*(?:IS\s*)?[:\-]?\s*"
+    )
+
+    verdict = re.search(verdict_re, text_upper)
+    if verdict:
+        match = re.search(choice_re, text_upper[verdict.end() :])
+        return choices_by_upper.get(match.group(1)) if match else None
+
+    matches = re.findall(choice_re, text_upper)
+    return choices_by_upper.get(matches[-1]) if matches else None
+
+
+def verify_boxed_math_answer(
+    response: str | None, answer: str, *, timeout_seconds: int = 5
+) -> float:
+    if not response or ("<think>" in response and "</think>" not in response):
+        return 0.0
+
+    prediction_text = response.rsplit("</think>", 1)[-1]
+    prediction = extract_boxed_answer(prediction_text, strict=True).strip()
+    gold = (extract_boxed_answer(answer, strict=True) or answer).strip()
+    if not prediction or not gold:
+        return 0.0
+
+    # math-verify expects both sides as boxed math expressions.
+    try:
+        parsed_gold = parse(f"\\boxed{{{gold}}}", parsing_timeout=timeout_seconds)
+        parsed_prediction = parse(
+            f"\\boxed{{{prediction}}}",
+            parsing_timeout=timeout_seconds,
+        )
+        return float(
+            verify(
+                parsed_gold,
+                parsed_prediction,
+                timeout_seconds=timeout_seconds,
+            )
+        )
+    except (Exception, MathVerifyTimeout):
+        return 0.0
+
+
+def compare_stdout_results(
+    exec_stdout: str, expected: str, *, tolerance: float = 1e-3
+) -> bool:
+    if exec_stdout.strip() == expected.strip():
+        return True
+
+    actual_lines = [line.strip() for line in exec_stdout.splitlines() if line.strip()]
+    expected_lines = [line.strip() for line in expected.splitlines() if line.strip()]
+    if actual_lines == expected_lines:
+        return True
+
+    actual_tokens = [token for line in actual_lines for token in line.split()]
+    expected_tokens = [token for line in expected_lines for token in line.split()]
+    if actual_tokens == expected_tokens:
+        return True
+    if len(actual_tokens) != len(expected_tokens) or not actual_tokens:
+        return False
+
+    # Code datasets often allow tiny floating-point drift in stdout.
+    try:
+        actual_numbers = [Decimal(token) for token in actual_tokens]
+        expected_numbers = [Decimal(token) for token in expected_tokens]
+    except (InvalidOperation, ValueError):
+        return False
+    if not all(number.is_finite() for number in actual_numbers + expected_numbers):
+        return False
+
+    limit = Decimal(str(tolerance))
+    return all(
+        abs(actual - expected) <= limit
+        for actual, expected in zip(actual_numbers, expected_numbers)
+    )
+
+
+def parse_pytest_outcomes(output: str | None) -> dict[str, str]:
+    outcomes: dict[str, str] = {}
+    for raw_line in ANSI_RE.sub("", output or "").splitlines():
+        parts = raw_line.strip().split(maxsplit=1)
+        if len(parts) != 2:
+            continue
+
+        outcome, test_id = parts
+        if outcome not in PYTEST_OUTCOMES:
+            continue
+
+        # SKIPPED short-summary rows use "[N] file.py:line: reason", not a node id.
+        if test_id.startswith("["):
+            continue
+
+        # These summary rows append " - <reason>"; passing node ids do not.
+        if outcome in ("FAILED", "ERROR", "XFAIL", "XPASS") and " - " in test_id:
+            parts = test_id.split(" - ")
+            for index in range(1, len(parts)):
+                node_id = " - ".join(parts[:index])
+                if node_id.count("[") == node_id.count("]"):
+                    test_id = node_id
+                    break
+        outcomes[test_id.rstrip()] = outcome
+    return outcomes