parallax-evals 2.0.0.dev1__tar.gz → 2.0.0.dev2__tar.gz

{parallax_evals-2.0.0.dev1 → parallax_evals-2.0.0.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: parallax-evals
-Version: 2.0.0.dev1
+Version: 2.0.0.dev2
 Summary: Agent evaluations on seeded MCP worlds, recorded and compared in Langfuse.
 License-Expression: MIT
 License-File: LICENSE
@@ -160,7 +160,7 @@ and how comparisons pair runs.
 
 ## Status
 
-`2.0.0.dev1`. The task format, world, checks, controls, statistics and Langfuse
+`2.0.0.dev2`. The task format, world, checks, controls, statistics and Langfuse
 recording are in place. No pilot has been run, so no cross-harness results are
 published yet.
 

{parallax_evals-2.0.0.dev1 → parallax_evals-2.0.0.dev2}/README.md

@@ -137,7 +137,7 @@ and how comparisons pair runs.
 
 ## Status
 
-`2.0.0.dev1`. The task format, world, checks, controls, statistics and Langfuse
+`2.0.0.dev2`. The task format, world, checks, controls, statistics and Langfuse
 recording are in place. No pilot has been run, so no cross-harness results are
 published yet.
 

{parallax_evals-2.0.0.dev1 → parallax_evals-2.0.0.dev2}/docs/adapters.md

@@ -32,8 +32,11 @@ class Harness(Protocol):
 - the per-trial `workspace` and `home`;
 - the meter's OpenAI and Anthropic base URLs and a per-trial key.
 
-Parallax sets no time or cost limit on the agent: a turn lasts until the
-product ends it, under the product's own limits, and the adapter waits for that.
+By default Parallax sets no time or cost limit on the agent: a turn lasts until
+the product ends it, under the product's own limits, and the adapter waits for
+that. With `--turn-timeout`, Parallax cancels a turn still running at the
+deadline and records a timeout. `stop` still runs afterwards, so release
+anything a cancelled turn holds there.
 
 When cost is metered, real provider keys never enter the box.
 
@@ -341,7 +344,7 @@ its provenance; unknown or unpriced usage is never silently treated as free.
 | codex | open | CLI `codex exec --json` | yes | metered | Verified: codex-cli 0.156.1 turns recorded |
 | hermes | open | CLI `hermes chat -Q -q` | yes | metered | 0.22.6 runtime protocol probes against a scripted upstream; see evidence below |
 | openclaw | open | CLI `openclaw agent --local --json` | yes | metered | 2026.10.10 runtime protocol probes; clean Docker install verified; see evidence below |
-| grok-build | product | CLI `grok -p --output-format streaming-json` | yes | harness | Unverified: not installed, synthetic fixture from public docs |
+| grok-build | product | CLI `grok -p --output-format streaming-json` | yes | metered | Flags and config verified against grok 1.0.41 (`fixtures/grok-build/help.txt`) |
 | oracle | control | Replays `solution/` | n/a | none | Control |
 | null-claim / null-refuse | control | Fixed answer, no calls | n/a | none | Control |
 

{parallax_evals-2.0.0.dev1 → parallax_evals-2.0.0.dev2}/docs/methodology.md

@@ -262,7 +262,7 @@ A human moves a flagged task by editing `task.toml` and bumping the task
 This section assesses Parallax against the Agentic Benchmark Checklist (Zhu et
 al., 2025, *Establishing Best Practices for Building Rigorous Agentic
 Benchmarks*). The items are grouped under the checklist's three headings. The
-status is as of `2.0.0.dev1`, before the first pilot:
+status is as of `2.0.0.dev2`, before the first pilot:
 
 - ✅ met by construction and enforced in CI;
 - ◐ designed but not yet exercised or complete;

{parallax_evals-2.0.0.dev1 → parallax_evals-2.0.0.dev2}/docs/quickstart.md

@@ -69,7 +69,11 @@ parallax run lite --harness codex --model openai/gpt-5-mini
 
 `--harness` and `--model` take comma-separated lists; every harness runs with
 every model, `--repeats` times (default 3). `--dry-run` lists the selected tasks
-and runs nothing. A run goes through these steps:
+and runs nothing. A turn waits for the harness to finish, however long it takes;
+`--turn-timeout SECONDS` kills a turn still running after that and records the
+trial as a timeout, so a hung harness cannot hold up a paid run. Parallax never
+caps spend itself, so size a paid run by its trial count. A run goes through
+these steps:
 
 1. It confirms Langfuse accepts writes for the configured project, before
    anything is spent.

{parallax_evals-2.0.0.dev1 → parallax_evals-2.0.0.dev2}/docs/roadmap.md

@@ -1,6 +1,6 @@
 # Roadmap
 
-Parallax is at `2.0.0.dev1`. The task format, the simulated world, the checks,
+Parallax is at `2.0.0.dev2`. The task format, the simulated world, the checks,
 the controls, the statistics and Langfuse recording are in place and exercised by CI.
 This page is directional; it is not a commitment to dates.
 

parallax_evals-2.0.0.dev2/parallax/__init__.py

@@ -0,0 +1 @@
+__version__ = "2.0.0.dev2"

{parallax_evals-2.0.0.dev1 → parallax_evals-2.0.0.dev2}/parallax/box.py

@@ -305,8 +305,8 @@ class ExecProc:
         self, stdin: bytes | None, timeout_s: float | None, *, quiet_s: float | None = None
     ) -> tuple[int | None, str, str]:
         """Returns `(None, out, err)` on timeout, after killing the process tree. `timeout_s` None
-        waits for the process to exit however long it takes (an agent's turn: its product owns the
-        limit). `quiet_s`: see `settle_pipes`."""
+        waits for the process to exit however long it takes, or until the wait is cancelled, which kills
+        the tree (an agent's turn: `--turn-timeout`). `quiet_s`: see `settle_pipes`."""
         out, err = bytearray(), bytearray()
         readers = asyncio.gather(drain(self.proc.stdout, out), drain(self.proc.stderr, err))
         readers.add_done_callback(observe_drains)
@@ -316,7 +316,11 @@ class ExecProc:
                     self.proc.stdin.write(stdin)
                     await self.proc.stdin.drain()
                 self.proc.stdin.close()
-        code = await wait_exit(self.proc, timeout_s)
+        try:
+            code = await wait_exit(self.proc, timeout_s)
+        except asyncio.CancelledError:  # the turn deadline or an interrupt: take the tree with it
+            await self.killer.kill()
+            raise
         if code is None:
             await self.killer.kill()
         self.exited_at = time.monotonic()
@@ -838,8 +842,10 @@ class Sidecar:
         net = f"px-{run_id}"
         name = f"{net}-sidecar"
         image = await build_base_image()
-        await docker("network", "create", "--internal", "--label", f"parallax.net={net}", net)
         self.network = net
+        await self.down()  # a resumed run reuses its id: clear what a killed attempt left behind
+        self.network = net  # before the create: an interrupt mid-create must still leave it to `down`
+        await docker("network", "create", "--internal", "--label", f"parallax.net={net}", net)
         env_names = [k for k in UPSTREAM_ENV if k in os.environ]
         serve = (
             f"python -m parallax.world serve --http 0.0.0.0:{WORLD_PORT} & "
@@ -904,8 +910,11 @@ class Sidecar:
             ids = (await docker("ps", "-aq", "--filter", f"label=parallax.net={self.network}")).split()
             if ids:
                 await docker("rm", "-f", *ids, timeout_s=120)
-        with contextlib.suppress(BoxError):
+        try:
             await docker("network", "rm", self.network, timeout_s=60)
+        except BoxError as exc:
+            if "not found" not in str(exc):  # none yet is the normal case before `up` creates it
+                logger.warning("box.network_rm network=%s: %s", self.network, exc)
         self.network = None
         self.container = None
 

{parallax_evals-2.0.0.dev1 → parallax_evals-2.0.0.dev2}/parallax/cli.py

@@ -101,7 +101,7 @@ def cmd_run(a: argparse.Namespace) -> int:
         out=out, repeats=a.repeats, isolation=a.isolation, concurrency=a.concurrency,
         world_public_url=a.world_public_url, world_port=a.world_port, judge=csv(a.judge) if a.judge else (),
         include_xfail=a.include_xfail, selection=" ".join(a.select), provenance_extra=extra,
-        pairs=pinned, retry_invalid=a.retry_invalid,
+        pairs=pinned, retry_invalid=a.retry_invalid, turn_timeout_s=a.turn_timeout,
     )  # fmt: skip
     run_dir = run(tasks, harnesses, models, opts)
     # Latest row per key: a transient-invalid trial's failed attempt is evidence, not a second trial.
@@ -275,6 +275,7 @@ def run_parser(sub: argparse._SubParsersAction[argparse.ArgumentParser]) -> None
         "--retry-invalid", type=int, default=1, metavar="N",
         help="re-run a trial that ended INVALID for a transient execution fault, up to N further attempts",
     )  # fmt: skip
+    p.add_argument("--turn-timeout", type=float, metavar="SECONDS", help="kill a turn still running after this")
     p.add_argument("--out", type=Path, help="run directory (default: runs/<timestamp>); deleted once uploaded")
     p.add_argument("--resume", type=Path, metavar="RUN_DIR", help="continue an interrupted run")
     p.add_argument("--concurrency", type=int, default=4)

{parallax_evals-2.0.0.dev1 → parallax_evals-2.0.0.dev2}/parallax/controls.py

@@ -39,6 +39,7 @@ from parallax.runner import (
     TrialRun,
     harness_config_hash,
     judge_setup,
+    run_to_teardown,
     services,
 )
 from parallax.semantic import grade_checks, record_grade
@@ -366,7 +367,7 @@ def control_plan(name: str) -> HarnessPlan:
 
 
 def check(tasks: Sequence[Task], isolation: Isolation) -> ControlsReport:
-    return asyncio.run(check_async(tasks, isolation))
+    return run_to_teardown(check_async(tasks, isolation))
 
 
 def render(report: ControlsReport) -> str:

{parallax_evals-2.0.0.dev1 → parallax_evals-2.0.0.dev2}/parallax/harnesses/base.py

@@ -25,7 +25,7 @@ from parallax.models import (
     TurnStatus,
     Usage,
 )
-from parallax.protocols import BoxHandle, Harness, Proc
+from parallax.protocols import BoxHandle, Harness
 
 STDERR_TAIL_CHARS = 4000
 RAW_OUTPUT_CHARS = 2 * 1024 * 1024  # the newest raw harness output kept per trial for diagnosis
@@ -167,7 +167,6 @@ class SubprocessHarness(ABC):
         self._info = info
         self._trial: Trial | None = None
         self._box: BoxHandle | None = None
-        self._proc: Proc | None = None
         self.session_id: str | None = None
         self.in_session = False
         self.stderr_tail = ""
@@ -228,12 +227,9 @@ class SubprocessHarness(ABC):
         prompt_file = self.trial.home / "turns" / f"{t.index:02d}.md"
         await self.box.write_file(prompt_file, t.text.encode())
         argv = self.argv(t, continuing=self.in_session, session_id=self.session_id, prompt_file=prompt_file)
-        self._proc = await self.box.spawn(argv, env=self.env(), cwd=self.trial.workspace)
-        try:
-            # No deadline: the turn ends when the product ends it, under its own limits.
-            code, out, err = await self._proc.communicate(self.stdin(t), None)
-        finally:
-            self._proc = None
+        proc = await self.box.spawn(argv, env=self.env(), cwd=self.trial.workspace)
+        # No deadline here: the runner's `--turn-timeout` cancels the wait, which kills the process tree.
+        code, out, err = await proc.communicate(self.stdin(t), None)
         self.stderr_tail = err[-STDERR_TAIL_CHARS:]
         self.output_tail.add(f"=== turn {t.index} exit {code} ===\n--- stdout ---\n{out}\n--- stderr ---\n{err}\n")
         self.in_session = True
@@ -258,9 +254,7 @@ class SubprocessHarness(ABC):
         return deltas
 
     async def stop(self) -> None:
-        proc, self._proc = self._proc, None
-        if proc is not None:
-            await proc.kill()
+        return None  # a turn's process never outlives its turn: a cancelled wait kills it
 
     async def version(self, box: BoxHandle) -> str:
         return await run_version(self.info, box)

{parallax_evals-2.0.0.dev1 → parallax_evals-2.0.0.dev2}/parallax/harnesses/oracle.py

@@ -143,7 +143,8 @@ class OracleHarness:
         self._saved = {}
 
     def step_name(self, index: int) -> str:
-        task = self._require_task()
+        if (task := self._task) is None:
+            raise OracleError("oracle has no task bound")
         names = [t.name for t in task.turns] + [f.name for f in task.meta.fires]
         if not 1 <= index <= len(names):
             raise OracleError(f"turn {index} is outside the task's {len(names)} steps and fires")
@@ -188,11 +189,6 @@ class OracleHarness:
                 self._saved[call.save] = result
         return TurnOutput(status=TurnStatus.COMPLETED, answer=step.answer, session_id=None, steps=tuple(steps))
 
-    def _require_task(self) -> Task:
-        if self._task is None:
-            raise OracleError("oracle has no task bound")
-        return self._task
-
     def _require_trial(self) -> Trial:
         if self._trial is None:
             raise OracleError("oracle turn() before start()")

{parallax_evals-2.0.0.dev1 → parallax_evals-2.0.0.dev2}/parallax/jev.py

@@ -166,11 +166,6 @@ class JevCall(Model):
     history: tuple[JevAttempt, ...] = ()
 
 
-def answers_match(answers: Mapping[str, BoolAnswer], questions: Mapping[str, BoolQuestion]) -> bool:
-    """Exactly the questions asked; the reply model already rejects any non-boolean answer."""
-    return set(answers) == set(questions)
-
-
 def state_sha(state: Mapping[str, JsonValue]) -> str:
     return hashlib.sha256(json.dumps(state, sort_keys=True).encode()).hexdigest()
 
@@ -321,11 +316,9 @@ def _parsed(
     try:
         wire = _Reply.model_validate_json(content)
         update["warnings"] = wire.warnings
-        error = (
-            None
-            if answers_match(wire.answers, questions)
-            else f"answers do not match the questions asked: {sorted(wire.answers)}"
-        )
+        # Exactly the questions asked; the reply model already rejects any non-boolean answer.
+        match = set(wire.answers) == set(questions)
+        error = None if match else f"answers do not match the questions asked: {sorted(wire.answers)}"
         if error is None:
             update["answers"] = dict(wire.answers)
     except ValidationError as exc:

{parallax_evals-2.0.0.dev1 → parallax_evals-2.0.0.dev2}/parallax/runner.py

@@ -16,10 +16,11 @@ import os
 import re
 import secrets
 import shutil
+import signal
 import socket
 import sys
 import time
-from collections.abc import AsyncIterator, Awaitable, Iterator, Mapping, Sequence
+from collections.abc import AsyncIterator, Awaitable, Coroutine, Iterator, Mapping, Sequence
 from dataclasses import dataclass, field
 from datetime import UTC, datetime
 from pathlib import Path
@@ -156,6 +157,7 @@ class RunOptions(Model):
     include_xfail: bool = False
     # Further attempts for a trial that ended INVALID for a transient execution fault (RETRYABLE_INVALID).
     retry_invalid: int = 1
+    turn_timeout_s: float | None = None  # None: a turn ends when the product ends it
     selection: str = ""
     quiet: bool = False
     # Recorded in provenance.json: values only the caller knows.
@@ -238,6 +240,21 @@ async def finish_uncancelled(cleanup: Awaitable[None]) -> None:
     task.result()
 
 
+def run_to_teardown[T](main: Coroutine[None, None, T]) -> T:
+    """`asyncio.run`, but every SIGINT only cancels `main`. A second one (a repeated Ctrl-C, or `uv run`
+    forwarding its copy of the terminal's) would raise KeyboardInterrupt mid-teardown and strand boxes."""
+
+    async def guarded() -> T:
+        if (task := asyncio.current_task()) is not None:
+            asyncio.get_running_loop().add_signal_handler(signal.SIGINT, task.cancel)
+        return await main
+
+    try:
+        return asyncio.run(guarded())
+    except asyncio.CancelledError:
+        raise KeyboardInterrupt from None
+
+
 # ---------------------------------------------------------------------------
 # redaction, paths, placement
 # ---------------------------------------------------------------------------
@@ -480,7 +497,7 @@ async def services(
         await svc.start()
         yield svc
     finally:
-        await svc.stop()
+        await finish_uncancelled(svc.stop())  # a Ctrl-C landing mid-teardown would strand the run network
 
 
 # ---------------------------------------------------------------------------
@@ -655,11 +672,13 @@ class Session:
         run_dir: Path,
         panel: JudgePanel | None,
         judge_config: JudgeConfig | None,
+        turn_timeout_s: float | None = None,
     ) -> None:
         self.svc = svc
         self.run_dir = run_dir
         self.panel = panel
         self.judge_config = judge_config
+        self.turn_timeout_s = turn_timeout_s
 
     async def trial(self, plan: TrialPlan) -> TrialRun:
         task, hp = plan.task, plan.harness
@@ -927,18 +946,23 @@ class TrialExecution:
         origin: RunOrigin = RunOrigin.MEMBER,
         approval: ApprovalDecision | None = None,
     ) -> bool:
-        """One turn, waited on until the harness returns: the product under test owns its limits.
-        False ends the trial (a non-completed turn)."""
+        """One turn, waited on until the harness returns or `--turn-timeout` passes (none by default:
+        the product under test owns its limits). False ends the trial (a non-completed turn)."""
         t_in = TurnInput(
             index=index, text=text, new_session=new_session, attachments=attached, origin=origin, approval=approval
         )
         self.t.delivered_turns.append(t_in)
         submitted = time.monotonic()
         out: TurnOutput | None = None
+        deadline = asyncio.timeout(self.s.turn_timeout_s)
         try:
-            out = await harness.turn(t_in)
+            async with deadline:
+                out = await harness.turn(t_in)
         except Exception as exc:  # an adapter crash mid-turn is the harness's failure, not the eval's
             self.t.status, self.t.error = TrajectoryStatus.HARNESS_ERROR, f"turn {index}: {type(exc).__name__}: {exc}"
+            if deadline.expired():  # status and error then come from the TIMEOUT turn, below
+                error = f"turn {index}: still running after {self.s.turn_timeout_s:.0f}s, killed"
+                out = TurnOutput(status=TurnStatus.TIMEOUT, answer="", session_id=None, error=error)
         # Every turn's time counts toward agent_ms, a failed one's included.
         measured = int((time.monotonic() - submitted) * 1000)
         if out is None:
@@ -1106,16 +1130,12 @@ class TrialExecution:
         world = ev.world
         self.write(self.dir / "world_evidence.json", world.model_dump_json(indent=2))
         self.write(self.dir / "world_log.jsonl", "".join(c.model_dump_json() + "\n" for c in world.calls))
-        self.write(self.dir / "world_final.json", _json_state(world))
+        self.write(self.dir / "world_final.json", json.dumps(world.final, indent=2, ensure_ascii=False))
         self.write(self.dir / "world_diff.json", world.diff.model_dump_json(indent=2))
         for check_id, outcome in ev.verify.items():
             self.write(self.dir / "verify" / f"{safe_segment(check_id)}.log", outcome.model_dump_json(indent=2))
 
 
-def _json_state(world: WorldEvidence) -> str:
-    return json.dumps(world.final, indent=2, ensure_ascii=False)
-
-
 def run_record(fire: Fire, job: JsonObject) -> WorldPatch:
     """The scheduler's own record of the delivery, so a later `list_runs` sees it."""
     row = {"id": f"run-{fire.name}", "job": str(job.get("id")), "at": fire.at.isoformat(), "fire": fire.name}
@@ -1295,31 +1315,27 @@ def manifest_for(opts: RunOptions, plans: Sequence[TrialPlan], harnesses: Sequen
     )  # fmt: skip
 
 
-def schedulable(tasks: Sequence[Task], include_xfail: bool) -> list[Task]:
-    return [t for t in tasks if include_xfail or t.meta.xfail is None]
-
-
 async def run_async(
     tasks: Sequence[Task], harnesses: Sequence[str], models: Sequence[str | None], opts: RunOptions
 ) -> Path:
-    chosen = schedulable(tasks, opts.include_xfail)
+    chosen = [t for t in tasks if opts.include_xfail or t.meta.xfail is None]
     extensions.check_lanes(chosen, harnesses, opts.concurrency)
     if not chosen:
         raise ValueError("no tasks to run (xfail tasks need --include-xfail)")
     panel, judge_config = judge_setup(opts.judge)
     opts = opts.model_copy(update={"judge": panel.models if panel else ()})
-    opts.out.mkdir(parents=True, exist_ok=True)
     infos = [load_harness(h).info for h in harnesses]
     need_meter = any(i.cost_source is CostSource.METERED for i in infos)
     async with services(opts.isolation, opts.world_public_url, need_meter, opts.out.name, opts.world_port) as svc:
         plans_h = [await plan_harness(h, chosen, opts.isolation, svc.network) for h in harnesses]
         plans = only_pairs(plan_trials(chosen, plans_h, models, opts), opts.pairs)
+        opts.out.mkdir(parents=True, exist_ok=True)  # only now: a run interrupted in setup leaves no husk
         (opts.out / "manifest.json").write_text(manifest_for(opts, plans, plans_h).model_dump_json(indent=2))
         prov = provenance.collect(opts.out, opts.provenance_extra)
         provenance.write(opts.out, prov)
         done = completed_keys(opts.out, plans)
         todo = start_order([p for p in plans if p.key not in done])
-        session = Session(svc, opts.out, panel, judge_config)
+        session = Session(svc, opts.out, panel, judge_config, opts.turn_timeout_s)
         runs = {r.result.key: r for r in await execute_all(session, todo, opts)}
         await retry_transient_invalid(session, todo, runs, opts)
     served = provenance.served_models(read_results(opts.out / RESULTS))
@@ -1384,7 +1400,7 @@ async def execute_all(session: Session, todo: Sequence[TrialPlan], opts: RunOpti
 
 
 def run(tasks: Sequence[Task], harnesses: Sequence[str], models: Sequence[str | None], opts: RunOptions) -> Path:
-    return asyncio.run(run_async(tasks, harnesses, models, opts))
+    return run_to_teardown(run_async(tasks, harnesses, models, opts))
 
 
 def invalid_share(results: Sequence[TrialResult]) -> float:

{parallax_evals-2.0.0.dev1 → parallax_evals-2.0.0.dev2}/parallax/track/projection.py

@@ -25,7 +25,6 @@ from pydantic import BaseModel
 
 from parallax.models import (
     CheckKind,
-    CheckResult,
     CostSource,
     FailureMode,
     LatencySplit,
@@ -192,19 +191,6 @@ def outcome_of(verdicts: Sequence[float]) -> Outcome:
     return Outcome.FLAKY
 
 
-def _failed_checks(trials: Sequence[TrialResult]) -> list[str]:
-    failed: dict[str, None] = {}
-    for trial in trials:
-        for check in trial.checks:
-            if _counts_against(check):
-                failed[check.id] = None
-    return list(failed)
-
-
-def _counts_against(check: CheckResult) -> bool:
-    return check.passed is False and not check.advisory
-
-
 def _mean(values: Sequence[float] | Sequence[int]) -> float | None:
     return statistics.fmean(values) if values else None
 
@@ -287,7 +273,9 @@ def project_trace(dataset: str, run_id: str, trials: Sequence[TrialResult]) -> T
         output=ItemOutput(
             outcome=outcome,
             verdicts=verdicts,
-            failed_checks=_failed_checks(trials),
+            failed_checks=list(
+                dict.fromkeys(c.id for t in trials for c in t.checks if c.passed is False and not c.advisory)
+            ),
             cost_usd=_mean(costs),
             agent_ms=_mean(agent_ms),
             harness_version=first.harness_version,

{parallax_evals-2.0.0.dev1 → parallax_evals-2.0.0.dev2}/parallax/track/sink.py

@@ -137,7 +137,10 @@ def _client(base_url: str, public_key: str, secret_key: str) -> Langfuse:
     # Our own provider, not `id_generator=`: Langfuse ignores that whenever a global provider is
     # already registered, and random ids would make a resumed upload write every item twice.
     provider = TracerProvider(id_generator=IDS)
-    return Langfuse(base_url=base_url, public_key=public_key, secret_key=secret_key, tracer_provider=provider)
+    # The SDK's 5 s API default fails a whole upload on one slow Langfuse Cloud read during verification.
+    return Langfuse(
+        base_url=base_url, public_key=public_key, secret_key=secret_key, tracer_provider=provider, timeout=30
+    )
 
 
 def connect(*, write: bool) -> tuple[Langfuse, Project]:

{parallax_evals-2.0.0.dev1 → parallax_evals-2.0.0.dev2}/tests/core/test_runner.py

@@ -4,6 +4,10 @@ from __future__ import annotations
 
 import asyncio
 import shutil
+import signal
+import subprocess
+import sys
+import time
 from datetime import UTC, datetime
 from pathlib import Path
 
@@ -253,6 +257,26 @@ async def test_a_turn_whose_adapter_raised_still_counts_toward_agent_time(
     assert row.agent_ms is not None
 
 
+class Hanging(Crashing):
+    async def turn(self, t: TurnInput) -> TurnOutput:
+        await asyncio.sleep(60)
+        raise AssertionError("the turn deadline should have cancelled this")
+
+
+async def test_a_turn_past_the_turn_timeout_ends_the_trial_as_timeout(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    # Enforced by the runner, so a plugin harness that never returns is bounded too.
+    monkeypatch.setattr(runner, "load_harness", lambda name: Hanging(""))
+    tasks = [t for t in discover([TASKS]) if t.id == "mini-mail"]
+    options = opts(tmp_path / "r").model_copy(update={"turn_timeout_s": 0.5})
+    started = time.monotonic()
+    run_dir = await asyncio.wait_for(run_async(tasks, ["crashing"], ["m/x"], options), 30)
+    (row,) = read_results(run_dir / "results.jsonl")
+    assert time.monotonic() - started < 30
+    assert row.status is TrajectoryStatus.TIMEOUT
+
+
 async def test_a_pinned_world_port_is_served_and_a_taken_one_is_an_environment_error() -> None:
     async with runner.services(Isolation.REMOTE, None, False, "port-test", world_port=0) as probe:
         port = int(probe.world_local.rsplit(":", 1)[1])
@@ -532,3 +556,35 @@ async def test_a_transient_invalid_trial_is_re_run_within_the_same_run(tmp_path:
     rows = read_results(tmp_path / "results.jsonl")
     assert [r.verdict for r in rows] == [Verdict.INVALID, Verdict.PASS]
     assert latest_results(tmp_path / "results.jsonl")[0].verdict is Verdict.PASS
+
+
+_TEARDOWN_PROBE = """
+import asyncio, sys
+from parallax.runner import finish_uncancelled, run_to_teardown
+async def cleanup():
+    await asyncio.sleep(1)
+    print("cleanup finished", flush=True)
+async def main():
+    try:
+        print("running", flush=True)
+        await asyncio.sleep(30)
+    finally:
+        await finish_uncancelled(cleanup())
+try:
+    run_to_teardown(main())
+except KeyboardInterrupt:
+    sys.exit(130)
+"""
+
+
+def test_a_second_sigint_does_not_cut_teardown_short() -> None:
+    # `uv run` forwards its own copy of the terminal's Ctrl-C about 200 ms after the group's, so one
+    # keypress arrives twice: plain `asyncio.run` raises on the second and strands the run's network.
+    proc = subprocess.Popen([sys.executable, "-c", _TEARDOWN_PROBE], stdout=subprocess.PIPE, text=True)
+    assert proc.stdout is not None
+    assert proc.stdout.readline().strip() == "running"
+    proc.send_signal(signal.SIGINT)
+    time.sleep(0.3)
+    proc.send_signal(signal.SIGINT)
+    out, _ = proc.communicate(timeout=30)
+    assert (proc.returncode, out.strip()) == (130, "cleanup finished")

{parallax_evals-2.0.0.dev1 → parallax_evals-2.0.0.dev2}/tests/harnesses/fixtures/runtime-verification/README.md

@@ -89,7 +89,7 @@ recorded `chat_read_channel` then `chat_post_message` (both ok) plus the posted
 - OpenClaw 2026.10.10 deferred `tool_search` -> `dispatch_action` (bridge ->
   world read/post).
 
-Requirements: `python3`, the parallax repo `.venv` (`.venv/bin/parallax` is
+Requirements: `python3`, the parallax repo `.venv` (`.venv/bin/python -m parallax.world` and `-m parallax.meter` are
 invoked directly, no `uv` wrapper), `node`/`npm` (tested Node 24.22.2 for OpenClaw
 2026.10.10), and the `hermes` / `openclaw` binaries on PATH (set `HERMES=` /
 `OPENCLAW=` to override). The script binds a unique `mktemp -d` scratch dir

{parallax_evals-2.0.0.dev1 → parallax_evals-2.0.0.dev2}/tests/harnesses/fixtures/runtime-verification/reproduce.sh

@@ -13,7 +13,7 @@ set -euo pipefail
 
 HERE="$(cd "$(dirname "$0")" && pwd)"
 REPO="$(cd "$HERE/../../../.." && pwd)"
-PARALLAX="$REPO/.venv/bin/parallax"          # direct bin, no uv child wrapper
+PYTHON="$REPO/.venv/bin/python"             # direct interpreter, no uv child wrapper
 SCRATCH="$(mktemp -d "${TMPDIR:-/tmp}/parallax-rv.XXXXXX")"
 if command -v keeper >/dev/null 2>&1; then keeper claim "$SCRATCH" >/dev/null 2>&1 || true; fi
 LOGS="$SCRATCH/logs"
@@ -92,7 +92,7 @@ wait_file_line() {  # $1 file  $2 pattern  $3 label
 }
 
 start_world() {  # sets WORLD_TOKEN, WORLD_CTRL, WORLD_PID
-  "$PARALLAX" world serve --task tasks/conformance/conf-mcp-echo --http "127.0.0.1:$WORLD_PORT" \
+  "$PYTHON" -m parallax.world serve --task tasks/conformance/conf-mcp-echo --http "127.0.0.1:$WORLD_PORT" \
     >"$LOGS/world.out" 2>"$LOGS/world.stderr" &
   WORLD_PID=$!
   record_pid "$WORLD_PID" "world"
@@ -113,7 +113,7 @@ PARALLAX_UPSTREAM_OPENAI="http://127.0.0.1:$FAKE_PORT/v1" \
 PARALLAX_UPSTREAM_ANTHROPIC="http://127.0.0.1:$FAKE_PORT/v1" \
 PARALLAX_UPSTREAM_OPENAI_KEY="$FAKE_OPENAI_KEY" \
 PARALLAX_UPSTREAM_ANTHROPIC_KEY="$FAKE_ANTHROPIC_KEY" \
-"$PARALLAX" meter serve --http "127.0.0.1:$METER_PORT" >"$LOGS/meter.out" 2>&1 &
+"$PYTHON" -m parallax.meter "127.0.0.1:$METER_PORT" >"$LOGS/meter.out" 2>&1 &
 record_pid "$!" "meter"
 wait_url "http://127.0.0.1:$METER_PORT/_health" "meter"
 

{parallax_evals-2.0.0.dev1 → parallax_evals-2.0.0.dev2}/tests/harnesses/test_manifest.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import asyncio
 import json
 import sys
 import tomllib
@@ -211,3 +212,15 @@ def test_native_credentials_forward_only_declared_keys_of_unmetered_harnesses()
     assert credential_env_problems(metered)
     for name in ("OPENAI_API_KEY", "PARALLAX_UPSTREAM_OPENAI_KEY"):
         assert credential_env_problems(gemini.model_copy(update={"credential_env": (name,)})), name
+
+
+async def test_a_cancelled_turn_kills_its_process(box: HostBox, fake_spec: ManifestSpec, tmp_path: Path) -> None:
+    # The runner's `--turn-timeout` cancels the turn; the CLI under it must not outlive that.
+    pid_file = tmp_path / "pid"
+    hang = f"import os, time; open({str(pid_file)!r}, 'w').write(str(os.getpid())); time.sleep(60)"
+    harness = ManifestHarness(fake_spec.model_copy(update={"argv_first": (sys.executable, "-c", hang)}))
+    await harness.start(make_trial(box), box)
+    with pytest.raises(TimeoutError):
+        await asyncio.wait_for(harness.turn(TurnInput(index=1, text="hi", new_session=False)), 1.0)
+    stat = Path(f"/proc/{pid_file.read_text()}/stat")
+    assert not stat.exists() or stat.read_text().split(") ")[1].startswith("Z")

parallax_evals-2.0.0.dev1/parallax/__init__.py

@@ -1 +0,0 @@
-__version__ = "2.0.0.dev1"