steppy-dev 0.3.0__py3-none-any.whl

steppy/_parallel/lifecycle.py

@@ -0,0 +1,90 @@
+"""Guaranteed-teardown context for the coordinator.
+
+Wraps an ``ExitStack`` plus signal handlers (SIGINT/SIGTERM on POSIX,
+SIGINT only on Windows — Windows doesn't deliver SIGTERM to console apps).
+The coordinator pushes a ``compose down -v`` callback for each stack as
+soon as ``compose up`` is invoked. Any of:
+
+  - normal completion (queue drained, all workers exited 0)
+  - user Ctrl-C
+  - uncaught exception in coordinator
+  - SIGTERM from outside
+
+…triggers ``ExitStack.close()`` which fires every registered teardown in
+LIFO order. This is the single place that guarantees no orphaned
+containers, networks, or volumes after a parallel run.
+
+The Ctrl-C path raises ``KeyboardInterrupt`` after first calling the
+default signal handler, so subsequent Ctrl-C presses still work normally
+(double Ctrl-C escape hatch — useful if a teardown itself hangs).
+"""
+
+from __future__ import annotations
+
+import contextlib
+import signal
+import sys
+from typing import Callable
+
+
+class CoordinatorLifecycle:
+    """Context manager: ``with CoordinatorLifecycle() as lc: lc.on_teardown(fn)``.
+
+    Restores the previous signal handlers on exit so this doesn't leak signal
+    state to a calling process.
+    """
+
+    def __init__(self):
+        self._stack = contextlib.ExitStack()
+        self._previous_handlers: dict[int, Callable | int | None] = {}
+        self._signals_to_handle: list[int] = [signal.SIGINT]
+        if hasattr(signal, "SIGTERM") and sys.platform != "win32":
+            # Windows console apps don't receive SIGTERM in the usual sense;
+            # skip it to avoid OSError on signal.signal().
+            self._signals_to_handle.append(signal.SIGTERM)
+
+    def __enter__(self) -> "CoordinatorLifecycle":
+        self._stack.__enter__()
+        for sig in self._signals_to_handle:
+            try:
+                self._previous_handlers[sig] = signal.getsignal(sig)
+                signal.signal(sig, self._on_signal)
+            except (ValueError, OSError):
+                # signal.signal raises ValueError off the main thread; OSError
+                # for unsupported signals on the platform. Either way, skip.
+                pass
+        return self
+
+    def __exit__(self, exc_type, exc, tb):
+        # Restore signal handlers BEFORE running teardown so a Ctrl-C during
+        # teardown gets handled by the default Python interrupt logic.
+        for sig, prev in self._previous_handlers.items():
+            try:
+                signal.signal(sig, prev)
+            except (ValueError, OSError):
+                pass
+        return self._stack.__exit__(exc_type, exc, tb)
+
+    def on_teardown(self, fn: Callable[[], None]) -> None:
+        """Register a callback to run on context exit (LIFO order)."""
+        self._stack.callback(self._safe_call, fn)
+
+    @staticmethod
+    def _safe_call(fn: Callable[[], None]) -> None:
+        """Run a teardown callback; never let an exception escape and abort
+        subsequent teardowns. Logs to stderr instead."""
+        try:
+            fn()
+        except Exception as e:  # noqa: BLE001 — teardown must be best-effort
+            print(f"steppy teardown error (continuing): {e}", file=sys.stderr)
+
+    def _on_signal(self, signum, frame):
+        """Convert SIGINT/SIGTERM into KeyboardInterrupt so the coordinator's
+        outer try/except can react and ExitStack runs teardowns. Restores
+        the previous handler immediately so a second Ctrl-C uses the default
+        behavior (panic-exit if a teardown hangs)."""
+        try:
+            signal.signal(signum, self._previous_handlers.get(signum, signal.SIG_DFL))
+        except (ValueError, OSError):
+            pass
+        raise KeyboardInterrupt()

steppy/_parallel/preauth.py

@@ -0,0 +1,109 @@
+"""In-network pre-authentication lock-wrapper for parallel mode.
+
+Runs INSIDE a one-off ``steppy-preauth`` container. The coordinator renders one
+such service per worker stack (so it can be gated, per-stack, before that
+stack's ``steppy-worker`` claims a scenario), which means N containers would
+otherwise all run the user's login command at once and race on the shared
+``.auth`` cache. Instead they share ``/steppy/coord`` (the coordinator tmp dir,
+bind-mounted into every stack) and use the SAME O_EXCL claim primitive as the
+work queue (`_parallel/queue.py`) to elect exactly ONE winner:
+
+  * winner  — runs the user's ``pre_auth.command``; on success drops a
+              ``preauth.done`` sentinel, on failure a ``preauth.failed`` one.
+  * losers  — block until they see the winner's sentinel, then exit with the
+              matching code (0 on done, 1 on failed / timeout).
+
+So the login happens once per run even though every stack starts a preauth
+container. The sentinels live in the per-run coord dir, so a *new* run re-runs
+the command (inter-run freshness is the command's own concern — e.g. the
+capture script skips roles whose session is still fresh).
+"""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+
+LOCK_NAME = "preauth.lock"
+DONE_NAME = "preauth.done"
+FAILED_NAME = "preauth.failed"
+_POLL_INTERVAL = 0.5
+
+
+def run_preauth(
+    command: list[str],
+    *,
+    timeout: int,
+    coord_dir: Path | None = None,
+) -> int:
+    """Elect one winner (O_EXCL) to run ``command``; losers wait for its
+    sentinel. Returns a process exit code (0 = auth ready for this stack).
+
+    ``coord_dir`` defaults to ``$STEPPY_COORD_DIR`` (``/steppy/coord`` in the
+    container). ``command`` is the argv to execute (a plain shell string is
+    wrapped as ``sh -c`` by the caller before it reaches here).
+    """
+    if not command:
+        print("[preauth] no command given — nothing to do", file=sys.stderr, flush=True)
+        return 2
+
+    coord = Path(coord_dir) if coord_dir is not None else Path(os.environ.get("STEPPY_COORD_DIR", "/steppy/coord"))
+    coord.mkdir(parents=True, exist_ok=True)
+    lock_path = coord / LOCK_NAME
+    done_path = coord / DONE_NAME
+    failed_path = coord / FAILED_NAME
+
+    # A previous stack already settled auth this run — short-circuit.
+    if done_path.exists():
+        return 0
+    if failed_path.exists():
+        return 1
+
+    try:
+        fd = os.open(str(lock_path), os.O_WRONLY | os.O_CREAT | os.O_EXCL)
+    except FileExistsError:
+        # Another stack is (or was) the winner — wait for its verdict.
+        return _wait_for_sentinel(done_path, failed_path, timeout)
+
+    # We won the claim: run the command exactly once for this run.
+    try:
+        os.write(fd, str(os.getpid()).encode("utf-8"))
+    finally:
+        os.close(fd)
+
+    print(f"[preauth] winner — running: {' '.join(command)}", flush=True)
+    try:
+        code = subprocess.run(command, timeout=timeout).returncode
+    except subprocess.TimeoutExpired:
+        print(f"[preauth] command timed out after {timeout}s", file=sys.stderr, flush=True)
+        code = 124
+    except FileNotFoundError as e:
+        print(f"[preauth] command not found: {e}", file=sys.stderr, flush=True)
+        code = 127
+
+    if code == 0:
+        done_path.write_text("ok", encoding="utf-8")
+        print("[preauth] done — auth cache is ready", flush=True)
+    else:
+        failed_path.write_text(str(code), encoding="utf-8")
+        print(f"[preauth] FAILED (exit {code})", file=sys.stderr, flush=True)
+    return code
+
+
+def _wait_for_sentinel(done_path: Path, failed_path: Path, timeout: int) -> int:
+    """Loser path: block until the winner signals done/failed, or timeout."""
+    print("[preauth] another stack is authenticating; waiting…", flush=True)
+    deadline = time.monotonic() + max(1, timeout)
+    while time.monotonic() < deadline:
+        if done_path.exists():
+            return 0
+        if failed_path.exists():
+            print("[preauth] auth winner reported failure", file=sys.stderr, flush=True)
+            return 1
+        time.sleep(_POLL_INTERVAL)
+    print("[preauth] timed out waiting for the auth winner", file=sys.stderr, flush=True)
+    return 1

steppy/_parallel/progress.py

@@ -0,0 +1,101 @@
+"""Tail each worker container's stdout and dispatch lines to the bus.
+
+For each Compose project, a thread runs
+``docker compose -p <project> logs -f --no-color steppy-worker`` and
+feeds every line into the shared :class:`EventBus`. The bus parses
+``STEPPY-EVT::*`` lines into typed events; everything else falls
+through to the raw-log channel (printed when ``--verbose`` is set,
+always written to a per-worker file for post-mortem).
+
+A second thread per worker captures the full-stack logs (every service
+in the project, not just steppy-worker) into a separate file so the
+user can dig in after a failure.
+"""
+
+from __future__ import annotations
+
+import subprocess
+import threading
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Optional
+
+from .event_bus import EventBus
+
+
+@dataclass
+class WorkerLogTailer:
+    """Background thread tailing one worker's logs into the EventBus."""
+
+    project: str
+    worker_id: str
+    bus: EventBus
+    app_log_file: Path  # full-stack logs go here for post-mortem
+    _proc: Optional[subprocess.Popen] = field(default=None, init=False, repr=False)
+    _thread: Optional[threading.Thread] = field(default=None, init=False, repr=False)
+    _stop: Optional[threading.Event] = field(default=None, init=False, repr=False)
+
+    def start(self) -> None:
+        self._stop = threading.Event()
+        self._thread = threading.Thread(target=self._run, name=f"log-{self.worker_id}", daemon=True)
+        self._thread.start()
+        # Also kick off a fire-and-forget capture of the entire stack's logs
+        # (worker + app) into a per-worker file so users have a post-mortem
+        # if the run fails.
+        self.app_log_file.parent.mkdir(parents=True, exist_ok=True)
+        threading.Thread(target=self._capture_full_stack, name=f"log-full-{self.worker_id}", daemon=True).start()
+
+    def _run(self) -> None:
+        cmd = [
+            "docker",
+            "compose",
+            "-p",
+            self.project,
+            "logs",
+            "-f",
+            "--no-color",
+            "--no-log-prefix",
+            "steppy-worker",
+        ]
+        try:
+            self._proc = subprocess.Popen(
+                cmd,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                text=True,
+                encoding="utf-8",
+                errors="replace",
+                bufsize=1,
+            )
+        except FileNotFoundError:
+            return
+        assert self._proc.stdout is not None
+        for line in self._proc.stdout:
+            if self._stop is not None and self._stop.is_set():
+                break
+            self.bus.feed_line(self.worker_id, line)
+
+    def _capture_full_stack(self) -> None:
+        """Capture every service's logs into ``self.app_log_file`` for
+        post-mortem. Runs ``compose logs -f`` over the whole project (no
+        service filter) and writes raw output. Best-effort; if it fails
+        we just lose the post-mortem file, never the run."""
+        cmd = ["docker", "compose", "-p", self.project, "logs", "-f", "--no-color"]
+        try:
+            with self.app_log_file.open("w", encoding="utf-8") as f:
+                subprocess.run(
+                    cmd, stdout=f, stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace", check=False
+                )
+        except (FileNotFoundError, OSError):
+            pass
+
+    def stop(self) -> None:
+        if self._stop is not None:
+            self._stop.set()
+        if self._proc is not None and self._proc.poll() is None:
+            try:
+                self._proc.terminate()
+            except OSError:
+                pass
+        if self._thread is not None:
+            self._thread.join(timeout=2)

steppy/_parallel/queue.py

@@ -0,0 +1,385 @@
+"""Filesystem work-stealing queue for parallel scenario execution.
+
+Layout under the run's tmp dir (mounted into every worker at /steppy/coord):
+    queue/<task_id>.task     — marker, written by the coordinator; never moved
+    claims/<task_id>.claim   — atomic O_EXCL create by a worker; content = worker_id
+    results/<worker_id>/<slug>.json — per-SCENARIO results, written by workers
+
+A task is one scenario by default, and ``task_id`` is then the bare slug and the
+marker file is empty — exactly as it has always been. An **exclusion group**
+(frontmatter ``exclusive: <name>``) collapses its members into a single task whose
+id is the group name and whose marker carries one line of JSON naming its members;
+the worker claiming it runs them sequentially, one pytest process each. That is the
+whole mechanism: mutual exclusion is a property of how the queue was seeded, not a
+run-time protocol, so there is no lock to leak and no worker ever waits on another.
+
+Results, logs and JUnit stay keyed **per scenario** even for a batch, which is what
+lets every downstream consumer (orphan detection, retry classification, failure
+artifacts, history, the renderer) keep its per-slug key unchanged.
+
+Claiming is atomic via ``os.open(claim_path, O_WRONLY|O_CREAT|O_EXCL)``,
+which is atomic on both POSIX and Windows (battle-tested mail-spool
+pattern). When two workers race for the same task, exactly one O_EXCL
+succeeds and the other gets ``FileExistsError`` and tries the next
+candidate.
+
+We chose O_EXCL over the rename-claim variant because ``os.rename`` on
+Windows under heavy concurrency does not provide the source-side mutual
+exclusion we need (observed ~20% double-claim rate in the test).
+
+Workers loop:
+
+    while True:
+        task = queue.claim_task(worker_id)
+        if task is None:
+            break  # queue empty
+        for slug in task.slugs:   # exactly one, unless it is an exclusion group
+            result = run_scenario(slug)
+            queue.write_result(result)
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional
+
+from ..exclusive import Task
+
+
+TASK_SUFFIX = ".task"
+CLAIM_SUFFIX = ".claim"
+RESULT_SUFFIX = ".json"
+
+# Version stamped into a batch marker's payload. Ungrouped tasks carry no payload
+# at all, so this only ever appears in a queue that declares an exclusion group.
+TASK_PAYLOAD_VERSION = 1
+
+# Worker key used to report a batch whose claim file was created but never written
+# — a worker killed between the claim's ``os.open`` and its ``os.write``. Its real
+# id is unrecoverable, and every member is unstarted by construction (the id is
+# written before any scenario runs), so they are all orphans of nobody.
+UNKNOWN_WORKER = "(unknown)"
+
+
+class TaskPayloadError(Exception):
+    """A batch marker exists but could not be read or parsed.
+
+    The claim already succeeded when this is raised, so the task is claimed with no
+    results: its members surface as orphans and are retried. The worker reports it
+    and moves on rather than dying, because one unreadable marker must not cost the
+    rest of the queue.
+    """
+
+    def __init__(self, task_id: str, reason: str):
+        super().__init__(f"unreadable task payload for {task_id!r}: {reason}")
+        self.task_id = task_id
+        self.reason = reason
+
+
+# Task files are named ``NNNN-<slug>.task``. Claiming walks the directory in
+# sorted order, so the prefix is what makes the queue honour the order the
+# coordinator seeded rather than the alphabet — which lets it put the slowest
+# scenarios first (LPT) so no worker is left running a 3-minute scenario alone
+# after everyone else has finished. Claim/result files stay keyed by the bare
+# slug, so nothing downstream has to know about this.
+_ORDER_PREFIX_RE = re.compile(r"^\d{4}-")
+_ORDER_PREFIX_WIDTH = 4
+
+
+@dataclass(frozen=True)
+class ScenarioResult:
+    """One scenario's outcome, written to results/<worker_id>/<slug>.json."""
+
+    slug: str
+    worker_id: str
+    exit_code: int
+    duration_s: float
+    started_at: float  # epoch seconds
+    finished_at: float
+
+    def to_dict(self) -> dict:
+        return {
+            "slug": self.slug,
+            "worker_id": self.worker_id,
+            "exit_code": self.exit_code,
+            "duration_s": self.duration_s,
+            "started_at": self.started_at,
+            "finished_at": self.finished_at,
+            "passed": self.exit_code == 0,
+        }
+
+
+class Queue:
+    """Filesystem-backed work-stealing queue.
+
+    The coordinator instantiates one Queue, calls ``seed(slugs)`` to populate
+    queue/, then bind-mounts the parent dir into every worker container.
+    Each worker instantiates a Queue against the in-container path and
+    loops on ``claim()``. queue/, claims/, results/ are siblings under
+    ``root`` on both sides — the coordinator writes to host:<tmp>/queue/,
+    the worker sees the same files at /steppy/coord/queue/.
+    """
+
+    def __init__(self, root: Path):
+        self.root = Path(root)
+        self.queue_dir = self.root / "queue"
+        self.claims_dir = self.root / "claims"
+        self.results_dir = self.root / "results"
+        # task_id -> member slugs, for the batches this Queue seeded. Held as
+        # instance state rather than passed to each reader on purpose: orphan
+        # detection has three call sites, and a reader that forgot the map would
+        # report a group NAME as a failed scenario and retry a test file that does
+        # not exist. State cannot be forgotten; a parameter can.
+        self._task_slugs: dict[str, list[str]] = {}
+
+    def init_dirs(self) -> None:
+        """Create the queue/claims/results directory skeleton."""
+        self.queue_dir.mkdir(parents=True, exist_ok=True)
+        self.claims_dir.mkdir(parents=True, exist_ok=True)
+        self.results_dir.mkdir(parents=True, exist_ok=True)
+
+    def seed(self, slugs: list[str], *, batches: Optional[dict[str, list[str]]] = None) -> None:
+        """Coordinator-only: write one task file per queue entry.
+
+        ``slugs`` is a list of **task ids** — bare scenario slugs unless an entry
+        names an exclusion group. Task files carry a zero-padded ordinal prefix so
+        ``claim()``'s sorted walk hands them out in **the order given**, not
+        alphabetically. The coordinator uses that to seed longest-first from
+        recorded durations; callers that don't care can keep passing any order.
+
+        ``batches`` maps a task id to its member slugs and is written into the
+        marker as one line of JSON. Ids absent from it keep today's empty
+        ``touch()``, so a run with no exclusion group produces a byte-identical
+        queue directory. Callers pass ``batches`` only when it is non-empty.
+        """
+        self.init_dirs()
+        batches = batches or {}
+        self._task_slugs = {task_id: list(members) for task_id, members in batches.items()}
+        for index, task_id in enumerate(slugs):
+            path = self.queue_dir / f"{index:0{_ORDER_PREFIX_WIDTH}d}-{task_id}{TASK_SUFFIX}"
+            members = batches.get(task_id)
+            if members:
+                payload = {"v": TASK_PAYLOAD_VERSION, "group": task_id, "slugs": list(members)}
+                path.write_text(json.dumps(payload), encoding="utf-8")
+            else:
+                path.touch()
+
+    @staticmethod
+    def _slug_from_task(name: str) -> str:
+        """``0007-checkout.task`` → ``checkout``. Tolerates un-prefixed names so
+        a queue dir seeded by an older steppy still drains.
+
+        Returns the *task id*, which is a scenario slug unless the entry is a
+        batch, in which case it is the exclusion group's name."""
+        stem = name[: -len(TASK_SUFFIX)] if name.endswith(TASK_SUFFIX) else name
+        return _ORDER_PREFIX_RE.sub("", stem, count=1)
+
+    def _read_task(self, entry: Path, task_id: str) -> Task:
+        """Turn a claimed marker file into a :class:`Task`.
+
+        Read **after** the claim succeeds: reading first would waste a read on
+        every lost race, and the file is written once at seed time and never
+        mutated, so reading it later is race-free.
+        """
+        try:
+            raw = entry.read_text(encoding="utf-8").strip()
+        except OSError as e:
+            raise TaskPayloadError(task_id, str(e))
+        if not raw:
+            return Task(task_id=task_id, slugs=(task_id,), group=None)
+        try:
+            payload = json.loads(raw)
+            members = [str(s) for s in payload["slugs"]]
+        except (json.JSONDecodeError, KeyError, TypeError) as e:
+            raise TaskPayloadError(task_id, str(e))
+        if not members:
+            raise TaskPayloadError(task_id, "payload declares no scenarios")
+        return Task(task_id=task_id, slugs=tuple(members), group=str(payload.get("group") or task_id))
+
+    def claim_task(self, worker_id: str) -> Optional[Task]:
+        """Worker-only: atomically claim the next pending task via O_EXCL on
+        a per-task claim file. Returns the :class:`Task`, or None if the
+        queue is empty for this worker.
+
+        Tasks are seeded once by the coordinator before any worker runs and
+        are never removed (task files stay put for orphan detection). A
+        single pass through the snapshot is therefore sufficient: if every
+        candidate raises ``FileExistsError`` then every queued task has
+        already been claimed by another worker and there is nothing for us
+        to do — return ``None`` so the caller exits.
+
+        Raises :class:`TaskPayloadError` when a batch marker cannot be read; the
+        claim has already succeeded by then, so the caller reports it and moves on
+        and the members surface as orphans.
+        """
+        self.init_dirs()
+        try:
+            candidates = sorted(e for e in self.queue_dir.iterdir() if e.name.endswith(TASK_SUFFIX))
+        except FileNotFoundError:
+            return None
+        for entry in candidates:
+            task_id = self._slug_from_task(entry.name)
+            claim_path = self.claims_dir / f"{task_id}{CLAIM_SUFFIX}"
+            try:
+                fd = os.open(
+                    str(claim_path),
+                    os.O_WRONLY | os.O_CREAT | os.O_EXCL,
+                )
+            except FileExistsError:
+                continue  # another worker got it; try the next candidate
+            try:
+                os.write(fd, worker_id.encode("utf-8"))
+            finally:
+                os.close(fd)
+            return self._read_task(entry, task_id)
+        return None  # every candidate already claimed → drained for us
+
+    def claim(self, worker_id: str) -> Optional[str]:
+        """Claim the next pending task and return its **task id**.
+
+        A thin wrapper over :meth:`claim_task` so there is one implementation of
+        the claim protocol and no drift between them.
+        """
+        task = self.claim_task(worker_id)
+        return task.task_id if task is not None else None
+
+    def write_result(self, result: ScenarioResult) -> None:
+        """Worker-only: persist the result to results/<worker_id>/<slug>.json."""
+        worker_results_dir = self.results_dir / result.worker_id
+        worker_results_dir.mkdir(parents=True, exist_ok=True)
+        result_path = worker_results_dir / f"{result.slug}{RESULT_SUFFIX}"
+        result_path.write_text(json.dumps(result.to_dict()), encoding="utf-8")
+
+    def unclaimed_tasks(self) -> set[str]:
+        """Task ids that were queued and never claimed by anybody.
+
+        Reachable only when every worker died with work left — but nothing else in
+        the system observes it: ``orphaned_running_tasks`` walks ``claims/`` and
+        ``collect_results`` walks ``results/``, so a task with neither file is
+        invisible to both. Without this the run can report success having silently
+        executed nothing.
+        """
+        try:
+            queued = {self._slug_from_task(p.name) for p in self.queue_dir.iterdir() if p.suffix == TASK_SUFFIX}
+        except FileNotFoundError:
+            return set()
+        try:
+            claimed = {p.stem for p in self.claims_dir.iterdir() if p.suffix == CLAIM_SUFFIX}
+        except FileNotFoundError:
+            claimed = set()
+        return queued - claimed
+
+    def _reported_slugs(self) -> set[str]:
+        """Every scenario that produced a result, by filename — no JSON parsing."""
+        if not self.results_dir.exists():
+            return set()
+        return {p.stem for p in self.results_dir.glob(f"*/*{RESULT_SUFFIX}")}
+
+    def unclaimed_slugs(self) -> list[str]:
+        """Scenarios with no evidence they ever ran.
+
+        :meth:`unclaimed_tasks` expanded through the batch map, minus anything
+        that reported a result. A result implies a claim in the real protocol
+        (the claim file is created first), so subtracting them changes nothing
+        about a live run — but it makes this incapable of a false positive, and
+        this set forces the run to fail.
+        """
+        reported = self._reported_slugs()
+        out: list[str] = []
+        for task_id in sorted(self.unclaimed_tasks()):
+            out.extend(s for s in (self._task_slugs.get(task_id) or [task_id]) if s not in reported)
+        return out
+
+    def is_empty(self) -> bool:
+        """Coordinator-only: True when no pending tasks remain (every queued
+        task has at least been claimed)."""
+        return not self.unclaimed_tasks()
+
+    def collect_results(self) -> list[ScenarioResult]:
+        """Coordinator-only: read all results written so far."""
+        out: list[ScenarioResult] = []
+        if not self.results_dir.exists():
+            return out
+        for worker_dir in self.results_dir.iterdir():
+            if not worker_dir.is_dir():
+                continue
+            for result_file in worker_dir.glob(f"*{RESULT_SUFFIX}"):
+                try:
+                    data = json.loads(result_file.read_text(encoding="utf-8"))
+                except (json.JSONDecodeError, OSError):
+                    continue
+                out.append(
+                    ScenarioResult(
+                        slug=data["slug"],
+                        worker_id=data["worker_id"],
+                        exit_code=int(data["exit_code"]),
+                        duration_s=float(data["duration_s"]),
+                        started_at=float(data["started_at"]),
+                        finished_at=float(data["finished_at"]),
+                    )
+                )
+        return out
+
+    def _assert_batch_map_present(self) -> None:
+        """Fail loudly if this Queue holds batch markers it cannot interpret.
+
+        A reader that expands claim stems without the map would report an exclusion
+        group's NAME as a failed scenario, retry a test file that does not exist,
+        and never retry the group's real members. That is a silent, plausible-looking
+        wrong answer, so turn the only way it can arise — querying a Queue instance
+        that did not seed this directory — into an immediate error.
+        """
+        if self._task_slugs:
+            return
+        try:
+            entries = [p for p in self.queue_dir.iterdir() if p.name.endswith(TASK_SUFFIX)]
+        except FileNotFoundError:
+            return
+        batched = sorted(self._slug_from_task(p.name) for p in entries if p.stat().st_size > 0)
+        if batched:
+            raise RuntimeError(
+                f"queue at {self.root} holds exclusion-group task(s) {batched} but this "
+                "Queue instance has no batch map. Read it through the Queue that seeded "
+                "it, or re-seed with `batches=`."
+            )
+
+    def orphaned_running_tasks(self) -> dict[str, list[str]]:
+        """Coordinator-only: scenarios claimed but never reported (worker crashed
+        before writing the result file). Returns ``{worker_id: [slug, ...]}``.
+
+        Always **scenario slugs**, never task ids: a claimed batch is expanded
+        through the map, so a worker that died after two of three members still
+        reports exactly the one that did not finish. A stem with no map entry is a
+        plain scenario and takes the original path verbatim.
+
+        v0.1: these are reported as failures, not requeued.
+        """
+        orphans: dict[str, list[str]] = {}
+        if not self.claims_dir.exists():
+            return orphans
+        self._assert_batch_map_present()
+        for claim_file in self.claims_dir.glob(f"*{CLAIM_SUFFIX}"):
+            task_id = claim_file.stem
+            members = self._task_slugs.get(task_id)
+            try:
+                worker_id = claim_file.read_text(encoding="utf-8").strip()
+            except OSError:
+                continue
+            if not worker_id:
+                # Killed between the claim's create and its write. For a single
+                # scenario that stays today's skip; for a batch, skipping would
+                # lose every member in silence — and the owner id is written
+                # before any scenario runs, so none of them started.
+                if members:
+                    orphans.setdefault(UNKNOWN_WORKER, []).extend(members)
+                continue
+            for slug in members if members else [task_id]:
+                result_path = self.results_dir / worker_id / f"{slug}{RESULT_SUFFIX}"
+                if not result_path.exists():
+                    orphans.setdefault(worker_id, []).append(slug)
+        # Keep slugs sorted for deterministic test output.
+        return {wid: sorted(slugs) for wid, slugs in orphans.items()}