benchkit-cli 0.1.0__py3-none-any.whl

benchkit/engine.py

@@ -0,0 +1,1270 @@
+"""Benchmark execution engine shared by the TUI and the headless runner.
+
+The engine is deliberately UI agnostic: it walks a list of jobs (one model x
+one benchmark x one optional task slice), emits events as it goes, and returns
+the same result dictionaries that :mod:`benchkit.report` already knows how to
+serialize.
+
+Only the stateful part lives here: the job loop, the request workers, verifier
+repairs and per-task finalization. Job planning is in :mod:`benchkit.jobs`, the
+events in :mod:`benchkit.events`, pause/skip/stop in :mod:`benchkit.controls`
+and result shaping in :mod:`benchkit.results`. Their public names are
+re-exported, so ``from benchkit.engine import ...`` keeps working.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import os
+import threading
+import time
+from collections.abc import Callable
+from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait
+from dataclasses import dataclass, field, replace
+
+import httpx
+
+from benchkit import decision
+from benchkit.benchmarks.base import Task
+from benchkit.client import GenerationUpdate, InferenceClient
+from benchkit.controls import RunControls
+from benchkit.evaluation import (
+    EvaluationResult,
+    combine_attempts,
+    evaluate_response,
+    standalone_repair_prompt,
+)
+from benchkit.events import (
+    EngineEvent,
+    GenerationProgress,
+    JobCompleted,
+    JobStarted,
+    ModelUnloaded,
+    RunCompleted,
+    RunFailed,
+    RunStarted,
+    Sink,
+    TaskCompleted,
+    TaskPhase,
+)
+from benchkit.jobs import (
+    _BENCHMARKS,
+    _TASKS,
+    MAX_REPAIR_ATTEMPTS,
+    JobSpec,
+    SliceError,
+    benchmark,
+    expand_jobs,
+    option_count,
+    parse_slice,
+    plan_total_tasks,
+    prompt_for,
+    repair_budget,
+    route_decision_jobs,
+    slice_label,
+    slice_task_count,
+    task_count,
+    tasks_for,
+    tasks_for_job,
+)
+from benchkit.looping import LoopAnalyzer
+from benchkit.perturbations import annotate_robustness, perturb_task
+from benchkit.pi_agent import PiAgentRunner
+from benchkit.results import (
+    JobTally,
+    TaskOutcome,
+    TaskRecord,
+    annotate_harness_effect,
+    empty_result,
+    job_result,
+)
+from benchkit.sandbox import cleanup_run_resources, run_resources_created
+
+# ``_BENCHMARKS`` and ``_TASKS`` are the same dict objects as in benchkit.jobs, so
+# patching the registry caches through this module still reaches the accessors.
+__all__ = [
+    "ERROR_GENERATION",
+    "MAX_REPAIR_ATTEMPTS",
+    "_BENCHMARKS",
+    "_TASKS",
+    "Engine",
+    "EngineEvent",
+    "GenerationProgress",
+    "JobCompleted",
+    "JobSpec",
+    "JobStarted",
+    "ModelUnloaded",
+    "RunCompleted",
+    "RunControls",
+    "RunFailed",
+    "RunStarted",
+    "Sink",
+    "SliceError",
+    "TaskCompleted",
+    "TaskPhase",
+    "TaskRecord",
+    "annotate_harness_effect",
+    "benchmark",
+    "expand_jobs",
+    "option_count",
+    "parse_slice",
+    "plan_total_tasks",
+    "prompt_for",
+    "repair_budget",
+    "route_decision_jobs",
+    "slice_label",
+    "slice_task_count",
+    "task_count",
+    "tasks_for",
+    "tasks_for_job",
+]
+
+
+class _GenerationCancelled(Exception):
+    """Unwind a streaming request from its own worker thread immediately."""
+
+
+class _DoomLoopKilled(Exception):
+    """Unwind a generation that exceeded the configured loop threshold."""
+
+
+def _env_float(name: str, default: float) -> float:
+    try:
+        return float(os.environ.get(name, str(default)))
+    except ValueError:
+        return default
+
+
+def _env_bool(name: str, default: bool) -> bool:
+    raw = os.environ.get(name)
+    if raw is None:
+        return default
+    value = raw.strip().lower()
+    if value in {"1", "true", "yes", "on"}:
+        return True
+    if value in {"0", "false", "no", "off"}:
+        return False
+    return default
+
+
+ERROR_GENERATION = {
+    "thinking": "",
+    "response": "",
+    "tok_s": 0.0,
+    "eval_count": 0,
+    "eval_duration_ns": 0,
+    "response_time_s": 0.0,
+    "done_reason": "error",
+    "timed_out": False,
+    "cancelled": False,
+    "loop_killed": False,
+    "length_exceeded": False,
+    "harness_error": False,
+    "trace_status": "unavailable",
+}
+
+
+@dataclass
+class _GeneratedTask:
+    """Generation state handed from a request worker to serial evaluation."""
+
+    position: int
+    task: Task
+    prompt: str
+    perturbation: dict
+    gen: dict
+    error: str
+    errors: int
+    analyzer: LoopAnalyzer
+    first_answer_at: float | None
+    loop_detected_at: float | None
+
+
+@dataclass
+class Engine:
+    """Runs a list of jobs, streaming events to a sink."""
+
+    client: InferenceClient
+    jobs: list[JobSpec]
+    sink: Sink | None = None
+    controls: RunControls = field(default_factory=RunControls)
+    loop_kill_enabled: bool = field(
+        default_factory=lambda: _env_bool("BENCHKIT_LOOP_KILL", True)
+    )
+    loop_kill_percent: float = field(
+        default_factory=lambda: _env_float("BENCHKIT_LOOP_KILL_PERCENT", 80.0)
+    )
+    loop_kill_seconds: float = field(
+        default_factory=lambda: _env_float("BENCHKIT_LOOP_KILL_SECONDS", 10.0)
+    )
+    failure: str | None = field(default=None, init=False)
+    _emit_lock: threading.Lock = field(
+        default_factory=threading.Lock, init=False, repr=False
+    )
+    _evaluation_lock: threading.Lock = field(
+        default_factory=threading.Lock, init=False, repr=False
+    )
+    _pi_runner: PiAgentRunner | None = field(default=None, init=False, repr=False)
+    _workspace_pi_runners: dict[str, PiAgentRunner] = field(
+        default_factory=dict, init=False, repr=False
+    )
+    # Every Pi image is built right before a task and removed right after it;
+    # the count keeps an image alive while a concurrent task still uses it.
+    _pi_image_users: dict[str, int] = field(
+        default_factory=dict, init=False, repr=False
+    )
+    _pi_lock: threading.Lock = field(
+        default_factory=threading.Lock, init=False, repr=False
+    )
+
+    def __post_init__(self) -> None:
+        self.loop_kill_percent = min(100.0, max(0.0, self.loop_kill_percent))
+        self.loop_kill_seconds = max(0.0, self.loop_kill_seconds)
+
+    def emit(self, event: EngineEvent) -> None:
+        if self.sink is not None:
+            # Generation progress originates from multiple request workers.
+            # Keep every sink callback atomic so front-ends can stay ordinary,
+            # single-threaded state machines.
+            with self._emit_lock:
+                self.sink(event)
+
+    def _pi_entry(
+        self, bench: object | None = None, task: Task | None = None
+    ) -> tuple[str, PiAgentRunner]:
+        """Return the runner for a task's image and the key it is cached under."""
+        name = str(getattr(bench, "name", type(bench).__name__))
+        task_image_factory = getattr(bench, "pi_image_for_task", None)
+        image_factory = getattr(bench, "pi_image", None)
+        if callable(task_image_factory) and task is not None:
+            image = task_image_factory(task)
+            key = f"{name}:{image.image}"
+        elif callable(image_factory):
+            image, key = None, name
+        else:
+            with self._pi_lock:
+                if self._pi_runner is None:
+                    if getattr(self.client, "provider", None) == "demo":
+                        raise RuntimeError("Pi harness is unavailable in demo mode")
+                    self._pi_runner = PiAgentRunner(self.client)
+                return "", self._pi_runner
+        with self._pi_lock:
+            if key not in self._workspace_pi_runners:
+                self._workspace_pi_runners[key] = PiAgentRunner(
+                    self.client, image=image if image is not None else image_factory()
+                )
+            return key, self._workspace_pi_runners[key]
+
+    def _pi(
+        self, bench: object | None = None, task: Task | None = None
+    ) -> PiAgentRunner:
+        return self._pi_entry(bench, task)[1]
+
+    def _acquire_pi_image(self, bench: object, task: Task) -> str:
+        """Claim the task's Pi image so no other task removes it mid-use."""
+        key, _runner = self._pi_entry(bench, task)
+        with self._pi_lock:
+            self._pi_image_users[key] = self._pi_image_users.get(key, 0) + 1
+        return key
+
+    def _release_pi_image(self, key: str) -> None:
+        """Remove a task's Pi image once no running task needs it any more."""
+        with self._pi_lock:
+            users = self._pi_image_users.get(key, 0) - 1
+            if users > 0:
+                self._pi_image_users[key] = users
+                return
+            self._pi_image_users.pop(key, None)
+            runner = (
+                self._pi_runner
+                if key == ""
+                else self._workspace_pi_runners.pop(key, None)
+            )
+        if runner is not None:
+            # The runner rebuilds on its next prepare(); a failed removal is
+            # retried by the run-level, label-scoped sweep.
+            with contextlib.suppress(Exception):
+                runner.cleanup()
+
+    def run(self) -> list[dict]:
+        """Run every job and return the results, including a partial run.
+
+        An unexpected failure ends the run through ``RunFailed`` rather than an
+        exception, so the caller still gets the jobs that already finished and
+        can write a report for them.
+        """
+        self.jobs = expand_jobs(self.jobs, self.client)
+        results: list[dict] = []
+        started = time.time()
+        overall_total = plan_total_tasks(self.jobs)
+        self.emit(RunStarted(list(self.jobs), overall_total))
+
+        try:
+            for index, job in enumerate(self.jobs):
+                if self.controls.stopped:
+                    break
+                try:
+                    result, skipped = self._run_job(index, job, overall_total)
+                except Exception as exc:
+                    self.failure = f"{type(exc).__name__}: {exc}"
+                    self.emit(RunFailed(self.failure))
+                    break
+                # Every job reports a terminal event, even an empty one, so the UI
+                # never leaves a queue row running.
+                self.emit(
+                    JobCompleted(index, job, result or empty_result(job), skipped)
+                )
+                if result is not None:
+                    results.append(result)
+                self._maybe_unload(index, job)
+        finally:
+            used_pi = self._pi_runner is not None or bool(self._workspace_pi_runners)
+            if self._pi_runner is not None:
+                with contextlib.suppress(Exception):
+                    self._pi_runner.cleanup()
+            for runner in self._workspace_pi_runners.values():
+                with contextlib.suppress(Exception):
+                    runner.cleanup()
+            # mc-arena builds on the same run builder without using Pi, so
+            # the sweep runs whenever anything was built, not only for Pi.
+            if used_pi or run_resources_created():
+                # Removes the run's shared builder, its build cache, and any
+                # image, container, network, or volume still carrying the run
+                # label. Scoped by label, never a global prune.
+                with contextlib.suppress(Exception):
+                    cleanup_run_resources()
+
+        annotate_robustness(results)
+        annotate_harness_effect(results)
+        elapsed = (
+            round(sum(result["total_time"] for result in results), 1)
+            if getattr(self.client, "simulated_timing", False)
+            else round(time.time() - started, 1)
+        )
+        self.emit(RunCompleted(results, self.controls.stopped, elapsed))
+        return results
+
+    def _maybe_unload(self, index: int, job: JobSpec) -> None:
+        """Free VRAM once the last job for a model finishes.
+
+        Jobs run strictly one model at a time and ``_run_job`` only returns
+        after every in-flight request has drained, so by the time this runs the
+        model's slots are idle. When force-unload is on, the model is evicted
+        here (and the unload request completes) before the next model's job
+        submits its first request.
+        """
+        upcoming = self.jobs[index + 1 :]
+        if any(other.model == job.model for other in upcoming):
+            return
+        if len(self.jobs) <= 1:
+            return
+        if self.controls.force_unload:
+            error = ""
+            try:
+                self.client.force_unload_model(job.model)
+            except Exception as exc:
+                error = str(exc)
+            self.emit(ModelUnloaded(job.model, error))
+        else:
+            with contextlib.suppress(Exception):
+                self.client.unload_model(job.model)
+
+    def _max_parallel_requests(self, job: JobSpec, total: int) -> int:
+        """Resolve a safe worker count, keeping unknown clients serial."""
+        if total <= 1:
+            return 1
+        discover = getattr(self.client, "max_parallel_requests", None)
+        if not callable(discover):
+            return 1
+        try:
+            capacity = discover(job.model)
+            if isinstance(capacity, bool):
+                return 1
+            return min(total, max(1, int(capacity)))
+        except Exception:
+            # Capacity discovery is an optional optimization. A missing or
+            # non-standard monitoring endpoint must never block a benchmark.
+            return 1
+
+    def _run_job(
+        self, index: int, job: JobSpec, overall_total: int
+    ) -> tuple[dict | None, bool]:
+        bench = benchmark(job.benchmark)
+        if job.harness == "decision" and not decision.supports(job.benchmark):
+            raise ValueError(f"{job.benchmark} cannot run on a decision model")
+        if getattr(bench, "workspace_task", False) and job.harness != "pi":
+            raise ValueError(
+                f"{job.benchmark} requires the Pi harness; use --harness pi"
+            )
+        if getattr(bench, "vision", False) and job.harness == "pi":
+            raise ValueError(
+                f"{job.benchmark} sends images, which the Pi harness cannot "
+                "forward; use --harness direct"
+            )
+        all_tasks = tasks_for_job(job)
+
+        slice_spec = job.slice_spec
+        try:
+            start, end = parse_slice(slice_spec, len(all_tasks))
+        except SliceError:
+            start, end, slice_spec = 0, len(all_tasks), None
+        tasks = all_tasks[start:end]
+
+        concurrency = self._max_parallel_requests(job, len(tasks))
+        self.emit(
+            JobStarted(
+                index,
+                job,
+                len(tasks),
+                overall_total,
+                concurrency=concurrency,
+            )
+        )
+
+        if not tasks:
+            return None, False
+
+        # Pi images are built per task: the build lands in the job wall time
+        # but never in a task's response time. Container startup and every
+        # agent/tool turn do count.
+        wall_start = time.perf_counter()
+        tally = JobTally()
+        skipped = False
+        paused_time = 0.0
+        next_position = 0
+        futures: set[Future[_GeneratedTask]] = set()
+
+        with ThreadPoolExecutor(
+            max_workers=concurrency,
+            thread_name_prefix=f"benchkit-{index}",
+        ) as pool:
+
+            def fill_workers() -> None:
+                nonlocal next_position
+                while (
+                    len(futures) < concurrency
+                    and next_position < len(tasks)
+                    and not self.controls.paused
+                    and not self.controls.stopped
+                    and not self.controls.skip_requested
+                ):
+                    position = next_position
+                    next_position += 1
+                    future = pool.submit(
+                        self._generate_task,
+                        index,
+                        job,
+                        position,
+                        len(tasks),
+                        tasks[position],
+                        bench,
+                    )
+                    futures.add(future)
+
+            fill_workers()
+            try:
+                while futures or next_position < len(tasks):
+                    if not futures:
+                        if self.controls.stopped or self.controls.skip_requested:
+                            break
+                        if self.controls.paused:
+                            pause_started = time.perf_counter()
+                            self.controls.wait_while_paused()
+                            paused_time += time.perf_counter() - pause_started
+                            if self.controls.stopped or self.controls.skip_requested:
+                                break
+                        fill_workers()
+                        if not futures:
+                            continue
+
+                    done, _ = wait(
+                        tuple(futures),
+                        timeout=0.05,
+                        return_when=FIRST_COMPLETED,
+                    )
+                    if not done:
+                        continue
+
+                    generated_batch: list[_GeneratedTask] = []
+                    for future in done:
+                        futures.discard(future)
+                        generated_batch.append(future.result())
+
+                    # Refill request slots before evaluation so the next
+                    # generations can overlap local benchmark scoring.
+                    fill_workers()
+
+                    if self.controls.stopped or self.controls.skip_requested:
+                        continue
+
+                    for generated in sorted(
+                        generated_batch, key=lambda item: item.position
+                    ):
+                        outcome = self._finalize_task(
+                            index,
+                            job,
+                            len(tasks),
+                            bench,
+                            generated,
+                        )
+                        if outcome is None:
+                            continue
+                        tally.add(outcome)
+                        self.emit(
+                            TaskCompleted(
+                                index,
+                                job,
+                                outcome.record,
+                                tally.passed,
+                                len(tally.records_by_position),
+                                tally.score_points,
+                                tally.scored_total,
+                            )
+                        )
+            except Exception:
+                self.controls.cancel_active()
+                for future in futures:
+                    future.cancel()
+                raise
+
+        if self.controls.skip_requested:
+            skipped = self.controls.take_skip()
+
+        if not tally.records_by_position:
+            return None, skipped
+
+        measured_wall_time_s = max(0.0, time.perf_counter() - wall_start - paused_time)
+        # Demo mode deliberately accelerates its fake generations. Use their
+        # simulated request durations so its reports still demonstrate
+        # internally consistent throughput rather than thousands of tok/s.
+        wall_time_s = (
+            tally.total_response_time / concurrency
+            if getattr(self.client, "simulated_timing", False)
+            and tally.total_response_time > 0
+            else measured_wall_time_s
+        )
+        result = job_result(
+            job,
+            bench,
+            tasks,
+            tally,
+            slice_spec=slice_spec,
+            concurrency=concurrency,
+            wall_time_s=wall_time_s,
+            loop_kill_enabled=self.loop_kill_enabled,
+            loop_kill_percent=self.loop_kill_percent,
+            loop_kill_seconds=self.loop_kill_seconds,
+        )
+        return result, skipped
+
+    def _verify_response(
+        self,
+        bench: object,
+        task: Task,
+        response: str,
+        *,
+        decision: bool = False,
+    ) -> EvaluationResult:
+        """Run evaluators serially and convert infrastructure errors to data."""
+        with self._evaluation_lock:
+            try:
+                return evaluate_response(bench, task, response, decision=decision)
+            except Exception as exc:
+                return EvaluationResult(
+                    score=0.0,
+                    error=f"evaluation failed: {type(exc).__name__}: {exc}",
+                )
+
+    def _generate_direct_with_repairs(
+        self,
+        job: JobSpec,
+        prompt: str,
+        verifier: Callable[[str], EvaluationResult],
+        on_progress: Callable[[GenerationUpdate], None],
+        images: list[str] | None = None,
+        repairs: int | None = None,
+    ) -> dict:
+        """Run stateless direct calls with an explicit feedback transcript."""
+        attempts: list[tuple[dict, EvaluationResult]] = []
+        feedback_sent: list[str] = []
+        request_prompt = prompt
+        elapsed_offset = 0.0
+
+        repairs = job.repair_attempts if repairs is None else repairs
+        for attempt in range(repairs + 1):
+            thinking_parts: list[str] = []
+            response_parts: list[str] = []
+
+            def forward(
+                update: GenerationUpdate,
+                *,
+                number: int = attempt,
+                offset: float = elapsed_offset,
+                thinking_parts: list[str] = thinking_parts,
+                response_parts: list[str] = response_parts,
+            ) -> None:
+                thinking_parts.append(update.thinking)
+                response_parts.append(update.response)
+                on_progress(
+                    replace(
+                        update,
+                        elapsed_s=offset + update.elapsed_s,
+                        attempt=number,
+                    )
+                )
+
+            gen = self.client.generate(
+                job.model,
+                request_prompt,
+                on_progress=forward,
+                cancel_event=self.controls.cancel_event,
+                **({"images": images} if images else {}),
+            )
+            terminal = bool(
+                gen.get("cancelled") or gen.get("timed_out") or gen.get("loop_killed")
+            )
+            # Recover per attempt: once attempts are combined, an earlier
+            # attempt's count would hide this one's missing usage.
+            self._recover_streamed_tokens(
+                job.model, gen, thinking_parts, response_parts
+            )
+            evaluation = (
+                EvaluationResult(score=0.0)
+                if terminal
+                else verifier(str(gen.get("response") or ""))
+            )
+            attempts.append((gen, evaluation))
+            if terminal or evaluation.passed or evaluation.error:
+                break
+            if attempt >= repairs:
+                break
+
+            feedback_sent.append(evaluation.feedback)
+            request_prompt = standalone_repair_prompt(
+                prompt,
+                str(gen.get("response") or ""),
+                evaluation.feedback,
+                attempt + 1,
+                repairs,
+            )
+            elapsed_offset += float(gen.get("response_time_s") or 0.0)
+
+        return combine_attempts(attempts, feedback_sent)
+
+    def _generate_task(
+        self,
+        index: int,
+        job: JobSpec,
+        position: int,
+        total: int,
+        task: Task,
+        bench: object,
+    ) -> _GeneratedTask:
+        """Generate one answer in a request worker, including live analysis."""
+        case = perturb_task(
+            job.benchmark,
+            task,
+            job.perturbation,
+            job.perturbation_seed,
+        )
+        task = case.evaluation_task
+        prompt = (
+            decision.render_request(bench, case.prompt_task)
+            if job.harness == "decision"
+            else prompt_for(bench, case.prompt_task, self.client, job.model)
+        )
+        # Vision suites attach local image files; only direct calls carry them.
+        images = list(case.prompt_task.metadata.get("images") or ())
+        error = ""
+        errors = 0
+        entry_point = str(task.metadata.get("entry_point", ""))
+        self.emit(
+            TaskPhase(
+                index=index,
+                job=job,
+                position=position + 1,
+                total=total,
+                task_id=task.id,
+                entry_point=entry_point,
+                phase="generating",
+                activity="waiting for model",
+            )
+        )
+        analyzer = LoopAnalyzer()
+        last_progress_emit = 0.0
+        last_progress_state = ""
+        reasoning_channel_seen = False
+        first_answer_at: float | None = None
+        loop_detected_at: float | None = None
+        loop_above_since: float | None = None
+        loop_killed_at_s: float | None = None
+        loop_kill_score = 0.0
+        current_attempt = 0
+        streamed_thinking_parts: list[str] = []
+        streamed_response_parts: list[str] = []
+
+        def on_progress(update: GenerationUpdate) -> None:
+            nonlocal analyzer
+            nonlocal current_attempt
+            nonlocal first_answer_at
+            nonlocal last_progress_emit
+            nonlocal last_progress_state
+            nonlocal loop_above_since
+            nonlocal loop_detected_at
+            nonlocal loop_killed_at_s
+            nonlocal loop_kill_score
+            nonlocal reasoning_channel_seen
+            nonlocal streamed_response_parts
+            nonlocal streamed_thinking_parts
+
+            if self.controls.cancel_requested:
+                raise _GenerationCancelled
+
+            if update.attempt != current_attempt:
+                current_attempt = update.attempt
+                analyzer = LoopAnalyzer()
+                reasoning_channel_seen = False
+                loop_above_since = None
+                loop_detected_at = None
+                last_progress_emit = 0.0
+                last_progress_state = ""
+                streamed_thinking_parts = []
+                streamed_response_parts = []
+                self.emit(
+                    TaskPhase(
+                        index=index,
+                        job=job,
+                        position=position + 1,
+                        total=total,
+                        task_id=task.id,
+                        entry_point=entry_point,
+                        phase="generating",
+                        activity=(
+                            f"running verifier-feedback repair {current_attempt}"
+                        ),
+                    )
+                )
+
+            streamed_thinking_parts.append(update.thinking)
+            streamed_response_parts.append(update.response)
+            analyzer.add(thinking=update.thinking, answer=update.response)
+            reasoning_channel_seen = (
+                reasoning_channel_seen or update.reasoning_channel_seen
+            )
+            if update.response and first_answer_at is None:
+                first_answer_at = update.elapsed_s
+
+            if analyzer.answer_chars:
+                live_phase = "answering"
+            elif analyzer.thinking_chars:
+                live_phase = "thinking"
+            else:
+                live_phase = "waiting"
+
+            trace_status = (
+                "observed"
+                if analyzer.thinking_chars
+                else "available_empty"
+                if reasoning_channel_seen
+                else "unavailable"
+            )
+            state_key = f"{live_phase}:{trace_status}"
+            now = time.monotonic()
+            if (
+                not update.done
+                and state_key == last_progress_state
+                and now - last_progress_emit < 0.25
+            ):
+                return
+            snapshot = analyzer.snapshot()
+            if snapshot.state == "looping" and loop_detected_at is None:
+                loop_detected_at = update.elapsed_s
+            threshold = self.loop_kill_percent / 100
+            if not self.loop_kill_enabled:
+                loop_above_since = None
+                remaining = None
+            elif snapshot.confirmed_cycle and snapshot.score >= threshold:
+                if loop_above_since is None:
+                    loop_above_since = now
+                above_for = now - loop_above_since
+                remaining = max(0.0, self.loop_kill_seconds - above_for)
+                if above_for >= self.loop_kill_seconds and not update.done:
+                    loop_killed_at_s = update.elapsed_s
+                    loop_kill_score = snapshot.score
+                    raise _DoomLoopKilled
+            else:
+                loop_above_since = None
+                remaining = None
+            last_progress_emit = now
+            last_progress_state = state_key
+            self.emit(
+                GenerationProgress(
+                    index=index,
+                    job=job,
+                    position=position + 1,
+                    total=total,
+                    task_id=task.id,
+                    entry_point=entry_point,
+                    phase=live_phase,
+                    elapsed_s=round(update.elapsed_s, 2),
+                    thinking_chars=analyzer.thinking_chars,
+                    response_chars=analyzer.answer_chars,
+                    trace_status=trace_status,
+                    loop_state=snapshot.state,
+                    loop_score=snapshot.score,
+                    loop_source=snapshot.source,
+                    loop_kill_remaining_s=(
+                        round(remaining, 2) if remaining is not None else None
+                    ),
+                    prompt=prompt,
+                    thinking=analyzer.thinking,
+                    response=analyzer.answer,
+                )
+            )
+
+        # Build this task's Pi image now, run the task, then remove the image:
+        # a run never holds more images than it has tasks in flight.
+        pi_image = self._acquire_pi_image(bench, task) if job.harness == "pi" else None
+        request_started = time.perf_counter()
+        try:
+            if pi_image is not None:
+                self.emit(
+                    TaskPhase(
+                        index=index,
+                        job=job,
+                        position=position + 1,
+                        total=total,
+                        task_id=task.id,
+                        entry_point=entry_point,
+                        phase="generating",
+                        activity="building Pi sandbox image",
+                    )
+                )
+                self._pi(bench, task).prepare()
+                # The build is setup, not generation: keep it out of the timing.
+                request_started = time.perf_counter()
+            generator = (
+                self._pi(bench, task)
+                if job.harness == "pi"
+                else decision.Decider(self.client)
+                if job.harness == "decision"
+                else self.client
+            )
+            workspace = bool(getattr(bench, "workspace_task", False))
+            workspace_setup = None
+            workspace_verifier = None
+            repair_prompt_builder = None
+            if workspace:
+
+                def workspace_setup(environment):
+                    return bench.prepare_workspace(task, environment)
+
+                def workspace_verifier(environment, tool_trace):
+                    return bench.verify_workspace(task, environment, tool_trace)
+
+                candidate_builder = getattr(bench, "build_repair_prompt", None)
+                if callable(candidate_builder):
+                    repair_prompt_builder = candidate_builder
+
+            # Decision models emit no text to repair; routing already zeroes
+            # their repairs, this also covers jobs built without routing.
+            repairs = (
+                0
+                if job.harness == "decision"
+                else repair_budget(bench, task, job.repair_attempts)
+            )
+            if repairs:
+
+                def verifier(response: str) -> EvaluationResult:
+                    return self._verify_response(bench, task, response)
+
+                if job.harness == "pi":
+                    gen = generator.generate(
+                        job.model,
+                        prompt,
+                        on_progress=on_progress,
+                        cancel_event=self.controls.cancel_event,
+                        verifier=verifier,
+                        repair_attempts=repairs,
+                        workspace_setup=workspace_setup,
+                        workspace_verifier=workspace_verifier,
+                        repair_prompt_builder=repair_prompt_builder,
+                    )
+                else:
+                    gen = self._generate_direct_with_repairs(
+                        job,
+                        prompt,
+                        verifier,
+                        on_progress,
+                        images,
+                        repairs,
+                    )
+            else:
+                gen = generator.generate(
+                    job.model,
+                    prompt,
+                    on_progress=on_progress,
+                    cancel_event=self.controls.cancel_event,
+                    **(
+                        {
+                            "workspace_setup": workspace_setup,
+                            "workspace_verifier": workspace_verifier,
+                        }
+                        if workspace
+                        else {}
+                    ),
+                    **(
+                        {"images": images} if images and job.harness == "direct" else {}
+                    ),
+                )
+        except _GenerationCancelled:
+            gen = dict(ERROR_GENERATION)
+            gen.update(done_reason="cancelled", cancelled=True)
+        except _DoomLoopKilled:
+            elapsed_s = loop_killed_at_s or 0.0
+            thinking = analyzer.thinking
+            response = analyzer.answer
+            gen = dict(ERROR_GENERATION)
+            gen.update(
+                thinking=thinking,
+                response=response,
+                response_time_s=elapsed_s,
+                done_reason="loop_killed",
+                loop_killed=True,
+                loop_kill_score=loop_kill_score,
+                loop_killed_at_s=elapsed_s,
+                trace_status=(
+                    "observed"
+                    if thinking
+                    else "available_empty"
+                    if reasoning_channel_seen
+                    else "unavailable"
+                ),
+            )
+        except Exception as exc:
+            gen = dict(ERROR_GENERATION)
+            if self.controls.cancel_requested:
+                gen.update(done_reason="cancelled", cancelled=True)
+            elif isinstance(exc, (TimeoutError, httpx.TimeoutException)):
+                error = f"generation timed out after {self.client.timeout:g}s"
+                gen.update(
+                    done_reason="timeout",
+                    timed_out=True,
+                    response_time_s=self.client.timeout,
+                )
+            else:
+                error = f"{type(exc).__name__}: {exc}"
+                gen.update(
+                    harness_error=True,
+                    response_time_s=time.perf_counter() - request_started,
+                )
+            if not gen.get("cancelled"):
+                errors += 1
+        finally:
+            # Workspace verification runs inside generate(), so the image is
+            # done once it returns, whichever way it ended.
+            if pi_image is not None:
+                self._release_pi_image(pi_image)
+
+        self._recover_streamed_tokens(
+            job.model, gen, streamed_thinking_parts, streamed_response_parts
+        )
+
+        return _GeneratedTask(
+            position=position,
+            task=task,
+            prompt=prompt,
+            perturbation=case.details,
+            gen=gen,
+            error=error,
+            errors=errors,
+            analyzer=analyzer,
+            first_answer_at=first_answer_at,
+            loop_detected_at=loop_detected_at,
+        )
+
+    def _recover_streamed_tokens(
+        self,
+        model: str,
+        gen: dict,
+        thinking_parts: list[str],
+        response_parts: list[str],
+    ) -> None:
+        """Fill in output tokens for a terminal stream that reported no usage.
+
+        Only the token count is recovered. Decode timing stays zero, so the
+        item remains outside throughput coverage.
+        """
+        if not (
+            gen.get("loop_killed") or gen.get("timed_out") or gen.get("length_exceeded")
+        ) or int(gen.get("eval_count") or 0):
+            return
+        token_count = self._count_streamed_tokens(model, thinking_parts, response_parts)
+        if token_count is None:
+            return
+        response_time_s = float(gen.get("response_time_s") or 0.0)
+        gen.update(
+            eval_count=token_count,
+            tokens_recovered=True,
+            tok_s=token_count / response_time_s if response_time_s > 0 else 0.0,
+        )
+
+    def _count_streamed_tokens(
+        self,
+        model: str,
+        thinking_parts: list[str],
+        response_parts: list[str],
+    ) -> int | None:
+        """Count terminal stream text when the final usage payload never arrived."""
+        tokenize = getattr(self.client, "tokenize", None)
+        if not callable(tokenize):
+            return None
+        total = 0
+        observed = False
+        for content in ("".join(thinking_parts), "".join(response_parts)):
+            if not content:
+                continue
+            try:
+                tokens = tokenize(model, content, add_special=False)
+            except (TypeError, ValueError):
+                return None
+            if tokens is None:
+                return None
+            total += len(tokens)
+            observed = True
+        return total if observed else None
+
+    def _finalize_task(
+        self,
+        index: int,
+        job: JobSpec,
+        total: int,
+        bench: object,
+        generated: _GeneratedTask,
+    ) -> TaskOutcome | None:
+        """Evaluate a generated answer and build its deterministic task record."""
+        gen = generated.gen
+        if (
+            self.controls.stopped
+            or self.controls.skip_requested
+            or gen.get("cancelled")
+        ):
+            return None
+
+        position = generated.position
+        task = generated.task
+        prompt = generated.prompt
+        error = generated.error
+        errors = generated.errors
+        entry_point = str(task.metadata.get("entry_point", ""))
+
+        final_analyzer = generated.analyzer
+        final_thinking = gen.get("thinking", "") or ""
+        final_answer = gen.get("response", "") or ""
+        if not final_analyzer.reconcile(
+            thinking=final_thinking,
+            answer=final_answer,
+        ):
+            final_analyzer = LoopAnalyzer()
+            final_analyzer.add(
+                thinking=final_thinking,
+                answer=final_answer,
+            )
+        loop = final_analyzer.snapshot(final=True)
+        loop_detected_at = generated.loop_detected_at
+        if loop.state == "looping" and loop_detected_at is None:
+            loop_detected_at = gen.get("response_time_s", 0.0)
+        first_answer_at = generated.first_answer_at
+        if first_answer_at is None and gen.get("response"):
+            first_answer_at = gen.get("response_time_s", 0.0)
+
+        timed_out = bool(gen.get("timed_out"))
+        loop_killed = bool(gen.get("loop_killed"))
+        length_exceeded = bool(gen.get("length_exceeded"))
+        harness_error = bool(gen.get("harness_error"))
+        terminal_workspace_evaluation = bool(gen.get("workspace_evaluated")) and (
+            "evaluation_score" in gen
+        )
+        evaluation_score = 0.0
+        evaluation_details = dict(gen.get("evaluation_details") or {})
+        contaminated = evaluation_details.get("contamination_verdict") == "CONTAMINATED"
+        if loop_killed:
+            ok = False
+            error = (
+                "doom loop killed after a confirmed active cycle stayed at or above "
+                f"{self.loop_kill_percent:g}% for "
+                f"{self.loop_kill_seconds:g}s"
+            )
+            errors += 1
+            phase = "loop_killed"
+            activity = "doom loop killed — skipping task"
+        elif timed_out and not terminal_workspace_evaluation:
+            ok = False
+            if not error:
+                timeout_s = float(gen.get("timeout_s") or self.client.timeout)
+                error = f"generation timed out after {timeout_s:g}s"
+                errors += 1
+            phase = "timed_out"
+            activity = "timed out — skipping task"
+        elif length_exceeded and not terminal_workspace_evaluation:
+            ok = False
+            if not error:
+                error = "generation reached the model output-token limit"
+                errors += 1
+            phase = "length_exceeded"
+            activity = "output-token limit reached — scoring task as failed"
+        elif harness_error:
+            ok = False
+            phase = "harness_error"
+            activity = "harness error — excluding task from score"
+        else:
+            phase = "evaluating"
+            activity = getattr(bench, "evaluation_activity", "evaluating response")
+
+        self.emit(
+            TaskPhase(
+                index=index,
+                job=job,
+                position=position + 1,
+                total=total,
+                task_id=task.id,
+                entry_point=entry_point,
+                phase=phase,
+                activity=activity,
+            )
+        )
+
+        if (
+            not loop_killed
+            and (not timed_out or terminal_workspace_evaluation)
+            and (not length_exceeded or terminal_workspace_evaluation)
+            and not harness_error
+        ):
+            if "evaluation_score" in gen:
+                evaluation_score = min(
+                    1.0, max(0.0, float(gen.get("evaluation_score") or 0.0))
+                )
+                evaluation_error = str(gen.get("evaluation_error") or "")
+                ok = evaluation_score >= 1.0 and not evaluation_error
+                if evaluation_error and not error:
+                    error = evaluation_error
+                    errors += 1
+                    harness_error = True
+            else:
+                evaluation = self._verify_response(
+                    bench, task, gen["response"], decision="decision" in gen
+                )
+                evaluation_score = evaluation.score
+                ok = evaluation.passed
+                # Verifiers that record artifacts (rendered pages, screenshots,
+                # workspace state) return them here; the repair path carries
+                # them on the generation instead.
+                evaluation_details = dict(evaluation.details)
+                if evaluation.error and not error:
+                    error = evaluation.error
+                    errors += 1
+                    harness_error = True
+            if not ok and not error and gen.get("evaluation_error"):
+                ok = False
+                error = str(gen["evaluation_error"])
+                errors += 1
+                harness_error = True
+
+        if contaminated:
+            ok = False
+            evaluation_score = 0.0
+
+        outcome = (
+            "harness_error"
+            if harness_error
+            else "contaminated"
+            if contaminated
+            else "pass"
+            if ok
+            else "length_exceeded"
+            if length_exceeded
+            else "loop_killed"
+            if loop_killed
+            else "timeout"
+            if timed_out
+            else "fail"
+        )
+
+        response_time_s = float(gen.get("response_time_s", 0.0))
+        record = TaskRecord(
+            index=position,
+            task_id=task.id,
+            passed=ok,
+            tok_s=round(float(gen.get("tok_s", 0.0)), 1),
+            response_time_s=round(response_time_s, 2),
+            prompt=prompt,
+            response=gen.get("response", ""),
+            outcome=outcome,
+            perturbation=generated.perturbation,
+            score=evaluation_score,
+            error=error,
+            entry_point=entry_point,
+            thinking=gen.get("thinking", ""),
+            output_tokens=int(gen.get("eval_count", 0)),
+            tokens_recovered=bool(gen.get("tokens_recovered")),
+            done_reason=gen.get("done_reason", ""),
+            timed_out=timed_out,
+            loop_killed=loop_killed,
+            length_exceeded=length_exceeded,
+            harness_error=harness_error,
+            contaminated=contaminated,
+            loop_kill_score=float(gen.get("loop_kill_score", 0.0)),
+            loop_killed_at_s=gen.get("loop_killed_at_s"),
+            trace_status=gen.get("trace_status", "unavailable"),
+            thinking_time_s=round(
+                min(
+                    first_answer_at if first_answer_at is not None else response_time_s,
+                    response_time_s,
+                )
+                if gen.get("thinking")
+                else 0.0,
+                2,
+            ),
+            time_to_first_answer_s=(
+                round(first_answer_at, 2) if first_answer_at is not None else None
+            ),
+            loop_state=loop.state,
+            loop_score=loop.score,
+            loop_source=loop.source,
+            loop_detected_at_s=(
+                round(loop_detected_at, 2) if loop_detected_at is not None else None
+            ),
+            repeated_ngram_coverage=loop.repeated_ngram_coverage,
+            max_window_similarity=loop.max_window_similarity,
+            low_novelty_windows=loop.low_novelty_windows,
+            max_repeated_block=loop.max_repeated_block,
+            loop_evidence=loop.evidence,
+            active_cycle=loop.active_cycle,
+            recovered_cycle=loop.recovered_cycle,
+            cycle_period_tokens=loop.cycle_period_tokens,
+            cycle_repetitions=loop.cycle_repetitions,
+            repeated_suffix_tokens=loop.repeated_suffix_tokens,
+            harness=str(gen.get("harness") or job.harness),
+            harness_version=str(gen.get("harness_version") or ""),
+            input_tokens=int(gen.get("input_tokens") or 0),
+            model_turns=int(gen.get("model_turns") or 1),
+            tool_calls=int(gen.get("tool_calls") or 0),
+            tool_trace=list(gen.get("tool_trace") or []),
+            pi_scaffold=dict(gen.get("pi_scaffold") or {}),
+            attempts=list(gen.get("attempts") or []),
+            repair_attempts_used=int(gen.get("repair_attempts_used") or 0),
+            repair_feedback=list(gen.get("repair_feedback") or []),
+            first_attempt_score=float(gen.get("first_attempt_score", evaluation_score)),
+            repaired=bool(gen.get("repaired")),
+            workspace=evaluation_details,
+            decision=decision.calibration(
+                gen.get("decision"), task.metadata.get("answer")
+            ),
+        )
+        return TaskOutcome(
+            record=record,
+            errors=errors,
+            eval_count=int(gen.get("eval_count", 0)),
+            eval_duration_ns=int(gen.get("eval_duration_ns", 0)),
+            response_time_s=response_time_s,
+        )