control-tower 6.0.0__py3-none-any.whl

control_tower/backends/fleet/controller.py

@@ -0,0 +1,2058 @@
+"""Fleet controller: orchestrates distributed EC2 eval.
+
+Runs on the user's machine. Creates SQS queues, enqueues jobs,
+launches EC2 workers, monitors progress, and returns S3 result locations.
+"""
+
+import contextlib
+import gzip
+import hashlib
+import json
+import logging
+import math
+import os
+import shutil
+import tempfile
+import time
+from collections import defaultdict
+from collections.abc import Collection
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from importlib.metadata import distribution
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+import boto3
+
+if TYPE_CHECKING:
+    from botocore.client import BaseClient
+import re
+import subprocess
+
+import botocore.config
+import botocore.exceptions
+from inspect_ai.log import EvalConfig, EvalLog, read_eval_log
+
+from control_tower.backends.fleet.ami import get_worker_ami_id
+from control_tower.backends.fleet.aws_identity import caller_identity_name
+from control_tower.backends.fleet.config import Ec2Config
+from control_tower.backends.fleet.job import (
+    FLEET_RUNNER_SCHEMA_VERSION,
+    SQS_MAX_VISIBILITY_S,
+    SQS_VISIBILITY_TIMEOUT_S,
+    FleetManifest,
+    InspectFleetJobWire,
+    JobDoneMessage,
+    JobPayload,
+    LifecycleMessage,
+    SampleDoneMessage,
+    compute_fleet_id,
+    enqueue_jobs,
+    list_eval_keys,
+)
+from control_tower.eval.log.merge import merge_eval_logs
+from control_tower.models.rates import default_rate_table
+from control_tower.sandbox.constants import FLEET_PLANNING_CONCURRENCY
+from control_tower.utils.env import EnvVar
+from control_tower.utils.paths import cache_dir, find_repo_root
+from control_tower.utils.versioning import git_changed_paths, setting_repo_dirs
+from ct_setting_sdk.errors import OperationError, ValidationError
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+
+INSTANCE_PROFILE_NAME = "linuxbench-worker"
+FLEET_SSM_PREFIX = "/linuxbench/fleet"
+WORKER_SECRET_ENV_VARS = (
+    "ANTHROPIC_API_KEY",
+    "OPENAI_API_KEY",
+    "OPENROUTER_API_KEY",
+    "WATCHER_API_KEY",
+    "WATCHER_API_URL",
+    "GH_TOKEN",
+    "DOCKERHUB_USERNAME",
+    "DOCKERHUB_TOKEN",
+)
+
+# Default fleet settings – imported from shared constants module
+FAILURE_RATE_THRESHOLD = 0.20  # pause if >20% failed
+STALE_WARN_S = 600  # warn after 10 min of no progress
+STALE_ABORT_S = 10800  # abort after 3 hours of no progress
+STALE_HARD_ABORT_S = 21600  # 6hr absolute ceiling even with lively workers
+ROOT_VOLUME_SIZE_GB = 300
+SQS_QUEUE_DELETE_COOLDOWN_S = 60
+SQS_PURGE_WINDOW_S = 60
+DLQ_RECONCILE_INTERVAL_S = 60
+# Extra receives per monitor tick when the done queue returns a full batch.
+DONE_QUEUE_RECEIVERS = 8
+
+
+@dataclass(frozen=True)
+class FleetRunResult:
+    """Generic result produced by the fleet runner after jobs are in S3."""
+
+    fleet_id: str
+    s3_bucket: str
+    s3_prefix: str
+    total_expected: int
+    total_completed: int
+    region: str
+    eval_log: EvalLog | None
+    setting_repos: dict[str, str]
+    shipped_dirty: dict[Path, bool]
+
+
+class FleetStreamingLog:
+    """Append fleet events to <log_dir>/events.jsonl, one JSON object per line (per-sample results arrive via run_job.FleetSampleStreamer)."""
+
+    def __init__(self, *, log_dir: Path, resume: bool = False):
+        log_dir.mkdir(parents=True, exist_ok=True)
+        self.path: Path = log_dir / "events.jsonl"
+        mode = "a" if resume else "w"
+        self._f = self.path.open(mode, buffering=1)
+
+    def event(self, event_type: str, **fields: Any) -> None:
+        record = {
+            "ts": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
+            "type": event_type,
+            **fields,
+        }
+        self._f.write(json.dumps(record, default=str) + "\n")
+        self._f.flush()
+
+    def load_seen_results(self) -> set[tuple[str, str]]:
+        """Load (job_id, status) tuples from existing result events."""
+        seen: set[tuple[str, str]] = set()
+        if not self.path.exists():
+            return seen
+        with self.path.open("r") as f:
+            for line in f:
+                try:
+                    event = json.loads(line)
+                    if event.get("type") == "result":
+                        job_id = event.get("job_id")
+                        status = event.get("status")
+                        if job_id and status:
+                            seen.add((job_id, status))
+                except (json.JSONDecodeError, KeyError):
+                    continue
+        return seen
+
+    def close(self) -> None:
+        self._f.close()
+
+
+class ProgressMonitor:
+    """Tracks fleet progress from SQS done-queue messages.
+
+    Encapsulates all monitor state: job statuses, durations, failure tracking,
+    instance liveness, and stale-progress detection. The controller drives the
+    loop; this class owns the bookkeeping.
+    """
+
+    def __init__(
+        self,
+        *,
+        sqs_client: "BaseClient",
+        done_queue_url: str,
+        work_queue_url: str,
+        dlq_url: str,
+        instance_ids: list[str],
+        concurrency_per_worker: int,
+        price_per_hour: float,
+        instance_type: str,
+        stale_abort_s: int,
+        stream_log: FleetStreamingLog,
+        pending_job_ids: set[str],
+        initial_seen_results: set[tuple[str, str]],
+    ):
+        self._sqs = sqs_client
+        self._done_queue_url = done_queue_url
+        self._work_queue_url = work_queue_url
+        self._dlq_url = dlq_url
+        self._total_expected = len(pending_job_ids)
+        self._pending_job_ids = pending_job_ids
+        self._expected_instances = set(instance_ids)
+        self._concurrency_per_worker = concurrency_per_worker
+        self._num_workers = len(instance_ids)
+        self._price_per_hour = price_per_hour
+        self._instance_type = instance_type
+        self._stale_abort_s = stale_abort_s
+        self._stream = stream_log
+
+        # Job tracking
+        self.seen_jobs: dict[str, str] = {}  # job_id -> last known status
+        # (job_id, status) keys of already-logged sample results. Keying on the
+        # status as well as the job_id lets a genuine retry that changes the
+        # outcome (e.g. failed -> completed) surface as a fresh result event,
+        # while still collapsing exact SQS redeliveries of the same message.
+        self._seen_sample_results = initial_seen_results
+        self.error_summaries: list[str] = []
+        self.durations: list[float] = []
+        # (eval_duration_s, upload_s) for completed jobs that reported upload_s.
+        self.upload_samples: list[tuple[float, float]] = []
+        self.total_retries: int = 0
+        self.retried_jobs: int = 0
+        self._repeated_failures_by_job: defaultdict[str, int] = defaultdict(int)
+
+        # Instance tracking
+        self.instance_last_seen: dict[str, float] = {}
+        self.failed_bootstraps: set[str] = set()
+        self._first_worker_time: float | None = None
+        self._first_completion_time: float | None = None
+
+        # Timing
+        self._start = time.monotonic()
+        self._last_progress = time.monotonic()
+        self._last_completion = time.monotonic()  # for hard abort (completions only)
+        self._stale_warned = False
+        self._lively_note_time: float = 0.0
+
+        # DLQ / work queue stats
+        self._last_dlq_check = 0.0
+        self._last_dlq_reconcile = 0.0
+        self._last_queue_check = 0.0
+        self.dlq_count: int = 0
+
+    @property
+    def completed(self) -> int:
+        return sum(1 for s in self.seen_jobs.values() if s == "completed")
+
+    @property
+    def failed(self) -> int:
+        return sum(1 for s in self.seen_jobs.values() if s == "failed")
+
+    @property
+    def repeated_failure_count(self) -> int:
+        return sum(self._repeated_failures_by_job.values())
+
+    @property
+    def permanent_failure_count(self) -> int:
+        return sum(1 for s in self.seen_jobs.values() if s == "permanent_failure")
+
+    @property
+    def terminal(self) -> int:
+        # DLQ jobs are counted via permanent_failure after explicit reconciliation.
+        # dlq_count (approximate) is display-only.
+        return self.completed + self.permanent_failure_count
+
+    @property
+    def in_flight(self) -> int:
+        return self._total_expected - self.terminal
+
+    @property
+    def is_done(self) -> bool:
+        return self.terminal >= self._total_expected
+
+    @property
+    def elapsed_s(self) -> float:
+        return time.monotonic() - self._start
+
+    @property
+    def stale_s(self) -> float:
+        return time.monotonic() - self._last_progress
+
+    def poll_and_process(self) -> bool:
+        """Poll done queue once, process messages. Returns True if progress was made."""
+        self._check_queue_stats()
+        self._reconcile_dlq()
+
+        try:
+            first = self._sqs.receive_message(
+                QueueUrl=self._done_queue_url,
+                MaxNumberOfMessages=10,
+                WaitTimeSeconds=20,
+            ).get("Messages", [])
+            batches = [first]
+            if len(first) == 10:
+                with ThreadPoolExecutor(max_workers=DONE_QUEUE_RECEIVERS) as pool:
+                    batches += pool.map(
+                        lambda _: self._sqs.receive_message(
+                            QueueUrl=self._done_queue_url,
+                            MaxNumberOfMessages=10,
+                            WaitTimeSeconds=0,
+                        ).get("Messages", []),
+                        range(DONE_QUEUE_RECEIVERS),
+                    )
+        except botocore.exceptions.ClientError as e:
+            error_code = e.response.get("Error", {}).get("Code", "")
+            if "NonExistentQueue" in error_code or "QueueDoesNotExist" in str(e):
+                raise RuntimeError(
+                    f"SQS queue no longer exists: {self._done_queue_url}. "
+                    "This usually means another eval with the same run-name "
+                    "already completed and cleaned up the queue."
+                ) from e
+            logger.warning("Failed to poll done queue", exc_info=True)
+            time.sleep(5)
+            return False
+        except Exception:
+            logger.warning("Failed to poll done queue", exc_info=True)
+            time.sleep(5)
+            return False
+
+        progress = False
+        completed_before = self.completed
+
+        for msg in (msg for batch in batches for msg in batch):
+            try:
+                made_progress = self._handle_message(msg)
+                if made_progress:
+                    progress = True
+            except Exception:
+                logger.warning(
+                    "Bad done message, deleting to prevent redelivery", exc_info=True
+                )
+        with ThreadPoolExecutor(max_workers=len(batches)) as pool:
+            list(pool.map(self._delete_done_messages, [b for b in batches if b]))
+
+        if progress:
+            self._last_progress = time.monotonic()
+            self._stale_warned = False
+        if self.completed > completed_before:
+            self._last_completion = time.monotonic()
+
+        return progress
+
+    def _has_lively_workers(self) -> bool:
+        """Check if any workers have sent a message recently (within 10 min)."""
+        if not self.instance_last_seen:
+            return False
+        now = time.monotonic()
+        for t in self.instance_last_seen.values():
+            if now - t < 600:  # heard from within last 10min
+                return True
+        return False
+
+    def check_stale(self, instance_ids: list[str]) -> str | None:
+        """Check for stale progress. Returns an action string or None.
+
+        Possible returns: "abort", "warn", or None.
+        """
+        # Hard abort: 6hr (or the stale-abort timeout, if longer) with no completions, regardless of liveness.
+        # Uses _last_completion (not _last_progress) so a stream of failures
+        # can't prevent the abort from firing.
+        no_completion_s = time.monotonic() - self._last_completion
+        if no_completion_s > max(STALE_HARD_ABORT_S, self._stale_abort_s):
+            print(f"\n\n  No completions for {no_completion_s / 60:.0f} minutes.")
+            return "abort"
+
+        if instance_ids and self.failed_bootstraps.issuperset(instance_ids):
+            print(
+                f"\n  Every worker failed to bootstrap ({len(instance_ids)} of {len(instance_ids)})."
+            )
+            return "abort"
+
+        lively = self._has_lively_workers()
+
+        if lively:
+            # Workers are alive but no completions — likely long-running evals
+            if self.stale_s > STALE_WARN_S:
+                now = time.monotonic()
+                if now - self._lively_note_time > 300:  # rate-limit to every 5min
+                    self._lively_note_time = now
+                    active = sum(
+                        1 for t in self.instance_last_seen.values() if now - t < 600
+                    )
+                    print(
+                        f"\n  No completions for {self.stale_s / 60:.0f}m but "
+                        f"{active} worker(s) active — long-running evals likely in progress"
+                    )
+            return None
+
+        # Workers are silent — use original escalation
+        if self.stale_s > self._stale_abort_s:
+            print(f"\n\n  No progress for {self.stale_s / 60:.0f} minutes.")
+            return "abort"
+        if not self._stale_warned and self.stale_s > STALE_WARN_S:
+            self._stale_warned = True
+            quiet = self._quiet_workers()
+            print(
+                "\n  Warning: no progress for 10+ minutes. Workers may have failed to bootstrap."
+                "\n  Check /var/log/lb-worker-bootstrap.log on worker instances via SSM or SSH."
+                f"\n  Instance IDs: {instance_ids}"
+            )
+            if quiet:
+                print(f"  Quiet workers (>5m): {', '.join(i[-8:] for i in quiet)}")
+            return "warn"
+        return None
+
+    def check_failure_rate(self) -> bool:
+        """Returns True if failure rate warrants aborting."""
+        grace_period = 180  # 3 min for workers to bootstrap
+        total_seen = len(self.seen_jobs)
+        if self.elapsed_s <= grace_period or total_seen < 5:
+            return False
+
+        if self.failed == total_seen:
+            return True
+        if (
+            self.repeated_failure_count >= 5
+            and self.repeated_failure_count / total_seen > FAILURE_RATE_THRESHOLD
+        ):
+            return True
+        return False
+
+    def _cost_estimate(self) -> float:
+        """Running EC2 cost estimate."""
+        return self._num_workers * self.elapsed_s * self._price_per_hour / 3600
+
+    def print_progress(self) -> None:
+        """Print the current progress line."""
+        pf = self.permanent_failure_count
+        dlq_str = f", {self.dlq_count} DLQ" if self.dlq_count else ""
+        pf_str = f", {pf} non-retryable" if pf else ""
+        eta_str = self._eta_string()
+        elapsed_min = self.elapsed_s / 60
+        print(
+            f"\r  [{self.terminal}/{self._total_expected}] "
+            f"{self.completed} done, {self.failed} retrying{dlq_str}{pf_str}, "
+            f"~{self.in_flight} in-flight{eta_str} [{elapsed_min:.0f}m] | ~${self._cost_estimate():.2f}",
+            end="",
+            flush=True,
+        )
+
+    def print_summary(self) -> None:
+        """Print final summary."""
+        pf = self.permanent_failure_count
+        dlq_str = (
+            f", {self.dlq_count} permanently failed (DLQ)" if self.dlq_count else ""
+        )
+        pf_str = f", {pf} non-retryable" if pf else ""
+        elapsed_min = self.elapsed_s / 60
+        print(
+            f"\n\nFleet complete: {self.completed} succeeded, {self.failed} failed"
+            f"{dlq_str}{pf_str}"
+            f" out of {self._total_expected} ({elapsed_min:.1f}m total)"
+        )
+        self._stream.event(
+            "fleet_summary",
+            completed=self.completed,
+            failed=self.failed,
+            permanent_failure=pf,
+            dlq=self.dlq_count,
+            total_expected=self._total_expected,
+            elapsed_s=round(self.elapsed_s, 1),
+        )
+        cost = self._cost_estimate()
+        print(
+            f"  EC2 cost: ~${cost:.2f} "
+            f"(~{cost / self._price_per_hour:.1f} {self._instance_type} worker-hours "
+            f"@ ${self._price_per_hour:.3f}/hr)"
+        )
+        if self._first_worker_time is not None:
+            print(
+                f"  Time to first worker: {self._first_worker_time - self._start:.0f}s"
+            )
+        if self._first_completion_time is not None:
+            print(
+                f"  Time to first completion: {self._first_completion_time - self._start:.0f}s"
+            )
+        if self.durations:
+            avg = sum(self.durations) / len(self.durations)
+            mn, mx = min(self.durations), max(self.durations)
+            print(f"  Task duration: avg={avg:.0f}s, min={mn:.0f}s, max={mx:.0f}s")
+        if self.upload_samples:
+            total_eval = sum(eval_s for eval_s, _ in self.upload_samples)
+            total_upload = sum(upload_s for _, upload_s in self.upload_samples)
+            avg_upload = total_upload / len(self.upload_samples)
+            split = ""
+            if total_eval + total_upload > 0:
+                upload_pct = total_upload / (total_eval + total_upload) * 100
+                split = f", {upload_pct:.1f}% of worker time on upload"
+            print(
+                f"  Result upload: avg={avg_upload:.1f}s/job{split} "
+                "(rest is eval exec: LLM + docker)"
+            )
+        if self.total_retries:
+            print(
+                f"  API retries: {self.total_retries} across "
+                f"{self.retried_jobs} jobs (rate-limit + transient)"
+            )
+
+    def print_failure_report(self) -> None:
+        """Print top errors for abort decision."""
+        total_seen = len(self.seen_jobs)
+        print(
+            f"\n\nHigh failure rate: {self.failed}/{total_seen} "
+            f"({self.failed / max(total_seen, 1):.0%})"
+            f", {self.repeated_failure_count} persistent"
+        )
+        print("Top errors:")
+        for err in self.error_summaries[-5:]:
+            print(f"  - {err[:200]}")
+
+    # -- Internal ----------------------------------------------------------
+
+    def _handle_message(self, msg: dict) -> bool:
+        """Process a single SQS message. Returns True if it's real progress."""
+        data = json.loads(msg["Body"])
+
+        # Track instance activity for all message types
+        if iid := data.get("instance_id"):
+            self.instance_last_seen[iid] = time.monotonic()
+
+        # Lifecycle messages: informational, never count as progress
+        if "type" in data:
+            if data.get("type") == "sample_done":
+                sample_done = SampleDoneMessage.model_validate(data)
+                # Skip stale sample_done messages for jobs not in the current pending set
+                if sample_done.job_id not in self._pending_job_ids:
+                    return False
+                result_key = (sample_done.job_id, sample_done.status)
+                if result_key not in self._seen_sample_results:
+                    self._stream.event(
+                        "result",
+                        job_id=sample_done.job_id,
+                        status=sample_done.status,
+                        task=sample_done.task,
+                        model=sample_done.model,
+                        scores=sample_done.scores,
+                        error=sample_done.error,
+                        duration_s=sample_done.duration_s,
+                        instance_id=sample_done.instance_id,
+                    )
+                    self._seen_sample_results.add(result_key)
+                return False
+
+            lifecycle = LifecycleMessage.model_validate(data)
+            if lifecycle.type == "worker_online":
+                if self._first_worker_time is None:
+                    self._first_worker_time = time.monotonic()
+                bootstrap_s = time.monotonic() - self._start
+                print(
+                    f"\n  Worker online: {lifecycle.instance_id} (+{bootstrap_s:.0f}s)"
+                )
+                self._stream.event(
+                    "worker_online",
+                    instance_id=lifecycle.instance_id,
+                    bootstrap_s=round(bootstrap_s, 1),
+                )
+            elif lifecycle.type == "bootstrap_failed":
+                self.failed_bootstraps.add(lifecycle.instance_id)
+                self.instance_last_seen.pop(lifecycle.instance_id, None)
+                print(
+                    f"\n  Worker {lifecycle.instance_id} failed to bootstrap:\n"
+                    + "\n".join(
+                        f"    {line}" for line in (lifecycle.detail or "").splitlines()
+                    )
+                )
+                self._stream.event(
+                    "bootstrap_failed",
+                    instance_id=lifecycle.instance_id,
+                    detail=lifecycle.detail,
+                )
+            return False
+
+        done = JobDoneMessage.model_validate(data)
+        job_id = done.job_id
+        status = done.status
+
+        # Skip stale messages for jobs not in the current pending set
+        if job_id not in self._pending_job_ids:
+            return False
+
+        # Track first completion
+        if status == "completed" and self._first_completion_time is None:
+            self._first_completion_time = time.monotonic()
+
+        prev = self.seen_jobs.get(job_id)
+
+        progress = True
+
+        if status == "permanent_failure":
+            if done.error_summary:
+                self.error_summaries.append(done.error_summary)
+                cat = done.failure_category or "unknown"
+                iid_short = done.instance_id[-8:]
+                logger.warning(
+                    "Job %s permanently failed [%s] on %s: %s",
+                    job_id,
+                    cat,
+                    iid_short,
+                    done.error_summary[:200],
+                )
+            if prev is None or prev == "failed":
+                self.seen_jobs[job_id] = "permanent_failure"
+            elif prev in ("permanent_failure", "completed"):
+                progress = False
+        elif prev is None:
+            self.seen_jobs[job_id] = status
+            if status == "failed":
+                if done.error_summary:
+                    self.error_summaries.append(done.error_summary)
+                    cat = done.failure_category or "unknown"
+                    iid_short = done.instance_id[-8:]
+                    logger.warning(
+                        "Job %s failed [%s] on %s: %s",
+                        job_id,
+                        cat,
+                        iid_short,
+                        done.error_summary[:200],
+                    )
+                if (done.receive_count or 1) >= 3:
+                    self._repeated_failures_by_job[job_id] += 1
+            if status == "completed":
+                if done.retries:
+                    self.total_retries += done.retries
+                    self.retried_jobs += 1
+                if done.duration_s:
+                    self.durations.append(done.duration_s)
+                    if done.upload_s is not None:
+                        self.upload_samples.append((done.duration_s, done.upload_s))
+        elif prev in ("failed", "permanent_failure") and status == "completed":
+            # Late completion overrides failure/DLQ status — the job actually succeeded
+            self.seen_jobs[job_id] = "completed"
+            self._repeated_failures_by_job.pop(job_id, None)
+            if done.retries:
+                self.total_retries += done.retries
+                self.retried_jobs += 1
+            if done.duration_s:
+                self.durations.append(done.duration_s)
+                if done.upload_s is not None:
+                    self.upload_samples.append((done.duration_s, done.upload_s))
+        elif prev == "failed" and status == "failed":
+            if (done.receive_count or 1) >= 3:
+                self._repeated_failures_by_job[job_id] += 1
+            if done.error_summary:
+                self.error_summaries.append(done.error_summary)
+        else:
+            progress = False
+
+        # Log job_done event only if this represents real progress (deduplication)
+        if progress:
+            self._stream.event(
+                "job_done",
+                job_id=job_id,
+                status=status,
+                instance_id=done.instance_id,
+                duration_s=done.duration_s,
+                failure_category=done.failure_category,
+                receive_count=done.receive_count,
+                error_summary=done.error_summary,
+            )
+
+        return progress
+
+    def _check_queue_stats(self) -> None:
+        """Periodically check DLQ and work queue stats (every 60s)."""
+        now = time.monotonic()
+
+        if now - self._last_dlq_check > 60:
+            self._last_dlq_check = now
+            try:
+                attrs = self._sqs.get_queue_attributes(
+                    QueueUrl=self._dlq_url,
+                    AttributeNames=["ApproximateNumberOfMessages"],
+                )
+                self.dlq_count = int(
+                    attrs["Attributes"].get("ApproximateNumberOfMessages", 0)
+                )
+            except Exception:
+                logger.warning(
+                    "Failed to read DLQ depth; dashboard count is stale",
+                    exc_info=True,
+                )
+
+        if now - self._last_queue_check > 60:
+            self._last_queue_check = now
+            try:
+                attrs = self._sqs.get_queue_attributes(
+                    QueueUrl=self._work_queue_url,
+                    AttributeNames=[
+                        "ApproximateNumberOfMessages",
+                        "ApproximateNumberOfMessagesNotVisible",
+                    ],
+                )
+                q_available = attrs["Attributes"].get(
+                    "ApproximateNumberOfMessages", "?"
+                )
+                q_inflight = attrs["Attributes"].get(
+                    "ApproximateNumberOfMessagesNotVisible", "?"
+                )
+                print(
+                    f"\n  [queue] available={q_available}, in-flight={q_inflight}",
+                    end="",
+                    flush=True,
+                )
+            except Exception:
+                logger.warning(
+                    "Failed to read work-queue depth; dashboard count is stale",
+                    exc_info=True,
+                )
+
+            quiet = self._quiet_workers()
+            if quiet:
+                print(
+                    f"\n  [quiet] {len(quiet)} worker(s) silent >5m: "
+                    + ", ".join(i[-8:] for i in quiet),
+                    end="",
+                    flush=True,
+                )
+
+    def _reconcile_dlq(self) -> None:
+        """Read DLQ message bodies and mark those job_ids as permanently failed."""
+        now = time.monotonic()
+        if now - self._last_dlq_reconcile < DLQ_RECONCILE_INTERVAL_S:
+            return
+        self._last_dlq_reconcile = now
+        while True:
+            try:
+                resp = self._sqs.receive_message(
+                    QueueUrl=self._dlq_url,
+                    MaxNumberOfMessages=10,
+                    WaitTimeSeconds=0,
+                )
+            except Exception:
+                logger.debug("Failed to read DLQ", exc_info=True)
+                return
+            msgs = resp.get("Messages", [])
+            if not msgs:
+                return
+            for msg in msgs:
+                try:
+                    data = json.loads(msg["Body"])
+                    job_id = data.get("job_id", "unknown")
+                    prev = self.seen_jobs.get(job_id)
+                    if prev != "completed" and prev != "permanent_failure":
+                        self.seen_jobs[job_id] = "permanent_failure"
+                        logger.warning(
+                            "Job %s found in DLQ — marking permanent failure", job_id
+                        )
+                except Exception:
+                    # Can't extract job_id — use receipt handle as a synthetic key
+                    # to ensure this counts toward terminalization
+                    synthetic_id = f"dlq-unknown-{msg['ReceiptHandle'][:16]}"
+                    if synthetic_id not in self.seen_jobs:
+                        self.seen_jobs[synthetic_id] = "permanent_failure"
+                    logger.warning(
+                        "Unparseable DLQ message %s, marking as permanent failure",
+                        synthetic_id,
+                    )
+                try:
+                    self._sqs.delete_message(
+                        QueueUrl=self._dlq_url, ReceiptHandle=msg["ReceiptHandle"]
+                    )
+                except Exception:
+                    logger.debug("Failed to delete DLQ message", exc_info=True)
+
+    def _quiet_workers(self) -> list[str]:
+        """Return instance IDs that haven't sent a message in 5+ minutes."""
+        if not self.instance_last_seen:
+            return []
+        now = time.monotonic()
+        return [iid for iid, t in self.instance_last_seen.items() if now - t > 300]
+
+    def _delete_done_messages(self, messages: list[dict]) -> None:
+        """Delete one received batch from the done queue (best-effort, idempotent processing)."""
+        try:
+            self._sqs.delete_message_batch(
+                QueueUrl=self._done_queue_url,
+                Entries=[
+                    {"Id": str(i), "ReceiptHandle": msg["ReceiptHandle"]}
+                    for i, msg in enumerate(messages)
+                ],
+            )
+        except Exception:
+            logger.debug(
+                "Failed to delete done messages (will be re-delivered)", exc_info=True
+            )
+
+    def _active_worker_count(self) -> int:
+        """Count workers heard from in the last 5 minutes."""
+        if not self.instance_last_seen:
+            return len(self._expected_instances)
+        now = time.monotonic()
+        active = sum(1 for t in self.instance_last_seen.values() if now - t < 300)
+        return active or len(self._expected_instances)
+
+    def _eta_string(self) -> str:
+        """Compute ETA string from recent durations (needs 3+ completions)."""
+        if len(self.durations) < 3:
+            return ""
+        recent = self.durations[-20:]
+        avg_dur = sum(recent) / len(recent)
+        active_slots = self._active_worker_count() * self._concurrency_per_worker
+        eta_min = self.in_flight * avg_dur / 60 / max(active_slots, 1)
+        return f" | est ~{eta_min:.0f}m left"
+
+
+class FleetController:
+    """Orchestrates a distributed eval run across EC2 workers."""
+
+    def __init__(self, config: Ec2Config):
+        self.config = config
+        self._worker_setup_s3_key = ""
+        env = EnvVar()
+        self.s3_bucket: str = env.fleet_s3_bucket
+
+        if env.aws_profile and env.aws_access_key_id:
+            raise ValidationError(
+                "Both AWS_PROFILE and AWS_ACCESS_KEY_ID are set — the static key would silently "
+                "take precedence over the profile. Unset one (usually: remove the "
+                "AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY lines from .env when using SSO)."
+            )
+        self._ec2 = boto3.client("ec2", region_name=config.region)
+        self._pricing = boto3.client("pricing", region_name="us-east-1")
+        self._sqs = boto3.client(
+            "sqs",
+            region_name=config.region,
+            config=botocore.config.Config(max_pool_connections=16),
+        )
+        self._s3 = boto3.client(
+            "s3",
+            region_name=config.region,
+            config=botocore.config.Config(max_pool_connections=64),
+        )
+        # Each instance costs 1 + one-per-setting-package PutParameter calls; standard mode backs off on throttling.
+        self._ssm = boto3.client(
+            "ssm",
+            region_name=config.region,
+            config=botocore.config.Config(
+                retries={"mode": "standard", "max_attempts": 10}
+            ),
+        )
+
+    def run(
+        self,
+        *,
+        inspect_jobs: list[InspectFleetJobWire],
+        eval_args: dict[str, Any],
+        setting_dirs: dict[str, Path],
+        env_dirs: list[Path],
+        shipped_files: Collection[Path],
+        extra_env_var_names: set[str],
+        setting_package_secrets: dict[str, str],
+        metadata: dict[str, Any] | None = None,
+    ) -> FleetRunResult | None:
+        """Run generic Inspect jobs on workers given the local checkouts of ``setting_dirs`` and of the repos holding ``env_dirs``, which may leave untracked only the ``shipped_files`` that reach workers another way; return the S3 result location."""
+        run_start = time.monotonic()
+        for package, values in setting_package_secrets.items():
+            if not re.fullmatch(r"[A-Za-z0-9_.-]+", package):
+                raise ValidationError(f"Invalid setting package name: {package!r}")
+            if len(values.encode()) > 4096:
+                raise ValidationError(
+                    f"Setting {package}'s {package}.env exceeds the 4 KB SSM parameter limit."
+                )
+
+        epoch_by_eval_name = {
+            job.output_key: epoch_index
+            for job in inspect_jobs
+            if isinstance(epoch_index := job.metadata.get("epoch_index"), int)
+        }
+
+        eval_args_wire = EvalConfig.model_validate(eval_args)
+        longest_job_s = SQS_MAX_VISIBILITY_S - SQS_VISIBILITY_TIMEOUT_S
+        if (eval_args_wire.time_limit or 0) > longest_job_s:
+            raise ValidationError(
+                f"`time_limit` is {eval_args_wire.time_limit}s, but a fleet job can run for at most {longest_job_s}s."
+            )
+        eval_args_dict = eval_args_wire.model_dump(mode="json", exclude_none=True)
+
+        # 1. Compute fleet identity
+
+        code_root = _code_root()
+        git_sha, _ = _git_snapshot(code_root)
+        setting_paths = setting_repo_dirs(setting_dirs, env_dirs)
+        snapshots = {dest: _git_snapshot(repo) for dest, repo in setting_paths.items()}
+        setting_repos = {
+            dest: f"{sha}+dirty" if dirty else sha
+            for dest, (sha, dirty) in snapshots.items()
+        }
+        shipped_dirty = {
+            setting_paths[dest]: dirty for dest, (_, dirty) in snapshots.items()
+        }
+        trees = {
+            label: subprocess.run(
+                ["git", "rev-parse", f"{sha}^{{tree}}"],
+                cwd=path,
+                capture_output=True,
+                text=True,
+                check=True,
+            ).stdout.strip()
+            for label, path, sha in [
+                ("control-tower", code_root, git_sha),
+                *((dest, setting_paths[dest], s) for dest, (s, _) in snapshots.items()),
+            ]
+        }
+        for label, path in [("control-tower", code_root), *setting_paths.items()]:
+            changed = {
+                file: tracked
+                for file, tracked in (git_changed_paths(path) or {}).items()
+                if not any(
+                    file.is_relative_to(repo)
+                    for repo in setting_paths.values()
+                    if path in repo.parents
+                )
+            }
+            modified = [file for file, tracked in changed.items() if tracked]
+            unshipped = [
+                file
+                for file, tracked in changed.items()
+                if not tracked and file.resolve() not in shipped_files
+            ]
+            if modified:
+                print(
+                    f"  WARNING: {label}: shipping {len(modified)} uncommitted file(s) from local worktree"
+                )
+            if unshipped:
+                raise ValidationError(
+                    f"{label} has {len(unshipped)} untracked file(s), which workers would not get: "
+                    "`git add` them to ship them, or gitignore or remove them. "
+                    f"First: {unshipped[0].relative_to(path)}"
+                )
+
+        worker_setup_script = b""
+        if self.config.worker_setup:
+            try:
+                worker_setup_script = Path(self.config.worker_setup).read_bytes()
+            except OSError as exc:
+                raise ValidationError(
+                    f"--ec2-arg worker_setup={self.config.worker_setup!r} is not a readable file: {exc}"
+                ) from exc
+        worker_setup_hash = (
+            hashlib.sha256(worker_setup_script).hexdigest()[:16]
+            if worker_setup_script
+            else ""
+        )
+        worker_bootstrap: dict[str, Any] = {
+            "settings": {dest: trees[dest] for dest in snapshots},
+            "instance_type": self.config.instance_type,
+            **({"worker_setup": worker_setup_hash} if worker_setup_hash else {}),
+            **({"ami": self.config.ami} if self.config.ami else {}),
+            **(
+                {"worker_user": self.config.worker_user}
+                if self.config.worker_user != "root"
+                else {}
+            ),
+        }
+        fleet_id = compute_fleet_id(
+            trees["control-tower"],
+            inspect_jobs,
+            eval_args_dict,
+            new_fleet=self.config.new_fleet,
+            worker_bootstrap=worker_bootstrap,
+        )
+        s3_prefix = f"{fleet_id}/"
+
+        # 2. Build job list
+        jobs = [
+            JobPayload(
+                job_id=inspect_job.job_id, fleet_id=fleet_id, inspect_job=inspect_job
+            )
+            for inspect_job in inspect_jobs
+        ]
+        total_jobs = len(jobs)  # Immutable: total across all invocations
+
+        if total_jobs == 0:
+            print("No jobs to run.")
+            return None
+
+        jobs = self._with_output_prefix(jobs, s3_prefix)
+        inspect_jobs = [job.inspect_job for job in jobs]
+
+        # 3. Check for existing fleet (resume)
+        existing_manifest = self._get_manifest(s3_prefix)
+        is_resume = existing_manifest is not None and not self.config.new_fleet
+        done_keys = set(list_eval_keys(self._s3, self.s3_bucket, s3_prefix))
+        pending_jobs = [j for j in jobs if j.inspect_job.output_key not in done_keys]
+        if existing_manifest and not self.config.new_fleet:
+            print(
+                f"Resuming existing fleet {fleet_id} ({existing_manifest.total_jobs} jobs)"
+            )
+            print(
+                f"  {total_jobs - len(pending_jobs)} done, {len(pending_jobs)} remaining"
+            )
+        elif existing_manifest:
+            logger.info(
+                "Existing manifest at s3://%s/%smanifest.json found but "
+                "--new-fleet was set; starting fresh fleet %s.",
+                self.s3_bucket,
+                s3_prefix,
+                fleet_id,
+            )
+        else:
+            logger.info(
+                "No existing manifest at s3://%s/%smanifest.json; "
+                "starting fresh fleet %s.",
+                self.s3_bucket,
+                s3_prefix,
+                fleet_id,
+            )
+        if not pending_jobs:
+            print(f"All {total_jobs} jobs already complete.")
+            try:
+                return self._collect_fleet_result(
+                    fleet_id,
+                    s3_prefix,
+                    total_jobs,
+                    epoch_by_eval_name=epoch_by_eval_name,
+                    setting_repos=setting_repos,
+                    shipped_dirty=shipped_dirty,
+                )
+            finally:
+                self._cleanup_fleet_resources(fleet_id, wait_for_workers=False)
+
+        instance_types = list(
+            dict.fromkeys(
+                [self.config.instance_type, *self.config.instance_type_fallbacks]
+            )
+        )
+        gpu_types = [
+            info["InstanceType"]
+            for info in self._ec2.describe_instance_types(InstanceTypes=instance_types)[
+                "InstanceTypes"
+            ]
+            if "GpuInfo" in info
+        ]
+        if gpu_types and self.config.concurrency_per_worker != 1:
+            raise ValidationError(
+                f"A {', '.join(gpu_types)} worker has a GPU and runs one sample at a time: "
+                "pass `--ec2-arg concurrency_per_worker=1`."
+            )
+        prices: dict[str, float] = {}
+        for instance_type in instance_types:
+            [product] = self._pricing.get_products(
+                ServiceCode="AmazonEC2",
+                Filters=[
+                    {"Type": "TERM_MATCH", "Field": field, "Value": value}
+                    for field, value in {
+                        "instanceType": instance_type,
+                        "regionCode": self.config.region,
+                        "operatingSystem": "Linux",
+                        "tenancy": "Shared",
+                        "preInstalledSw": "NA",
+                        "capacitystatus": "Used",
+                        "licenseModel": "No License required",
+                    }.items()
+                ],
+            )["PriceList"]
+            [on_demand] = json.loads(product)["terms"]["OnDemand"].values()
+            [dimension] = on_demand["priceDimensions"].values()
+            prices[instance_type] = float(dimension["pricePerUnit"]["USD"])
+
+        # 4. Size the fleet so every pending job starts at once, up to max_workers
+        planning_cpw = min(
+            self.config.concurrency_per_worker, FLEET_PLANNING_CONCURRENCY
+        )
+        num_workers = max(
+            1,
+            min(self.config.max_workers, math.ceil(len(pending_jobs) / planning_cpw)),
+        )
+
+        # 5. Print cost estimate (for pending work only)
+        fleet_cost_per_hour = num_workers * max(prices.values())
+        self._print_estimate(len(pending_jobs), num_workers, fleet_cost_per_hour)
+
+        if self.config.estimate:
+            return None
+
+        # 6. Wait for confirmation
+        if not self.config.auto_confirm:
+            self._wait_for_confirm()
+
+        stream_log: FleetStreamingLog | None = None
+        wait_for_workers = is_resume
+        try:
+            # 7. Create/reuse queues
+            t0 = time.monotonic()
+            work_queue_url, work_created = self._ensure_queue(
+                f"lb-fleet-{fleet_id}-work",
+                VisibilityTimeout=str(SQS_VISIBILITY_TIMEOUT_S),
+                ReceiveMessageWaitTimeSeconds="20",
+            )
+            done_queue_url, done_created = self._ensure_queue(
+                f"lb-fleet-{fleet_id}-done",
+                ReceiveMessageWaitTimeSeconds="20",
+            )
+            dlq_url, dlq_created = self._ensure_queue(f"lb-fleet-{fleet_id}-dlq")
+            self._configure_dlq(work_queue_url, dlq_url)
+            print(f"  Queues ready ({time.monotonic() - t0:.1f}s)")
+
+            # 8. Write manifest
+            if not existing_manifest:
+                manifest = FleetManifest(
+                    fleet_id=fleet_id,
+                    total_jobs=len(jobs),
+                    runner_schema_version=FLEET_RUNNER_SCHEMA_VERSION,
+                    inspect_jobs=inspect_jobs,
+                    eval_args=eval_args_wire,
+                    metadata=metadata or {},
+                    git_sha=git_sha,
+                    setting_repos=setting_repos,
+                    created_at=time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+                )
+                self._put_manifest(manifest, s3_prefix)
+
+            # 9. Enqueue the pending jobs
+            purges: list[float] = []
+            if not work_created:
+                purges.append(self._purge_queue(work_queue_url, "work"))
+            if not done_created and not is_resume:
+                purges.append(self._purge_queue(done_queue_url, "done"))
+            if not dlq_created:
+                purges.append(self._purge_queue(dlq_url, "DLQ"))
+
+            last_purge = max(purges, default=None)
+            if last_purge is not None:
+                remaining = SQS_PURGE_WINDOW_S - (time.monotonic() - last_purge)
+                if remaining > 0:
+                    print(f"Waiting {remaining:.0f}s for the queue purges to settle...")
+                    time.sleep(remaining)
+
+            t0 = time.monotonic()
+            print(f"Enqueueing {len(pending_jobs)} jobs...")
+            enqueue_jobs(self._sqs, work_queue_url, pending_jobs)
+            print(f"  Enqueued {len(pending_jobs)} ({time.monotonic() - t0:.1f}s)")
+
+            # 10. Upload code artifact to S3
+            t0 = time.monotonic()
+            print("Uploading code artifact to S3...")
+            code_s3_key = self._upload_git_artifact(code_root, git_sha)
+            print(f"  Code artifact ready ({time.monotonic() - t0:.1f}s)")
+
+            setting_archives: list[tuple[str, str]] = []
+            if setting_repos:
+                t0 = time.monotonic()
+                print(f"Uploading {len(setting_repos)} setting repo(s) to S3...")
+                setting_archives = [
+                    (self._upload_git_artifact(setting_paths[dest], sha), dest)
+                    for dest, (sha, _) in snapshots.items()
+                ]
+                print(f"  Setting artifacts ready ({time.monotonic() - t0:.1f}s)")
+
+            rates_s3_key = (
+                "" if EnvVar().pricing == "off" else self._upload_rate_table()
+            )
+            if worker_setup_hash:
+                self._worker_setup_s3_key = f"worker-setup/{worker_setup_hash}.sh"
+                self._s3.put_object(
+                    Bucket=self.s3_bucket,
+                    Key=self._worker_setup_s3_key,
+                    Body=worker_setup_script,
+                )
+
+            # 11. Launch workers
+            t0 = time.monotonic()
+            instance_ids, fleet_price_per_hour = self._launch_workers(
+                num_workers,
+                prices=prices,
+                fleet_id=fleet_id,
+                user_data=self._build_user_data(
+                    code_s3_key=code_s3_key,
+                    rates_s3_key=rates_s3_key,
+                    s3_prefix=s3_prefix,
+                    work_queue_url=work_queue_url,
+                    done_queue_url=done_queue_url,
+                    setting_archives=setting_archives,
+                    env_secret_packages=sorted(setting_package_secrets),
+                    fleet_secrets_prefix=f"{FLEET_SSM_PREFIX}/{fleet_id}",
+                ),
+                secrets=self._worker_secrets(extra_env_var_names),
+                setting_package_secrets=setting_package_secrets,
+            )
+            wait_for_workers = True
+            print(
+                f"Launched {len(instance_ids)} workers ({time.monotonic() - t0:.1f}s): "
+                f"{instance_ids}"
+            )
+
+            setup_s = time.monotonic() - run_start
+            print(f"\n  Setup complete in {setup_s:.1f}s")
+
+            # 12. Monitor progress
+            # Scale the stale-abort timeout based on time_limit so that
+            # individual sample timeouts have a chance to fire before the fleet
+            # controller gives up. Default: max(STALE_ABORT_S, time_limit * 1.5).
+            sample_tl = eval_args_wire.time_limit
+            if sample_tl is not None and sample_tl > 0:
+                stale_abort_s = max(STALE_ABORT_S, int(sample_tl * 1.5))
+            else:
+                stale_abort_s = STALE_ABORT_S
+            if stale_abort_s != STALE_ABORT_S:
+                logger.info(
+                    "Stale-abort timeout scaled to %ds (time_limit=%ds)",
+                    stale_abort_s,
+                    sample_tl,
+                )
+            stream_log_dir = Path("logs/fleet") / fleet_id
+            stream_log = FleetStreamingLog(log_dir=stream_log_dir, resume=is_resume)
+            monitor = ProgressMonitor(
+                sqs_client=self._sqs,
+                done_queue_url=done_queue_url,
+                work_queue_url=work_queue_url,
+                dlq_url=dlq_url,
+                instance_ids=instance_ids,
+                concurrency_per_worker=planning_cpw,
+                price_per_hour=fleet_price_per_hour / len(instance_ids),
+                instance_type="/".join(instance_types),
+                stale_abort_s=stale_abort_s,
+                stream_log=stream_log,
+                pending_job_ids={j.job_id for j in pending_jobs},
+                initial_seen_results=stream_log.load_seen_results(),
+            )
+            print(f"\nMonitoring fleet {fleet_id}: 0/{len(pending_jobs)}")
+            print(f"  Streaming: tail -f {stream_log.path}")
+            stream_log.event(
+                "fleet_start",
+                fleet_id=fleet_id,
+                expected=len(pending_jobs),
+                workers=len(instance_ids),
+                instance_type=self.config.instance_type,
+            )
+
+            self._run_monitor_loop(monitor, fleet_id, instance_ids)
+            monitor.print_summary()
+            return self._collect_fleet_result(
+                fleet_id,
+                s3_prefix,
+                total_jobs,
+                epoch_by_eval_name=epoch_by_eval_name,
+                setting_repos=setting_repos,
+                shipped_dirty=shipped_dirty,
+            )
+        finally:
+            if stream_log is not None:
+                stream_log.close()
+            # 13. Tear down fleet resources. S3 results remain for the caller to consume.
+            self._cleanup_fleet_resources(fleet_id, wait_for_workers=wait_for_workers)
+
+    def _collect_fleet_result(
+        self,
+        fleet_id: str,
+        s3_prefix: str,
+        total_expected: int,
+        *,
+        epoch_by_eval_name: dict[str, int] | None = None,
+        setting_repos: dict[str, str],
+        shipped_dirty: dict[Path, bool],
+    ) -> FleetRunResult:
+        job_order = {name: index for index, name in enumerate(epoch_by_eval_name or {})}
+        eval_keys = sorted(
+            list_eval_keys(self._s3, self.s3_bucket, s3_prefix),
+            key=lambda key: (job_order.get(Path(key).name, len(job_order)), key),
+        )
+        print(f"\nResults: found {len(eval_keys)}/{total_expected} .eval files in S3")
+        eval_log = (
+            self._download_and_merge_eval_logs(eval_keys, epoch_by_eval_name or {})
+            if eval_keys
+            else None
+        )
+        completed = len(eval_log.samples or []) if eval_log is not None else 0
+        if completed < total_expected:
+            print(
+                f"  Warning: {total_expected - completed}/{total_expected} jobs "
+                "produced no sample (may have failed)"
+            )
+
+        return FleetRunResult(
+            fleet_id=fleet_id,
+            s3_bucket=self.s3_bucket,
+            s3_prefix=s3_prefix,
+            total_expected=total_expected,
+            total_completed=completed,
+            region=self.config.region,
+            eval_log=eval_log,
+            setting_repos=setting_repos,
+            shipped_dirty=shipped_dirty,
+        )
+
+    def _download_and_merge_eval_logs(
+        self, eval_keys: list[str], epoch_by_eval_name: dict[str, int]
+    ) -> EvalLog | None:
+        with tempfile.TemporaryDirectory(prefix="lb-fleet-merge-") as tmpdir:
+            tmppath = Path(tmpdir)
+            print(f"Downloading {len(eval_keys)} .eval files...")
+            paths_by_key: dict[str, Path] = {}
+
+            max_workers = min(64, len(eval_keys))
+            with ThreadPoolExecutor(max_workers=max_workers) as pool:
+                futures = {}
+                for idx, key in enumerate(eval_keys):
+                    local_path = tmppath / f"{idx}-{Path(key).name}"
+                    future = pool.submit(
+                        self._s3.download_file,
+                        self.s3_bucket,
+                        key,
+                        str(local_path),
+                    )
+                    futures[future] = (key, local_path)
+                for future in as_completed(futures):
+                    key, path = futures[future]
+                    try:
+                        future.result()
+                        paths_by_key[key] = path
+                    except Exception as e:
+                        logger.error("Failed to download %s: %s", key, e)
+
+                present_keys = [key for key in eval_keys if key in paths_by_key]
+                eval_paths = [paths_by_key[key] for key in present_keys]
+                print(f"  Downloaded {len(eval_paths)} files")
+                eval_logs = list(pool.map(read_eval_log, map(str, eval_paths)))
+
+        # Fleet epoch jobs all emit epoch 1 with the same deterministic sample
+        # id; stamp the job's true epoch so merged (id, epoch) keys are stable
+        # across re-merges.
+        for key, log in zip(present_keys, eval_logs, strict=True):
+            epoch_index = epoch_by_eval_name.get(Path(key).name)
+            if epoch_index is not None:
+                for sample in log.samples or []:
+                    sample.epoch = epoch_index + 1
+
+        combined = merge_eval_logs(eval_logs)
+        if combined is not None:
+            print(f"  Merged {len(combined.samples or [])} eval samples")
+        return combined
+
+    def _run_monitor_loop(
+        self,
+        monitor: ProgressMonitor,
+        fleet_id: str,
+        instance_ids: list[str],
+    ) -> None:
+        """Drive the progress monitor until completion or abort."""
+        last_liveness_check = time.monotonic()
+        try:
+            while not monitor.is_done:
+                action = monitor.check_stale(instance_ids)
+                if action is None and time.monotonic() - last_liveness_check > 60:
+                    last_liveness_check = time.monotonic()
+                    resp = self._ec2.describe_instances(
+                        Filters=[
+                            {"Name": "tag:FleetId", "Values": [fleet_id]},
+                            {
+                                "Name": "instance-state-name",
+                                "Values": ["pending", "running"],
+                            },
+                        ]
+                    )
+                    if not any(res["Instances"] for res in resp["Reservations"]):
+                        print(
+                            f"\n\n  Every worker is gone: none of the {len(instance_ids)} "
+                            "launched instances is running. Bootstrap logs, if any: "
+                            f"s3://{self.s3_bucket}/{fleet_id}/logs/"
+                        )
+                        action = "abort"
+                if action == "abort":
+                    print("  Terminating fleet and aborting.")
+                    self._terminate_fleet_instances(fleet_id)
+                    raise OperationError(
+                        f"Fleet {fleet_id} aborted due to lack of progress. "
+                        "S3 results preserved. Fix the issue and re-run to resume."
+                    )
+
+                # Poll and process
+                monitor.poll_and_process()
+                monitor.print_progress()
+
+                # Check failure rate
+                if monitor.check_failure_rate():
+                    monitor.print_failure_report()
+                    print("\nTerminating workers to stop the bleeding...")
+                    self._terminate_fleet_instances(fleet_id)
+                    raise OperationError(
+                        f"Fleet {fleet_id} stopped on its failure rate. "
+                        "S3 results preserved. Fix the issue and re-run to resume."
+                    )
+        except KeyboardInterrupt:
+            # Best effort: stop workers from pulling new jobs so they can drain
+            # in-flight work and self-terminate on idle.
+            purge_failure: botocore.exceptions.ClientError | None = None
+            try:
+                self._purge_queue(monitor._work_queue_url, "work")
+            except self._sqs.exceptions.PurgeQueueInProgress:
+                logger.info("Work queue purge already in progress")
+            except botocore.exceptions.ClientError as exc:
+                purge_failure = exc
+            monitor.print_summary()
+            queue_state = (
+                "Work queue purged; workers will self-terminate after idle timeout."
+                if purge_failure is None
+                else f"Work queue purge FAILED ({purge_failure}); workers keep pulling jobs."
+            )
+            print(
+                f"\nInterrupted. {queue_state}"
+                f"\nRe-run the same command to resume, or terminate instances:"
+                f"\n  aws ec2 describe-instances --filters Name=tag:FleetId,Values={fleet_id} "
+                f"Name=instance-state-name,Values=running --query Reservations[].Instances[].InstanceId"
+            )
+            raise SystemExit(1)
+
+    def _with_output_prefix(
+        self, jobs: list[JobPayload], s3_prefix: str
+    ) -> list[JobPayload]:
+        """Fill each generic job's output key after the fleet ID is known."""
+        updated: list[JobPayload] = []
+        for job in jobs:
+            base_output_key = job.inspect_job.output_key
+            inspect_job = job.inspect_job.model_copy(
+                update={"output_key": f"{s3_prefix}{base_output_key.lstrip('/')}"}
+            )
+            updated.append(job.model_copy(update={"inspect_job": inspect_job}))
+        return updated
+
+    def _ensure_queue(self, name: str, **extra_attrs: str) -> tuple[str, bool]:
+        """Create the SQS queue if it does not exist.
+
+        Returns the URL and whether this call created it: the only case a caller
+        can skip purging on, and one AWS's message counts cannot answer.
+        """
+        try:
+            resp = self._sqs.get_queue_url(QueueName=name)
+            return resp["QueueUrl"], False
+        except self._sqs.exceptions.QueueDoesNotExist:
+            pass
+
+        attrs = {"MessageRetentionPeriod": "86400"}
+        attrs.update(extra_attrs)
+
+        deadline = time.monotonic() + SQS_QUEUE_DELETE_COOLDOWN_S + 10
+        while True:
+            try:
+                resp = self._sqs.create_queue(QueueName=name, Attributes=attrs)
+                break
+            except self._sqs.exceptions.QueueDeletedRecently:
+                if time.monotonic() >= deadline:
+                    raise
+                logger.info("Queue %s deleted recently; retrying after cooldown", name)
+                time.sleep(5)
+        logger.info("Created queue: %s", name)
+        return resp["QueueUrl"], True
+
+    def _configure_dlq(self, work_queue_url: str, dlq_url: str) -> None:
+        """Attach a dead-letter queue to the work queue."""
+        dlq_arn = self._sqs.get_queue_attributes(
+            QueueUrl=dlq_url, AttributeNames=["QueueArn"]
+        )["Attributes"]["QueueArn"]
+
+        self._sqs.set_queue_attributes(
+            QueueUrl=work_queue_url,
+            Attributes={
+                "RedrivePolicy": json.dumps(
+                    {"deadLetterTargetArn": dlq_arn, "maxReceiveCount": 3}
+                )
+            },
+        )
+
+    def _delete_queues(self, fleet_id: str) -> None:
+        """Clean up queues for a fleet."""
+        for suffix in ("work", "done", "dlq"):
+            name = f"lb-fleet-{fleet_id}-{suffix}"
+            try:
+                url = self._sqs.get_queue_url(QueueName=name)["QueueUrl"]
+                self._sqs.delete_queue(QueueUrl=url)
+                logger.info("Deleted queue: %s", name)
+            except Exception:
+                logger.warning(
+                    "Failed to delete queue %s; it may be orphaned", name, exc_info=True
+                )
+
+    def _purge_queue(self, queue_url: str, label: str = "") -> float:
+        """Purge an SQS queue.
+
+        Returns when the purge was accepted so callers can wait out the window
+        in which SQS may still delete newly sent messages.
+        """
+        self._sqs.purge_queue(QueueUrl=queue_url)
+        if label:
+            logger.info("Purged %s queue", label)
+        return time.monotonic()
+
+    def _put_manifest(self, manifest: FleetManifest, s3_prefix: str) -> None:
+        self._s3.put_object(
+            Bucket=self.s3_bucket,
+            Key=f"{s3_prefix}manifest.json",
+            Body=manifest.model_dump_json(indent=2),
+            ContentType="application/json",
+        )
+
+    def _get_manifest(self, s3_prefix: str) -> FleetManifest | None:
+        key = f"{s3_prefix}manifest.json"
+        try:
+            resp = self._s3.get_object(Bucket=self.s3_bucket, Key=key)
+            data = json.loads(resp["Body"].read())
+            return FleetManifest.model_validate(data)
+        except self._s3.exceptions.NoSuchKey:
+            return None
+
+    def _ensure_fleet_security_group(self) -> str:
+        """Create or find a no-inbound security group for fleet workers."""
+        sg_name = "linuxbench-fleet-worker-sg"
+
+        vpcs = self._ec2.describe_vpcs(
+            Filters=[{"Name": "is-default", "Values": ["true"]}]
+        )["Vpcs"]
+        vpc_id = vpcs[0]["VpcId"] if vpcs else None
+
+        filters = [{"Name": "group-name", "Values": [sg_name]}]
+        if vpc_id:
+            filters.append({"Name": "vpc-id", "Values": [vpc_id]})
+        groups = self._ec2.describe_security_groups(Filters=filters)["SecurityGroups"]
+        if groups:
+            return groups[0]["GroupId"]
+
+        create_kwargs: dict = dict(
+            GroupName=sg_name,
+            Description="LinuxBench fleet workers - outbound only, no inbound",
+        )
+        if vpc_id:
+            create_kwargs["VpcId"] = vpc_id
+        resp = self._ec2.create_security_group(**create_kwargs)
+        sg_id = resp["GroupId"]
+        logger.info(
+            "Created fleet security group: %s (%s) in VPC %s", sg_name, sg_id, vpc_id
+        )
+        return sg_id
+
+    def _upload_git_artifact(self, repo: Path, git_sha: str) -> str:
+        """Upload a tarball of ``git_sha`` from ``repo``, keyed by that SHA."""
+
+        s3_key = f"artifacts/{git_sha}.tar.gz"
+
+        # Check if already uploaded (shared across fleets with same SHA)
+        try:
+            self._s3.head_object(Bucket=self.s3_bucket, Key=s3_key)
+        except botocore.exceptions.ClientError as e:
+            if e.response["Error"]["Code"] != "404":
+                raise
+        else:
+            logger.info("Artifact already exists at s3://%s/%s", self.s3_bucket, s3_key)
+            return s3_key
+
+        common_dir = subprocess.run(
+            ["git", "rev-parse", "--path-format=absolute", "--git-common-dir"],
+            cwd=repo,
+            check=True,
+            capture_output=True,
+            text=True,
+        ).stdout.strip()
+        with (
+            tempfile.NamedTemporaryFile(suffix=".tar.gz", delete=True) as tmp,
+            tempfile.TemporaryDirectory(prefix="ct-archive-") as git_dir,
+        ):
+            # $GIT_DIR/info/attributes outranks in-tree .gitattributes, so this disables export-ignore.
+            subprocess.run(
+                ["git", "init", "--bare", "-q", git_dir],
+                check=True,
+                capture_output=True,
+            )
+            (Path(git_dir) / "objects" / "info" / "alternates").write_text(
+                f"{common_dir}/objects\n"
+            )
+            (Path(git_dir) / "info" / "attributes").write_text(
+                "* -export-ignore -export-subst\n"
+            )
+            subprocess.run(
+                ["git", "--git-dir", git_dir, "archive", "--format=tar.gz"]
+                + ["-o", tmp.name, git_sha],
+                cwd=git_dir,
+                check=True,
+                capture_output=True,
+            )
+
+            self._s3.upload_file(tmp.name, self.s3_bucket, s3_key)
+
+        logger.info("Uploaded %s to s3://%s/%s", repo, self.s3_bucket, s3_key)
+        return s3_key
+
+    def _upload_rate_table(self) -> str:
+        """Upload the driver's rate table so workers price with the same rates."""
+        table = default_rate_table().model_dump_json()
+        s3_key = f"rates/{hashlib.sha256(table.encode()).hexdigest()[:16]}.json"
+        self._s3.put_object(Bucket=self.s3_bucket, Key=s3_key, Body=table.encode())
+        logger.info("Uploaded rate table to s3://%s/%s", self.s3_bucket, s3_key)
+        return s3_key
+
+    def upload_recording(self, path: Path) -> str:
+        """Upload a local recording for workers to replay; return the ``s3://`` URL they load it from."""
+        if path.is_dir():
+            raise ValidationError(
+                f"--ec2 ships a recording as one file, but {path} is a directory; pass the .eval file in it."
+            )
+        with path.open("rb") as file:
+            digest = hashlib.file_digest(file, "sha256").hexdigest()
+        s3_key = f"recordings/{digest[:16]}{path.suffix}"
+        if not self.config.estimate:
+            self._s3.upload_file(str(path), self.s3_bucket, s3_key)
+        return f"s3://{self.s3_bucket}/{s3_key}"
+
+    def _launch_workers(
+        self,
+        num_workers: int,
+        *,
+        prices: dict[str, float],
+        fleet_id: str,
+        user_data: str,
+        secrets: dict[str, str],
+        setting_package_secrets: dict[str, str],
+    ) -> tuple[list[str], float]:
+        """Launch EC2 worker instances."""
+        ami_id = get_worker_ami_id(self.config.region, self.config.ami)
+        sg_id = self._ensure_fleet_security_group()
+
+        try:
+            launched_by = caller_identity_name(self.config.region)
+        except Exception as e:
+            raise RuntimeError(
+                f"Cannot determine AWS identity: {e}\n"
+                "The scoped fleet policy requires a LaunchedBy tag matching your identity on RunInstances. "
+                "Check your AWS credentials: AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY in .env, or "
+                "AWS_PROFILE pointing at a logged-in SSO profile (aws sso login --profile <name>)."
+            ) from e
+
+        fleet_tags = [
+            {"Key": "ManagedBy", "Value": "linuxbench"},
+            {"Key": "FleetId", "Value": fleet_id},
+            {"Key": "LaunchedBy", "Value": launched_by},
+        ]
+
+        # Resolve instance profile ARN
+        iam = boto3.client("iam", region_name=self.config.region)
+        try:
+            profile_resp = iam.get_instance_profile(
+                InstanceProfileName=INSTANCE_PROFILE_NAME
+            )
+            profile_arn = profile_resp["InstanceProfile"]["Arn"]
+        except Exception:
+            raise SystemExit(
+                f"IAM instance profile '{INSTANCE_PROFILE_NAME}' not found.\n"
+                "Create it with SQS, S3, and SSM permissions for fleet workers."
+            )
+
+        launch_kwargs: dict = dict(
+            ImageId=ami_id,
+            InstanceType=self.config.instance_type,
+            MinCount=1,
+            MaxCount=num_workers,
+            UserData=gzip.compress(user_data.encode()),
+            IamInstanceProfile={"Arn": profile_arn},
+            InstanceInitiatedShutdownBehavior="terminate",
+            MetadataOptions={"HttpTokens": "required", "HttpPutResponseHopLimit": 1},
+            BlockDeviceMappings=[
+                {
+                    "DeviceName": "/dev/sda1",
+                    "Ebs": {
+                        "VolumeSize": ROOT_VOLUME_SIZE_GB,
+                        "VolumeType": "gp3",
+                        "DeleteOnTermination": True,
+                    },
+                }
+            ],
+            TagSpecifications=[
+                {
+                    "ResourceType": "instance",
+                    "Tags": [
+                        {"Key": "Name", "Value": f"lb-fleet-{fleet_id[:8]}"},
+                        *fleet_tags,
+                    ],
+                },
+            ],
+            NetworkInterfaces=[
+                {
+                    "DeviceIndex": 0,
+                    "AssociatePublicIpAddress": True,
+                    "Groups": [sg_id],
+                }
+            ],
+        )
+        if self.config.spot:
+            launch_kwargs["InstanceMarketOptions"] = {
+                "MarketType": "spot",
+                "SpotOptions": {
+                    "SpotInstanceType": "one-time",
+                    "InstanceInterruptionBehavior": "terminate",
+                },
+            }
+
+        instance_ids: list[str] = []
+        fleet_price_per_hour = 0.0
+        for instance_type, price_per_hour in prices.items():
+            if (shortfall := num_workers - len(instance_ids)) == 0:
+                break
+            launched = self._launch_with_fallback(
+                {**launch_kwargs, "InstanceType": instance_type, "MaxCount": shortfall},
+                quota_fatal=not instance_ids,
+            )
+            instance_ids += launched
+            fleet_price_per_hour += len(launched) * price_per_hour
+        if not instance_ids:
+            raise SystemExit(
+                f"No EC2 capacity for any of {', '.join(prices)} in {self.config.region}. "
+                "Retry later or add `--ec2-arg instance_type_fallbacks=...`."
+            )
+        try:
+            self._put_instance_secrets(
+                fleet_id,
+                instance_ids,
+                secrets,
+                setting_package_secrets,
+                fleet_tags,
+            )
+        except Exception:
+            self._ec2.terminate_instances(InstanceIds=instance_ids)
+            raise
+
+        if len(instance_ids) < num_workers:
+            logger.warning(
+                "Requested %d workers but only %d launched (AWS capacity). "
+                "Fleet will run slower but still complete.",
+                num_workers,
+                len(instance_ids),
+            )
+            print(
+                f"  Warning: only {len(instance_ids)}/{num_workers} workers launched "
+                f"(insufficient capacity). Fleet will run slower."
+            )
+
+        # Print console link for debugging
+        console_url = (
+            f"https://{self.config.region}.console.aws.amazon.com/ec2/home"
+            f"?region={self.config.region}#Instances:tag:FleetId={fleet_id};instanceState=running,pending"
+        )
+        print(f"  EC2 console: {console_url}")
+
+        return instance_ids, fleet_price_per_hour
+
+    def _launch_with_fallback(
+        self, launch_kwargs: dict, *, quota_fatal: bool
+    ) -> list[str]:
+        """Launch instances, falling back from spot→on-demand."""
+        try:
+            response = self._ec2.run_instances(**launch_kwargs)
+            instance_ids = [inst["InstanceId"] for inst in response["Instances"]]
+            if self.config.spot and instance_ids:
+                print(f"  Launched {len(instance_ids)} spot instances")
+            return instance_ids
+        except botocore.exceptions.ClientError as e:
+            error_code = e.response["Error"]["Code"]
+
+            # Spot capacity unavailable — fall back to on-demand
+            if self.config.spot and error_code in (
+                "InsufficientInstanceCapacity",
+                "SpotMaxPriceTooLow",
+                "MaxSpotInstanceCountExceeded",
+            ):
+                print(
+                    f"  Spot capacity unavailable ({error_code}), falling back to on-demand..."
+                )
+                fallback_kwargs = {
+                    k: v
+                    for k, v in launch_kwargs.items()
+                    if k != "InstanceMarketOptions"
+                }
+                try:
+                    response = self._ec2.run_instances(**fallback_kwargs)
+                    instance_ids = [
+                        inst["InstanceId"] for inst in response["Instances"]
+                    ]
+                    print(f"  Launched {len(instance_ids)} on-demand instances")
+                    return instance_ids
+                except botocore.exceptions.ClientError as e2:
+                    error_code = e2.response["Error"]["Code"]
+                    # Fall through to common error handling below
+                    e = e2
+
+            if error_code == "InsufficientInstanceCapacity":
+                print(f"  No {launch_kwargs['InstanceType']} capacity")
+                return []
+            if (
+                "VcpuLimitExceeded" in error_code
+                or "InstanceLimitExceeded" in error_code
+            ):
+                if not quota_fatal:
+                    print(
+                        f"  vCPU quota reached before {launch_kwargs['InstanceType']}"
+                    )
+                    return []
+                raise SystemExit(
+                    f"AWS vCPU quota exceeded for {launch_kwargs['InstanceType']} in {self.config.region}.\n"
+                    f"Request a quota increase at: https://console.aws.amazon.com/servicequotas/\n"
+                    f"Or reduce --ec2-arg max_workers=N (currently {self.config.max_workers})"
+                ) from e
+            if "PendingVerification" in error_code:
+                raise SystemExit(
+                    f"AWS account is pending verification in {self.config.region}.\n"
+                    "You cannot launch EC2 instances until verification completes.\n"
+                    "This usually resolves within minutes but can take up to 4 hours.\n"
+                    "Fleet state in S3 is preserved. Re-run the same command once "
+                    "verification completes; queues will be recreated."
+                ) from e
+            raise
+
+    def _worker_secrets(self, extra_env_var_names: set[str]) -> dict[str, str]:
+        names = sorted(set(WORKER_SECRET_ENV_VARS) | extra_env_var_names)
+        for name in names:
+            if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", name) or name.startswith(
+                "AWS_"
+            ):
+                raise ValueError(f"Invalid worker env var name: {name!r}")
+        secrets = {name: value for name in names if (value := os.environ.get(name))}
+        if len(json.dumps(secrets)) > 4096:
+            largest = sorted(secrets, key=lambda n: -len(secrets[n]))[:3]
+            raise ValueError(
+                "Worker secrets exceed the 4 KB SSM parameter limit; largest: "
+                + ", ".join(f"{n} ({len(secrets[n])} chars)" for n in largest)
+            )
+        return secrets
+
+    def _put_instance_secrets(
+        self,
+        fleet_id: str,
+        instance_ids: list[str],
+        secrets: dict[str, str],
+        setting_package_secrets: dict[str, str],
+        tags: list[dict[str, str]],
+    ) -> None:
+        """Store one SecureString per instance and one per instance and setting package, each tagged with the ARN that may read it.
+
+        Bootstrap polls for the instance's own parameter, so the setting packages' are written first.
+        """
+        account_id = boto3.client(
+            "sts", region_name=self.config.region
+        ).get_caller_identity()["Account"]
+        for instance_id in instance_ids:
+            instance_arn = (
+                f"arn:aws:ec2:{self.config.region}:{account_id}:instance/{instance_id}"
+            )
+            name = f"{FLEET_SSM_PREFIX}/{fleet_id}/{instance_id}"
+            values = {
+                **{
+                    f"{name}.env.{package}": package_values
+                    for package, package_values in setting_package_secrets.items()
+                },
+                name: json.dumps(secrets),
+            }
+            for parameter, value in values.items():
+                self._ssm.put_parameter(
+                    Name=parameter,
+                    Value=value,
+                    Type="SecureString",
+                    Tags=[*tags, {"Key": "InstanceArn", "Value": instance_arn}],
+                )
+
+    def _delete_fleet_secrets(self, fleet_id: str) -> None:
+        pages = self._ssm.get_paginator("get_parameters_by_path").paginate(
+            Path=f"{FLEET_SSM_PREFIX}/{fleet_id}/"
+        )
+        names = [p["Name"] for page in pages for p in page["Parameters"]]
+        for i in range(0, len(names), 10):
+            self._ssm.delete_parameters(Names=names[i : i + 10])
+
+    def _build_user_data(
+        self,
+        *,
+        code_s3_key: str,
+        rates_s3_key: str,
+        s3_prefix: str,
+        work_queue_url: str,
+        done_queue_url: str,
+        setting_archives: list[tuple[str, str]],
+        env_secret_packages: list[str],
+        fleet_secrets_prefix: str,
+    ) -> str:
+        """Fill the bootstrap.sh template into EC2 user-data for a worker.
+
+        Secrets are never embedded: the script carries the SSM prefix under
+        which the instance role reads its own instance's parameter.
+
+        Args:
+            setting_archives: Each archive's S3 key and the worker directory it is extracted into.
+        """
+
+        template_path = Path(__file__).parent / "bootstrap.sh"
+        template = template_path.read_text()
+
+        def _escape_bash_sq(s: str) -> str:
+            """Escape a string for embedding in bash single quotes."""
+            return s.replace("'", "'\\''")
+
+        return template.format(
+            s3_bucket=self.s3_bucket,
+            s3_prefix=s3_prefix,
+            code_s3_key=code_s3_key,
+            rates_s3_key=_escape_bash_sq(rates_s3_key),
+            worker_setup_s3_key=_escape_bash_sq(self._worker_setup_s3_key),
+            region=self.config.region,
+            work_queue_url=work_queue_url,
+            done_queue_url=done_queue_url,
+            concurrency=self.config.concurrency_per_worker,
+            idle_timeout=300,
+            fleet_secrets_prefix=_escape_bash_sq(fleet_secrets_prefix),
+            setting_archives=" ".join(
+                f"{key}={dest}" for key, dest in setting_archives
+            ),
+            env_secret_packages=_escape_bash_sq(" ".join(env_secret_packages)),
+            worker_user=_escape_bash_sq(self.config.worker_user),
+            inspect_log_level=_escape_bash_sq(
+                os.environ.get("INSPECT_LOG_LEVEL", "warning")
+            ),
+            docker_org_name_ssm_param=_escape_bash_sq(
+                EnvVar().dockerhub_username_ssm_param
+            ),
+            docker_org_pat_ssm_param=_escape_bash_sq(
+                EnvVar().dockerhub_token_ssm_param
+            ),
+        )
+
+    def _terminate_fleet_instances(self, fleet_id: str) -> None:
+        """Terminate EC2 instances for this fleet owned by the current user.
+
+        Always filters by LaunchedBy tag to prevent cross-user termination.
+        If we can't determine the caller's AWS identity, we skip termination
+        entirely rather than risk terminating another user's instances.
+        """
+        try:
+            username = caller_identity_name(self.config.region)
+        except Exception:
+            logger.warning(
+                "Cannot determine AWS identity — skipping fleet termination "
+                "to avoid cross-user impact. Instances will self-terminate on idle."
+            )
+            return
+
+        filters = [
+            {"Name": "tag:FleetId", "Values": [fleet_id]},
+            {"Name": "tag:LaunchedBy", "Values": [username]},
+            {"Name": "instance-state-name", "Values": ["running", "pending"]},
+        ]
+
+        resp = self._ec2.describe_instances(Filters=filters)
+        instance_ids = [
+            inst["InstanceId"]
+            for res in resp["Reservations"]
+            for inst in res["Instances"]
+        ]
+        if instance_ids:
+            self._ec2.terminate_instances(InstanceIds=instance_ids)
+            print(f"Terminated {len(instance_ids)} instances (owner: {username})")
+
+    def _cleanup_fleet_resources(
+        self, fleet_id: str, *, wait_for_workers: bool = True
+    ) -> None:
+        """Best-effort cleanup for queues and tagged workers after terminal results."""
+        if wait_for_workers:
+            logger.info("Waiting 30s for workers to drain before cleanup...")
+            time.sleep(30)
+        try:
+            self._delete_queues(fleet_id)
+        except Exception as e:
+            logger.warning("Failed to delete fleet queues for %s: %s", fleet_id, e)
+        try:
+            self._terminate_fleet_instances(fleet_id)
+        except Exception as e:
+            logger.warning(
+                "Failed to terminate fleet instances for %s: %s", fleet_id, e
+            )
+        try:
+            self._delete_fleet_secrets(fleet_id)
+        except Exception as e:
+            logger.warning("Failed to delete fleet secrets for %s: %s", fleet_id, e)
+
+    def _print_estimate(
+        self, total_jobs: int, num_workers: int, fleet_cost_per_hour: float
+    ) -> None:
+        print(f"\nDistributed EC2 eval: {total_jobs} samples")
+        spot_label = " (spot)" if self.config.spot else ""
+        print(
+            f"Launching {num_workers}x {self.config.instance_type}{spot_label} ({min(self.config.concurrency_per_worker, FLEET_PLANNING_CONCURRENCY)} per worker) in {self.config.region}"
+        )
+        price_note = (
+            "" if self.config.region == "us-east-1" else " (prices assume us-east-1)"
+        )
+        if self.config.spot:
+            price_note += (
+                " (spot pricing varies; estimate uses on-demand rates as ceiling)"
+            )
+        print(
+            f"Est compute: ~${fleet_cost_per_hour:.2f}/hr  |  1h: ~${fleet_cost_per_hour:.0f}  |  3h: ~${fleet_cost_per_hour * 3:.0f}  |  6h: ~${fleet_cost_per_hour * 6:.0f}{price_note}"
+        )
+
+    def _wait_for_confirm(self) -> None:
+        """Wait 5 seconds, abort on Ctrl+C."""
+        print("Ctrl+C in 5s to cancel...")
+        try:
+            for i in range(5, 0, -1):
+                print(f"  {i}...", end=" ", flush=True)
+                time.sleep(1)
+            print()
+        except KeyboardInterrupt:
+            print("\nAborted.")
+            raise SystemExit(0)
+
+
+def _code_root() -> Path:
+    """The control-tower checkout fleet workers run: for a git or package-index install, its commit or release tag fetched into the cache; otherwise the checkout this runs from."""
+    dist = distribution("control-tower")
+    direct_url = json.loads(dist.read_text("direct_url.json") or "{}")
+    vcs_info = direct_url.get("vcs_info") or {}
+    if vcs_info.get("vcs") == "git":
+        url, ref = direct_url["url"], vcs_info["commit_id"]
+    elif not direct_url:
+        url, ref = "https://github.com/linuxarena/control-tower", f"v{dist.version}"
+    else:
+        return find_repo_root(Path(__file__).resolve().parent)
+    checkout = cache_dir("fleet-checkouts") / ref
+    if not checkout.is_dir():
+        with tempfile.TemporaryDirectory(dir=checkout.parent) as scratch:
+            for args in (
+                ["init", "-q"],
+                ["fetch", "-q", "--depth", "1", url, ref],
+                ["checkout", "-q", "FETCH_HEAD"],
+            ):
+                try:
+                    subprocess.run(
+                        ["git", *args],
+                        cwd=scratch,
+                        check=True,
+                        capture_output=True,
+                        text=True,
+                    )
+                except subprocess.CalledProcessError as exc:
+                    raise OperationError(
+                        f"Cannot fetch {ref} from {url} for fleet workers ({exc.stderr.strip()})."
+                    ) from exc
+            # Another driver fetching the same ref may have put its checkout there first.
+            with contextlib.suppress(OSError):
+                os.rename(scratch, checkout)
+    return checkout
+
+
+def _git_snapshot(path: Path) -> tuple[str, bool]:
+    """The git object to archive for ``path``: HEAD if clean, else the dirty worktree's tree, narrowed to ``path`` inside its repo; and whether it differs from HEAD's."""
+
+    def git(*args: str, env: dict[str, str] | None = None) -> str:
+        return subprocess.run(
+            ["git", *args],
+            cwd=path,
+            capture_output=True,
+            text=True,
+            check=True,
+            env=env,
+        ).stdout.strip()
+
+    try:
+        prefix = git("rev-parse", "--show-prefix")
+        # A private copy of the index, so a concurrent git command holding index.lock can't fail this.
+        with tempfile.TemporaryDirectory(prefix="ct-index-") as tmp:
+            index = Path(tmp) / "index"
+            real_index = Path(path, git("rev-parse", "--git-path", "index"))
+            if real_index.exists():
+                shutil.copy2(real_index, index)
+            env = {**os.environ, "GIT_INDEX_FILE": str(index)}
+            git("add", "--update", "--", ":/", env=env)
+            tree = git("write-tree", env=env)
+        if tree == git("rev-parse", "HEAD^{tree}"):
+            return git("rev-parse", f"HEAD:{prefix}" if prefix else "HEAD"), False
+        tree = git("rev-parse", f"{tree}:{prefix}")
+        head_tree = subprocess.run(
+            ["git", "rev-parse", "--verify", "--quiet", f"HEAD:{prefix}"],
+            cwd=path,
+            capture_output=True,
+            text=True,
+        ).stdout.strip()
+        return tree, tree != head_tree
+    except subprocess.CalledProcessError as exc:
+        raise ValidationError(
+            f"Cannot ship {path} to fleet workers: git cannot archive it "
+            f"({exc.stderr.strip()}). Workers get only tracked files from a git checkout."
+        ) from exc