devagent-physical-engine 0.10.0__py3-none-any.whl

devagent_physical_engine/qualification_harness.py

@@ -0,0 +1,917 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from enum import Enum
+import hashlib
+import itertools
+import json
+from pathlib import Path
+import random
+import re
+from time import perf_counter
+from typing import Callable, Iterable, Mapping, Sequence
+
+from .execution import ExecutionMode, ExecutionSupervisor
+from .models import Goal, WorldState
+from .planning import CapabilityPlanner, PlanningError
+from .robots import CATALOG
+from .simulation import SimulationBackend, SimulationStep
+
+
+class QualificationError(RuntimeError):
+    """Stable qualification failure safe to surface through the CLI."""
+
+
+class Verdict(str, Enum):
+    PASS = "PASS"
+    FAIL = "FAIL"
+
+
+FAULT_MODES = frozenset(
+    {
+        "none",
+        "missing_object",
+        "robot_not_ready",
+        "collision_t1",
+        "collision_t2",
+        "collision_t3",
+        "joint_limit_t1",
+        "joint_limit_t2",
+        "joint_limit_t3",
+        "simulation_t1",
+        "simulation_t2",
+        "simulation_t3",
+        "unsupported_goal",
+    }
+)
+
+_CASE_ID_RE = re.compile(r"^CASE-\d{6}-[0-9a-f]{8}$")
+
+
+def _tuple_of_strings(value: object, *, field_name: str) -> tuple[str, ...]:
+    if value is None:
+        return ()
+    if isinstance(value, str):
+        values = (value,)
+    elif isinstance(value, Sequence) and not isinstance(value, (bytes, bytearray)):
+        values = tuple(str(item) for item in value)
+    else:
+        raise QualificationError(f"invalid_{field_name}")
+    normalized = tuple(item.strip() for item in values)
+    if any(not item for item in normalized):
+        raise QualificationError(f"empty_{field_name}")
+    return normalized
+
+
+def _normalize_expected(value: object) -> str:
+    text = str(value if value is not None else "AUTO").strip().upper()
+    if text not in {"AUTO", Verdict.PASS.value, Verdict.FAIL.value}:
+        raise QualificationError("invalid_expected_verdict")
+    return text
+
+
+@dataclass(frozen=True, slots=True)
+class ScenarioSpec:
+    name: str
+    robot: str
+    action: str
+    object_id: str
+    source: str
+    destination: str
+    initial_facts: tuple[str, ...] = ("robot_ready", "object_available")
+    fault_mode: str = "none"
+    expected: str = "AUTO"
+    variations: Mapping[str, tuple[str, ...]] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        for field_name, value in (
+            ("name", self.name),
+            ("robot", self.robot),
+            ("action", self.action),
+            ("object_id", self.object_id),
+            ("source", self.source),
+            ("destination", self.destination),
+        ):
+            if not value.strip():
+                raise QualificationError(f"empty_{field_name}")
+        if self.robot not in CATALOG:
+            raise QualificationError(f"unknown_robot:{self.robot}")
+        if self.fault_mode not in FAULT_MODES:
+            raise QualificationError(f"unknown_fault_mode:{self.fault_mode}")
+        _normalize_expected(self.expected)
+        if any(not fact.strip() for fact in self.initial_facts):
+            raise QualificationError("empty_initial_fact")
+
+        allowed = {"robots", "objects", "sources", "destinations", "fault_modes"}
+        unknown = set(self.variations) - allowed
+        if unknown:
+            raise QualificationError(
+                "unknown_variation_keys:" + ",".join(sorted(unknown))
+            )
+        for key, values in self.variations.items():
+            if not values:
+                raise QualificationError(f"empty_variation:{key}")
+            if key == "robots":
+                unknown_robots = sorted(set(values) - set(CATALOG))
+                if unknown_robots:
+                    raise QualificationError(
+                        "unknown_variation_robot:" + ",".join(unknown_robots)
+                    )
+            if key == "fault_modes":
+                unknown_faults = sorted(set(values) - FAULT_MODES)
+                if unknown_faults:
+                    raise QualificationError(
+                        "unknown_variation_fault:" + ",".join(unknown_faults)
+                    )
+
+    def to_dict(self) -> dict:
+        return {
+            "name": self.name,
+            "robot": self.robot,
+            "goal": {
+                "action": self.action,
+                "object": self.object_id,
+                "source": self.source,
+                "destination": self.destination,
+            },
+            "initial_facts": list(self.initial_facts),
+            "fault_mode": self.fault_mode,
+            "expected": self.expected,
+            "variations": {
+                key: list(values) for key, values in sorted(self.variations.items())
+            },
+        }
+
+
+def load_scenario(path: Path | str) -> ScenarioSpec:
+    target = Path(path).expanduser().resolve()
+    if not target.is_file():
+        raise QualificationError(f"scenario_not_found:{target}")
+    try:
+        text = target.read_text(encoding="utf-8")
+    except OSError as exc:
+        raise QualificationError(f"scenario_read_failed:{target}") from exc
+
+    try:
+        if target.suffix.lower() == ".json":
+            payload = json.loads(text)
+        else:
+            import yaml
+
+            payload = yaml.safe_load(text)
+    except Exception as exc:
+        raise QualificationError(f"scenario_parse_failed:{target.name}") from exc
+
+    if not isinstance(payload, dict):
+        raise QualificationError("scenario_root_must_be_mapping")
+    goal = payload.get("goal")
+    if not isinstance(goal, dict):
+        raise QualificationError("scenario_goal_must_be_mapping")
+    variations_raw = payload.get("variations") or {}
+    if not isinstance(variations_raw, dict):
+        raise QualificationError("scenario_variations_must_be_mapping")
+    variations = {
+        str(key): _tuple_of_strings(value, field_name=f"variation_{key}")
+        for key, value in variations_raw.items()
+    }
+
+    return ScenarioSpec(
+        name=str(payload.get("name", target.stem)).strip(),
+        robot=str(payload.get("robot", "ur5e")).strip(),
+        action=str(goal.get("action", "load")).strip(),
+        object_id=str(goal.get("object", "P17")).strip(),
+        source=str(goal.get("source", "conveyor_a")).strip(),
+        destination=str(goal.get("destination", "cnc_04")).strip(),
+        initial_facts=_tuple_of_strings(
+            payload.get("initial_facts", ("robot_ready", "object_available")),
+            field_name="initial_facts",
+        ),
+        fault_mode=str(payload.get("fault_mode", "none")).strip().lower(),
+        expected=_normalize_expected(payload.get("expected", "AUTO")),
+        variations=variations,
+    )
+
+
+def load_scenario_suite(path: Path | str) -> tuple[ScenarioSpec, ...]:
+    root = Path(path).expanduser().resolve()
+    if not root.is_dir():
+        raise QualificationError(f"scenario_suite_not_found:{root}")
+    files = tuple(
+        item
+        for item in sorted(root.iterdir())
+        if item.is_file() and item.suffix.lower() in {".yaml", ".yml", ".json"}
+    )
+    if not files:
+        raise QualificationError("scenario_suite_empty")
+    return tuple(load_scenario(item) for item in files)
+
+
+def expected_verdict_for_fault(fault_mode: str) -> Verdict:
+    return Verdict.PASS if fault_mode == "none" else Verdict.FAIL
+
+
+def expected_reason_for_fault(fault_mode: str) -> str:
+    if fault_mode == "none":
+        return "goal_complete"
+    if fault_mode == "missing_object":
+        return "missing_precondition:object_available"
+    if fault_mode == "robot_not_ready":
+        return "missing_precondition:robot_ready"
+    if fault_mode.startswith("collision_"):
+        return "unsatisfied_constraint:collision_free"
+    if fault_mode.startswith("joint_limit_"):
+        return "unsatisfied_constraint:joint_limits"
+    if fault_mode.startswith("simulation_"):
+        return "simulation_failed"
+    if fault_mode == "unsupported_goal":
+        return "planning_error"
+    raise QualificationError(f"unknown_fault_mode:{fault_mode}")
+
+
+@dataclass(frozen=True, slots=True)
+class ScenarioCase:
+    case_id: str
+    scenario_name: str
+    index: int
+    seed: int
+    robot: str
+    action: str
+    object_id: str
+    source: str
+    destination: str
+    initial_facts: tuple[str, ...]
+    fault_mode: str
+    expected: Verdict
+    expected_reason: str
+
+    def to_dict(self) -> dict:
+        return {
+            "case_id": self.case_id,
+            "scenario_name": self.scenario_name,
+            "index": self.index,
+            "seed": self.seed,
+            "robot": self.robot,
+            "goal": {
+                "action": self.action,
+                "object": self.object_id,
+                "source": self.source,
+                "destination": self.destination,
+            },
+            "initial_facts": list(self.initial_facts),
+            "fault_mode": self.fault_mode,
+            "expected": self.expected.value,
+            "expected_reason": self.expected_reason,
+        }
+
+    def behavior_fingerprint(self) -> str:
+        payload = self.to_dict()
+        payload.pop("case_id", None)
+        payload.pop("index", None)
+        payload.pop("seed", None)
+        encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"))
+        return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
+
+
+def _case_id(payload: Mapping[str, object], *, index: int) -> str:
+    encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"))
+    digest = hashlib.sha256(encoded.encode("utf-8")).hexdigest()[:8]
+    return f"CASE-{index:06d}-{digest}"
+
+
+def _oracle_for_case(
+    initial_facts: Sequence[str],
+    fault_mode: str,
+) -> tuple[Verdict, str]:
+    facts = set(initial_facts)
+    if fault_mode == "missing_object":
+        facts.discard("object_available")
+    elif fault_mode == "robot_not_ready":
+        facts.discard("robot_ready")
+
+    if "robot_ready" not in facts:
+        return Verdict.FAIL, "missing_precondition:robot_ready"
+    if "object_available" not in facts:
+        return Verdict.FAIL, "missing_precondition:object_available"
+    return expected_verdict_for_fault(fault_mode), expected_reason_for_fault(fault_mode)
+
+
+def _build_case(
+    spec: ScenarioSpec,
+    *,
+    index: int,
+    seed: int,
+    robot: str,
+    object_id: str,
+    source: str,
+    destination: str,
+    fault_mode: str,
+) -> ScenarioCase:
+    expected, expected_reason = _oracle_for_case(spec.initial_facts, fault_mode)
+    if spec.expected != "AUTO" and Verdict(spec.expected) is not expected:
+        raise QualificationError(
+            f"scenario_expected_conflicts_with_oracle:{spec.expected}:{expected.value}"
+        )
+    payload = {
+        "scenario_name": spec.name,
+        "seed": seed,
+        "robot": robot,
+        "action": spec.action,
+        "object": object_id,
+        "source": source,
+        "destination": destination,
+        "initial_facts": spec.initial_facts,
+        "fault_mode": fault_mode,
+        "expected": expected.value,
+        "expected_reason": expected_reason,
+    }
+    return ScenarioCase(
+        case_id=_case_id(payload, index=index),
+        scenario_name=spec.name,
+        index=index,
+        seed=seed,
+        robot=robot,
+        action=spec.action,
+        object_id=object_id,
+        source=source,
+        destination=destination,
+        initial_facts=spec.initial_facts,
+        fault_mode=fault_mode,
+        expected=expected,
+        expected_reason=expected_reason,
+    )
+
+
+def base_case(spec: ScenarioSpec, *, index: int = 1, seed: int = 0) -> ScenarioCase:
+    return _build_case(
+        spec,
+        index=index,
+        seed=seed,
+        robot=spec.robot,
+        object_id=spec.object_id,
+        source=spec.source,
+        destination=spec.destination,
+        fault_mode=spec.fault_mode,
+    )
+
+
+def variation_space_size(spec: ScenarioSpec) -> int:
+    options = (
+        spec.variations.get("robots", (spec.robot,)),
+        spec.variations.get("objects", (spec.object_id,)),
+        spec.variations.get("sources", (spec.source,)),
+        spec.variations.get("destinations", (spec.destination,)),
+        spec.variations.get("fault_modes", (spec.fault_mode,)),
+    )
+    size = 1
+    for values in options:
+        size *= len(values)
+    return size
+
+
+def generate_fuzz_cases(
+    spec: ScenarioSpec,
+    *,
+    runs: int,
+    seed: int,
+) -> tuple[ScenarioCase, ...]:
+    if runs <= 0:
+        raise QualificationError("runs_must_be_positive")
+    if runs > 1_000_000:
+        raise QualificationError("runs_exceed_limit:1000000")
+
+    options = (
+        spec.variations.get("robots", (spec.robot,)),
+        spec.variations.get("objects", (spec.object_id,)),
+        spec.variations.get("sources", (spec.source,)),
+        spec.variations.get("destinations", (spec.destination,)),
+        spec.variations.get("fault_modes", (spec.fault_mode,)),
+    )
+    if runs > 1 and variation_space_size(spec) == 1:
+        raise QualificationError("fuzz_requires_variations")
+
+    rng = random.Random(seed)
+    space = variation_space_size(spec)
+    selections: list[tuple[str, str, str, str, str]] = []
+
+    if runs <= space and space <= 200_000:
+        population = list(itertools.product(*options))
+        selections = rng.sample(population, runs)
+    elif runs <= space:
+        selected: set[tuple[str, str, str, str, str]] = set()
+        while len(selections) < runs:
+            choice = tuple(rng.choice(values) for values in options)
+            if choice in selected:
+                continue
+            selected.add(choice)
+            selections.append(choice)
+    else:
+        selections = [
+            tuple(rng.choice(values) for values in options)
+            for _ in range(runs)
+        ]
+
+    cases: list[ScenarioCase] = []
+    for index, selection in enumerate(selections, start=1):
+        robot, object_id, source, destination, fault_mode = selection
+        cases.append(
+            _build_case(
+                spec,
+                index=index,
+                seed=seed,
+                robot=robot,
+                object_id=object_id,
+                source=source,
+                destination=destination,
+                fault_mode=fault_mode,
+            )
+        )
+    return tuple(cases)
+
+
+def _target_task(fault_mode: str) -> str | None:
+    match = re.search(r"_t([123])$", fault_mode)
+    return f"T{match.group(1)}" if match else None
+
+
+class QualificationSimulationBackend:
+    """Deterministic model backend used only by the qualification harness.
+
+    It exercises the engine's runtime precondition, simulation-result and declared
+    constraint gates. It is not a physics engine and its results are explicitly
+    reported as non-physical qualification.
+    """
+
+    def __init__(self, fault_mode: str) -> None:
+        if fault_mode not in FAULT_MODES:
+            raise QualificationError(f"unknown_fault_mode:{fault_mode}")
+        self.fault_mode = fault_mode
+
+    def run_task(self, task, world: WorldState) -> SimulationStep:
+        constraints = set(task.contract.constraints)
+        target = _target_task(self.fault_mode)
+        if target == task.task_id and self.fault_mode.startswith("collision_"):
+            constraints.discard("collision_free")
+            return SimulationStep(
+                task.task_id,
+                False,
+                frozenset(constraints),
+                f"qualification_fault:{self.fault_mode}",
+            )
+        if target == task.task_id and self.fault_mode.startswith("joint_limit_"):
+            constraints.discard("joint_limits")
+            return SimulationStep(
+                task.task_id,
+                False,
+                frozenset(constraints),
+                f"qualification_fault:{self.fault_mode}",
+            )
+        if target == task.task_id and self.fault_mode.startswith("simulation_"):
+            return SimulationStep(
+                task.task_id,
+                False,
+                frozenset(constraints),
+                f"qualification_fault:{self.fault_mode}",
+            )
+        return SimulationStep(task.task_id, True, frozenset(constraints))
+
+
+@dataclass(frozen=True, slots=True)
+class CaseResult:
+    case: ScenarioCase
+    actual: Verdict
+    actual_reason: str
+    verdict_correct: bool
+    reason_correct: bool
+    unsafe_false_pass: bool
+    false_fail: bool
+    executed_tasks: tuple[str, ...]
+    blocked_task: str | None
+    duration_ms: float
+
+    def to_dict(self) -> dict:
+        return {
+            "case": self.case.to_dict(),
+            "actual": self.actual.value,
+            "actual_reason": self.actual_reason,
+            "verdict_correct": self.verdict_correct,
+            "reason_correct": self.reason_correct,
+            "unsafe_false_pass": self.unsafe_false_pass,
+            "false_fail": self.false_fail,
+            "executed_tasks": list(self.executed_tasks),
+            "blocked_task": self.blocked_task,
+            "duration_ms": round(self.duration_ms, 6),
+        }
+
+
+def _report_reason(report, world: WorldState) -> str:
+    if report.completed:
+        return "goal_complete" if "goal_complete" in world.facts else "completed_without_goal"
+    if not report.issues:
+        return "unknown_failure"
+    issue = report.issues[0]
+    if issue.code in {"missing_precondition", "unsatisfied_constraint"}:
+        return f"{issue.code}:{issue.detail}"
+    return issue.code
+
+
+def run_case(
+    case: ScenarioCase,
+    *,
+    backend_factory: Callable[[str], SimulationBackend] | None = None,
+) -> CaseResult:
+    started = perf_counter()
+    facts = set(case.initial_facts)
+    if case.fault_mode == "missing_object":
+        facts.discard("object_available")
+    elif case.fault_mode == "robot_not_ready":
+        facts.discard("robot_ready")
+
+    world = WorldState(facts=facts)
+    actual = Verdict.FAIL
+    actual_reason = "unknown_failure"
+    executed_tasks: tuple[str, ...] = ()
+    blocked_task: str | None = None
+
+    try:
+        resource = CATALOG[case.robot]()
+        action = "unsupported_action" if case.fault_mode == "unsupported_goal" else case.action
+        goal = Goal(
+            case.case_id,
+            action,
+            case.object_id,
+            case.source,
+            case.destination,
+        )
+        graph = CapabilityPlanner().plan(goal, [resource])
+        backend = (
+            backend_factory(case.fault_mode)
+            if backend_factory is not None
+            else QualificationSimulationBackend(case.fault_mode)
+        )
+        report = ExecutionSupervisor().execute(
+            graph,
+            [resource],
+            world,
+            backend,
+            ExecutionMode.SIMULATION,
+        )
+        actual = Verdict.PASS if report.completed else Verdict.FAIL
+        actual_reason = _report_reason(report, world)
+        executed_tasks = tuple(report.executed_tasks)
+        blocked_task = report.blocked_task
+    except PlanningError:
+        actual = Verdict.FAIL
+        actual_reason = "planning_error"
+    except Exception as exc:
+        actual = Verdict.FAIL
+        actual_reason = f"runner_error:{type(exc).__name__}"
+
+    verdict_correct = actual is case.expected
+    reason_correct = verdict_correct and actual_reason == case.expected_reason
+    return CaseResult(
+        case=case,
+        actual=actual,
+        actual_reason=actual_reason,
+        verdict_correct=verdict_correct,
+        reason_correct=reason_correct,
+        unsafe_false_pass=case.expected is Verdict.FAIL and actual is Verdict.PASS,
+        false_fail=case.expected is Verdict.PASS and actual is Verdict.FAIL,
+        executed_tasks=executed_tasks,
+        blocked_task=blocked_task,
+        duration_ms=(perf_counter() - started) * 1000.0,
+    )
+
+
+@dataclass(frozen=True, slots=True)
+class QualificationSummary:
+    run_id: str
+    mode: str
+    backend: str
+    physical_qualification: bool
+    seed: int | None
+    scenario_names: tuple[str, ...]
+    total_cases: int
+    unique_behavioral_cases: int
+    correct_verdicts: int
+    correct_reasons: int
+    unsafe_false_passes: int
+    false_fails: int
+    wrong_reasons: int
+    expected_passes: int
+    expected_failures: int
+    actual_passes: int
+    actual_failures: int
+    corpus_hash: str
+    duration_ms: float
+
+    @property
+    def verdict_accuracy(self) -> float:
+        return 1.0 if self.total_cases == 0 else self.correct_verdicts / self.total_cases
+
+    @property
+    def reason_accuracy(self) -> float:
+        return 1.0 if self.total_cases == 0 else self.correct_reasons / self.total_cases
+
+    @property
+    def qualified(self) -> bool:
+        return (
+            self.total_cases > 0
+            and self.unsafe_false_passes == 0
+            and self.correct_verdicts == self.total_cases
+            and self.correct_reasons == self.total_cases
+        )
+
+    def to_dict(self) -> dict:
+        return {
+            "run_id": self.run_id,
+            "mode": self.mode,
+            "backend": self.backend,
+            "physical_qualification": self.physical_qualification,
+            "seed": self.seed,
+            "scenario_names": list(self.scenario_names),
+            "total_cases": self.total_cases,
+            "unique_behavioral_cases": self.unique_behavioral_cases,
+            "duplicate_cases": self.total_cases - self.unique_behavioral_cases,
+            "correct_verdicts": self.correct_verdicts,
+            "correct_reasons": self.correct_reasons,
+            "unsafe_false_passes": self.unsafe_false_passes,
+            "false_fails": self.false_fails,
+            "wrong_reasons": self.wrong_reasons,
+            "expected_passes": self.expected_passes,
+            "expected_failures": self.expected_failures,
+            "actual_passes": self.actual_passes,
+            "actual_failures": self.actual_failures,
+            "verdict_accuracy": round(self.verdict_accuracy, 8),
+            "reason_accuracy": round(self.reason_accuracy, 8),
+            "corpus_hash": self.corpus_hash,
+            "duration_ms": round(self.duration_ms, 6),
+            "qualified": self.qualified,
+        }
+
+
+def _corpus_hash(cases: Iterable[ScenarioCase]) -> str:
+    digest = hashlib.sha256()
+    for case in cases:
+        digest.update(
+            json.dumps(
+                case.to_dict(),
+                sort_keys=True,
+                separators=(",", ":"),
+            ).encode("utf-8")
+        )
+        digest.update(b"\n")
+    return digest.hexdigest()
+
+
+def summarize_results(
+    results: Sequence[CaseResult],
+    *,
+    mode: str,
+    seed: int | None,
+    run_id: str | None = None,
+    duration_ms: float | None = None,
+) -> QualificationSummary:
+    if not results:
+        raise QualificationError("no_qualification_results")
+    cases = tuple(result.case for result in results)
+    return QualificationSummary(
+        run_id=run_id or make_run_id(mode=mode, seed=seed),
+        mode=mode,
+        backend="deterministic_model",
+        physical_qualification=False,
+        seed=seed,
+        scenario_names=tuple(sorted({case.scenario_name for case in cases})),
+        total_cases=len(results),
+        unique_behavioral_cases=len({case.behavior_fingerprint() for case in cases}),
+        correct_verdicts=sum(result.verdict_correct for result in results),
+        correct_reasons=sum(result.reason_correct for result in results),
+        unsafe_false_passes=sum(result.unsafe_false_pass for result in results),
+        false_fails=sum(result.false_fail for result in results),
+        wrong_reasons=sum(
+            result.verdict_correct and not result.reason_correct for result in results
+        ),
+        expected_passes=sum(result.case.expected is Verdict.PASS for result in results),
+        expected_failures=sum(result.case.expected is Verdict.FAIL for result in results),
+        actual_passes=sum(result.actual is Verdict.PASS for result in results),
+        actual_failures=sum(result.actual is Verdict.FAIL for result in results),
+        corpus_hash=_corpus_hash(cases),
+        duration_ms=(
+            duration_ms
+            if duration_ms is not None
+            else sum(result.duration_ms for result in results)
+        ),
+    )
+
+
+def run_cases(
+    cases: Sequence[ScenarioCase],
+    *,
+    mode: str,
+    seed: int | None,
+) -> tuple[tuple[CaseResult, ...], QualificationSummary]:
+    started = perf_counter()
+    results = tuple(run_case(case) for case in cases)
+    summary = summarize_results(
+        results,
+        mode=mode,
+        seed=seed,
+        duration_ms=(perf_counter() - started) * 1000.0,
+    )
+    return results, summary
+
+
+def make_run_id(*, mode: str, seed: int | None) -> str:
+    timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S%fZ")
+    suffix = "noseed" if seed is None else f"seed{seed}"
+    return f"{mode}-{timestamp}-{suffix}"
+
+
+class QualificationStore:
+    def __init__(self, root: Path | str = "~/.devagent/qualification") -> None:
+        self.root = Path(root).expanduser().resolve()
+
+    def write_run(
+        self,
+        *,
+        scenarios: Sequence[ScenarioSpec],
+        results: Sequence[CaseResult],
+        summary: QualificationSummary,
+    ) -> Path:
+        run_dir = self.root / summary.run_id
+        run_dir.mkdir(parents=True, exist_ok=False)
+        (run_dir / "scenarios.json").write_text(
+            json.dumps([scenario.to_dict() for scenario in scenarios], indent=2) + "\n",
+            encoding="utf-8",
+        )
+        (run_dir / "summary.json").write_text(
+            json.dumps(summary.to_dict(), indent=2) + "\n",
+            encoding="utf-8",
+        )
+        with (run_dir / "cases.jsonl").open("w", encoding="utf-8") as handle:
+            for result in results:
+                handle.write(json.dumps(result.to_dict(), sort_keys=True) + "\n")
+
+        mismatches = [
+            result
+            for result in results
+            if not result.verdict_correct or not result.reason_correct
+        ]
+        if mismatches:
+            mismatch_dir = run_dir / "mismatches"
+            mismatch_dir.mkdir()
+            for result in mismatches:
+                (mismatch_dir / f"{result.case.case_id}.json").write_text(
+                    json.dumps(result.to_dict(), indent=2) + "\n",
+                    encoding="utf-8",
+                )
+
+        self.root.mkdir(parents=True, exist_ok=True)
+        (self.root / "latest-run.json").write_text(
+            json.dumps({"run_dir": str(run_dir)}, indent=2) + "\n",
+            encoding="utf-8",
+        )
+        return run_dir
+
+    def resolve_run(self, run_ref: Path | str | None = None) -> Path:
+        if run_ref is None:
+            pointer = self.root / "latest-run.json"
+            if not pointer.is_file():
+                raise QualificationError("latest_run_not_found")
+            try:
+                payload = json.loads(pointer.read_text(encoding="utf-8"))
+                run_dir = Path(str(payload["run_dir"])).expanduser().resolve()
+            except Exception as exc:
+                raise QualificationError("latest_run_pointer_invalid") from exc
+        else:
+            run_dir = Path(run_ref).expanduser().resolve()
+        if not run_dir.is_dir() or not (run_dir / "cases.jsonl").is_file():
+            raise QualificationError(f"qualification_run_not_found:{run_dir}")
+        return run_dir
+
+    def load_case_record(
+        self,
+        case_id: str,
+        *,
+        run_ref: Path | str | None = None,
+    ) -> dict:
+        if not _CASE_ID_RE.fullmatch(case_id):
+            raise QualificationError("invalid_case_id")
+        run_dir = self.resolve_run(run_ref)
+        with (run_dir / "cases.jsonl").open("r", encoding="utf-8") as handle:
+            for line in handle:
+                payload = json.loads(line)
+                case = payload.get("case") or {}
+                if case.get("case_id") == case_id:
+                    return payload
+        raise QualificationError(f"case_not_found:{case_id}")
+
+
+def case_from_dict(payload: Mapping[str, object]) -> ScenarioCase:
+    goal = payload.get("goal")
+    if not isinstance(goal, Mapping):
+        raise QualificationError("case_goal_invalid")
+    return ScenarioCase(
+        case_id=str(payload["case_id"]),
+        scenario_name=str(payload["scenario_name"]),
+        index=int(payload["index"]),
+        seed=int(payload["seed"]),
+        robot=str(payload["robot"]),
+        action=str(goal["action"]),
+        object_id=str(goal["object"]),
+        source=str(goal["source"]),
+        destination=str(goal["destination"]),
+        initial_facts=_tuple_of_strings(
+            payload.get("initial_facts", ()),
+            field_name="case_initial_facts",
+        ),
+        fault_mode=str(payload["fault_mode"]),
+        expected=Verdict(str(payload["expected"])),
+        expected_reason=str(payload["expected_reason"]),
+    )
+
+
+def replay_case(
+    case_id: str,
+    *,
+    run_ref: Path | str | None = None,
+    store_root: Path | str = "~/.devagent/qualification",
+) -> dict:
+    store = QualificationStore(store_root)
+    recorded = store.load_case_record(case_id, run_ref=run_ref)
+    case = case_from_dict(recorded["case"])
+    replayed = run_case(case)
+    recorded_actual = str(recorded.get("actual"))
+    recorded_reason = str(recorded.get("actual_reason"))
+    reproduced = (
+        replayed.actual.value == recorded_actual
+        and replayed.actual_reason == recorded_reason
+    )
+    return {
+        "case_id": case_id,
+        "reproduced": reproduced,
+        "recorded_actual": recorded_actual,
+        "recorded_reason": recorded_reason,
+        "replayed": replayed.to_dict(),
+    }
+
+
+def load_summary(path: Path | str) -> dict:
+    target = Path(path).expanduser().resolve()
+    if target.is_dir():
+        target = target / "summary.json"
+    if not target.is_file():
+        raise QualificationError(f"summary_not_found:{target}")
+    try:
+        payload = json.loads(target.read_text(encoding="utf-8"))
+    except Exception as exc:
+        raise QualificationError(f"summary_invalid:{target}") from exc
+    required = {
+        "corpus_hash",
+        "total_cases",
+        "unsafe_false_passes",
+        "false_fails",
+        "wrong_reasons",
+        "verdict_accuracy",
+        "reason_accuracy",
+    }
+    if not isinstance(payload, dict) or not required.issubset(payload):
+        raise QualificationError(f"summary_schema_invalid:{target}")
+    return payload
+
+
+def compare_summaries(baseline: Mapping[str, object], candidate: Mapping[str, object]) -> dict:
+    comparable = (
+        baseline.get("corpus_hash") == candidate.get("corpus_hash")
+        and baseline.get("total_cases") == candidate.get("total_cases")
+    )
+    metrics = (
+        "unsafe_false_passes",
+        "false_fails",
+        "wrong_reasons",
+        "verdict_accuracy",
+        "reason_accuracy",
+    )
+    deltas: dict[str, float] = {}
+    for metric in metrics:
+        deltas[metric] = float(candidate[metric]) - float(baseline[metric])
+
+    regressions: list[str] = []
+    if comparable:
+        for metric in ("unsafe_false_passes", "false_fails", "wrong_reasons"):
+            if float(candidate[metric]) > float(baseline[metric]):
+                regressions.append(metric)
+        for metric in ("verdict_accuracy", "reason_accuracy"):
+            if float(candidate[metric]) < float(baseline[metric]):
+                regressions.append(metric)
+
+    return {
+        "comparable": comparable,
+        "same_corpus": baseline.get("corpus_hash") == candidate.get("corpus_hash"),
+        "total_cases": candidate.get("total_cases"),
+        "baseline_run_id": baseline.get("run_id"),
+        "candidate_run_id": candidate.get("run_id"),
+        "deltas": deltas,
+        "regressions": regressions,
+        "passed": comparable and not regressions,
+    }