flowspec2 1.0.0__py3-none-any.whl

flowspec2/authoring/benchmark.py

@@ -0,0 +1,1409 @@
+"""Deterministic, provider-neutral benchmarks for AI-authored flow sources.
+
+The benchmark deliberately does not call a model or estimate model quality.  A
+caller injects an author and a source adapter.  Every adapted document is judged
+by the same :func:`flowspec2.checker.check_flow` structural, semantic, and
+compilation contract.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from collections.abc import Callable, Iterable, Mapping, Sequence
+from dataclasses import dataclass, field
+from typing import Any, Final, Protocol, cast
+
+from flowspec2.checker import check_flow
+from flowspec2.diagnostics import CompilationStatus, FlowDiagnostic
+from flowspec2.ir import normalize_flow
+from flowspec2.json_codec import StrictJsonError, strict_json_loads
+from flowspec2.profiles import FlowProfile, reference_profile
+
+_IDENTIFIER_PATTERN: Final[re.Pattern[str]] = re.compile(r"^[a-z0-9][a-z0-9_-]*$")
+TOKEN_PROXY_METHOD: Final[str] = "utf8_byte_quartets"
+TOKEN_PROXY_BYTES_PER_UNIT: Final[int] = 4
+AUTHORING_ACCEPTANCE_FORMAT: Final[str] = "flowspec2/authoring-acceptance@2"
+DEFAULT_VARIABLE_POINTER_PATTERNS: Final[tuple[str, ...]] = (
+    "/flow",
+    "/route/description",
+    "/route/trigger_phrases",
+    "/version",
+)
+_VARIABLE_PATH_PROMPT_PATTERN: Final[re.Pattern[str]] = re.compile(
+    r"^/path/[0-9]+/prompt/(?:extract_hint|text)$"
+)
+
+
+def _require_identifier(identifier: str, contract_name: str) -> None:
+    if not _IDENTIFIER_PATTERN.fullmatch(identifier):
+        raise ValueError(
+            f"{contract_name} must contain only lowercase letters, digits, hyphens, "
+            f"and underscores: {identifier!r}"
+        )
+
+
+def _compact_json(json_document: object) -> str:
+    return json.dumps(
+        json_document,
+        ensure_ascii=False,
+        allow_nan=False,
+        separators=(",", ":"),
+        sort_keys=True,
+    )
+
+
+def _validate_variable_pointer_patterns(pointer_patterns: tuple[str, ...]) -> None:
+    if len(set(pointer_patterns)) != len(pointer_patterns):
+        raise ValueError("authoring variable pointer patterns must be unique")
+    if any(not pointer_pattern.startswith("/") for pointer_pattern in pointer_patterns):
+        raise ValueError("authoring variable pointer patterns must start with '/'")
+    if any(
+        pointer_pattern not in DEFAULT_VARIABLE_POINTER_PATTERNS
+        and not _VARIABLE_PATH_PROMPT_PATTERN.fullmatch(pointer_pattern)
+        for pointer_pattern in pointer_patterns
+    ):
+        raise ValueError(
+            "authoring variable pointers are restricted to versioned presentation paths"
+        )
+
+
+def _normalized_flow_json(flow_document: object) -> str:
+    if not isinstance(flow_document, dict):
+        raise ValueError("an authoring flow must be a JSON object")
+    return _compact_json(normalize_flow(cast(dict[str, Any], flow_document)))
+
+
+def _json_pointer(path_segments: Iterable[str | int]) -> str:
+    encoded_segments = (
+        str(path_segment).replace("~", "~0").replace("/", "~1") for path_segment in path_segments
+    )
+    return "".join(f"/{encoded_segment}" for encoded_segment in encoded_segments)
+
+
+def _diagnostic_sort_key(flow_diagnostic: FlowDiagnostic) -> tuple[str, str, str, str, str]:
+    return (
+        flow_diagnostic.path,
+        flow_diagnostic.code,
+        flow_diagnostic.severity,
+        flow_diagnostic.message,
+        _compact_json(flow_diagnostic.to_dict()),
+    )
+
+
+def _deterministic_diagnostics(
+    flow_diagnostics: Iterable[FlowDiagnostic],
+) -> tuple[FlowDiagnostic, ...]:
+    return tuple(sorted(set(flow_diagnostics), key=_diagnostic_sort_key))
+
+
+@dataclass(frozen=True)
+class SourceAdaptation:
+    """Immutable source and executable documents emitted by one adapter."""
+
+    format_identifier: str
+    authored_document_json: str | None
+    executable_flow_json: str | None
+    canonical_authored_source: str | None = None
+    diagnostics: tuple[FlowDiagnostic, ...] = ()
+
+    def __post_init__(self) -> None:
+        if not self.format_identifier:
+            raise ValueError("source adaptation format identifier must be non-empty")
+        for field_name, serialized_document in (
+            ("authored_document_json", self.authored_document_json),
+            ("executable_flow_json", self.executable_flow_json),
+        ):
+            if serialized_document is None:
+                continue
+            parsed_document = strict_json_loads(serialized_document)
+            if _compact_json(parsed_document) != serialized_document:
+                raise ValueError(f"source adaptation {field_name} must be canonical compact JSON")
+        if self.executable_flow_json is not None and self.authored_document_json is None:
+            raise ValueError("an executable flow requires an authored source document")
+        if (self.authored_document_json is None) != (self.canonical_authored_source is None):
+            raise ValueError(
+                "an authored document and its canonical source must be present together"
+            )
+        if self.canonical_authored_source is not None and not isinstance(
+            self.canonical_authored_source, str
+        ):
+            raise TypeError("source adaptation canonical authored source must be a string")
+        if self.canonical_authored_source == "":
+            raise ValueError("source adaptation canonical authored source must be non-empty")
+        object.__setattr__(
+            self,
+            "diagnostics",
+            _deterministic_diagnostics(self.diagnostics),
+        )
+
+    @classmethod
+    def from_documents(
+        cls,
+        *,
+        format_identifier: str,
+        authored_document: object,
+        executable_flow: object,
+        canonical_authored_source: str | None = None,
+        diagnostics: tuple[FlowDiagnostic, ...] = (),
+    ) -> SourceAdaptation:
+        """Create an adaptation with canonical documents and authored syntax."""
+
+        authored_document_json = _compact_json(authored_document)
+
+        return cls(
+            format_identifier=format_identifier,
+            authored_document_json=authored_document_json,
+            executable_flow_json=_compact_json(executable_flow),
+            canonical_authored_source=(
+                authored_document_json
+                if canonical_authored_source is None
+                else canonical_authored_source
+            ),
+            diagnostics=diagnostics,
+        )
+
+
+class AuthoringSourceAdapter(Protocol):
+    """Convert an authored source string into the executable flowspec contract."""
+
+    @property
+    def format_identifier(self) -> str: ...
+
+    def adapt(self, authored_source: str) -> SourceAdaptation: ...
+
+
+@dataclass(frozen=True)
+class FlowSpec2JsonAdapter:
+    """Identity adapter for JSON-authored ``flowspec/2`` documents."""
+
+    format_identifier: str = "flowspec/2"
+
+    def adapt(self, authored_source: str) -> SourceAdaptation:
+        try:
+            authored_document: object = strict_json_loads(authored_source)
+        except (json.JSONDecodeError, StrictJsonError) as decoding_error:
+            decoding_message = (
+                f"Invalid JSON at line {decoding_error.lineno}, "
+                f"column {decoding_error.colno}: {decoding_error.msg}"
+                if isinstance(decoding_error, json.JSONDecodeError)
+                else f"Invalid JSON: {decoding_error}"
+            )
+            return SourceAdaptation(
+                format_identifier=self.format_identifier,
+                authored_document_json=None,
+                executable_flow_json=None,
+                diagnostics=(
+                    FlowDiagnostic(
+                        code="AUTHORING_SOURCE_JSON_INVALID",
+                        severity="error",
+                        path="",
+                        message=decoding_message,
+                    ),
+                ),
+            )
+        return SourceAdaptation.from_documents(
+            format_identifier=self.format_identifier,
+            authored_document=authored_document,
+            executable_flow=authored_document,
+        )
+
+
+@dataclass(frozen=True)
+class ForbiddenConstruct:
+    """A JSON Pointer pattern for one intentionally unsupported construct."""
+
+    identifier: str
+    pointer_pattern: str
+    message: str
+
+    def __post_init__(self) -> None:
+        _require_identifier(self.identifier, "forbidden construct identifier")
+        if not self.pointer_pattern.startswith("/"):
+            raise ValueError("forbidden construct pointer pattern must start with '/'")
+        if not self.message:
+            raise ValueError("forbidden construct message must be non-empty")
+
+
+@dataclass(frozen=True)
+class RequiredFlowConstruct:
+    """A required executable-flow location and optional exact JSON value."""
+
+    identifier: str
+    pointer_pattern: str
+    expected_json: str | None = None
+
+    def __post_init__(self) -> None:
+        _require_identifier(self.identifier, "required flow construct identifier")
+        if not self.pointer_pattern.startswith("/"):
+            raise ValueError("required flow construct pointer pattern must start with '/'")
+        if self.expected_json is not None:
+            expected_value = strict_json_loads(self.expected_json)
+            if _compact_json(expected_value) != self.expected_json:
+                raise ValueError("required flow construct expected value must be canonical JSON")
+
+    @classmethod
+    def expecting(
+        cls,
+        identifier: str,
+        pointer_pattern: str,
+        expected_value: object,
+    ) -> RequiredFlowConstruct:
+        """Create a required construct with an exact JSON-compatible value."""
+
+        return cls(
+            identifier=identifier,
+            pointer_pattern=pointer_pattern,
+            expected_json=_compact_json(expected_value),
+        )
+
+    def to_dict(self) -> dict[str, object]:
+        """Return the public presence or exact-value requirement."""
+
+        required_construct: dict[str, object] = {
+            "identifier": self.identifier,
+            "pointer_pattern": self.pointer_pattern,
+        }
+        if self.expected_json is not None:
+            required_construct["expected"] = strict_json_loads(self.expected_json)
+        return required_construct
+
+
+@dataclass(frozen=True)
+class AcceptanceExpectation:
+    """One public semantic observation required from an executable flow."""
+
+    path: str
+    expected_json: str
+
+    def __post_init__(self) -> None:
+        if not self.path.startswith("/"):
+            raise ValueError("acceptance expectation path must start with '/'")
+        expected_value = strict_json_loads(self.expected_json)
+        if _compact_json(expected_value) != self.expected_json:
+            raise ValueError("acceptance expectation value must be canonical JSON")
+
+    @classmethod
+    def expecting(cls, path: str, expected_value: object) -> AcceptanceExpectation:
+        """Create an expectation from a JSON-compatible value."""
+
+        return cls(path=path, expected_json=_compact_json(expected_value))
+
+    def to_dict(self) -> dict[str, object]:
+        """Return the public expectation contract."""
+
+        return {
+            "path": self.path,
+            "expected": strict_json_loads(self.expected_json),
+        }
+
+
+@dataclass(frozen=True)
+class AuthoringAcceptanceContract:
+    """Complete public grading contract without a reference source document."""
+
+    expectations: tuple[AcceptanceExpectation, ...]
+    required_constructs: tuple[RequiredFlowConstruct, ...] = ()
+    forbidden_constructs: tuple[ForbiddenConstruct, ...] = field(
+        default_factory=lambda: DEFAULT_FORBIDDEN_CONSTRUCTS
+    )
+    variable_pointer_patterns: tuple[str, ...] = DEFAULT_VARIABLE_POINTER_PATTERNS
+    format_identifier: str = AUTHORING_ACCEPTANCE_FORMAT
+
+    def __post_init__(self) -> None:
+        if self.format_identifier != AUTHORING_ACCEPTANCE_FORMAT:
+            raise ValueError("unknown authoring acceptance format")
+        expectations = tuple(sorted(self.expectations, key=lambda expectation: expectation.path))
+        if not expectations:
+            raise ValueError("authoring acceptance contract requires expectations")
+        expectation_paths = tuple(expectation.path for expectation in expectations)
+        if len(set(expectation_paths)) != len(expectation_paths):
+            raise ValueError("authoring acceptance expectation paths must be unique")
+        required_constructs = tuple(
+            sorted(self.required_constructs, key=lambda construct: construct.identifier)
+        )
+        forbidden_constructs = tuple(
+            sorted(self.forbidden_constructs, key=lambda construct: construct.identifier)
+        )
+        variable_pointer_patterns = tuple(sorted(self.variable_pointer_patterns))
+        _validate_variable_pointer_patterns(variable_pointer_patterns)
+        object.__setattr__(self, "expectations", expectations)
+        object.__setattr__(self, "required_constructs", required_constructs)
+        object.__setattr__(self, "forbidden_constructs", forbidden_constructs)
+        object.__setattr__(self, "variable_pointer_patterns", variable_pointer_patterns)
+
+    def to_dict(self) -> dict[str, object]:
+        """Return the closed author-visible acceptance contract."""
+
+        return {
+            "format": self.format_identifier,
+            "expectations": [expectation.to_dict() for expectation in self.expectations],
+            "required_constructs": [
+                required_construct.to_dict() for required_construct in self.required_constructs
+            ],
+            "forbidden_constructs": [
+                {
+                    "identifier": forbidden_construct.identifier,
+                    "pointer_pattern": forbidden_construct.pointer_pattern,
+                    "message": forbidden_construct.message,
+                }
+                for forbidden_construct in self.forbidden_constructs
+            ],
+            "variable_pointer_patterns": list(self.variable_pointer_patterns),
+        }
+
+
+def _forbidden_construct(
+    identifier: str,
+    pointer_pattern: str,
+    construct_name: str,
+) -> ForbiddenConstruct:
+    return ForbiddenConstruct(
+        identifier=identifier,
+        pointer_pattern=pointer_pattern,
+        message=(
+            f"The authored source uses unsupported {construct_name}; "
+            "keep the flow on the closed ordered conversational rail."
+        ),
+    )
+
+
+DEFAULT_FORBIDDEN_CONSTRUCTS: Final[tuple[ForbiddenConstruct, ...]] = (
+    _forbidden_construct("top-code", "/code", "arbitrary code"),
+    _forbidden_construct("top-cron", "/cron", "cron scheduling"),
+    _forbidden_construct("top-expressions", "/expressions", "arbitrary expressions"),
+    _forbidden_construct("top-loop", "/loop", "loops"),
+    _forbidden_construct("top-loops", "/loops", "loops"),
+    _forbidden_construct("top-parallel", "/parallel", "parallel execution"),
+    _forbidden_construct("top-schedule", "/schedule", "scheduling"),
+    _forbidden_construct("top-script", "/script", "arbitrary scripts"),
+    _forbidden_construct("top-transitions", "/transitions", "manual transition tables"),
+    _forbidden_construct("path-code", "/path/*/code", "arbitrary step code"),
+    _forbidden_construct("path-expression", "/path/*/expression", "arbitrary expressions"),
+    _forbidden_construct("path-for-each", "/path/*/for_each", "iteration"),
+    _forbidden_construct("path-goto", "/path/*/goto", "manual path transitions"),
+    _forbidden_construct("path-loop", "/path/*/loop", "loops"),
+    _forbidden_construct("path-next", "/path/*/next", "manual path transitions"),
+    _forbidden_construct("path-parallel", "/path/*/parallel", "parallel execution"),
+    _forbidden_construct("path-script", "/path/*/script", "arbitrary scripts"),
+    _forbidden_construct("path-while", "/path/*/while", "loops"),
+)
+
+
+@dataclass(frozen=True)
+class AuthoringTask:
+    """The source-answer-free task and complete public grading contract."""
+
+    identifier: str
+    prompt: str
+    acceptance: AuthoringAcceptanceContract
+
+    def __post_init__(self) -> None:
+        _require_identifier(self.identifier, "authoring task identifier")
+        if not self.prompt.strip():
+            raise ValueError(f"authoring task {self.identifier!r} prompt must be non-empty")
+        object.__setattr__(self, "prompt", self.prompt.strip())
+
+
+@dataclass(frozen=True)
+class AuthoringBenchmarkCase:
+    """One private fixture projected into a public semantic task contract."""
+
+    identifier: str
+    prompt: str
+    expected_flow_json: str
+    feature_tags: tuple[str, ...] = ()
+    required_constructs: tuple[RequiredFlowConstruct, ...] = ()
+    forbidden_constructs: tuple[ForbiddenConstruct, ...] = DEFAULT_FORBIDDEN_CONSTRUCTS
+
+    def __post_init__(self) -> None:
+        _require_identifier(self.identifier, "benchmark case identifier")
+        if not self.prompt.strip():
+            raise ValueError(f"benchmark case {self.identifier!r} prompt must be non-empty")
+        expected_flow = strict_json_loads(self.expected_flow_json)
+        feature_tags = tuple(self.feature_tags)
+        required_constructs = tuple(self.required_constructs)
+        forbidden_constructs = tuple(self.forbidden_constructs)
+        if len(set(feature_tags)) != len(feature_tags):
+            raise ValueError(f"benchmark case {self.identifier!r} feature tags must be unique")
+        if len({rule.identifier for rule in forbidden_constructs}) != len(forbidden_constructs):
+            raise ValueError(
+                f"benchmark case {self.identifier!r} forbidden construct identifiers must be unique"
+            )
+        if len({rule.identifier for rule in required_constructs}) != len(required_constructs):
+            raise ValueError(
+                f"benchmark case {self.identifier!r} required construct identifiers must be unique"
+            )
+        object.__setattr__(self, "prompt", self.prompt.strip())
+        object.__setattr__(self, "expected_flow_json", _normalized_flow_json(expected_flow))
+        object.__setattr__(self, "feature_tags", tuple(sorted(feature_tags)))
+        object.__setattr__(
+            self,
+            "required_constructs",
+            tuple(sorted(required_constructs, key=lambda rule: rule.identifier)),
+        )
+        object.__setattr__(
+            self,
+            "forbidden_constructs",
+            tuple(sorted(forbidden_constructs, key=lambda rule: rule.identifier)),
+        )
+
+    @classmethod
+    def expecting_flow(
+        cls,
+        *,
+        identifier: str,
+        prompt: str,
+        expected_flow: object,
+        feature_tags: tuple[str, ...] = (),
+        required_constructs: tuple[RequiredFlowConstruct, ...] = (),
+        forbidden_constructs: tuple[ForbiddenConstruct, ...] = DEFAULT_FORBIDDEN_CONSTRUCTS,
+    ) -> AuthoringBenchmarkCase:
+        """Create a closed case from a reviewed private fixture flow."""
+
+        return cls(
+            identifier=identifier,
+            prompt=prompt,
+            expected_flow_json=_compact_json(expected_flow),
+            feature_tags=feature_tags,
+            required_constructs=required_constructs,
+            forbidden_constructs=forbidden_constructs,
+        )
+
+    def authoring_task(self) -> AuthoringTask:
+        """Project the evaluator-private case into its author-facing contract."""
+
+        return AuthoringTask(
+            identifier=self.identifier,
+            prompt=self.prompt,
+            acceptance=authoring_acceptance_for_flow(
+                strict_json_loads(self.expected_flow_json),
+                required_constructs=self.required_constructs,
+                forbidden_constructs=self.forbidden_constructs,
+            ),
+        )
+
+
+@dataclass(frozen=True)
+class AuthoringBenchmarkLimits:
+    """Bound the correction interaction for every benchmark case."""
+
+    max_correction_rounds: int = 2
+
+    def __post_init__(self) -> None:
+        if self.max_correction_rounds < 0:
+            raise ValueError("max_correction_rounds must not be negative")
+
+
+@dataclass(frozen=True)
+class AuthoringRequest:
+    """One initial or diagnostic-guided request delivered to an injected author."""
+
+    task: AuthoringTask
+    format_identifier: str
+    profile_identifier: str
+    profile_contract_json: str
+    correction_round: int
+    previous_source: str | None
+    previous_diagnostics: tuple[FlowDiagnostic, ...]
+
+    def __post_init__(self) -> None:
+        if not self.format_identifier:
+            raise ValueError("authoring request format identifier must be non-empty")
+        if not self.profile_identifier:
+            raise ValueError("authoring request profile identifier must be non-empty")
+        profile_contract = strict_json_loads(self.profile_contract_json)
+        if not isinstance(profile_contract, dict):
+            raise ValueError("authoring request profile contract must be a JSON object")
+        if _compact_json(profile_contract) != self.profile_contract_json:
+            raise ValueError("authoring request profile contract must be canonical JSON")
+        if profile_contract.get("identifier") != self.profile_identifier:
+            raise ValueError("authoring request profile contract identifier does not match")
+        if self.correction_round < 0:
+            raise ValueError("authoring request correction round must not be negative")
+        if self.correction_round == 0 and (
+            self.previous_source is not None or self.previous_diagnostics
+        ):
+            raise ValueError("the initial authoring request cannot contain a previous attempt")
+        if self.correction_round > 0 and self.previous_source is None:
+            raise ValueError("a correction request requires the previous source")
+        object.__setattr__(
+            self,
+            "previous_diagnostics",
+            _deterministic_diagnostics(self.previous_diagnostics),
+        )
+
+
+@dataclass(frozen=True)
+class AuthoredSource:
+    """One exact authored source and optional effective provider model version."""
+
+    source: str
+    effective_model_version: str | None = None
+
+    def __post_init__(self) -> None:
+        if not isinstance(self.source, str):
+            raise TypeError("authored source must be a string")
+        if self.effective_model_version is not None:
+            normalized_model_version = self.effective_model_version.strip()
+            if not normalized_model_version:
+                raise ValueError("effective model version must be non-empty when present")
+            object.__setattr__(self, "effective_model_version", normalized_model_version)
+
+
+AuthoringAuthor = Callable[[AuthoringRequest], AuthoredSource]
+
+
+@dataclass(frozen=True)
+class AuthoringAttempt:
+    """Machine-readable outcome for one emitted source document."""
+
+    correction_round: int
+    source_sha256: str
+    raw_source_bytes: int
+    canonical_source_bytes: int | None
+    token_proxy_units: int | None
+    compilation: CompilationStatus
+    diagnostics: tuple[FlowDiagnostic, ...]
+    succeeded: bool
+
+    def __post_init__(self) -> None:
+        if self.correction_round < 0:
+            raise ValueError("authoring attempt correction round must not be negative")
+        if not re.fullmatch(r"[0-9a-f]{64}", self.source_sha256):
+            raise ValueError("authoring attempt source digest must be lowercase SHA-256")
+        if self.raw_source_bytes < 0:
+            raise ValueError("authoring attempt raw source size must not be negative")
+        if (self.canonical_source_bytes is None) != (self.token_proxy_units is None):
+            raise ValueError("canonical source size and token proxy must be present together")
+        if self.canonical_source_bytes is not None:
+            expected_proxy = (
+                self.canonical_source_bytes + TOKEN_PROXY_BYTES_PER_UNIT - 1
+            ) // TOKEN_PROXY_BYTES_PER_UNIT
+            if self.canonical_source_bytes < 0 or self.token_proxy_units != expected_proxy:
+                raise ValueError(
+                    "authoring attempt token proxy does not match canonical source size"
+                )
+        deterministic_diagnostics = _deterministic_diagnostics(self.diagnostics)
+        object.__setattr__(self, "diagnostics", deterministic_diagnostics)
+        if self.succeeded and (
+            self.compilation != "succeeded"
+            or any(
+                flow_diagnostic.severity == "error" for flow_diagnostic in deterministic_diagnostics
+            )
+        ):
+            raise ValueError("a successful authoring attempt must compile without errors")
+
+    def to_dict(self) -> dict[str, object]:
+        """Return a deterministic JSON-compatible representation."""
+
+        return {
+            "correction_round": self.correction_round,
+            "source_sha256": self.source_sha256,
+            "raw_source_bytes": self.raw_source_bytes,
+            "canonical_source_bytes": self.canonical_source_bytes,
+            "token_proxy_units": self.token_proxy_units,
+            "compilation": self.compilation,
+            "succeeded": self.succeeded,
+            "diagnostics": [diagnostic.to_dict() for diagnostic in self.diagnostics],
+        }
+
+
+@dataclass(frozen=True)
+class AuthoringBenchmarkResult:
+    """All bounded attempts for one benchmark case."""
+
+    case_identifier: str
+    feature_tags: tuple[str, ...]
+    format_identifier: str
+    attempts: tuple[AuthoringAttempt, ...]
+
+    def __post_init__(self) -> None:
+        _require_identifier(self.case_identifier, "benchmark result case identifier")
+        if not self.format_identifier:
+            raise ValueError("benchmark result format identifier must be non-empty")
+        feature_tags = tuple(self.feature_tags)
+        if len(set(feature_tags)) != len(feature_tags):
+            raise ValueError(
+                f"benchmark result {self.case_identifier!r} feature tags must be unique"
+            )
+        authoring_attempts = tuple(self.attempts)
+        object.__setattr__(self, "feature_tags", tuple(sorted(feature_tags)))
+        object.__setattr__(self, "attempts", authoring_attempts)
+        if not authoring_attempts:
+            raise ValueError(f"benchmark result {self.case_identifier!r} requires an attempt")
+        expected_rounds = tuple(range(len(authoring_attempts)))
+        actual_rounds = tuple(attempt.correction_round for attempt in authoring_attempts)
+        if actual_rounds != expected_rounds:
+            raise ValueError(
+                f"benchmark result {self.case_identifier!r} attempts must use contiguous rounds"
+            )
+        if any(attempt.succeeded for attempt in authoring_attempts[:-1]):
+            raise ValueError(
+                f"benchmark result {self.case_identifier!r} cannot continue after success"
+            )
+
+    @property
+    def succeeded(self) -> bool:
+        return self.attempts[-1].succeeded
+
+    @property
+    def correction_rounds(self) -> int:
+        return len(self.attempts) - 1
+
+    def to_dict(self) -> dict[str, object]:
+        """Return a deterministic JSON-compatible representation."""
+
+        return {
+            "case_identifier": self.case_identifier,
+            "feature_tags": list(self.feature_tags),
+            "format_identifier": self.format_identifier,
+            "succeeded": self.succeeded,
+            "correction_rounds": self.correction_rounds,
+            "attempts": [attempt.to_dict() for attempt in self.attempts],
+        }
+
+
+@dataclass(frozen=True)
+class AuthoringBenchmarkReport:
+    """Deterministic aggregate suitable for cross-adapter comparison."""
+
+    benchmark_identifier: str
+    format_identifier: str
+    profile_identifier: str
+    case_results: tuple[AuthoringBenchmarkResult, ...]
+
+    def __post_init__(self) -> None:
+        _require_identifier(self.benchmark_identifier, "benchmark identifier")
+        if not self.format_identifier:
+            raise ValueError("benchmark report format identifier must be non-empty")
+        if not self.profile_identifier:
+            raise ValueError("benchmark report profile identifier must be non-empty")
+        case_results = tuple(self.case_results)
+        object.__setattr__(self, "case_results", case_results)
+        if not case_results:
+            raise ValueError("benchmark report requires at least one case")
+        case_identifiers = tuple(case_result.case_identifier for case_result in case_results)
+        if case_identifiers != tuple(sorted(case_identifiers)):
+            raise ValueError("benchmark report cases must be sorted by identifier")
+        if len(set(case_identifiers)) != len(case_identifiers):
+            raise ValueError("benchmark report case identifiers must be unique")
+        if any(
+            case_result.format_identifier != self.format_identifier for case_result in case_results
+        ):
+            raise ValueError("benchmark report case formats must match the report format")
+
+    @property
+    def successful_cases(self) -> int:
+        return sum(case_result.succeeded for case_result in self.case_results)
+
+    @property
+    def total_cases(self) -> int:
+        return len(self.case_results)
+
+    @property
+    def total_attempts(self) -> int:
+        return sum(len(case_result.attempts) for case_result in self.case_results)
+
+    @property
+    def total_correction_rounds(self) -> int:
+        return sum(case_result.correction_rounds for case_result in self.case_results)
+
+    def to_dict(self) -> dict[str, object]:
+        """Return a deterministic JSON-compatible representation."""
+
+        return {
+            "benchmark_identifier": self.benchmark_identifier,
+            "format_identifier": self.format_identifier,
+            "profile_identifier": self.profile_identifier,
+            "successful_cases": self.successful_cases,
+            "total_cases": self.total_cases,
+            "total_attempts": self.total_attempts,
+            "total_correction_rounds": self.total_correction_rounds,
+            "token_proxy": {
+                "method": TOKEN_PROXY_METHOD,
+                "bytes_per_unit": TOKEN_PROXY_BYTES_PER_UNIT,
+            },
+            "case_results": [case_result.to_dict() for case_result in self.case_results],
+        }
+
+    def to_json(self) -> str:
+        """Serialize the report as canonical compact JSON."""
+
+        return _compact_json(self.to_dict())
+
+
+def _document_locations(
+    json_document: object,
+    path_segments: tuple[str | int, ...] = (),
+) -> Iterable[tuple[tuple[str | int, ...], object]]:
+    if isinstance(json_document, Mapping):
+        for property_name in sorted(json_document, key=str):
+            property_path = (*path_segments, str(property_name))
+            property_value = json_document[property_name]
+            yield property_path, property_value
+            yield from _document_locations(property_value, property_path)
+    elif isinstance(json_document, list):
+        for element_index, element_value in enumerate(json_document):
+            element_path = (*path_segments, element_index)
+            yield element_path, element_value
+            yield from _document_locations(element_value, element_path)
+
+
+def _pattern_segments(pointer_pattern: str) -> tuple[str, ...]:
+    return tuple(
+        encoded_segment.replace("~1", "/").replace("~0", "~")
+        for encoded_segment in pointer_pattern[1:].split("/")
+    )
+
+
+def _matches_pointer_pattern(
+    path_segments: tuple[str | int, ...],
+    pointer_pattern: str,
+) -> bool:
+    expected_segments = _pattern_segments(pointer_pattern)
+    return len(path_segments) == len(expected_segments) and all(
+        expected_segment == "*" or expected_segment == str(path_segment)
+        for path_segment, expected_segment in zip(path_segments, expected_segments, strict=True)
+    )
+
+
+def _semantic_observation_items(
+    json_document: object,
+    path_segments: tuple[str | int, ...] = (),
+) -> Iterable[tuple[str, object]]:
+    if isinstance(json_document, Mapping):
+        if not json_document and path_segments:
+            yield _json_pointer(path_segments), {}
+        for property_name in sorted(json_document, key=str):
+            property_path = (*path_segments, str(property_name))
+            yield from _semantic_observation_items(
+                json_document[property_name],
+                property_path,
+            )
+        return
+    if isinstance(json_document, list):
+        if not json_document and path_segments:
+            yield _json_pointer(path_segments), []
+        for element_index, element_value in enumerate(json_document):
+            yield from _semantic_observation_items(
+                element_value,
+                (*path_segments, element_index),
+            )
+        return
+    if not path_segments:
+        raise ValueError("an authoring semantic observation requires a JSON object")
+    yield _json_pointer(path_segments), json_document
+
+
+def _referenced_domain_names(json_document: object) -> frozenset[str]:
+    referenced_names: set[str] = set()
+    if isinstance(json_document, Mapping):
+        for property_name, property_value in json_document.items():
+            if property_name in {"domain", "from_domain"} and isinstance(property_value, str):
+                referenced_names.add(property_value)
+            referenced_names.update(_referenced_domain_names(property_value))
+    elif isinstance(json_document, list):
+        for element_value in json_document:
+            referenced_names.update(_referenced_domain_names(element_value))
+    return frozenset(referenced_names)
+
+
+def _step_references(normalized_flow: Mapping[str, object]) -> dict[str, str]:
+    step_references: dict[str, str] = {}
+    normalized_path = normalized_flow.get("path")
+    if isinstance(normalized_path, list):
+        for path_index, path_step in enumerate(normalized_path):
+            if isinstance(path_step, Mapping) and isinstance(path_step.get("step"), str):
+                step_references[cast(str, path_step["step"])] = f"path[{path_index}]"
+    for contract_name in ("confirm", "terminal"):
+        contract = normalized_flow.get(contract_name)
+        if isinstance(contract, Mapping) and isinstance(contract.get("step"), str):
+            step_references[cast(str, contract["step"])] = contract_name
+    return step_references
+
+
+def _semantic_prompt(
+    prompt_contract: Mapping[str, object],
+    domains: Mapping[str, object],
+    step_references: Mapping[str, str],
+    path_segments: tuple[str | int, ...],
+) -> dict[str, object]:
+    verbatim = prompt_contract.get("verbatim") is True
+    semantic_prompt: dict[str, object] = {
+        "present": True,
+        "verbatim": verbatim,
+    }
+    if verbatim:
+        semantic_prompt["text"] = prompt_contract.get("text")
+    for property_name, property_value in prompt_contract.items():
+        if property_name in {"extract_hint", "text", "verbatim"}:
+            continue
+        semantic_prompt[property_name] = _semantic_flow_value(
+            property_value,
+            domains,
+            step_references,
+            (*path_segments, property_name),
+        )
+    return semantic_prompt
+
+
+def _semantic_flow_value(
+    json_document: object,
+    domains: Mapping[str, object],
+    step_references: Mapping[str, str],
+    path_segments: tuple[str | int, ...] = (),
+) -> object:
+    if isinstance(json_document, Mapping):
+        semantic_mapping: dict[str, object] = {}
+        for property_name, property_value in json_document.items():
+            property_path = (*path_segments, property_name)
+            if not path_segments and property_name in {"domains", "flow", "version"}:
+                continue
+            if path_segments == ("route",) and property_name in {
+                "description",
+                "trigger_phrases",
+            }:
+                continue
+            if property_name == "prompt" and isinstance(property_value, Mapping):
+                semantic_mapping[property_name] = _semantic_prompt(
+                    property_value,
+                    domains,
+                    step_references,
+                    property_path,
+                )
+                continue
+            if property_name in {"domain", "from_domain"} and isinstance(property_value, str):
+                semantic_mapping[f"{property_name}_contract"] = _semantic_flow_value(
+                    domains[property_value],
+                    domains,
+                    step_references,
+                    property_path,
+                )
+                continue
+            is_step_declaration = property_name == "step" and (
+                (len(path_segments) == 2 and path_segments[0] == "path")
+                or path_segments in {("confirm",), ("terminal",)}
+            )
+            if is_step_declaration:
+                continue
+            if property_name in {"goto", "next_step", "on_confirm", "step"} and isinstance(
+                property_value, str
+            ):
+                semantic_mapping[property_name] = step_references.get(
+                    property_value,
+                    property_value,
+                )
+                continue
+            semantic_mapping[property_name] = _semantic_flow_value(
+                property_value,
+                domains,
+                step_references,
+                property_path,
+            )
+        return semantic_mapping
+    if isinstance(json_document, list):
+        return [
+            _semantic_flow_value(
+                element_value,
+                domains,
+                step_references,
+                (*path_segments, element_index),
+            )
+            for element_index, element_value in enumerate(json_document)
+        ]
+    return json_document
+
+
+def _semantic_flow_document(normalized_flow: Mapping[str, object]) -> dict[str, object]:
+    raw_domains = normalized_flow.get("domains")
+    if not isinstance(raw_domains, Mapping):
+        raise ValueError("a normalized authoring flow requires domain contracts")
+    domains = cast(Mapping[str, object], raw_domains)
+    semantic_flow = cast(
+        dict[str, object],
+        _semantic_flow_value(
+            normalized_flow,
+            domains,
+            _step_references(normalized_flow),
+        ),
+    )
+    referenced_domains = _referenced_domain_names(normalized_flow)
+    semantic_flow["unused_domain_contracts"] = sorted(
+        (
+            _semantic_flow_value(domain_contract, domains, {}, ("unused_domain_contracts",))
+            for domain_name, domain_contract in domains.items()
+            if domain_name not in referenced_domains
+        ),
+        key=_compact_json,
+    )
+    return semantic_flow
+
+
+def observe_authoring_flow(
+    executable_flow: object,
+    *,
+    variable_pointer_patterns: tuple[str, ...] = DEFAULT_VARIABLE_POINTER_PATTERNS,
+) -> dict[str, object]:
+    """Project one flow into complete, source-syntax-free semantic observations."""
+
+    normalized_flow = strict_json_loads(_normalized_flow_json(executable_flow))
+    if not isinstance(normalized_flow, dict):
+        raise AssertionError("normalized authoring flows must be JSON objects")
+    _validate_variable_pointer_patterns(tuple(sorted(variable_pointer_patterns)))
+    return dict(_semantic_observation_items(_semantic_flow_document(normalized_flow)))
+
+
+def authoring_acceptance_for_flow(
+    executable_flow: object,
+    *,
+    required_constructs: tuple[RequiredFlowConstruct, ...] = (),
+    forbidden_constructs: tuple[ForbiddenConstruct, ...] = DEFAULT_FORBIDDEN_CONSTRUCTS,
+    variable_pointer_patterns: tuple[str, ...] | None = None,
+) -> AuthoringAcceptanceContract:
+    """Build the public semantic contract for one private fixture source."""
+
+    normalized_flow = strict_json_loads(_normalized_flow_json(executable_flow))
+    if not isinstance(normalized_flow, dict):
+        raise AssertionError("normalized authoring flows must be JSON objects")
+    resolved_variable_pointer_patterns = list(
+        DEFAULT_VARIABLE_POINTER_PATTERNS
+        if variable_pointer_patterns is None
+        else variable_pointer_patterns
+    )
+    normalized_path = normalized_flow.get("path")
+    if variable_pointer_patterns is None and isinstance(normalized_path, list):
+        for path_index, path_step in enumerate(normalized_path):
+            if not isinstance(path_step, dict):
+                continue
+            prompt_contract = path_step.get("prompt")
+            if isinstance(prompt_contract, dict) and prompt_contract.get("verbatim") is not True:
+                resolved_variable_pointer_patterns.append(f"/path/{path_index}/prompt/text")
+            if isinstance(prompt_contract, dict):
+                resolved_variable_pointer_patterns.append(f"/path/{path_index}/prompt/extract_hint")
+    resolved_patterns = tuple(sorted(resolved_variable_pointer_patterns))
+    observations = observe_authoring_flow(
+        normalized_flow,
+        variable_pointer_patterns=resolved_patterns,
+    )
+    return AuthoringAcceptanceContract(
+        expectations=tuple(
+            AcceptanceExpectation.expecting(path, expected_value)
+            for path, expected_value in observations.items()
+        ),
+        required_constructs=required_constructs,
+        forbidden_constructs=forbidden_constructs,
+        variable_pointer_patterns=resolved_patterns,
+    )
+
+
+def _forbidden_diagnostics(
+    authored_document: object,
+    forbidden_constructs: tuple[ForbiddenConstruct, ...],
+) -> tuple[FlowDiagnostic, ...]:
+    flow_diagnostics = (
+        FlowDiagnostic(
+            code=f"AUTHORING_FORBIDDEN_{forbidden_construct.identifier.upper().replace('-', '_')}",
+            severity="error",
+            path=_json_pointer(path_segments),
+            message=forbidden_construct.message,
+        )
+        for path_segments, _property_value in _document_locations(authored_document)
+        for forbidden_construct in forbidden_constructs
+        if _matches_pointer_pattern(path_segments, forbidden_construct.pointer_pattern)
+    )
+    return _deterministic_diagnostics(flow_diagnostics)
+
+
+def _matching_locations(
+    json_document: object,
+    pointer_pattern: str,
+) -> tuple[tuple[tuple[str | int, ...], object], ...]:
+    return tuple(
+        (path_segments, property_value)
+        for path_segments, property_value in _document_locations(json_document)
+        if _matches_pointer_pattern(path_segments, pointer_pattern)
+    )
+
+
+def _missing_construct_path(pointer_pattern: str) -> str:
+    fixed_segments = []
+    for pattern_segment in _pattern_segments(pointer_pattern):
+        if pattern_segment == "*":
+            break
+        fixed_segments.append(pattern_segment)
+    return _json_pointer(fixed_segments)
+
+
+def _required_construct_diagnostics(
+    executable_flow: object,
+    required_constructs: tuple[RequiredFlowConstruct, ...],
+) -> tuple[FlowDiagnostic, ...]:
+    flow_diagnostics: list[FlowDiagnostic] = []
+    for required_construct in required_constructs:
+        matching_locations = _matching_locations(
+            executable_flow,
+            required_construct.pointer_pattern,
+        )
+        diagnostic_code = required_construct.identifier.upper().replace("-", "_")
+        if not matching_locations:
+            expected_contract = (
+                ""
+                if required_construct.expected_json is None
+                else f" with the exact JSON value {required_construct.expected_json}"
+            )
+            flow_diagnostics.append(
+                FlowDiagnostic(
+                    code=f"AUTHORING_REQUIRED_{diagnostic_code}_MISSING",
+                    severity="error",
+                    path=_missing_construct_path(required_construct.pointer_pattern),
+                    message=(
+                        "The executable flow is missing required construct "
+                        f"{required_construct.pointer_pattern!r}{expected_contract}."
+                    ),
+                    suggested_fix=(
+                        f"Add a location matching {required_construct.pointer_pattern!r}"
+                        f"{expected_contract}."
+                    ),
+                )
+            )
+            continue
+        if required_construct.expected_json is None or any(
+            _compact_json(property_value) == required_construct.expected_json
+            for _path_segments, property_value in matching_locations
+        ):
+            continue
+        first_path, first_property_value = matching_locations[0]
+        actual_json_value = _compact_json(first_property_value)
+        flow_diagnostics.append(
+            FlowDiagnostic(
+                code=f"AUTHORING_REQUIRED_{diagnostic_code}_MISMATCH",
+                severity="error",
+                path=_json_pointer(first_path),
+                message=(
+                    f"Required construct {required_construct.pointer_pattern!r} matched "
+                    f"{actual_json_value}, but must have the exact JSON value "
+                    f"{required_construct.expected_json}."
+                ),
+                suggested_fix=(
+                    f"Set a location matching {required_construct.pointer_pattern!r} to "
+                    f"{required_construct.expected_json}."
+                ),
+            )
+        )
+    return _deterministic_diagnostics(flow_diagnostics)
+
+
+def acceptance_diagnostics(
+    executable_flow: object,
+    acceptance_contract: AuthoringAcceptanceContract,
+    *,
+    covered_paths: frozenset[str] = frozenset(),
+) -> tuple[FlowDiagnostic, ...]:
+    """Return every deterministic mismatch against the public task contract."""
+
+    expected_observations = {
+        expectation.path: strict_json_loads(expectation.expected_json)
+        for expectation in acceptance_contract.expectations
+    }
+    observed_observations = observe_authoring_flow(
+        executable_flow,
+        variable_pointer_patterns=acceptance_contract.variable_pointer_patterns,
+    )
+    diagnostic_paths = sorted(set(expected_observations) | set(observed_observations))
+    return _deterministic_diagnostics(
+        FlowDiagnostic(
+            code=(
+                "AUTHORING_ACCEPTANCE_UNEXPECTED"
+                if diagnostic_path not in expected_observations
+                else (
+                    "AUTHORING_ACCEPTANCE_MISSING"
+                    if diagnostic_path not in observed_observations
+                    else "AUTHORING_ACCEPTANCE_MISMATCH"
+                )
+            ),
+            severity="error",
+            path=diagnostic_path,
+            message=(
+                f"Unexpected semantic observation {_compact_json(observed_observations[diagnostic_path])}."
+                if diagnostic_path not in expected_observations
+                else (
+                    f"Missing required semantic observation {_compact_json(expected_observations[diagnostic_path])}."
+                    if diagnostic_path not in observed_observations
+                    else (
+                        "Semantic observation must equal "
+                        f"{_compact_json(expected_observations[diagnostic_path])}, but observed "
+                        f"{_compact_json(observed_observations[diagnostic_path])}."
+                    )
+                )
+            ),
+            suggested_fix=(
+                f"Remove the behavior at {diagnostic_path!r}."
+                if diagnostic_path not in expected_observations
+                else f"Set the semantic observation at {diagnostic_path!r} to the public expected value."
+            ),
+        )
+        for diagnostic_path in diagnostic_paths
+        if (
+            diagnostic_path not in expected_observations
+            or diagnostic_path not in observed_observations
+            or expected_observations[diagnostic_path] != observed_observations[diagnostic_path]
+        )
+        if diagnostic_path not in covered_paths
+    )
+
+
+def _evaluate_source(
+    authored_source: str,
+    source_adapter: AuthoringSourceAdapter,
+    acceptance_contract: AuthoringAcceptanceContract,
+    correction_round: int,
+    flow_profile: FlowProfile,
+) -> AuthoringAttempt:
+    encoded_source = authored_source.encode("utf-8")
+    try:
+        source_adaptation = source_adapter.adapt(authored_source)
+        if source_adaptation.format_identifier != source_adapter.format_identifier:
+            raise ValueError(
+                "source adapter returned a format identifier that differs from its declaration"
+            )
+        if source_adaptation.canonical_authored_source is not None:
+            canonical_adaptation = source_adapter.adapt(source_adaptation.canonical_authored_source)
+            source_projection = (
+                source_adaptation.format_identifier,
+                source_adaptation.authored_document_json,
+                source_adaptation.executable_flow_json,
+                source_adaptation.canonical_authored_source,
+            )
+            canonical_projection = (
+                canonical_adaptation.format_identifier,
+                canonical_adaptation.authored_document_json,
+                canonical_adaptation.executable_flow_json,
+                canonical_adaptation.canonical_authored_source,
+            )
+            if canonical_projection != source_projection:
+                raise ValueError(
+                    "source adapter canonical syntax must round-trip to the same authored "
+                    "and executable projections"
+                )
+    except Exception as adaptation_error:
+        adaptation_details = str(adaptation_error).strip() or "no details"
+        return AuthoringAttempt(
+            correction_round=correction_round,
+            source_sha256=hashlib.sha256(encoded_source).hexdigest(),
+            raw_source_bytes=len(encoded_source),
+            canonical_source_bytes=None,
+            token_proxy_units=None,
+            compilation="skipped",
+            diagnostics=(
+                FlowDiagnostic(
+                    code="AUTHORING_SOURCE_ADAPTATION_FAILED",
+                    severity="error",
+                    path="",
+                    message=(
+                        f"The {source_adapter.format_identifier!r} source adapter failed with "
+                        f"{type(adaptation_error).__name__}: {adaptation_details}."
+                    ),
+                ),
+            ),
+            succeeded=False,
+        )
+
+    flow_diagnostics = list(source_adaptation.diagnostics)
+    canonical_source_bytes: int | None = None
+    token_proxy_units: int | None = None
+    if source_adaptation.authored_document_json is not None:
+        authored_document = strict_json_loads(source_adaptation.authored_document_json)
+        canonical_source_bytes = len(
+            cast(str, source_adaptation.canonical_authored_source).encode("utf-8")
+        )
+        token_proxy_units = (
+            canonical_source_bytes + TOKEN_PROXY_BYTES_PER_UNIT - 1
+        ) // TOKEN_PROXY_BYTES_PER_UNIT
+        flow_diagnostics.extend(
+            _forbidden_diagnostics(
+                authored_document,
+                acceptance_contract.forbidden_constructs,
+            )
+        )
+
+    compilation: CompilationStatus = "skipped"
+    executable_flow_available = source_adaptation.executable_flow_json is not None
+    if executable_flow_available:
+        executable_flow = strict_json_loads(cast(str, source_adaptation.executable_flow_json))
+        required_diagnostics = _required_construct_diagnostics(
+            executable_flow,
+            acceptance_contract.required_constructs,
+        )
+        flow_diagnostics.extend(required_diagnostics)
+        flow_check_report = check_flow(executable_flow, profile=flow_profile)
+        compilation = flow_check_report.compilation
+        flow_diagnostics.extend(flow_check_report.diagnostics)
+        if compilation == "succeeded":
+            flow_diagnostics.extend(
+                acceptance_diagnostics(
+                    executable_flow,
+                    acceptance_contract,
+                    covered_paths=frozenset(diagnostic.path for diagnostic in required_diagnostics),
+                )
+            )
+
+    deterministic_diagnostics = _deterministic_diagnostics(flow_diagnostics)
+    succeeded = (
+        executable_flow_available
+        and compilation == "succeeded"
+        and not any(
+            flow_diagnostic.severity == "error" for flow_diagnostic in deterministic_diagnostics
+        )
+    )
+    return AuthoringAttempt(
+        correction_round=correction_round,
+        source_sha256=hashlib.sha256(encoded_source).hexdigest(),
+        raw_source_bytes=len(encoded_source),
+        canonical_source_bytes=canonical_source_bytes,
+        token_proxy_units=token_proxy_units,
+        compilation=compilation,
+        diagnostics=deterministic_diagnostics,
+        succeeded=succeeded,
+    )
+
+
+def _run_benchmark_case(
+    benchmark_case: AuthoringBenchmarkCase,
+    author: AuthoringAuthor,
+    source_adapter: AuthoringSourceAdapter,
+    benchmark_limits: AuthoringBenchmarkLimits,
+    flow_profile: FlowProfile,
+) -> AuthoringBenchmarkResult:
+    authoring_attempts: list[AuthoringAttempt] = []
+    previous_source: str | None = None
+    previous_diagnostics: tuple[FlowDiagnostic, ...] = ()
+    authoring_task = benchmark_case.authoring_task()
+    for correction_round in range(benchmark_limits.max_correction_rounds + 1):
+        authoring_request = AuthoringRequest(
+            task=authoring_task,
+            format_identifier=source_adapter.format_identifier,
+            profile_identifier=flow_profile.identifier,
+            profile_contract_json=flow_profile.canonical_json(),
+            correction_round=correction_round,
+            previous_source=previous_source,
+            previous_diagnostics=previous_diagnostics,
+        )
+        authored_response = author(authoring_request)
+        if not isinstance(authored_response, AuthoredSource):
+            raise TypeError("the injected author must return AuthoredSource")
+        authoring_attempt = _evaluate_source(
+            authored_response.source,
+            source_adapter,
+            authoring_task.acceptance,
+            correction_round,
+            flow_profile,
+        )
+        authoring_attempts.append(authoring_attempt)
+        if authoring_attempt.succeeded:
+            break
+        previous_source = authored_response.source
+        previous_diagnostics = authoring_attempt.diagnostics
+
+    return AuthoringBenchmarkResult(
+        case_identifier=benchmark_case.identifier,
+        feature_tags=benchmark_case.feature_tags,
+        format_identifier=source_adapter.format_identifier,
+        attempts=tuple(authoring_attempts),
+    )
+
+
+def run_authoring_benchmark(
+    benchmark_identifier: str,
+    benchmark_cases: Sequence[AuthoringBenchmarkCase],
+    author: AuthoringAuthor,
+    *,
+    source_adapter: AuthoringSourceAdapter | None = None,
+    limits: AuthoringBenchmarkLimits | None = None,
+    profile: FlowProfile | None = None,
+) -> AuthoringBenchmarkReport:
+    """Run all cases through the injected author, adapter, and exact checker."""
+
+    _require_identifier(benchmark_identifier, "benchmark identifier")
+    resolved_source_adapter = source_adapter or FlowSpec2JsonAdapter()
+    resolved_limits = limits or AuthoringBenchmarkLimits()
+    resolved_profile = profile or reference_profile()
+    sorted_cases = tuple(
+        sorted(benchmark_cases, key=lambda benchmark_case: benchmark_case.identifier)
+    )
+    case_identifiers = tuple(benchmark_case.identifier for benchmark_case in sorted_cases)
+    if len(set(case_identifiers)) != len(case_identifiers):
+        raise ValueError("benchmark case identifiers must be unique")
+
+    return AuthoringBenchmarkReport(
+        benchmark_identifier=benchmark_identifier,
+        format_identifier=resolved_source_adapter.format_identifier,
+        profile_identifier=resolved_profile.identifier,
+        case_results=tuple(
+            _run_benchmark_case(
+                benchmark_case,
+                author,
+                resolved_source_adapter,
+                resolved_limits,
+                resolved_profile,
+            )
+            for benchmark_case in sorted_cases
+        ),
+    )
+
+
+def replay_authoring_benchmark(
+    benchmark_identifier: str,
+    benchmark_cases: Sequence[AuthoringBenchmarkCase],
+    captured_sources: Mapping[str, Sequence[str]],
+    *,
+    source_adapter: AuthoringSourceAdapter | None = None,
+    profile: FlowProfile | None = None,
+) -> AuthoringBenchmarkReport:
+    """Re-evaluate an exact closed capture set without invoking an author."""
+
+    _require_identifier(benchmark_identifier, "benchmark identifier")
+    resolved_source_adapter = source_adapter or FlowSpec2JsonAdapter()
+    resolved_profile = profile or reference_profile()
+    sorted_cases = tuple(
+        sorted(benchmark_cases, key=lambda benchmark_case: benchmark_case.identifier)
+    )
+    case_identifiers = tuple(benchmark_case.identifier for benchmark_case in sorted_cases)
+    if len(set(case_identifiers)) != len(case_identifiers):
+        raise ValueError("benchmark case identifiers must be unique")
+    if set(captured_sources) != set(case_identifiers):
+        raise ValueError("captured source cases do not match the benchmark corpus")
+
+    case_results: list[AuthoringBenchmarkResult] = []
+    for benchmark_case in sorted_cases:
+        case_sources = tuple(captured_sources[benchmark_case.identifier])
+        if not case_sources:
+            raise ValueError(
+                f"captured source case {benchmark_case.identifier!r} requires an attempt"
+            )
+        acceptance_contract = benchmark_case.authoring_task().acceptance
+        case_results.append(
+            AuthoringBenchmarkResult(
+                case_identifier=benchmark_case.identifier,
+                feature_tags=benchmark_case.feature_tags,
+                format_identifier=resolved_source_adapter.format_identifier,
+                attempts=tuple(
+                    _evaluate_source(
+                        authored_source,
+                        resolved_source_adapter,
+                        acceptance_contract,
+                        correction_round,
+                        resolved_profile,
+                    )
+                    for correction_round, authored_source in enumerate(case_sources)
+                ),
+            )
+        )
+
+    return AuthoringBenchmarkReport(
+        benchmark_identifier=benchmark_identifier,
+        format_identifier=resolved_source_adapter.format_identifier,
+        profile_identifier=resolved_profile.identifier,
+        case_results=tuple(case_results),
+    )