convergent-cli 0.1.0__py3-none-any.whl

convergent_cli/calibration_runner.py

@@ -0,0 +1,1413 @@
+"""Pure checks for ``cv calibration``.
+
+The platform is the record of what already ran. These helpers decide whether
+a new run is needed, why a tenant is left out, and how a contribution
+response should be read. They do not call the network.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+import re
+from collections.abc import Mapping, Sequence
+from decimal import Decimal
+from typing import Literal
+
+from convergent_cli.errors import InvalidResponseError, UsageError
+
+SESSION_MINIMUM = 15
+MONTHLY_BUDGET_USD = "2500"
+ERROR_RATE_LIMIT = 0.10
+_EXTRACTION_USD = Decimal("0.056")
+_SAMPLE_USD = Decimal("0.114")
+_MINUTES_PER_SAMPLE = Decimal("1.6")
+
+_TAG = re.compile(r"^[a-z0-9_]{1,64}$")
+_HEX64 = re.compile(r"^[0-9a-f]{64}$")
+_LABEL = re.compile(r"^[a-z0-9][a-z0-9._@-]{0,63}$")
+_ORG = re.compile(r"^org_[0-9a-f]{16}$")
+_WORKSPACE = re.compile(r"^ws_[0-9a-f]{16}$")
+_ENVIRONMENT = re.compile(r"^env_[0-9a-f]{16}$")
+_DATASET = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_:.-]{0,127}$")
+_VERDICTS = frozenset({"passed", "failed", "error", "skipped"})
+_RUN_KIND = {"input": "cin_", "treatment": "ctr_"}
+_FAILED_INPUT = frozenset({"failed", "cancelled"})
+_MAX_ATTEMPT = 20
+
+BUSY_SCOPE = "none"
+BUSY_SCOPE_NOTE = "This plan does not check whether a customer evaluation is already running."
+
+
+class TenantSpec:
+    """One tenant the operator named on the command line."""
+
+    def __init__(
+        self,
+        *,
+        tenant_tag: str,
+        organization_id: str,
+        workspace_id: str,
+        environment_id: str,
+        dataset_id: str | None,
+    ) -> None:
+        self.tenant_tag = tenant_tag
+        self.organization_id = organization_id
+        self.workspace_id = workspace_id
+        self.environment_id = environment_id
+        self.dataset_id = dataset_id
+
+
+def parse_tenant(raw: str) -> TenantSpec:
+    parts = raw.split(",")
+    if len(parts) not in (4, 5):
+        raise UsageError(
+            "--tenant must be tag,organization_id,workspace_id,environment_id[,dataset_id]"
+        )
+    tag, organization_id, workspace_id, environment_id, *rest = parts
+    dataset_id = rest[0] if rest else None
+    if dataset_id == "":
+        raise UsageError("--tenant dataset id is empty")
+    dataset = None
+    if dataset_id is not None:
+        dataset = _match(dataset_id, _DATASET, "dataset id is not usable")
+    return TenantSpec(
+        tenant_tag=_match(tag, _TAG, "tenant tag must be 1-64 characters of a-z, 0-9, or _"),
+        organization_id=_match(
+            organization_id,
+            _ORG,
+            "organization id must look like org_ and 16 hex characters",
+        ),
+        workspace_id=_match(
+            workspace_id,
+            _WORKSPACE,
+            "workspace id must look like ws_ and 16 hex characters",
+        ),
+        environment_id=_match(
+            environment_id,
+            _ENVIRONMENT,
+            "environment id must look like env_ and 16 hex characters",
+        ),
+        dataset_id=dataset,
+    )
+
+
+def require_treatment(value: str, *, label: str = "treatment") -> str:
+    return _match(value, _TAG, f"{label} must be 1-64 characters of a-z, 0-9, or _")
+
+
+def suite_digest_from_run(run: Mapping[str, object]) -> str | None:
+    """Suite digest written after grading. A run that is still open has none."""
+    metadata = run.get("metadata")
+    if not isinstance(metadata, Mapping):
+        return None
+    summary = metadata.get("baseline_summary")
+    if not isinstance(summary, Mapping):
+        return None
+    inner = summary.get("metadata")
+    if not isinstance(inner, Mapping):
+        return None
+    digest = inner.get("grader_suite_digest")
+    if not isinstance(digest, str) or _HEX64.fullmatch(digest) is None:
+        return None
+    return digest
+
+
+def require_manifest_id(value: str) -> str:
+    return _match(value, _HEX64, "corpus manifest id must be 64 hex characters")
+
+
+def require_label_release(value: str) -> str:
+    return _match(
+        value,
+        _LABEL,
+        "label release must start with a letter or digit and stay within 64 characters",
+    )
+
+
+def require_digest(value: str, label: str) -> str:
+    return _match(value, _HEX64, f"{label} must be 64 hex characters")
+
+
+_INPUT_RUN = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_.:-]{0,127}$")
+_INPUT_LABEL = re.compile(r"^[A-Za-z0-9][A-Za-z0-9-]{0,31}$")
+
+
+def default_input_setting_run_id(
+    label_release: str,
+    tenant_tag: str,
+    *,
+    eval_dataset_version_id: str,
+    preprocessing_digest: str | None,
+) -> str:
+    """Characters a label release may contain, and a run id may not, are replaced."""
+    label = require_label_release(label_release)
+    tag = require_treatment(tenant_tag, label="tenant tag")
+    pin = hashlib.sha256(
+        f"{eval_dataset_version_id}\n{preprocessing_digest or ''}".encode()
+    ).hexdigest()[:8]
+    cleaned = re.sub(r"[^A-Za-z0-9_.:-]", "_", f"input_{tag}_{label}_{pin}")
+    return cleaned[:128]
+
+
+def require_input_label(value: str) -> str:
+    return _match(
+        value,
+        _INPUT_LABEL,
+        "input label must use letters, digits, and hyphens, and stay within 32 characters",
+    )
+
+
+def labeled_input_run_id(base: str, label: str | None) -> str:
+    """The unlabeled input keeps the manifest run id. A label names another one.
+
+    Later attempts still add ``_2``, ``_3``, and so on, to this id.
+    """
+    if label is None:
+        return base
+    suffix = f"_in_{require_input_label(label)}"
+    room = 128 - len(suffix)
+    trimmed = base if len(base) <= room else base[:room]
+    labeled = trimmed + suffix
+    if labeled == base:
+        raise UsageError("input label does not fit in a run id")
+    return labeled
+
+
+def require_input_run_id(value: str) -> str:
+    run_id = _match(value, _INPUT_RUN, "input-setting run id is not usable")
+    if run_id.startswith("ctr_"):
+        raise UsageError("input-setting run id cannot be a treatment run")
+    return run_id
+
+
+def document_digest(document: Mapping[str, object]) -> str:
+    """SHA-256 of the document's canonical JSON. Matches the server digest."""
+    return hashlib.sha256(
+        json.dumps(
+            dict(document),
+            allow_nan=False,
+            ensure_ascii=False,
+            separators=(",", ":"),
+            sort_keys=True,
+        ).encode("utf-8")
+    ).hexdigest()
+
+
+def is_input_setting_run(run: Mapping[str, object]) -> bool:
+    if run.get("purpose") != "calibration":
+        return False
+    return _metadata(run).get("calibration_treatment") is None
+
+
+def require_k(value: int) -> int:
+    if isinstance(value, bool) or not isinstance(value, int) or not 1 <= value <= 30:
+        raise UsageError("k must be an integer from 1 to 30")
+    return value
+
+
+def require_json_object(value: object, label: str) -> dict[str, object]:
+    if not isinstance(value, dict) or not value:
+        raise UsageError(f"{label} must be a non-empty JSON object")
+    return value
+
+
+def exclusion_reason(
+    *,
+    eligible: bool,
+    dataset_id: str | None,
+    dataset_missing: bool,
+    session_count: int | None,
+) -> str | None:
+    """Why this tenant is left out, or None when it can run.
+
+    An organization that is absent from the eligible list is not minted, so
+    this cannot tell a bring-your-own-key organization from one that is off
+    the shared plane.
+    """
+    if not eligible:
+        return "ineligible"
+    if dataset_id is None or dataset_missing:
+        return "no_dataset"
+    if session_count is None:
+        raise UsageError("session count is missing for a tenant with a dataset")
+    if session_count < SESSION_MINIMUM:
+        return "below_session_minimum"
+    return None
+
+
+def session_minimum_waived(session_count: int, *, allowed: bool) -> bool:
+    """True when the operator asked for the waiver and the tenant has 1 to 14 sessions."""
+    return allowed and 1 <= session_count < SESSION_MINIMUM
+
+
+def session_minimum_warning(subject: str, session_count: int) -> str:
+    """The warning for a result under the floor. It names only the subject and the count."""
+    return (
+        f"warning: {subject} has {session_count} independent root sessions, under the "
+        f"{SESSION_MINIMUM} minimum. Its results can reveal single-session outcomes and "
+        "its intervals are wide; use them only to judge agreement on these samples, not "
+        "for a grader swap decision. The waiver is accepted only on staging."
+    )
+
+
+def estimate(*, trace_count: int, k: int) -> dict[str, str | int]:
+    """Cost and time for one input run plus one treatment, run one after the other."""
+    require_k(k)
+    if trace_count < 0:
+        raise UsageError("trace count must be zero or more")
+    traces = Decimal(trace_count)
+    samples = traces * Decimal(k)
+    return {
+        "trace_count": trace_count,
+        "k": k,
+        "extraction_usd": _money(traces * _EXTRACTION_USD),
+        "input_sample_usd": _money(traces * _SAMPLE_USD),
+        "treatment_sample_usd": _money(samples * _SAMPLE_USD),
+        "input_minutes": _minutes(traces * _MINUTES_PER_SAMPLE),
+        "treatment_minutes": _minutes(samples * _MINUTES_PER_SAMPLE),
+    }
+
+
+def error_verdict_rate(payload: Mapping[str, object]) -> float | None:
+    """Share of criterion verdicts that are errors. No verdicts is not zero."""
+    counts = payload.get("verdict_counts")
+    if not isinstance(counts, Mapping):
+        raise InvalidResponseError("verdict counts are missing")
+    errors = 0
+    total = 0
+    for status in _VERDICTS:
+        value = counts.get(status, 0)
+        if isinstance(value, bool) or not isinstance(value, int) or value < 0:
+            raise InvalidResponseError("verdict counts must be zero or more")
+        total += value
+        if status == "error":
+            errors = value
+    if total == 0:
+        return None
+    return errors / total
+
+
+def error_rate_exceeds_limit(rate: float | None) -> bool:
+    return rate is not None and rate > ERROR_RATE_LIMIT
+
+
+def spend_report(costs: Sequence[object]) -> dict[str, object]:
+    """Sum known run costs. A missing cost is not zero."""
+    known = Decimal(0)
+    known_count = 0
+    missing = 0
+    for cost in costs:
+        if isinstance(cost, bool) or not isinstance(cost, (int, float)):
+            missing += 1
+            continue
+        known += Decimal(str(cost))
+        known_count += 1
+    return {
+        "budget_usd": MONTHLY_BUDGET_USD,
+        "known_spend_usd": None if known_count == 0 else format(known, "f"),
+        "runs_missing_cost": missing,
+    }
+
+
+_GRADES_NOT_READY = (
+    "The run's grades have not all reached the data plane's store yet, so nothing was sent. "
+    "Re-run cv calibration stats in a few minutes."
+)
+_STORE_UNAVAILABLE = (
+    "The data plane was unavailable, so nothing was sent. "
+    "Re-run cv calibration stats once it recovers."
+)
+# Receipts that sent nothing for sure and are safe to retry, or may have landed.
+RETRY_RECEIPTS = frozenset({"unknown", "not_ready", "store_unavailable"})
+
+
+def contribution_receipt(
+    *,
+    status_code: int,
+    outcome: str | None,
+    message: str | None,
+    code: str | None = None,
+) -> tuple[str, str | None]:
+    if status_code == 200 and outcome == "stored":
+        return "accepted", None
+    if status_code == 200 and outcome == "duplicate":
+        return "duplicate", None
+    if status_code == 422:
+        reason = _refusal_reason(message or "")
+        if reason == "counting_revision_changed":
+            reason += ": counting changed; grade the run again before stats."
+        return "refused", reason
+    if status_code == 503 and code == "grades_not_ready":
+        return "not_ready", _GRADES_NOT_READY
+    if status_code == 503:
+        return "store_unavailable", _STORE_UNAVAILABLE
+    raise InvalidResponseError(f"unexpected contribution response {status_code}")
+
+
+def step_event(
+    *,
+    step: str,
+    outcome: str,
+    duration_seconds: float,
+    tenant_tag: str | None = None,
+    run_id: str | None = None,
+    reason: str | None = None,
+) -> dict[str, object]:
+    event: dict[str, object] = {
+        "event": "calibration.runner_step",
+        "step": step,
+        "outcome": outcome,
+        "duration_seconds": duration_seconds,
+    }
+    if tenant_tag is not None:
+        event["tenant_tag"] = tenant_tag
+    if run_id is not None:
+        event["run_id"] = run_id
+    if reason is not None:
+        event["reason"] = reason
+    return event
+
+
+def stable_run_id(kind: Literal["input", "treatment"], parts: Sequence[str]) -> str:
+    prefix = _RUN_KIND.get(kind)
+    if prefix is None:
+        raise UsageError("run id kind must be input or treatment")
+    digest = hashlib.sha256("\n".join(parts).encode()).hexdigest()
+    return prefix + digest[:32]
+
+
+def attempt_run_id(base: str, attempt: int) -> str:
+    """The first attempt keeps the manifest's run id. Later attempts add a suffix."""
+    if isinstance(attempt, bool) or not isinstance(attempt, int):
+        raise UsageError(f"run attempt must be an integer from 1 to {_MAX_ATTEMPT}")
+    if not 1 <= attempt <= _MAX_ATTEMPT:
+        raise UsageError(f"run attempt must be an integer from 1 to {_MAX_ATTEMPT}")
+    if attempt == 1:
+        return base
+    suffix = f"_{attempt}"
+    room = 128 - len(suffix)
+    trimmed = base if len(base) <= room else base[:room]
+    return trimmed + suffix
+
+
+def input_attempt_exhausted(attempt: int) -> bool:
+    return attempt > _MAX_ATTEMPT
+
+
+def preprocessing_digest(
+    document: Mapping[str, object],
+    *,
+    purpose: str = "calibration",
+) -> str:
+    """SHA-256 of the same canonical fields the server stores.
+
+    A test locks this to ``grader_preprocessing_digest``. A document
+    the server would reject is refused here, before a token is minted.
+    """
+    schema_version = document.get("schema_version", "grader_preprocessing.v1")
+    if schema_version not in {"grader_preprocessing.v1", "grader_preprocessing.v2"}:
+        raise UsageError(
+            "grader preprocessing schema_version must be grader_preprocessing.v1 "
+            "or grader_preprocessing.v2"
+        )
+    v2 = schema_version == "grader_preprocessing.v2"
+    if purpose != "calibration" and v2:
+        raise UsageError("grader_preprocessing.v2 is accepted only on a calibration run")
+    agent_context = document.get("agent_context")
+    if agent_context not in {"production_recorded", "none"}:
+        raise UsageError("grader preprocessing agent_context must be production_recorded or none")
+    guidance = document.get("guidance", "excluded")
+    hierarchy = document.get("hierarchy_context", "excluded")
+    if guidance != "excluded" or hierarchy != "excluded":
+        raise UsageError("grader preprocessing guidance and hierarchy_context must be excluded")
+    extraction = document.get("extraction")
+    if not isinstance(extraction, Mapping):
+        raise UsageError("grader preprocessing needs an extraction object")
+    selector = _extraction_selector(extraction)
+    allowed = _CALIBRATION_EXTRACTION_SELECTORS if v2 else _PRODUCTION_EXTRACTION_SELECTORS
+    if selector not in allowed:
+        raise UsageError("extraction model selector is not a platform extraction model")
+    if selector not in _PRODUCTION_EXTRACTION_SELECTORS:
+        if purpose != "calibration":
+            raise UsageError("a non-default extraction model is accepted only on a calibration run")
+        if _extraction_settings(extraction) is None:
+            raise UsageError("a non-default extraction model requires reasoning_effort")
+    payload: dict[str, object] = {
+        "schema_version": schema_version,
+        "extraction": _extraction_digest_form(extraction, include_revision=v2),
+        "agent_context": agent_context,
+        "guidance": guidance,
+        "hierarchy_context": hierarchy,
+    }
+    if v2:
+        variant = document.get("variant")
+        if not isinstance(variant, str) or not variant.strip():
+            raise UsageError("grader_preprocessing.v2 needs a variant")
+        produces = _produces_digest_form(document.get("produces"))
+        payload["produces"] = produces
+        payload["variant"] = variant
+        if _compresses(produces):
+            payload["compression_revision"] = _COMPRESSION_REVISION
+    elif "produces" in document or "variant" in document:
+        raise UsageError("grader_preprocessing.v1 does not include produces or variant")
+    return hashlib.sha256(_canonical_bytes(payload)).hexdigest()
+
+
+def default_preprocessing_document(runtime: object) -> dict[str, object] | None:
+    """The preprocessing document the server derives from a runtime pin.
+
+    Both document flags off hash extraction as null, with production-recorded
+    agent context. ``revision`` stays off this document because the server's
+    preprocessing model rejects that field.
+    """
+    if not isinstance(runtime, Mapping):
+        return None
+    extraction = runtime.get("grader_extraction")
+    if not isinstance(extraction, Mapping):
+        stored_extraction: dict[str, object] = {
+            "format_contracts": False,
+            "claim_evidence": False,
+        }
+    else:
+        stored_extraction = {
+            key: extraction[key]
+            for key in ("format_contracts", "claim_evidence", "model", "timeout_seconds")
+            if key in extraction
+        }
+        stored_extraction.setdefault("format_contracts", False)
+        stored_extraction.setdefault("claim_evidence", False)
+    return {
+        "schema_version": "grader_preprocessing.v1",
+        "extraction": stored_extraction,
+        "agent_context": "production_recorded",
+        "guidance": "excluded",
+        "hierarchy_context": "excluded",
+    }
+
+
+def default_preprocessing_digest(runtime: object) -> str | None:
+    """Digest the server derives from a runtime pin when no document was stored.
+
+    A test locks this to ``read_grader_preprocessing``.
+    """
+    document = default_preprocessing_document(runtime)
+    if document is None:
+        return None
+    return preprocessing_digest(document)
+
+
+def resolved_runtime_pin(
+    runtime: Mapping[str, object],
+    preprocessing: Mapping[str, object] | None = None,
+) -> dict[str, object]:
+    """The runtime pin the server stores for this request.
+
+    Admission fills defaults, copies extraction from the preprocessing document,
+    and sets medium reasoning on an unset ``gpt-5.6-luna`` verifier. Comparing
+    the operator's JSON to the stored pin always disagrees.
+    """
+    judge_model = runtime.get("judge_model")
+    if not isinstance(judge_model, str) or not judge_model.strip():
+        raise UsageError("platform grader runtime is missing judge_model")
+    verifier_model = runtime.get("verifier_model")
+    if not isinstance(verifier_model, Mapping):
+        raise UsageError("platform grader runtime is missing verifier_model")
+    _refuse_decisions_verifier_reasoning(verifier_model)
+    extraction: object = runtime.get("grader_extraction")
+    if preprocessing is not None:
+        declared = preprocessing.get("extraction")
+        if (
+            "grader_extraction" in runtime
+            and isinstance(extraction, Mapping)
+            and isinstance(declared, Mapping)
+            and _extraction_digest_form(extraction) != _extraction_digest_form(declared)
+        ):
+            raise UsageError(
+                "platform grader runtime grader_extraction does not match "
+                "the declared preprocessing extraction"
+            )
+        extraction = declared
+    pin: dict[str, object] = {
+        "judge_model": judge_model,
+        "verifier_model": _luna_verifier(_stored_verifier(verifier_model)),
+        "allow_customer_private_llm_judge": bool(
+            runtime.get("allow_customer_private_llm_judge", False)
+        ),
+        "judge_timeout_seconds": _judge_timeout(runtime.get("judge_timeout_seconds", 60.0)),
+        "judge_max_retries": _judge_retries(runtime.get("judge_max_retries", 3)),
+        "finding_audit": _finding_audit(runtime.get("finding_audit")),
+        "max_occurrences_per_finding": _occurrences(runtime.get("max_occurrences_per_finding", 10)),
+        "agent_handoff_quality": bool(runtime.get("agent_handoff_quality", True)),
+        "grader_extraction": _stored_extraction(extraction),
+    }
+    batching_pin = _batching(runtime["batching"]) if "batching" in runtime else None
+    threshold = _decision_threshold(runtime.get("decision_threshold", 0.5))
+    verifier_selector = verifier_model.get("selector")
+    batch_selectors = _batching_selectors(batching_pin)
+    decisions_arm = (
+        judge_model.strip().startswith("openai-decisions/")
+        or (
+            isinstance(verifier_selector, str) and verifier_selector.startswith("openai-decisions/")
+        )
+        or any(
+            isinstance(selector, str) and selector.startswith("openai-decisions/")
+            for selector in batch_selectors
+        )
+    )
+    if threshold != 0.5:
+        if not decisions_arm:
+            raise UsageError(
+                "decision_threshold is only valid for an openai-decisions judge or verifier"
+            )
+        pin["decision_threshold"] = threshold
+    cutoffs = _decision_threshold_by_criterion_id(
+        runtime.get("decision_threshold_by_criterion_id", {})
+    )
+    if cutoffs:
+        if not (
+            _names_probability_route(judge_model)
+            or _names_probability_route(verifier_model.get("selector"))
+            or any(_names_probability_route(selector) for selector in batch_selectors)
+        ):
+            raise UsageError(
+                "decision_threshold_by_criterion_id is only valid for a jev/ or "
+                "openai-decisions/ route"
+            )
+        pin["decision_threshold_by_criterion_id"] = cutoffs
+    if batching_pin is not None:
+        pin["batching"] = batching_pin
+    return pin
+
+
+def runtime_has_decider(runtime: Mapping[str, object]) -> bool:
+    """Whether any grading route on this pin requires an explicit gate choice."""
+    verifier = runtime.get("verifier_model")
+    selectors = (
+        runtime.get("judge_model"),
+        verifier.get("selector") if isinstance(verifier, Mapping) else None,
+        *_batching_selectors(runtime.get("batching")),
+    )
+    return any(_names_probability_route(selector) for selector in selectors)
+
+
+def run_request_mismatch(
+    run: Mapping[str, object],
+    *,
+    dataset_id: str,
+    version_id: str,
+    environment_id: str,
+    workspace_id: str,
+    samples: int,
+    preprocessing_digest: str | None,
+    runtime: Mapping[str, object] | None = None,
+    preprocessing: Mapping[str, object] | None = None,
+    criterion_ids: Sequence[str] | None = None,
+    treatment: str | None = None,
+    tenant_tag: str | None = None,
+    manifest_id: str | None = None,
+    label_release: str | None = None,
+    source_run_id: str | None = None,
+    criterion_versions: Mapping[str, str] | None = None,
+) -> str | None:
+    """Why this stored run is not the run this command asked for, or None."""
+    if run.get("purpose") != "calibration":
+        return "purpose"
+    metadata = _metadata(run)
+    if metadata.get("environment_id") != environment_id:
+        return "environment_id"
+    if metadata.get("workspace_id") != workspace_id:
+        return "workspace_id"
+    if metadata.get("eval_dataset_id") != dataset_id:
+        return "dataset_id"
+    if _version_id(metadata) != version_id:
+        return "dataset_version"
+    if _samples(run) != samples:
+        return "samples_per_task"
+    if not _preprocessing_matches(run, preprocessing_digest):
+        return "preprocessing"
+    if runtime is not None and _canonical_text(
+        metadata.get("platform_grader_runtime")
+    ) != _canonical_text(resolved_runtime_pin(runtime, preprocessing)):
+        return "runtime"
+    if criterion_ids is not None and metadata.get("platform_grader_criterion_ids") != list(
+        criterion_ids
+    ):
+        return "criterion_ids"
+    if criterion_versions is not None:
+        stored_pins = _criterion_version_map(metadata.get("platform_grader_criterion_versions"))
+        if stored_pins is None or stored_pins != dict(criterion_versions):
+            return "criterion_versions"
+    if treatment is None:
+        if metadata.get("calibration_treatment") is not None:
+            return "treatment"
+        if metadata.get("production_input_source_run_id") is not None:
+            return "source_run_id"
+        return None
+    if metadata.get("calibration_treatment") != treatment:
+        return "treatment"
+    if metadata.get("calibration_tenant_tag") != tenant_tag:
+        return "tenant_tag"
+    if metadata.get("calibration_corpus_manifest_id") != manifest_id:
+        return "corpus_manifest_id"
+    if metadata.get("calibration_label_release") != label_release:
+        return "label_release"
+    if metadata.get("production_input_source_run_id") != source_run_id:
+        return "source_run_id"
+    return None
+
+
+def input_run_is_reusable(run: Mapping[str, object]) -> bool:
+    """Resume existing inputs, including a failed run with certified complete evidence."""
+    metadata = _metadata(run)
+    return run.get("status") not in _FAILED_INPUT or (
+        run.get("status") == "failed"
+        and metadata.get("evidence_complete") is True
+        and metadata.get("failure_stage") == "after_evidence"
+    )
+
+
+def _criterion_version_map(raw: object) -> dict[str, str] | None:
+    if raw is None:
+        return {}
+    if not isinstance(raw, Mapping):
+        return None
+    parsed: dict[str, str] = {}
+    for key, value in raw.items():
+        if not isinstance(key, str) or not isinstance(value, str):
+            return None
+        parsed[key] = value
+    return parsed
+
+
+def _metadata(run: Mapping[str, object]) -> Mapping[str, object]:
+    metadata = run.get("metadata")
+    if isinstance(metadata, Mapping):
+        return metadata
+    return {}
+
+
+def _version_id(metadata: Mapping[str, object]) -> str | None:
+    version = metadata.get("eval_dataset_version")
+    if isinstance(version, Mapping):
+        value = version.get("version_id")
+        return value if isinstance(value, str) else None
+    if isinstance(version, str):
+        return version
+    return None
+
+
+def _preprocessing_matches(run: Mapping[str, object], expected: str | None) -> bool:
+    actual = _preprocessing_digest(run)
+    if expected is not None:
+        return actual == expected
+    # No manifest digest means the pin-derived default. A read with no object matches too.
+    if actual is None:
+        return True
+    return actual == default_preprocessing_digest(_metadata(run).get("platform_grader_runtime"))
+
+
+def _preprocessing_digest(run: Mapping[str, object]) -> str | None:
+    for source in (_metadata(run), run):
+        raw = source.get("grader_preprocessing")
+        if not isinstance(raw, Mapping):
+            continue
+        digest = raw.get("grader_preprocessing_digest")
+        if isinstance(digest, str):
+            return digest
+    return None
+
+
+def _samples(run: Mapping[str, object]) -> int | None:
+    raw = _metadata(run).get("samples_per_task", 1)
+    if isinstance(raw, bool) or not isinstance(raw, int) or raw < 1:
+        return None
+    return raw
+
+
+def _extraction_digest_form(
+    extraction: Mapping[str, object],
+    *,
+    include_revision: bool = False,
+) -> dict[str, object] | None:
+    format_contracts = extraction.get("format_contracts", False) is True
+    claim_evidence = extraction.get("claim_evidence", False) is True
+    if not format_contracts and not claim_evidence:
+        return None
+    form: dict[str, object] = {
+        "format_contracts": format_contracts,
+        "model": _model_digest_form(extraction.get("model")),
+        "timeout_seconds": _timeout_seconds(extraction.get("timeout_seconds", 600.0)),
+    }
+    if claim_evidence:
+        form["claim_evidence"] = True
+    if include_revision:
+        form["revision"] = _EXTRACTION_REVISION
+    return form
+
+
+def _extraction_selector(extraction: Mapping[str, object]) -> str:
+    model = extraction.get("model")
+    if model is None:
+        return _DEFAULT_EXTRACTION_SELECTOR
+    if isinstance(model, str):
+        return model
+    if isinstance(model, Mapping):
+        selector = model.get("selector", _DEFAULT_EXTRACTION_SELECTOR)
+        if isinstance(selector, str) and selector:
+            return selector
+    raise UsageError("grader extraction model must be an object or a selector")
+
+
+def _extraction_settings(extraction: Mapping[str, object]) -> Mapping[str, object] | None:
+    model = extraction.get("model")
+    if not isinstance(model, Mapping):
+        return None
+    settings = model.get("settings")
+    if settings is None:
+        return None
+    if not isinstance(settings, Mapping):
+        raise UsageError("grader extraction model settings must be an object")
+    return settings
+
+
+def _produces_digest_form(value: object) -> list[dict[str, object]]:
+    if not isinstance(value, list):
+        raise UsageError("grader_preprocessing.v2 needs produces")
+    items: list[dict[str, object]] = []
+    names: list[str] = []
+    unknown: list[str] = []
+    for item in value:
+        if not isinstance(item, Mapping):
+            raise UsageError("grader preprocessing produces items must be objects")
+        name = item.get("name")
+        shape = item.get("shape", 1)
+        source = item.get("source")
+        if (
+            not isinstance(name, str)
+            or not name
+            or name in {".", ".."}
+            or "/" in name
+            or "\\" in name
+        ):
+            raise UsageError("grader preprocessing produces name must be a file name")
+        if isinstance(shape, bool) or not isinstance(shape, int) or shape < 1:
+            raise UsageError("grader preprocessing produces shape must be a positive integer")
+        if source not in {"code", "model"}:
+            raise UsageError("grader preprocessing produces source must be code or model")
+        if name in names:
+            raise UsageError("grader preprocessing produces names must be unique")
+        names.append(name)
+        if source == "code" and name not in _KNOWN_CODE_DOCUMENTS:
+            unknown.append(name)
+        form: dict[str, object] = {"name": name, "shape": shape, "source": source}
+        recipe = item.get("recipe")
+        if recipe is not None:
+            form["recipe"] = _recipe_digest_form(name, source, recipe)
+        items.append(form)
+    if unknown:
+        raise UsageError(
+            "code_document_unknown: no code builder writes " + ", ".join(sorted(unknown))
+        )
+    unbuilt = sorted(
+        str(item["name"])
+        for item in items
+        if item["source"] == "model"
+        and item["name"] not in _EXTRACTION_DOCUMENTS
+        and not (isinstance(item.get("recipe"), Mapping) and "model_document" in item["recipe"])
+    )
+    if unbuilt:
+        raise UsageError(
+            "model_document_unknown: no model builder writes "
+            + ", ".join(unbuilt)
+            + "; add a model_document recipe"
+        )
+    _require_model_document_sources(items)
+    return items
+
+
+def _require_model_document_sources(items: list[dict[str, object]]) -> None:
+    """The server's ``_require_model_document_sources`` refusals."""
+
+    model_names = {item["name"] for item in items if item["source"] == "model"}
+    model_names |= _EXTRACTION_DOCUMENTS
+    listed_code = {item["name"] for item in items if item["source"] == "code"}
+    for item in items:
+        recipe = item.get("recipe")
+        model_document = recipe.get("model_document") if isinstance(recipe, Mapping) else None
+        sources = model_document.get("sources") if isinstance(model_document, Mapping) else None
+        for source in sources if isinstance(sources, list) else ():
+            if source in model_names:
+                raise UsageError(f"{item['name']} cannot read another model document: {source}")
+            if source != "trajectory" and source not in _KNOWN_CODE_DOCUMENTS:
+                raise UsageError(f"{item['name']} names an unknown source: {source}")
+            if source != "trajectory" and source not in listed_code:
+                raise UsageError(
+                    f"{item['name']} reads {source}, which is not listed with source code"
+                )
+
+
+def _compresses(produces: list[dict[str, object]]) -> bool:
+    for item in produces:
+        recipe = item.get("recipe")
+        clip_slots = recipe.get("clip_slots") if isinstance(recipe, Mapping) else None
+        if isinstance(clip_slots, Mapping) and any(
+            isinstance(rule, Mapping) and rule.get("mode") == "compress"
+            for rule in clip_slots.values()
+        ):
+            return True
+    return False
+
+
+def preprocessing_slot_lines(document: Mapping[str, object]) -> list[str]:
+    """One line per clip slot: its document and how it clips. Never the prompt text."""
+
+    produces = document.get("produces")
+    listed: dict[str, Mapping[str, object]] | None = None
+    if document.get("schema_version") == "grader_preprocessing.v2" and isinstance(produces, list):
+        listed = {
+            str(item["name"]): item
+            for item in produces
+            if isinstance(item, Mapping) and item.get("source") == "code"
+        }
+    lines = []
+    for slot, (name, limit, _ceiling) in _CLIP_SLOTS.items():
+        line = f"{slot} document={name}"
+        if listed is not None and name not in listed:
+            lines.append(f"{line} mode=not_written")
+            continue
+        recipe = (listed or {}).get(name, {}).get("recipe")
+        clip_slots = recipe.get("clip_slots") if isinstance(recipe, Mapping) else None
+        rule = clip_slots.get(slot) if isinstance(clip_slots, Mapping) else None
+        if not isinstance(rule, Mapping):
+            lines.append(f"{line} mode=clip limit={limit}")
+        elif rule.get("mode") == "clip":
+            lines.append(f"{line} mode=clip limit={rule.get('limit', limit)}")
+        else:
+            lines.append(
+                f"{line} mode=compress limit={rule['limit']} budget={rule['budget']} "
+                f"model={rule['model']} reasoning_effort={rule['reasoning_effort']} "
+                f"prompt_chars={len(str(rule['prompt']))}"
+            )
+    return lines
+
+
+def _recipe_digest_form(name: str, source: object, recipe: object) -> dict[str, object]:
+    """The server's ``ExtractionRecipe`` refusals, and its canonical form."""
+
+    if not isinstance(recipe, Mapping):
+        raise UsageError("a recipe must be an object")
+    unknown = sorted(str(key) for key in recipe if key not in _RECIPE_OPERATORS)
+    if unknown:
+        raise UsageError("unknown recipe operator: " + ", ".join(unknown))
+    if not any(recipe.get(operator) is not None for operator in _RECIPE_OPERATORS):
+        raise UsageError("a recipe needs at least one operator")
+    if "model_document" in recipe and recipe["model_document"] is None:
+        raise UsageError("model_document cannot be null; leave it out")
+    form: dict[str, object] = {}
+    if recipe.get("model_document") is not None:
+        form["model_document"] = _model_document_form(name, source, recipe["model_document"])
+    if recipe.get("clip_slots") is not None:
+        if source != "code":
+            raise UsageError("a recipe applies only to a code document")
+        form["clip_slots"] = _clip_slots_form(name, recipe["clip_slots"])
+    return form
+
+
+def _clip_slots_form(name: str, clip_slots: object) -> dict[str, object]:
+    if not isinstance(clip_slots, Mapping) or not clip_slots:
+        raise UsageError("recipe clip_slots must be a non-empty object")
+    rules: dict[str, object] = {}
+    for slot, rule in clip_slots.items():
+        known = _CLIP_SLOTS.get(slot)
+        if known is None:
+            raise UsageError(f"unknown clip slot {slot!r}")
+        owner, _limit, ceiling = known
+        if owner != name:
+            raise UsageError(f"clip slot {slot} belongs to {owner}")
+        rules[slot] = _slot_rule_form(slot, rule, ceiling)
+    return rules
+
+
+def _model_document_form(name: str, source: object, recipe: object) -> dict[str, object]:
+    """The server's ``ModelDocumentRecipe`` refusals, and its canonical form."""
+
+    if not isinstance(recipe, Mapping) or set(recipe) != set(_MODEL_DOCUMENT_FIELDS):
+        raise UsageError(f"model_document needs exactly {', '.join(_MODEL_DOCUMENT_FIELDS)}")
+    if source != "model":
+        raise UsageError("a model_document recipe needs source model")
+    if name in _KNOWN_CODE_DOCUMENTS | _EXTRACTION_DOCUMENTS | {"extraction.json"}:
+        raise UsageError(f"{name} has its own builder")
+    sources = recipe["sources"]
+    if (
+        not isinstance(sources, list)
+        or not sources
+        or not all(isinstance(item, str) and item for item in sources)
+        or len(sources) != len(set(sources))
+    ):
+        raise UsageError("model document sources must be unique non-empty names")
+    prompt = recipe["prompt"]
+    if not isinstance(prompt, str) or not prompt:
+        raise UsageError("a model document needs a prompt")
+    if len(prompt.encode("utf-8")) > 16 * 1024:
+        raise UsageError("a recipe prompt must be at most 16 KB")
+    form: dict[str, object] = {"sources": sources, "prompt": prompt}
+    for key, ceiling in (
+        ("max_input_chars", _MODEL_DOCUMENT_INPUT_CEILING),
+        ("budget", _MODEL_DOCUMENT_BUDGET_CEILING),
+    ):
+        value = recipe[key]
+        if isinstance(value, bool) or not isinstance(value, int) or value < 1:
+            raise UsageError(f"model document {key} must be a positive integer")
+        if value > ceiling:
+            raise UsageError(f"model document {key} is above the ceiling of {ceiling}")
+        form[key] = value
+    if recipe["model"] not in _CALIBRATION_EXTRACTION_SELECTORS:
+        raise UsageError("extraction model selector is not a platform extraction model")
+    if recipe["reasoning_effort"] not in _REASONING_EFFORTS:
+        raise UsageError("model document reasoning_effort is not a known effort")
+    return {**form, "model": recipe["model"], "reasoning_effort": recipe["reasoning_effort"]}
+
+
+def _slot_rule_form(slot: str, rule: object, ceiling: int) -> dict[str, object]:
+    if not isinstance(rule, Mapping):
+        raise UsageError(f"clip slot {slot} rule must be an object")
+    mode = rule.get("mode")
+    fields = {"clip": {"mode", "limit"}, "compress": set(_COMPRESS_FIELDS)}.get(str(mode))
+    if fields is None:
+        raise UsageError(f"clip slot {slot} mode must be clip or compress")
+    if not set(rule) <= fields:
+        raise UsageError(f"clip slot {slot} has unknown fields")
+    form: dict[str, object] = {"mode": mode}
+    amounts = ("limit",) if mode == "clip" else ("limit", "budget")
+    for key in amounts:
+        if key not in rule and mode == "clip":
+            continue
+        value = rule.get(key)
+        if isinstance(value, bool) or not isinstance(value, int) or value < 1:
+            raise UsageError(f"clip slot {slot} {key} must be a positive integer")
+        if value > ceiling:
+            raise UsageError(f"{slot} {key} is above the ceiling of {ceiling}")
+        form[key] = value
+    if mode == "clip":
+        return form
+    prompt = rule.get("prompt")
+    if not isinstance(prompt, str) or not prompt:
+        raise UsageError(f"clip slot {slot} needs a prompt")
+    if len(prompt.encode("utf-8")) > 16 * 1024:
+        raise UsageError("a recipe prompt must be at most 16 KB")
+    model = rule.get("model")
+    if model not in _CALIBRATION_EXTRACTION_SELECTORS:
+        raise UsageError("extraction model selector is not a platform extraction model")
+    effort = rule.get("reasoning_effort")
+    if effort not in _REASONING_EFFORTS:
+        raise UsageError(f"clip slot {slot} reasoning_effort is not a known effort")
+    return {**form, "prompt": prompt, "model": model, "reasoning_effort": effort}
+
+
+def _model_digest_form(model: object) -> str | dict[str, object]:
+    if model is None:
+        model = {
+            "selector": "gpt-5.6-luna",
+            "settings": {"reasoning_effort": "high"},
+        }
+    if isinstance(model, str):
+        return _canonical_selector(model)
+    if not isinstance(model, Mapping):
+        raise UsageError("grader extraction model must be an object or a selector")
+    selector = model.get("selector", "gpt-5.6-luna")
+    if not isinstance(selector, str) or not selector:
+        raise UsageError("grader extraction model selector is empty")
+    settings = model.get("settings")
+    canonical = _canonical_selector(selector)
+    if settings is None:
+        return canonical
+    if not isinstance(settings, Mapping):
+        raise UsageError("grader extraction model settings must be an object")
+    return {"selector": canonical, "settings": dict(settings)}
+
+
+def _canonical_selector(selector: str) -> str:
+    provider, separator, model_name = selector.partition("/")
+    if separator:
+        return f"{provider}/{model_name}"
+    return f"openai/{selector}"
+
+
+def _timeout_seconds(value: object) -> float:
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        raise UsageError("grader extraction timeout_seconds must be a positive number")
+    if value <= 0:
+        raise UsageError("grader extraction timeout_seconds must be a positive number")
+    return float(value)
+
+
+_EXTRACTION_REVISION = 16
+# Mirrors GRADER_COMPRESSION_REVISION; a test locks the two.
+_COMPRESSION_REVISION = 1
+_DEFAULT_EXTRACTION_SELECTOR = "gpt-5.6-luna"
+_PRODUCTION_EXTRACTION_SELECTORS = frozenset({"gpt-5.6-luna", "openai/gpt-5.6-luna"})
+_CALIBRATION_EXTRACTION_SELECTORS = _PRODUCTION_EXTRACTION_SELECTORS | {
+    "gpt-6-luna",
+    "openai/gpt-6-luna",
+}
+# Mirrors the server's CLIP_SLOTS:
+# slot -> (document, today's limit, recipe ceiling). A test locks the two.
+_CLIP_SLOTS: dict[str, tuple[str, int, int]] = {
+    "context.args": ("aggregate_context.md", 2_000, 8_000),
+    "context.error": ("aggregate_context.md", 500, 2_000),
+    "answers.item": ("aggregate_answers.md", 30_000, 120_000),
+    "answers.final": ("aggregate_answers.md", 30_000, 120_000),
+    "input_output.final_reply": ("input_output.md", 2_000, 8_000),
+    "facts.scalar": ("extracted_facts.md", 300, 1_200),
+    "facts.line": ("extracted_facts.md", 400, 1_600),
+    "revision.request": ("revision_diff.md", 2_000, 8_000),
+    "revision.sentence": ("revision_diff.md", 400, 1_600),
+    "pairs.result_excerpt": ("output_reasoning_pairs.json", 4_000, 16_000),
+}
+_KNOWN_CODE_DOCUMENTS = frozenset(
+    {
+        *(document for document, _limit, _ceiling in _CLIP_SLOTS.values()),
+        "thought_action_pairs.json",
+        "aggregate_fields.md",
+        "action_ledger.jsonl",
+        "task.json",
+        "output.json",
+        "run_status.json",
+        "turns.md",
+    }
+)
+# Mirrors ExtractionRecipe's fields: the operators a recipe may name.
+_RECIPE_OPERATORS = ("clip_slots", "model_document")
+_COMPRESS_FIELDS = ("mode", "limit", "budget", "prompt", "model", "reasoning_effort")
+# Mirrors the server's EXTRACTION_DOCUMENTS and MODEL_DOCUMENT_* ceilings;
+# a test locks them.
+_EXTRACTION_DOCUMENTS = frozenset({"claim_evidence.md", "format_contracts.md"})
+_MODEL_DOCUMENT_INPUT_CEILING = 800_000
+_MODEL_DOCUMENT_BUDGET_CEILING = 120_000
+_MODEL_DOCUMENT_FIELDS = (
+    "sources",
+    "prompt",
+    "model",
+    "reasoning_effort",
+    "max_input_chars",
+    "budget",
+)
+_REASONING_EFFORTS = frozenset({"none", "minimal", "low", "medium", "high", "xhigh", "max"})
+_LUNA_MEDIUM = "medium"
+_MAX_OCCURRENCES = 100
+
+
+def _refuse_decisions_verifier_reasoning(verifier_model: Mapping[str, object]) -> None:
+    selector = verifier_model.get("selector")
+    if not isinstance(selector, str) or not selector.strip().startswith("openai-decisions/"):
+        return
+    if verifier_model.get("settings") is not None:
+        raise UsageError("reasoning settings are not valid for an openai-decisions verifier")
+
+
+def _stored_verifier(model: Mapping[str, object]) -> dict[str, object]:
+    selector = model.get("selector")
+    if not isinstance(selector, str) or not selector.strip():
+        raise UsageError("platform grader runtime is missing verifier_model")
+    return {"selector": selector, "settings": _reasoning_settings(model.get("settings"))}
+
+
+def _luna_verifier(verifier: dict[str, object]) -> dict[str, object]:
+    """An unset Luna verifier is stored with medium reasoning, as the suite does."""
+    selector = verifier["selector"]
+    if not isinstance(selector, str) or verifier["settings"] is not None:
+        return verifier
+    provider, separator, model_name = selector.partition("/")
+    if not separator:
+        provider, model_name = "openai", selector
+    if provider == "openai" and model_name == _DEFAULT_EXTRACTION_SELECTOR:
+        return {"selector": selector, "settings": {"reasoning_effort": _LUNA_MEDIUM}}
+    return verifier
+
+
+def _stored_extraction(raw: object) -> dict[str, object]:
+    if raw is None:
+        raw = {}
+    if not isinstance(raw, Mapping):
+        raise UsageError("grader extraction must be an object")
+    format_contracts = raw.get("format_contracts", False) is True
+    claim_evidence = raw.get("claim_evidence", False) is True
+    model = raw.get("model")
+    if model is None:
+        stored_model: dict[str, object] = {
+            "selector": _DEFAULT_EXTRACTION_SELECTOR,
+            "settings": {"reasoning_effort": "high"},
+        }
+    elif isinstance(model, str):
+        stored_model = {"selector": model, "settings": None}
+    elif isinstance(model, Mapping):
+        stored_model = _stored_verifier(model)
+    else:
+        raise UsageError("grader extraction model must be an object or a selector")
+    stored: dict[str, object] = {
+        "format_contracts": format_contracts,
+        "claim_evidence": claim_evidence,
+        "model": stored_model,
+        "timeout_seconds": _timeout_seconds(raw.get("timeout_seconds", 600.0)),
+    }
+    if format_contracts or claim_evidence:
+        stored["revision"] = _EXTRACTION_REVISION
+    return stored
+
+
+def _reasoning_settings(settings: object) -> dict[str, str] | None:
+    if settings is None:
+        return None
+    if not isinstance(settings, Mapping) or set(settings) != {"reasoning_effort"}:
+        raise UsageError("model settings must name reasoning_effort")
+    effort = settings.get("reasoning_effort")
+    if not isinstance(effort, str) or not effort:
+        raise UsageError("model settings must name reasoning_effort")
+    return {"reasoning_effort": effort}
+
+
+def _finding_audit(raw: object) -> dict[str, object]:
+    if not isinstance(raw, Mapping):
+        raw = {}
+    enabled = raw.get("enabled", False)
+    if not isinstance(enabled, bool):
+        raise UsageError("finding audit enabled must be a boolean")
+    model = raw.get("model", None)
+    if model is not None and not isinstance(model, str):
+        raise UsageError("finding audit model must be a string")
+    if enabled and (not isinstance(model, str) or not model.strip()):
+        raise UsageError("enabled finding audit requires a non-empty model")
+    return {"enabled": enabled, "model": model}
+
+
+def _decision_threshold(value: object) -> float:
+    if (
+        isinstance(value, bool)
+        or not isinstance(value, (int, float))
+        or not math.isfinite(value)
+        or not 0 <= value <= 1
+    ):
+        raise UsageError("decision_threshold must be a number from 0 to 1")
+    return float(value)
+
+
+def _decision_threshold_by_criterion_id(value: object) -> dict[str, float]:
+    if not isinstance(value, Mapping) or not all(
+        isinstance(key, str)
+        and key
+        and not isinstance(cutoff, bool)
+        and isinstance(cutoff, (int, float))
+        and math.isfinite(cutoff)
+        and 0 <= cutoff <= 1
+        for key, cutoff in value.items()
+    ):
+        raise UsageError(
+            "decision_threshold_by_criterion_id must map criterion ids to numbers from 0 to 1"
+        )
+    return {key: float(value[key]) for key in sorted(value)}
+
+
+_BATCHING_FIELDS = frozenset({"groups", "verifier_by_batch", "judge_by_criterion"})
+
+
+def _batching(value: object) -> dict[str, object]:
+    if not isinstance(value, Mapping) or not value or not set(value) <= _BATCHING_FIELDS:
+        raise UsageError("batching must be an object with groups")
+    groups = value.get("groups", {})
+    if not isinstance(groups, Mapping):
+        raise UsageError("batching.groups must map rubric groups to batch keys")
+    cleaned_groups: dict[str, str] = {}
+    for key, item in groups.items():
+        if not _bare_rubric_group(key):
+            raise UsageError("batching.groups keys must be bare rubric group names")
+        if not _batch_key(item):
+            raise UsageError("batching.groups values must be non-empty batch keys")
+        cleaned_groups[key] = item
+    stored: dict[str, object] = {
+        "groups": {key: cleaned_groups[key] for key in sorted(cleaned_groups)}
+    }
+    verifiers = value.get("verifier_by_batch", {})
+    if verifiers:
+        if not isinstance(verifiers, Mapping):
+            raise UsageError("batching.verifier_by_batch must map batch keys to verifier models")
+        cleaned_verifiers: dict[str, dict[str, object]] = {}
+        for key, item in verifiers.items():
+            if not _batch_key(key):
+                raise UsageError("batching.verifier_by_batch keys must be non-empty batch keys")
+            if not isinstance(item, Mapping):
+                raise UsageError("batching.verifier_by_batch values must be verifier models")
+            _refuse_decisions_verifier_reasoning(item)
+            stored_verifier = _luna_verifier(_stored_verifier(item))
+            _require_batch_verifier_settings(key, stored_verifier)
+            cleaned_verifiers[key] = stored_verifier
+        stored["verifier_by_batch"] = {
+            key: cleaned_verifiers[key] for key in sorted(cleaned_verifiers)
+        }
+    judges = value.get("judge_by_criterion", {})
+    if judges:
+        if not isinstance(judges, Mapping):
+            raise UsageError("batching.judge_by_criterion must map criterion ids to judge models")
+        cleaned_judges: dict[str, str] = {}
+        for key, item in judges.items():
+            if not _criterion_id(key):
+                raise UsageError("batching.judge_by_criterion keys must be criterion ids")
+            if not _provider_model_selector(item):
+                raise UsageError(
+                    "batching.judge_by_criterion values must be provider/model selectors"
+                )
+            cleaned_judges[key] = item
+        stored["judge_by_criterion"] = {key: cleaned_judges[key] for key in sorted(cleaned_judges)}
+    return stored
+
+
+def _criterion_id(value: object) -> bool:
+    return (
+        isinstance(value, str)
+        and bool(value)
+        and not any(character.isspace() for character in value)
+    )
+
+
+def _provider_model_selector(value: object) -> bool:
+    if not isinstance(value, str) or not value or any(character.isspace() for character in value):
+        return False
+    provider, separator, model = value.partition("/")
+    return bool(provider and separator and model)
+
+
+def _require_batch_verifier_settings(key: str, verifier: Mapping[str, object]) -> None:
+    if verifier.get("settings") is not None:
+        return
+    selector = verifier.get("selector")
+    if isinstance(selector, str) and (
+        selector.strip().startswith(("jev/", "openai-decisions/"))
+        or _luna_verifier({"selector": selector, "settings": None})["settings"] is not None
+    ):
+        return
+    raise UsageError(f"batching.verifier_by_batch[{key}] requires settings")
+
+
+def _batching_selectors(block: object) -> tuple[str, ...]:
+    if not isinstance(block, Mapping):
+        return ()
+    found: list[str] = []
+    judges = block.get("judge_by_criterion")
+    if isinstance(judges, Mapping):
+        found.extend(item for item in judges.values() if isinstance(item, str))
+    verifiers = block.get("verifier_by_batch")
+    if isinstance(verifiers, Mapping):
+        for item in verifiers.values():
+            if isinstance(item, Mapping) and isinstance(item.get("selector"), str):
+                found.append(item["selector"])
+    return tuple(found)
+
+
+def _bare_rubric_group(value: object) -> bool:
+    return (
+        isinstance(value, str)
+        and bool(value)
+        and "." not in value
+        and not any(character.isspace() for character in value)
+    )
+
+
+def _batch_key(value: object) -> bool:
+    return (
+        isinstance(value, str)
+        and bool(value)
+        and not any(character.isspace() for character in value)
+    )
+
+
+def _names_probability_route(selector: object) -> bool:
+    return isinstance(selector, str) and selector.strip().startswith(("jev/", "openai-decisions/"))
+
+
+def _judge_timeout(value: object) -> float:
+    if isinstance(value, bool) or not isinstance(value, (int, float)) or value <= 0:
+        raise UsageError("judge_timeout_seconds must be a positive number")
+    return float(value)
+
+
+def _judge_retries(value: object) -> int:
+    if isinstance(value, bool) or not isinstance(value, int) or value < 0:
+        raise UsageError("judge_max_retries must be a non-negative integer")
+    return value
+
+
+def _occurrences(value: object) -> int:
+    if isinstance(value, bool) or not isinstance(value, int) or not 1 <= value <= _MAX_OCCURRENCES:
+        raise UsageError(
+            f"max_occurrences_per_finding must be an integer from 1 to {_MAX_OCCURRENCES}"
+        )
+    return value
+
+
+def _canonical_bytes(payload: Mapping[str, object]) -> bytes:
+    return _canonical_text(dict(payload)).encode("utf-8")
+
+
+def _canonical_text(value: object) -> str:
+    return json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(",", ":"))
+
+
+def _refusal_reason(message: str) -> str:
+    first = message.splitlines()[0] if message else ""
+    if "—" in first:
+        return first.split("—", 1)[1].strip()
+    return first or "refused"
+
+
+def _money(value: Decimal) -> str:
+    return format(value.quantize(Decimal("0.001")), "f")
+
+
+def _minutes(value: Decimal) -> str:
+    return format(value.quantize(Decimal("0.1")), "f")
+
+
+def _match(value: str, pattern: re.Pattern[str], message: str) -> str:
+    if pattern.fullmatch(value) is None:
+        raise UsageError(message)
+    return value
+
+
+__all__ = [
+    "BUSY_SCOPE",
+    "BUSY_SCOPE_NOTE",
+    "MONTHLY_BUDGET_USD",
+    "SESSION_MINIMUM",
+    "TenantSpec",
+    "UsageError",
+    "attempt_run_id",
+    "contribution_receipt",
+    "default_input_setting_run_id",
+    "default_preprocessing_digest",
+    "default_preprocessing_document",
+    "document_digest",
+    "error_rate_exceeds_limit",
+    "error_verdict_rate",
+    "estimate",
+    "exclusion_reason",
+    "input_attempt_exhausted",
+    "input_run_is_reusable",
+    "is_input_setting_run",
+    "parse_tenant",
+    "preprocessing_digest",
+    "require_digest",
+    "labeled_input_run_id",
+    "require_input_label",
+    "require_input_run_id",
+    "require_json_object",
+    "require_k",
+    "require_label_release",
+    "require_manifest_id",
+    "require_treatment",
+    "resolved_runtime_pin",
+    "run_request_mismatch",
+    "spend_report",
+    "stable_run_id",
+    "step_event",
+]