sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/pytest_plugin.py

@@ -0,0 +1,62 @@
+"""Opt-in pytest fixture over the existing synchronous Plan/Run/Report path.
+
+Load with ``pytest -p sphynx.pytest_plugin``. Importing sphynx does not load
+pytest. Every test supplies its own check; no acceptance rule is selected here.
+"""
+from __future__ import annotations
+
+from collections.abc import Callable
+import inspect
+from pathlib import Path
+
+import pytest
+
+from .workflow_contracts import WorkflowContractError
+from .workflows import Plan, Report
+
+
+class EvaluationExecutionError(RuntimeError):
+    """Execution/grading did not complete. The permitted report is retained."""
+
+    def __init__(self, report: Report):
+        self.report = report
+        super().__init__(f"Evaluation is {report.status}; inspect run {report.run_id} "
+                         f"and grading {report.grading_id} before checking quality")
+
+
+class EvaluationCheckFailed(AssertionError):
+    """The test's explicit check returned False for a completed report."""
+
+    def __init__(self, report: Report):
+        self.report = report
+        super().__init__(f"Evaluation check returned False for run {report.run_id}, "
+                         f"grading {report.grading_id}")
+
+
+@pytest.fixture
+def sphynx_evaluate(tmp_path_factory) -> Callable[..., Report]:
+    """Run and grade a Plan, then apply the test's explicit boolean check.
+
+    Configuration/authorization errors keep their existing exception types.
+    Incomplete reports raise EvaluationExecutionError without invoking check.
+    Checks must return exactly bool. A False raises EvaluationCheckFailed.
+    Both exceptions retain the report; no new candidate retry is introduced.
+    """
+    def evaluate(plan: Plan, *, check: Callable[[Report], bool],
+                 output_dir: str | Path | None = None) -> Report:
+        if (not isinstance(plan, Plan) or not callable(check)
+                or inspect.iscoroutinefunction(check)
+                or inspect.iscoroutinefunction(getattr(check, "__call__", None))):
+            raise WorkflowContractError("sphynx_evaluate needs a Plan and a synchronous report check")
+        destination = output_dir if output_dir is not None else tmp_path_factory.mktemp("sx")
+        report = plan.run(destination).assess()
+        if report.status != "completed":
+            raise EvaluationExecutionError(report)
+        passed = check(report)
+        if type(passed) is not bool:
+            raise WorkflowContractError("the pytest report check must return exactly True or False")
+        if not passed:
+            raise EvaluationCheckFailed(report)
+        return report
+
+    return evaluate

sphynx/question_robustness.py

@@ -0,0 +1,437 @@
+"""Paired question wording, observed query calls and owner-reviewed query fidelity.
+
+Local adapters are trusted Python code. The pilot remains provisional until a
+new owner-reviewed release passes every review flag; hiding Python data is not
+a security boundary against the machine's owner.
+"""
+from __future__ import annotations
+
+import copy
+import io
+import time
+from contextlib import redirect_stdout, redirect_stderr
+from pathlib import Path
+from typing import Any, Callable, Mapping
+
+from ._local_eval import (LocalEvaluationRef, LocalRunRef, LocalScorecard, capture, clustered_interval,
+                         compare_scorecards, digest, encoded, identity, metric, pinned_file, read_json,
+                         report_evaluation, require, save_evaluation, text, verified_capture)
+from .contracts import ContractError
+from .retrieval import _strict_json
+
+QUESTION_ROBUSTNESS_PACK_ID = "rag.question_robustness.v1"
+QUESTION_ROBUSTNESS_INTERFACE = "sphynx.rag.question_robustness.v1"
+PILOT_MANIFEST_SHA256 = "cbb64ec35a6798f3f500ef61adbd48434b7433ea6b18e8cdc1674eb3295c5b78"
+METRIC_RELEASE = "rag.question_robustness.metric.v1"
+GRADER_RELEASE = "rag.question_robustness.grader.v1"
+VARIANTS = ("direct", "embedded", "implicit", "distractor")
+ACTIONS = ("CALL_RAG", "ASK_BUYER", "ANSWER_FROM_CONTEXT")
+_FAMILY_FLAGS = ("anchor_identity", "intent_constraints", "routing_label", "semantic_evidence_sufficiency",
+                 "temporal_scope", "variant_equivalence")
+
+
+class QuestionRobustnessPack:
+    def __init__(self, manifest_digest, cases, corpus, inputs, reviewed):
+        self.cases, self.corpus, self.inputs = cases, corpus, inputs
+        self.identity = {"pack_id": QUESTION_ROBUSTNESS_PACK_ID, "pack_release": "v1.external",
+                         "dataset_digest": manifest_digest, "corpus_digest": identity(corpus),
+                         "cohort_digest": identity(cases), "label_status": "owner_reviewed" if reviewed else "provisional",
+                         "development_only": True, "human_review_complete": reviewed}
+
+
+def load_question_robustness_pack(data_dir: str | Path, *, expected_sha256: str = PILOT_MANIFEST_SHA256) -> QuestionRobustnessPack:
+    root = Path(data_dir).expanduser().resolve(strict=True)
+    manifest_raw = (root / "manifest.json").read_bytes()
+    require(digest(manifest_raw) == "sha256:" + expected_sha256.removeprefix("sha256:"), "robustness manifest SHA-256 differs")
+    manifest = _strict_json(manifest_raw)
+    require(manifest.get("schema_version") == "sphynx.question_robustness.pilot.manifest.v1", "unsupported robustness manifest")
+    files = manifest.get("files")
+    required = {"owner-dataset.json", "candidate-input.jsonl", "corpus.json", "review/decisions.json"}
+    require(isinstance(files, dict) and required <= set(files), "robustness manifest lacks required files")
+    payloads = {name: pinned_file(root, name, entry) for name, entry in files.items()}
+    owner = _strict_json(payloads["owner-dataset.json"])
+    corpus = _strict_json(payloads["corpus.json"])
+    review = _strict_json(payloads["review/decisions.json"])
+    require(owner.get("schema_version") == "sphynx.rag.question_robustness.pilot.draft.v1"
+            and corpus.get("schema_version") == "sphynx.rag.question_robustness.pilot.corpus.v1"
+            and review.get("schema_version") == "sphynx.question_robustness.owner_review.v1", "unsupported pilot schemas")
+    chunks = corpus.get("chunks")
+    require(isinstance(chunks, list) and chunks, "pilot corpus must contain chunks")
+    chunk_map = {}
+    for chunk in chunks:
+        require(isinstance(chunk, dict) and set(chunk) == {"chunk_id", "customer", "document_id", "source_class", "source_uri", "text", "text_sha256"},
+                "invalid corpus chunk fields")
+        for key in chunk:
+            text(chunk[key], "chunk." + key)
+        require(digest(chunk["text"].encode("utf-8")) == "sha256:" + chunk["text_sha256"].removeprefix("sha256:"), "chunk text digest differs")
+        require(chunk["chunk_id"] not in chunk_map, "duplicate corpus chunk")
+        chunk_map[chunk["chunk_id"]] = chunk
+    cases = owner.get("cases")
+    families = owner.get("families")
+    require(isinstance(cases, list) and cases and isinstance(families, list), "pilot must contain cases and families")
+    case_map, family_map = {}, {}
+    for case in cases:
+        cid = text(case.get("case_id"), "case_id")
+        require(cid not in case_map, "duplicate robustness case")
+        case_map[cid] = case
+        text(case.get("customer"), "customer")
+        text(case.get("user_turn"), "user_turn")
+        require(isinstance(case.get("conversation_context"), list), "context must be the frozen JSON list")
+        refs = case.get("references")
+        require(isinstance(refs, dict) and refs.get("expected_action") in ACTIONS, "case requires an owner routing reference")
+        groups = refs.get("evidence_groups")
+        require(isinstance(groups, list), "reference evidence groups must be a list")
+        group_ids = set()
+        for group in groups:
+            gid = text(group.get("evidence_group_id"), "evidence_group_id")
+            anchors = group.get("acceptable_chunk_ids")
+            require(gid not in group_ids and type(group.get("required")) is bool
+                    and isinstance(anchors, list) and anchors and len(set(anchors)) == len(anchors), "invalid evidence group")
+            group_ids.add(gid)
+            require(all(a in chunk_map and chunk_map[a]["customer"] == case["customer"] for a in anchors), "reference crosses customer scope")
+        if case.get("family_id"):
+            require(case.get("variant") in VARIANTS and refs["expected_action"] == "CALL_RAG"
+                    and any(group["required"] for group in groups), "positive family must be corpus-answerable CALL_RAG")
+            rubric = refs.get("intent_rubric")
+            require(isinstance(rubric, dict) and isinstance(rubric.get("required_constraints"), list), "family requires an intent rubric")
+            text(rubric.get("information_need"), "information_need")
+            constraints = rubric["required_constraints"]
+            require(len({text(c.get("constraint_id"), "constraint_id") for c in constraints}) == len(constraints), "duplicate constraint")
+            qrels = refs.get("graded_relevance")
+            require(isinstance(qrels, dict) and all(a in chunk_map and chunk_map[a]["customer"] == case["customer"]
+                    and type(g) is int and 0 <= g <= 3 for a, g in qrels.items()), "invalid same-customer graded references")
+            require(all(qrels.get(a, 0) > 0 for group in groups for a in group["acceptable_chunk_ids"]), "evidence anchors need positive relevance references")
+            family_map.setdefault(case["family_id"], []).append(case)
+        else:
+            require(case.get("variant") is None and refs["expected_action"] != "CALL_RAG", "controls must be separate non-lookup cases")
+    require(len(families) == len(family_map) and len({f.get("family_id") for f in families}) == len(families), "family inventory differs")
+    for family in families:
+        siblings = family_map.get(family.get("family_id"), [])
+        require(len(siblings) == 4 and {c["variant"] for c in siblings} == set(VARIANTS)
+                and set(family.get("case_ids", [])) == {c["case_id"] for c in siblings}, "family must contain exactly four equivalent variants")
+        direct = next(c for c in siblings if c["variant"] == "direct")
+        if "references" in family:
+            require(family["references"] == direct["references"], "family reference differs from its variants")
+        require(all((c["customer"], c["conversation_context"], c["references"], c["split"])
+                    == (direct["customer"], direct["conversation_context"], direct["references"], direct["split"]) for c in siblings),
+                "paired variants differ in context, scope, split or references")
+    inputs = [_strict_json(line) for line in payloads["candidate-input.jsonl"].splitlines() if line.strip()]
+    require(len(inputs) == len(cases) and {r.get("case_id") for r in inputs} == set(case_map), "candidate input cohort differs")
+    for row in inputs:
+        require(set(row) == {"case_id", "customer", "user_turn", "conversation_context"}
+                and row == {key: case_map[row["case_id"]][key] for key in row}, "candidate inputs differ or contain references")
+    # Reordering follows the frozen, shuffled candidate file, never family order.
+    ordered = [case_map[row["case_id"]] for row in inputs]
+    require(type(review.get("human_review_complete")) is bool, "review completeness must be boolean")
+    require(len(review.get("families", [])) == len(family_map)
+            and len(review.get("controls", [])) == sum(not c.get("family_id") for c in cases)
+            and {r.get("family_id") for r in review.get("families", [])} == set(family_map)
+            and {r.get("case_id") for r in review.get("controls", [])} == {c["case_id"] for c in cases if not c.get("family_id")}, "review inventory differs")
+    reviewed = review["human_review_complete"]
+    if reviewed:
+        require(all(c["references"].get("review_status") == "owner_reviewed" for c in cases), "case references still have draft review status")
+        for row in review["families"]:
+            require(all(row.get(flag) == "approved" for flag in _FAMILY_FLAGS), "family human review is incomplete")
+        for row in review["controls"]:
+            require(all(row.get(flag) == "approved" for flag in ("context_sufficiency_or_clarification", "non_lookup_label")), "control human review is incomplete")
+        for row in review["families"] + review["controls"]:
+            text(row.get("reviewed_by"), "reviewed_by")
+            text(row.get("reviewed_at"), "reviewed_at")
+    return QuestionRobustnessPack(digest(manifest_raw), ordered, chunks, inputs, reviewed)
+
+
+def _policy(mode: str, max_context_bytes: int) -> dict[str, Any]:
+    require(mode in ("routing_only", "observed_pipeline"), "mode must be routing_only or observed_pipeline")
+    require(type(max_context_bytes) is int and max_context_bytes > 0, "context budget must be a positive integer")
+    return {"policy_version": "rag.question_robustness.policy.v1", "mode": mode, "max_search_calls": 4,
+            "candidate_depth": 50, "max_context_bytes": max_context_bytes, "context_separator": "\n\n",
+            "trials": 1, "timeout_seconds": None, "isolation": "trusted_local_in_process",
+            "cache_policy": "caller_managed_not_observed", "paired_weights": "equal_family_equal_variant"}
+
+
+def build_question_robustness_request(pack: QuestionRobustnessPack, case_id: str, *,
+                                      mode: str = "routing_only", max_context_bytes: int = 16000) -> dict[str, Any]:
+    row = next((row for row in pack.inputs if row["case_id"] == case_id), None)
+    require(row is not None, "unknown robustness case")
+    request = {"schema_version": "sphynx.rag.question_robustness.request.v1", "pack_id": QUESTION_ROBUSTNESS_PACK_ID,
+               **copy.deepcopy(row), "corpus_digest": pack.identity["corpus_digest"],
+               "execution_policy": _policy(mode, max_context_bytes)}
+    if mode == "observed_pipeline":
+        request["corpus"] = [copy.deepcopy(c) for c in pack.corpus if c["customer"] == row["customer"]]
+    request["request_sha256"] = identity(request)
+    return request
+
+
+def _stages(value: Any, request: dict[str, Any]) -> dict[str, Any]:
+    require(isinstance(value, dict) and set(value) == {"stages", "context_text"}, "retrieval evidence requires stages and exact context_text")
+    stages = value["stages"]
+    require(isinstance(stages, dict) and set(stages) == {"candidate", "fused", "reranked", "context"}, "all four retrieval stages must be captured")
+    allowed = {c["chunk_id"]: c for c in request.get("corpus", [])}
+    previous = None
+    for stage in ("candidate", "fused", "reranked", "context"):
+        ids = stages[stage]
+        require(isinstance(ids, list) and all(isinstance(cid, str) for cid in ids) and len(set(ids)) == len(ids), "stage IDs must be unique strings")
+        require(set(ids) <= set(allowed) and (previous is None or set(ids) <= previous), "stage contains unknown, wrong-customer or unobserved chunks")
+        require(len(ids) <= request["execution_policy"]["candidate_depth"], "stage exceeds candidate budget")
+        previous = set(ids)
+    expected = "\n\n".join(allowed[cid]["text"] for cid in stages["context"])
+    require(value["context_text"] == expected and len(expected.encode("utf-8")) <= request["execution_policy"]["max_context_bytes"], "final context differs from source bytes or exceeds budget")
+    return copy.deepcopy(value)
+
+
+class ObservedSearch:
+    """SDK wrapper around actual local search calls, distinct from claimed queries."""
+    def __init__(self, request: dict[str, Any]):
+        self._request = copy.deepcopy(request)
+        self._calls: list[dict[str, Any]] = []
+
+    def search(self, query: str, retrieve: Callable[[str, list[dict[str, Any]]], dict[str, Any]]) -> dict[str, Any]:
+        require(self._request["execution_policy"]["mode"] == "observed_pipeline", "routing-only run cannot observe retrieval")
+        text(query, "observed query")
+        require(len(self._calls) < self._request["execution_policy"]["max_search_calls"], "search-call budget exceeded")
+        record = {"query": query, "result": None, "failure": None}
+        self._calls.append(record)
+        start = time.perf_counter()
+        try:
+            raw = retrieve(query, copy.deepcopy(self._request["corpus"]))
+            # A malformed backend result must still leave a terminal capture;
+            # unserializable Python objects/NaN cannot be put in the manifest.
+            record["raw_result"] = _strict_json(encoded(raw))
+            record["result"] = _stages(raw, self._request)
+            return copy.deepcopy(record["result"])
+        except Exception as exc:
+            record["failure"] = type(exc).__name__
+            raise
+        finally:
+            record["latency_ms"] = (time.perf_counter() - start) * 1000
+
+
+def run_question_robustness(candidate: Mapping[str, Any], adapter: Callable, data_dir: str | Path,
+                            output_dir: str | Path, *, mode: str = "routing_only", max_context_bytes: int = 16000,
+                            expected_sha256: str = PILOT_MANIFEST_SHA256) -> LocalRunRef:
+    """Call adapter(request, observed_search). No providers are chosen or called by the SDK."""
+    require(isinstance(candidate, Mapping) and callable(adapter), "candidate metadata and adapter callable are required")
+    candidate = {"name": text(candidate.get("name"), "candidate.name"), "version": text(candidate.get("version"), "candidate.version"),
+                 "code_revision": text(candidate.get("code_revision"), "candidate.code_revision"),
+                 "configuration": copy.deepcopy(candidate.get("configuration")), "interface_version": QUESTION_ROBUSTNESS_INTERFACE,
+                 "identity_authority": "caller_declared_not_code_attestation"}
+    require(isinstance(candidate["configuration"], dict) and candidate["configuration"], "pin query/model/index/reranker configuration")
+    encoded(candidate)  # Reject unserializable configuration before invoking any candidate.
+    pack = load_question_robustness_pack(data_dir, expected_sha256=expected_sha256)
+    policy = _policy(mode, max_context_bytes)
+    requests = [build_question_robustness_request(pack, row["case_id"], mode=mode, max_context_bytes=max_context_bytes) for row in pack.inputs]
+    def execute(raw_request, request):
+        observer = ObservedSearch(request)
+        out, err = io.StringIO(), io.StringIO()
+        failure = None
+        start = time.perf_counter()
+        try:
+            with redirect_stdout(out), redirect_stderr(err):
+                raw = encoded(adapter(copy.deepcopy(request), observer))
+        except Exception as exc:
+            failure = "adapter_error:" + type(exc).__name__
+            raw = b""
+        latency = (time.perf_counter() - start) * 1000
+        return raw, (out.getvalue() + err.getvalue()).encode("utf-8"), 0 if failure is None else 1, failure, latency, {"search_calls": observer._calls}
+    return capture(pack.identity, candidate, policy, requests, execute, output_dir)
+
+
+def _outcome(record: dict[str, Any], request: dict[str, Any]) -> dict[str, Any]:
+    raw = record["response"]
+    require(record["failure"] is None and isinstance(raw, dict), "candidate failed or response missing")
+    envelope = {"schema_version", "case_id", "request_sha256", "action"}
+    require(raw.get("schema_version") == "sphynx.rag.question_robustness.response.v1"
+            and raw.get("case_id") == request["case_id"] and raw.get("request_sha256") == request["request_sha256"], "response must echo its request identity")
+    require(raw.get("action") in ACTIONS and envelope <= set(raw) <= envelope | {"queries", "stages", "context_text", "diagnostics"}, "invalid robustness response")
+    if "queries" in raw:
+        require(isinstance(raw["queries"], list) and all(isinstance(q, str) and q.strip() for q in raw["queries"]), "claimed queries must be a list of text")
+    calls = record["observed"].get("search_calls")
+    require(isinstance(calls, list) and len(calls) <= request["execution_policy"]["max_search_calls"], "invalid observed search inventory")
+    if request["execution_policy"]["mode"] == "routing_only":
+        require(not calls and "stages" not in raw and "context_text" not in raw, "routing-only response cannot claim observed retrieval")
+    for call in calls:
+        text(call.get("query"), "observed query")
+        require(call.get("failure") is None and call.get("result") == _stages(call.get("raw_result"), request), "observed retrieval failed or changed")
+    require(not calls or raw["action"] == "CALL_RAG", "observed lookup contradicts routing action")
+    if calls and "queries" in raw:
+        require(raw["queries"] == [c["query"] for c in calls], "claimed query set differs from actual search calls")
+    final = None
+    if "stages" in raw or "context_text" in raw:
+        require(bool(calls), "final retrieval evidence needs an observed search call")
+        final = _stages({"stages": raw.get("stages"), "context_text": raw.get("context_text")}, request)
+        recovered = set(cid for c in calls for cid in c["result"]["stages"]["candidate"])
+        require(set(final["stages"]["candidate"]) <= recovered, "final stages include chunks never returned by observed search")
+    return {"action": raw["action"], "queries": [c["query"] for c in calls],
+            "query_set_sha256": identity([c["query"] for c in calls]) if calls else None,
+            "context_ids": final["stages"]["context"] if final else None}
+
+
+def build_question_robustness_response(request: Mapping[str, Any], action: str, **evidence: Any) -> dict[str, Any]:
+    """Construct an echoed response; the grader independently validates all evidence."""
+    require(action in ACTIONS and set(evidence) <= {"queries", "stages", "context_text", "diagnostics"}, "unsupported response fields")
+    return {"schema_version": "sphynx.rag.question_robustness.response.v1", "case_id": request["case_id"],
+            "request_sha256": request["request_sha256"], "action": action, **evidence}
+
+
+def _judgments(annotations: Any, run: LocalRunRef, pack: QuestionRobustnessPack, outcomes: dict[str, Any]) -> dict[str, Any]:
+    if annotations is None:
+        return {}
+    require(isinstance(annotations, dict) and set(annotations) == {"schema_version", "review_protocol", "run_id", "dataset_digest", "judgments"}
+            and annotations["schema_version"] == "sphynx.question-query-review.v1"
+            and annotations["review_protocol"] == "whole-query-set-owner-review.v1"
+            and annotations["run_id"] == run.run_id and annotations["dataset_digest"] == pack.identity["dataset_digest"], "query judgments must bind this capture and review protocol")
+    require(isinstance(annotations["judgments"], list), "judgments must be a list")
+    indexed = {}
+    case_map = {c["case_id"]: c for c in pack.cases}
+    for judgment in annotations["judgments"]:
+        require(isinstance(judgment, dict) and set(judgment) == {"case_id", "query_set_sha256", "intent_preserved", "constraints_preserved", "reviewed_by", "reviewed_at"}, "invalid query judgment fields")
+        cid = judgment["case_id"]
+        outcome = outcomes.get(cid)
+        require(cid not in indexed and outcome is not None and outcome["action"] == "CALL_RAG" and bool(outcome["queries"])
+                and case_map[cid]["references"]["expected_action"] == "CALL_RAG"
+                and judgment["query_set_sha256"] == outcome["query_set_sha256"], "judgment does not bind an eligible observed query set")
+        constraint_ids = {c["constraint_id"] for c in case_map[cid]["references"]["intent_rubric"]["required_constraints"]}
+        constraints = judgment["constraints_preserved"]
+        require(type(judgment["intent_preserved"]) is bool and isinstance(constraints, dict) and set(constraints) == constraint_ids
+                and all(type(v) is bool for v in constraints.values()), "judgment must decide intent and every required constraint")
+        text(judgment["reviewed_by"], "reviewed_by")
+        text(judgment["reviewed_at"], "reviewed_at")
+        indexed[cid] = judgment
+    return indexed
+
+
+def _paired(cases: list[dict[str, Any]], field: str) -> dict[str, Any]:
+    families = {}
+    for case in cases:
+        if case["family_id"]:
+            families.setdefault(case["family_id"], {})[case["variant"]] = case
+    slices, combined, bootstrap_rows = {}, [], []
+    for variant in VARIANTS[1:]:
+        values, rows = [], []
+        for fid, siblings in families.items():
+            direct, alternate = siblings["direct"][field], siblings[variant][field]
+            value = (direct - alternate) * 100 if direct is not None and alternate is not None else None
+            values.append(value)
+            combined.append(value)
+            if value is not None:
+                rows.append((siblings["direct"]["customer"], fid, value))
+                bootstrap_rows.append(rows[-1])
+        slices[variant] = {**metric(values), "ci95_family_bootstrap_pp": clustered_interval(rows) if None not in values else None}
+    return {**metric(combined), "weighting": "equal_family_equal_variant", "variant_slices": slices,
+            "ci95_family_bootstrap_pp": clustered_interval(bootstrap_rows) if None not in combined else None}
+
+
+def _compute(run: LocalRunRef, pack: QuestionRobustnessPack, annotations: Any) -> dict[str, Any]:
+    manifest = read_json(run.path / "manifest.json")
+    policy = manifest.get("execution_policy", {})
+    policy = _policy(policy.get("mode"), policy.get("max_context_bytes"))
+    requests = [build_question_robustness_request(pack, c["case_id"], mode=policy["mode"], max_context_bytes=policy["max_context_bytes"]) for c in pack.cases]
+    manifest, captures = verified_capture(run, pack.identity, requests, policy)
+    outcomes, errors = {}, {}
+    for case, record, request in zip(pack.cases, captures, requests):
+        try:
+            outcomes[case["case_id"]] = _outcome(record, request)
+        except (ContractError, ValueError, TypeError, KeyError):
+            errors[case["case_id"]] = record["failure"] or "invalid_response_or_observed_evidence"
+    judgments = _judgments(annotations, run, pack, outcomes)
+    rows = []
+    query_capture_missing, required_without_query = 0, 0
+    for case, record in zip(pack.cases, captures):
+        refs = case["references"]
+        positive = refs["expected_action"] == "CALL_RAG"
+        outcome = outcomes.get(case["case_id"])
+        action = outcome["action"] if outcome else None
+        lookup = float(action == "CALL_RAG") if outcome and positive else None
+        evidence = None
+        if positive and outcome:
+            if action != "CALL_RAG":
+                evidence = 0.0
+            elif outcome["queries"] and outcome["context_ids"] is not None:
+                evidence = float(all(set(g["acceptable_chunk_ids"]) & set(outcome["context_ids"]) for g in refs["evidence_groups"] if g["required"]))
+        intent, constraints = None, None
+        query_eligible = positive and (action == "CALL_RAG" or outcome is None)
+        constraint_count = len(refs.get("intent_rubric", {}).get("required_constraints", []))
+        if positive and (not outcome or not outcome["queries"]):
+            required_without_query += 1
+        if query_eligible and (outcome is None or not outcome["queries"]):
+            query_capture_missing += 1
+        judgment = judgments.get(case["case_id"])
+        if judgment:
+            intent = float(judgment["intent_preserved"])
+            constraints = sum(judgment["constraints_preserved"].values()) / constraint_count if constraint_count else None
+        rows.append({"case_id": case["case_id"], "customer": case["customer"], "family_id": case.get("family_id"),
+                     "variant": case.get("variant"), "positive": positive, "action": action, "failure": errors.get(case["case_id"]),
+                     "lookup": lookup, "evidence": evidence, "intent": intent, "constraints": constraints,
+                     "query_eligible": query_eligible, "constraint_count": constraint_count,
+                     "query_set_sha256": outcome["query_set_sha256"] if outcome else None,
+                     "observed_queries": outcome["queries"] if outcome else [],
+                     "runner_latency_ms": record["runner_latency_ms"]})
+    metrics = {"required_lookup_recall": metric([r["lookup"] for r in rows if r["positive"]]),
+               "unnecessary_call_rate": metric([float(r["action"] == "CALL_RAG") if r["action"] else None for r in rows if not r["positive"]]),
+               "query_intent_preservation": metric([r["intent"] for r in rows if r["query_eligible"]]),
+               "required_constraint_coverage": metric([r["constraints"] for r in rows if r["query_eligible"] and r["constraint_count"]]),
+               "complete_evidence_success": metric([r["evidence"] for r in rows if r["positive"]]),
+               "paired_lookup_drop_pp": _paired(rows, "lookup"), "paired_evidence_drop_pp": _paired(rows, "evidence"),
+               "valid_capture_coverage": metric([float(r["action"] is not None) for r in rows])}
+    unknown = sum(r["positive"] and r["action"] is None for r in rows)
+    for key in ("query_intent_preservation", "required_constraint_coverage"):
+        metrics[key]["eligibility_unknown_count"] = unknown
+    if policy["mode"] == "routing_only":
+        for key in ("query_intent_preservation", "required_constraint_coverage", "complete_evidence_success", "paired_evidence_drop_pp"):
+            if metrics[key]["missing_count"]:
+                metrics[key]["status"] = "unavailable"
+    elif query_capture_missing:
+        for key in ("query_intent_preservation", "required_constraint_coverage"):
+            metrics[key]["status"] = "incomplete"
+    slices = {variant: {"required_lookup_recall": metric([r["lookup"] for r in rows if r["variant"] == variant]),
+                        "complete_evidence_success": metric([r["evidence"] for r in rows if r["variant"] == variant])} for variant in VARIANTS}
+    customers = {customer: {"required_lookup_recall": metric([r["lookup"] for r in rows if r["customer"] == customer and r["positive"]]),
+                           "valid_capture_coverage": metric([float(r["action"] is not None) for r in rows if r["customer"] == customer])}
+                 for customer in sorted({r["customer"] for r in rows})}
+    return {"schema_version": "sphynx.question-robustness.scorecard.v1", "pack_id": QUESTION_ROBUSTNESS_PACK_ID,
+            "track": "rag.question_robustness", "pack": pack.identity, "candidate": manifest["candidate"],
+            "execution_policy": policy, "runtime": manifest["runtime"], "trust_tier": "local_unverified", "development_only": True,
+            "label_status": pack.identity["label_status"], "human_review_complete": pack.identity["human_review_complete"],
+            "label_basis": "owner_reviewed" if pack.identity["human_review_complete"] else "provisional_source_anchor_recovery_and_assistant_authored_controls",
+            "metric_semantics": "Paired development wording; evidence recovery is a provisional source-anchor proxy until owner review.",
+            "metric_release": METRIC_RELEASE, "grader_release": GRADER_RELEASE, "metrics": metrics,
+            "variant_slices": slices, "customer_slices": customers, "cases": rows,
+            "diagnostics": {"required_opportunities_without_observed_query": required_without_query,
+                            "claimed_lookup_without_observed_query": query_capture_missing,
+                            "query_eligibility_unknown_count": unknown,
+                            "query_review_count": len(judgments)},
+            "counts": {"attempted_count": len(rows), "success_count": len(outcomes), "failure_count": len(errors)}}
+
+
+def grade_question_robustness(run: LocalRunRef, data_dir: str | Path, *, query_judgments_path: str | Path | None = None,
+                              expected_sha256: str = PILOT_MANIFEST_SHA256) -> LocalEvaluationRef:
+    pack = load_question_robustness_pack(data_dir, expected_sha256=expected_sha256)
+    annotations = read_json(Path(query_judgments_path)) if query_judgments_path is not None else None
+    return save_evaluation(run, _compute(run, pack, annotations), annotations)
+
+
+def report_question_robustness(evaluation: LocalEvaluationRef, data_dir: str | Path, *, expected_sha256: str = PILOT_MANIFEST_SHA256) -> LocalScorecard:
+    pack = load_question_robustness_pack(data_dir, expected_sha256=expected_sha256)
+    return report_evaluation(evaluation, lambda annotations: _compute(evaluation.run, pack, annotations))
+
+
+def compare_question_robustness(a: LocalEvaluationRef, b: LocalEvaluationRef, data_dir: str | Path, *, expected_sha256: str = PILOT_MANIFEST_SHA256) -> dict[str, Any]:
+    return compare_scorecards(report_question_robustness(a, data_dir, expected_sha256=expected_sha256),
+                             report_question_robustness(b, data_dir, expected_sha256=expected_sha256))
+
+
+def create_query_review_template(run: LocalRunRef, data_dir: str | Path, output_path: str | Path, *, expected_sha256: str = PILOT_MANIFEST_SHA256) -> Path:
+    """Write owner-only unfilled judgments; None decisions never confer credit."""
+    pack = load_question_robustness_pack(data_dir, expected_sha256=expected_sha256)
+    data = _compute(run, pack, None)
+    case_map = {c["case_id"]: c for c in pack.cases}
+    judgments = [{"case_id": r["case_id"], "query_set_sha256": r["query_set_sha256"], "intent_preserved": None,
+                  "constraints_preserved": {c["constraint_id"]: None for c in case_map[r["case_id"]]["references"]["intent_rubric"]["required_constraints"]},
+                  "reviewed_by": None, "reviewed_at": None} for r in data["cases"] if r["query_eligible"] and r["observed_queries"]]
+    template = {"schema_version": "sphynx.question-query-review.v1", "review_protocol": "whole-query-set-owner-review.v1",
+                "run_id": run.run_id, "dataset_digest": pack.identity["dataset_digest"], "judgments": judgments}
+    target = Path(output_path).expanduser().resolve()
+    target.parent.mkdir(parents=True, exist_ok=True)
+    with target.open("xb") as stream:
+        stream.write(encoded(template))
+    return target