sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/invocation.py

@@ -0,0 +1,791 @@
+"""Local, externally supplied customer RAG invocation evaluation.
+
+The corpus and labels are never package resources. This development pack is
+intentionally separate from the frozen-pool reranker evidence format.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+import os
+import platform
+import re
+import subprocess
+import sys
+import threading
+import time
+import uuid
+from collections import Counter
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from .contracts import CandidateSpec, ContractError, canonical_json
+from .runner import MAX_STREAM_BYTES, _BoundedStream, _InputWrite, _candidate_environment
+
+PACK_ID = "rag.invoke.v1"
+INTERFACE_VERSION = "sphynx.rag.invoke.v1"
+REQUEST_VERSION = "sphynx.rag.invoke.request.v1"
+RESPONSE_VERSION = "sphynx.rag.invoke.response.v1"
+METRIC_RELEASE = "rag.invoke.metric.v1"
+GRADER_RELEASE = "rag.invoke.grader.v1"
+INPUT_SHA256 = "7316862d37a7185ed7c995278a5d5f9ae34ff0ae05afe5c4d281e2c00303d2bf"
+GOLD_SHA256 = "ee66b8193deceb4c82f5fe87a4344278b94cca9436724e4b2bc7d1b1f51dedef"
+_COMPANIES = frozenset({"figma", "monday", "notion", "tipalti", "vanta"})
+_ACTIONS = frozenset({"CALL_RAG", "ASK_BUYER", "ANSWER_FROM_CONTEXT"})
+_SOURCE_ACTIONS = {
+    "call_rag": "CALL_RAG",
+    "clarify_first": "ASK_BUYER",
+    "do_not_call_rag": "ANSWER_FROM_CONTEXT",
+}
+_ID = re.compile(r"^(?:invq|nhq|dbq)-[a-z0-9]+$")
+_RUN_ID = re.compile(r"^invoke-run-[0-9a-f]{32}$")
+_EVAL_ID = re.compile(r"^invoke-eval-[0-9a-f]{32}$")
+_METRICS = (
+    "three_way_accuracy", "required_lookup_recall", "unnecessary_call_rate",
+    "clarification_accuracy", "answer_from_context_accuracy",
+)
+
+
+class InvocationError(ValueError):
+    """The private corpus, candidate response, or saved result is invalid."""
+
+
+@dataclass(frozen=True)
+class InvocationBank:
+    """Content-free identity of a supported owner-held question release."""
+
+    bank_id: str
+    release: str
+    title: str
+    case_count: int
+    source_case_count: int
+    cases_per_company: int
+    action_counts: tuple[tuple[str, int], ...]
+    input_sha256: str
+    gold_sha256: str
+    source_schema: str
+    manifest_sha256: str | None
+    id_prefix: str
+    population: str
+    contains_source_excerpt_controls: bool = False
+    companies: tuple[str, ...] = tuple(sorted(_COMPANIES))
+
+
+_BANKS = (
+    InvocationBank(
+        "excerpt-free-200", "v1-development-200", "Excerpt-free development slice",
+        200, 200, 40, (("CALL_RAG", 119), ("ASK_BUYER", 81)),
+        INPUT_SHA256, GOLD_SHA256, "sphynx.jev-hybrid-safe200.v1", None, "invq-",
+        "authored_development",
+    ),
+    InvocationBank(
+        "development-300", "v1-20260918-r3-five-customer", "Full five-customer development bank",
+        300, 360, 60,
+        (("CALL_RAG", 100), ("ASK_BUYER", 100), ("ANSWER_FROM_CONTEXT", 100)),
+        "80336bade48c6bc1acadbae8c66e321d68008e0ee6d2d18fd473e56d566319a6",
+        "b886cc3cb572249018e94656ceeff796c576dc7bc5162b30e90d9ed039a02472",
+        "sable.invocation-question-corpus.v1",
+        "fca2e2236016983de5f8cf579d488b20b07a8a2f32e1fc9cb459e6ef124e6f92", "invq-",
+        "authored_development", True,
+    ),
+    InvocationBank(
+        "narrow-policy-90", "v1-20260925", "Narrow-policy regression bank",
+        90, 90, 18,
+        (("CALL_RAG", 30), ("ASK_BUYER", 30), ("ANSWER_FROM_CONTEXT", 30)),
+        "ecde7c3141fa8b508386e5b59468296acba55e14aa4f64e1e2f8143de977878e",
+        "01430c66f73a58562834c7521186bace303d8a2ecc52e46a96da64e93cd769a9",
+        "sphynx.invocation-holdout.v1",
+        "eae4d921391dad58c73507a129eb20fbe86fd5109d37f5e19626cdb3a061d937", "nhq-",
+        "previously_exercised_regression",
+    ),
+    InvocationBank(
+        "decision-boundary-80", "v1-20260925", "Decision-boundary regression bank",
+        80, 80, 16,
+        (("CALL_RAG", 40), ("ASK_BUYER", 20), ("ANSWER_FROM_CONTEXT", 20)),
+        "f1aea9ab88c55586ed01a6bf5aee4ac78661ae11727dfea12ecee14da09ea2b6",
+        "fc3f39b79435eab66033b9ac2d919a2931b26568f848542cce2f6a27dc15eb44",
+        "sphynx.decision-boundary-holdout.v1",
+        "307d5bb41d7c649062b0da8bdf9966ef3479c4d1484958bf331b69300a18ada9", "dbq-",
+        "previously_exercised_regression",
+    ),
+)
+
+
+# Revised bytes always retain distinct releases. Six-customer banks require explicit selection.
+_BANKS += (
+    InvocationBank('narrow-gate-20260925-r2', 'narrow-gate-20260925-r2', 'narrow-gate-20260925-r2', 90, 90, 18, (('CALL_RAG', 30), ('ASK_BUYER', 30), ('ANSWER_FROM_CONTEXT', 30)),
+        'fea544f918248f710be172bcadfdf1e6506ca2f4e08ab97c6099bd3afed1d316', '45dc2201b9e92efb2713282781ece6605f208855240b7f3087892275fcc5e4c1',
+        'sphynx.invocation-holdout.v1', '0a1b1a81e58db5c4b442c834fe40f458b738d3fdd296b5be873fd6af269aa32b', 'nhq-', 'previously_exercised_regression', True),
+    InvocationBank('narrow-gate-20260925-r4', 'narrow-gate-20260925-r4', 'narrow-gate-20260925-r4', 90, 90, 18, (('CALL_RAG', 30), ('ASK_BUYER', 30), ('ANSWER_FROM_CONTEXT', 30)),
+        '2af3d98cb92146511e85de1a6c038d01f4bf7c117b622c61c6425a03f6e4a391', '7b593b9838e0bbfd3a74f359ddefe9eed7af476f0a914040ccff8889352750d5',
+        'sphynx.invocation-holdout.v1', 'bd945b7961e662af377b3a933cc7c92cbbe448442724d3b9fac0965bb4a9cedf', 'nhq-', 'previously_exercised_regression'),
+    InvocationBank('narrow-gate-20260925-r5', 'narrow-gate-20260925-r5', 'narrow-gate-20260925-r5', 90, 90, 18, (('CALL_RAG', 30), ('ASK_BUYER', 30), ('ANSWER_FROM_CONTEXT', 30)),
+        'c230ad79a9aba6cef17266b83345d273e93eba0871408ce23f87d59e27543194', '7b593b9838e0bbfd3a74f359ddefe9eed7af476f0a914040ccff8889352750d5',
+        'sphynx.invocation-holdout.v1', '66638d4679cb8858eec24d170a5f36ac60954718dca29c8c7b6d7f472feb0146', 'nhq-', 'previously_exercised_regression'),
+    InvocationBank('six-customer-v1-20260918', 'v1-20260918', 'six-customer-v1-20260918', 360, 360, 60, (('CALL_RAG', 120), ('ASK_BUYER', 120), ('ANSWER_FROM_CONTEXT', 120)),
+        '797cf8793b901a1f87460bf079216a48385ac31bcb965210e588251ef430694b', '92db1243cd6decaa2ca84ee1302ec07b22d08af4235c8946fcd86522e0ea4977',
+        'sable.invocation-question-corpus.v1', 'dbce75ef669d09156598d0bfc822495eeb8c2572802521ff7b4ab1903a44d5d3', 'invq-', 'six_customer_authored_development', True, tuple(sorted(_COMPANIES | {"atlassian"}))),
+    InvocationBank('six-customer-v1-20260918-r2', 'v1-20260918-r2', 'six-customer-v1-20260918-r2', 360, 360, 60, (('CALL_RAG', 120), ('ASK_BUYER', 120), ('ANSWER_FROM_CONTEXT', 120)),
+        'd4f60c0046b9c55fc992dacc699ffbb8743c2c331dfc6ce73f5601be1c6c78fb', 'c21dd65089f2a1a06f891e9f2c618fda4f69ec8b6d67ea42a83afac7bdff9e29',
+        'sable.invocation-question-corpus.v1', 'dbce75ef669d09156598d0bfc822495eeb8c2572802521ff7b4ab1903a44d5d3', 'invq-', 'six_customer_authored_development', True, tuple(sorted(_COMPANIES | {"atlassian"}))),
+    InvocationBank('six-customer-v1-20260918-r3', 'v1-20260918-r3', 'six-customer-v1-20260918-r3', 360, 360, 60, (('CALL_RAG', 120), ('ASK_BUYER', 120), ('ANSWER_FROM_CONTEXT', 120)),
+        '80336bade48c6bc1acadbae8c66e321d68008e0ee6d2d18fd473e56d566319a6', 'b886cc3cb572249018e94656ceeff796c576dc7bc5162b30e90d9ed039a02472',
+        'sable.invocation-question-corpus.v1', 'fca2e2236016983de5f8cf579d488b20b07a8a2f32e1fc9cb459e6ef124e6f92', 'invq-', 'six_customer_authored_development', True, tuple(sorted(_COMPANIES | {"atlassian"}))),
+)
+
+def list_invocation_banks() -> tuple[InvocationBank, ...]:
+    """List supported releases without loading or downloading private data."""
+
+    return _BANKS
+
+
+def _sha(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def _write_json(path: Path, value: object) -> str:
+    data = (canonical_json(value) + "\n").encode("utf-8")
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("xb") as stream:
+        stream.write(data)
+    return _sha(data)
+
+
+def _read_json(path: Path) -> dict[str, Any]:
+    try:
+        value = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, UnicodeError, json.JSONDecodeError) as exc:
+        raise InvocationError(f"cannot read {path.name}: {exc}") from exc
+    if not isinstance(value, dict):
+        raise InvocationError(f"{path.name} must contain an object")
+    return value
+
+
+def _read_jsonl_bytes(data: bytes, label: str) -> list[dict[str, Any]]:
+    try:
+        rows = [json.loads(line) for line in data.decode("utf-8").splitlines() if line]
+    except (UnicodeError, json.JSONDecodeError) as exc:
+        raise InvocationError(f"{label} is not valid UTF-8 JSONL") from exc
+    if any(not isinstance(row, dict) for row in rows):
+        raise InvocationError(f"{label} must contain JSON objects")
+    return rows
+
+
+@dataclass(frozen=True)
+class InvocationCase:
+    case_id: str
+    company_id: str
+    user_turn: str
+    conversation_context: tuple[dict[str, str], ...]
+
+    def request(self) -> dict[str, Any]:
+        # Explicit allowlist: never serialize the gold row or a source anchor.
+        return {
+            "schema_version": REQUEST_VERSION,
+            "pack_id": PACK_ID,
+            "case_id": self.case_id,
+            "company_id": self.company_id,
+            "user_turn": self.user_turn,
+            "conversation_context": [dict(turn) for turn in self.conversation_context],
+        }
+
+
+@dataclass(frozen=True)
+class InvocationPack:
+    cases: tuple[InvocationCase, ...]
+    gold: dict[str, str]
+    input_sha256: str
+    gold_sha256: str
+    bank_id: str = "excerpt-free-200"
+    bank_release: str = "v1-development-200"
+    population: str = "authored_development"
+    contains_source_excerpt_controls: bool = False
+
+    @property
+    def cohort_sha256(self) -> str:
+        return _sha(canonical_json([
+            {"request": case.request(), "gold_action": self.gold[case.case_id]}
+            for case in self.cases
+        ]).encode("utf-8"))
+
+    def identity(self) -> dict[str, Any]:
+        return {
+            "bank_id": self.bank_id, "bank_release": self.bank_release,
+            "cohort_sha256": self.cohort_sha256, "population": self.population,
+            "contains_source_excerpt_controls": self.contains_source_excerpt_controls,
+            "label_status": "provisional_no_independent_human_review",
+        }
+
+
+@dataclass(frozen=True)
+class InvocationRunRef:
+    run_id: str
+    path: Path
+    pack_id: str = PACK_ID
+
+
+@dataclass(frozen=True)
+class InvocationEvaluationRef:
+    evaluation_id: str
+    path: Path
+    run_id: str
+
+
+@dataclass(frozen=True)
+class InvocationScorecard:
+    data: dict[str, Any]
+
+    def to_json(self) -> str:
+        return canonical_json(self.data)
+
+    def to_dict(self) -> dict[str, Any]:
+        return json.loads(self.to_json())
+
+    def to_markdown(self) -> str:
+        data = self.data
+        lines = [
+            "# Sphynx RAG invocation development scorecard", "",
+            "**LOCAL_UNVERIFIED — authored questions; no independent human label review.**", "",
+            f"- Pack: `{PACK_ID}`; candidate: `{data['candidate']['name']}` `{data['candidate']['version']}`",
+            f"- Cases: {data['case_count']}; successful: {data['success_count']}; failed: {data['failure_count']}",
+            f"- Bank: `{data.get('bank_id', 'excerpt-free-200')}`; population: `{data.get('population', 'authored_development')}`",
+            ("- No ANSWER_FROM_CONTEXT gold cases; that metric is undefined."
+             if data['metrics']['answer_from_context_accuracy']['eligible_count'] == 0
+             else "- Includes ANSWER_FROM_CONTEXT gold cases; that metric is graded."),
+            ("- Source-excerpt controls are supplied conversation context, not retrieved evidence."
+             if data.get('contains_source_excerpt_controls') else "- Labels and source anchors stay outside candidate requests."), "",
+            "| Metric | Value | Eligible | Observed | Missing |", "| --- | ---: | ---: | ---: | ---: |",
+        ]
+        for metric, item in data["metrics"].items():
+            value = "null" if item["value"] is None else f"{item['value']:.6f}"
+            lines.append(f"| {metric} | {value} | {item['eligible_count']} | {item['observed_count']} | {item['missing_count']} |")
+        lines.extend(["", "| Customer | Three-way accuracy | Observed | Missing |", "| --- | ---: | ---: | ---: |"])
+        for customer, item in data["customer_slices"].items():
+            value = "null" if item["value"] is None else f"{item['value']:.6f}"
+            lines.append(f"| {customer} | {value} | {item['observed_count']} | {item['missing_count']} |")
+        return "\n".join(lines) + "\n"
+
+
+@dataclass(frozen=True)
+class InvocationComparison:
+    data: dict[str, Any]
+
+    def to_json(self) -> str:
+        return canonical_json(self.data)
+
+    def to_dict(self) -> dict[str, Any]:
+        return json.loads(self.to_json())
+
+    def to_markdown(self) -> str:
+        data = self.data
+        value = data["mean_delta_b_minus_a"]
+        return (
+            "# Sphynx RAG invocation development comparison\n\n"
+            "**LOCAL_UNVERIFIED — paired authored development cases.**\n\n"
+            f"- A: `{data['evaluation_a']}`; B: `{data['evaluation_b']}`\n"
+            f"- Bank: `{data.get('bank_id', 'excerpt-free-200')}`\n"
+            f"- Paired: {data['paired_count']}/{data['eligible_count']}; missing: {data['missing_pair_count']}\n"
+            f"- Three-way accuracy mean B − A: {'null' if value is None else format(value, '.6f')}\n"
+        )
+
+
+def load_invocation_pack(
+    data_dir: str | os.PathLike[str], *, bank_id: str | None = None,
+) -> InvocationPack:
+    """Load a supported hash-pinned bank; infer its identity when omitted.
+
+    The 300-case bank selects the five customers from the pinned 360-row source.
+    Six-customer releases require explicit bank_id selection. Every revised
+    source uses its own pinned release; arbitrary subsets are unsupported.
+    """
+
+    root = Path(data_dir).expanduser().resolve()
+    try:
+        input_bytes = (root / "model-input.jsonl").read_bytes()
+        gold_bytes = (root / "gold.jsonl").read_bytes()
+    except OSError as exc:
+        raise InvocationError(f"local invocation corpus is unavailable: {exc}") from exc
+    if bank_id is not None and bank_id not in {bank.bank_id for bank in _BANKS}:
+        raise InvocationError(f"unknown invocation bank {bank_id!r}; use list_invocation_banks()")
+    matches = [bank for bank in _BANKS
+               if ((bank_id is None and not bank.bank_id.startswith("six-customer-")) or bank.bank_id == bank_id)
+               and _sha(input_bytes) == bank.input_sha256 and _sha(gold_bytes) == bank.gold_sha256]
+    if len(matches) != 1:
+        raise InvocationError("local invocation corpus differs from a supported pinned bank or selected bank_id")
+    bank = matches[0]
+    manifest = _read_json(root / "manifest.json")
+    if bank.manifest_sha256 is not None:
+        try:
+            manifest_digest = _sha((root / "manifest.json").read_bytes())
+        except OSError as exc:
+            raise InvocationError("invocation manifest is unavailable") from exc
+        if manifest_digest != bank.manifest_sha256:
+            raise InvocationError("invocation manifest differs from the pinned bank release")
+    count_field = "question_count" if bank.source_schema == "sable.invocation-question-corpus.v1" else "case_count"
+    if (manifest.get("schema_version") != bank.source_schema
+            or type(manifest.get(count_field)) is not int
+            or manifest.get(count_field) != bank.source_case_count
+            or (count_field == "case_count" and (
+                manifest.get("model_input_sha256") != bank.input_sha256
+                or manifest.get("gold_sha256") != bank.gold_sha256))):
+        raise InvocationError("invocation manifest does not match the pinned release")
+    inputs = _read_jsonl_bytes(input_bytes, "model input")
+    labels = _read_jsonl_bytes(gold_bytes, "gold")
+    if len(inputs) != bank.source_case_count or len(labels) != bank.source_case_count:
+        raise InvocationError("input and gold counts differ from the pinned source release")
+    cases: list[InvocationCase] = []
+    input_by_id: dict[str, dict[str, Any]] = {}
+    selected_companies = frozenset(bank.companies)
+    source_companies = selected_companies | {"atlassian"} if bank.source_case_count != bank.case_count else selected_companies
+    for row in inputs:
+        if set(row) != {"question_id", "company_id", "user_turn", "conversation_context"}:
+            raise InvocationError("model input fields differ from the allowlisted contract")
+        case_id, company, question, context = (row[key] for key in ("question_id", "company_id", "user_turn", "conversation_context"))
+        if (not isinstance(case_id, str) or not _ID.fullmatch(case_id)
+                or not case_id.startswith(bank.id_prefix) or case_id in input_by_id):
+            raise InvocationError("invalid question ID")
+        if not isinstance(company, str) or company not in source_companies or not isinstance(question, str) or not question.strip():
+            raise InvocationError("invalid company or question")
+        if not isinstance(context, list) or any(
+            not isinstance(turn, dict) or set(turn) != {"role", "content"}
+            or not isinstance(turn["role"], str) or not isinstance(turn["content"], str)
+            for turn in context
+        ):
+            raise InvocationError("invalid conversation context")
+        input_by_id[case_id] = row
+        if company in selected_companies:
+            cases.append(InvocationCase(case_id, company, question, tuple(context)))
+    ids = [case.case_id for case in cases]
+    if (len(cases) != bank.case_count
+            or Counter(case.company_id for case in cases) != {company: bank.cases_per_company for company in selected_companies}
+            or Counter(row["company_id"] for row in inputs) != {company: bank.cases_per_company for company in source_companies}):
+        raise InvocationError("invocation case IDs or customer distribution differ from release")
+    gold: dict[str, str] = {}
+    source_gold_ids: set[str] = set()
+    for row in labels:
+        if bank.bank_id == "excerpt-free-200" and set(row) != {"question_id", "company_id", "family_id", "final_action", "provenance"}:
+            raise InvocationError("gold row fields differ from the pinned contract")
+        case_id = row.get("question_id")
+        if (not isinstance(case_id, str) or case_id in source_gold_ids
+                or case_id not in input_by_id or row.get("company_id") != input_by_id[case_id]["company_id"]):
+            raise InvocationError("invalid, duplicate, or misaligned gold identity")
+        source_gold_ids.add(case_id)
+        if count_field == "question_count":
+            reference = row.get("gold")
+            if not isinstance(reference, dict) or any(
+                row.get(key) != input_by_id[case_id][key] for key in ("user_turn", "conversation_context")
+            ):
+                raise InvocationError("gold context differs from the pinned model input")
+            source_action = reference.get("invocation_action")
+        else:
+            source_action = row.get("final_action")
+        if not isinstance(source_action, str) or source_action not in _SOURCE_ACTIONS.keys() | _ACTIONS:
+            raise InvocationError("invalid or duplicate gold action")
+        if row["company_id"] in selected_companies:
+            gold[case_id] = _SOURCE_ACTIONS.get(source_action, source_action)
+    if (source_gold_ids != set(input_by_id) or set(gold) != set(ids)
+            or Counter(gold.values()) != dict(bank.action_counts)):
+        raise InvocationError("gold IDs or action counts differ from release")
+    return InvocationPack(
+        tuple(cases), gold, bank.input_sha256, bank.gold_sha256,
+        bank.bank_id, bank.release, bank.population, bank.contains_source_excerpt_controls,
+    )
+
+
+def _parse_response(stdout: bytes, case_id: str) -> str:
+    if len(stdout) > MAX_STREAM_BYTES:
+        raise InvocationError("candidate output exceeds 1 MiB")
+    def unique_fields(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
+        result: dict[str, Any] = {}
+        for key, item in pairs:
+            if key in result:
+                raise ValueError(f"duplicate response field: {key}")
+            result[key] = item
+        return result
+
+    def invalid_constant(value: str) -> None:
+        raise ValueError(f"non-finite JSON constant: {value}")
+
+    try:
+        value = json.loads(stdout.decode("utf-8"), object_pairs_hook=unique_fields, parse_constant=invalid_constant)
+    except (UnicodeError, json.JSONDecodeError, ValueError) as exc:
+        raise InvocationError("candidate output is not one JSON object") from exc
+    if not isinstance(value, dict) or set(value) != {"schema_version", "case_id", "action"}:
+        raise InvocationError("candidate response fields must be schema_version, case_id, action")
+    if (value["schema_version"] != RESPONSE_VERSION or value["case_id"] != case_id
+            or not isinstance(value["action"], str) or value["action"] not in _ACTIONS):
+        raise InvocationError("candidate response version, case ID, or action is invalid")
+    return value["action"]
+
+
+def _execute_candidate(candidate: CandidateSpec, request_bytes: bytes, timeout_seconds: float) -> tuple[bytes, bytes, int | None, str | None, float]:
+    """Bound stdout/stderr retained in memory while draining to avoid pipe stalls."""
+
+    start = time.monotonic_ns()
+    stdout = _BoundedStream()
+    stderr = _BoundedStream()
+    sent = _InputWrite()
+    process: subprocess.Popen[bytes] | None = None
+    exit_code: int | None = None
+    failure: str | None = None
+    try:
+        process = subprocess.Popen(
+            candidate.argv, shell=False, stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+            env=_candidate_environment(), bufsize=0,
+        )
+        assert process.stdin is not None and process.stdout is not None and process.stderr is not None
+        readers = [
+            threading.Thread(target=stdout.drain, args=(process.stdout,), daemon=True),
+            threading.Thread(target=stderr.drain, args=(process.stderr,), daemon=True),
+        ]
+        for reader in readers:
+            reader.start()
+        writer = threading.Thread(target=sent.send, args=(process.stdin, request_bytes), daemon=True)
+        writer.start()
+        try:
+            elapsed = (time.monotonic_ns() - start) / 1_000_000_000
+            exit_code = process.wait(timeout=max(0.0, timeout_seconds - elapsed))
+        except subprocess.TimeoutExpired:
+            failure = "timeout"
+            process.kill()
+            process.wait(timeout=5)
+        writer.join(timeout=1)
+        for reader in readers:
+            reader.join(timeout=1)
+        if failure is None:
+            if not sent.complete:
+                failure = "input_write_error"
+            elif exit_code != 0:
+                failure = f"candidate_exit_{exit_code}"
+            elif stdout.capped or not stdout.eof or stdout.error:
+                failure = "output_truncated"
+    except OSError as exc:
+        failure = f"spawn_error:{type(exc).__name__}"
+    finally:
+        if process is not None:
+            for stream in (process.stdin, process.stdout, process.stderr):
+                if stream is not None:
+                    try:
+                        stream.close()
+                    except OSError:
+                        pass
+    return bytes(stdout.retained), bytes(stderr.retained), exit_code, failure, (time.monotonic_ns() - start) / 1_000_000
+
+
+def run_invocation(
+    candidate: CandidateSpec,
+    data_dir: str | os.PathLike[str],
+    output_dir: str | os.PathLike[str],
+    *,
+    timeout_seconds: float = 30.0,
+    bank_id: str | None = None,
+) -> InvocationRunRef:
+    """Run every case in the selected pinned bank through a local subprocess."""
+
+    if not isinstance(candidate, CandidateSpec) or candidate.interface_version != INTERFACE_VERSION:
+        raise ContractError(f"candidate must use {INTERFACE_VERSION}")
+    if type(timeout_seconds) not in (int, float) or not math.isfinite(timeout_seconds) or timeout_seconds <= 0:
+        raise InvocationError("timeout_seconds must be positive and finite")
+    pack = load_invocation_pack(data_dir, bank_id=bank_id)
+    run_id = "invoke-run-" + uuid.uuid4().hex
+    run_path = Path(output_dir).expanduser().resolve() / "runs" / run_id
+    run_path.mkdir(parents=True, exist_ok=False)
+    captures: list[dict[str, str]] = []
+    for case in pack.cases:
+        request = case.request()
+        request_bytes = (canonical_json(request) + "\n").encode("utf-8")
+        action: str | None = None
+        stdout, stderr, exit_code, failure, elapsed_ms = _execute_candidate(candidate, request_bytes, timeout_seconds)
+        if failure is None:
+            try:
+                action = _parse_response(stdout, case.case_id)
+            except InvocationError as exc:
+                failure = str(exc)
+        capture = {
+            "schema_version": "sphynx.rag.invoke.capture.v1",
+            "case_id": case.case_id,
+            "request": request,
+            "request_sha256": _sha(request_bytes),
+            "stdout_utf8": stdout.decode("utf-8", errors="replace"),
+            "stderr_utf8": stderr.decode("utf-8", errors="replace"),
+            "exit_code": exit_code,
+            "action": action,
+            "status": "success" if action is not None else "failure",
+            "failure": failure,
+            "runner_latency_ms": elapsed_ms,
+        }
+        digest = _write_json(run_path / "captures" / f"{case.case_id}.json", capture)
+        captures.append({"case_id": case.case_id, "sha256": digest})
+    manifest = {
+        "schema_version": "sphynx.rag.invoke.run.v1",
+        "run_id": run_id,
+        "pack_id": PACK_ID,
+        "input_sha256": pack.input_sha256,
+        "gold_sha256": pack.gold_sha256,
+        "candidate": {
+            "name": candidate.name,
+            "version": candidate.version,
+            "interface_version": candidate.interface_version,
+            "argv_sha256": _sha(canonical_json(list(candidate.argv)).encode("utf-8")),
+        },
+        "execution": {"trials": 1, "timeout_seconds": timeout_seconds, "environment_policy": "minimal-candidate-runtime.v1"},
+        "runner": {"release": "rag.invoke.runner.v1", "python_version": platform.python_version(), "platform": sys.platform},
+        "trust_tier": "local_unverified",
+        "case_count": len(pack.cases),
+        "captures": captures,
+        **pack.identity(),
+    }
+    _write_json(run_path / "manifest.json", manifest)
+    return InvocationRunRef(run_id, run_path)
+
+
+def _verified_run(ref: InvocationRunRef | str | os.PathLike[str], pack: InvocationPack) -> tuple[dict[str, Any], list[dict[str, Any]], Path]:
+    path = Path(ref.path if isinstance(ref, InvocationRunRef) else ref).expanduser().resolve()
+    if isinstance(ref, InvocationRunRef) and (ref.run_id != path.name or ref.pack_id != PACK_ID):
+        raise InvocationError("run reference differs from its directory")
+    manifest = _read_json(path / "manifest.json")
+    if (not _RUN_ID.fullmatch(path.name) or manifest.get("run_id") != path.name
+            or manifest.get("schema_version") != "sphynx.rag.invoke.run.v1"
+            or manifest.get("pack_id") != PACK_ID
+            or manifest.get("input_sha256") != pack.input_sha256
+            or manifest.get("gold_sha256") != pack.gold_sha256
+            or manifest.get("case_count") != len(pack.cases)
+            or manifest.get("trust_tier") != "local_unverified"):
+        raise InvocationError("run identity differs from the pinned invocation pack")
+    identity = pack.identity()
+    if any(key in manifest for key in identity):
+        if any(manifest.get(key) != value for key, value in identity.items()):
+            raise InvocationError("run bank identity differs from the pinned invocation cohort")
+    elif pack.bank_id != "excerpt-free-200":
+        raise InvocationError("run is missing its bank identity")
+    candidate = manifest.get("candidate")
+    execution = manifest.get("execution")
+    runner = manifest.get("runner")
+    if (not isinstance(candidate, dict) or set(candidate) != {"name", "version", "interface_version", "argv_sha256"}
+            or candidate.get("interface_version") != INTERFACE_VERSION
+            or not isinstance(execution, dict)
+            or set(execution) != {"trials", "timeout_seconds", "environment_policy"}
+            or execution.get("trials") != 1
+            or type(execution.get("timeout_seconds")) not in (int, float)
+            or not math.isfinite(execution["timeout_seconds"])
+            or execution["timeout_seconds"] <= 0
+            or execution.get("environment_policy") != "minimal-candidate-runtime.v1"
+            or not isinstance(runner, dict)
+            or set(runner) != {"release", "python_version", "platform"}
+            or runner.get("release") != "rag.invoke.runner.v1"):
+        raise InvocationError("run candidate, execution, or runner identity is invalid")
+    entries = manifest.get("captures")
+    if not isinstance(entries, list) or [item.get("case_id") for item in entries if isinstance(item, dict)] != [case.case_id for case in pack.cases]:
+        raise InvocationError("run capture inventory differs from the pack")
+    if {item.name for item in (path / "captures").iterdir()} != {f"{case.case_id}.json" for case in pack.cases}:
+        raise InvocationError("run capture directory has missing or unindexed files")
+    captures: list[dict[str, Any]] = []
+    for case, entry in zip(pack.cases, entries):
+        capture_path = path / "captures" / f"{case.case_id}.json"
+        try:
+            raw = capture_path.read_bytes()
+        except OSError as exc:
+            raise InvocationError(f"capture missing for {case.case_id}") from exc
+        if set(entry) != {"case_id", "sha256"} or _sha(raw) != entry["sha256"]:
+            raise InvocationError(f"capture hash mismatch for {case.case_id}")
+        capture = _read_json(capture_path)
+        expected = case.request()
+        if (capture.get("schema_version") != "sphynx.rag.invoke.capture.v1"
+                or capture.get("case_id") != case.case_id or capture.get("request") != expected
+                or capture.get("request_sha256") != _sha((canonical_json(expected) + "\n").encode("utf-8"))):
+            raise InvocationError(f"capture request differs from pinned case {case.case_id}")
+        if not isinstance(capture.get("stdout_utf8"), str) or not isinstance(capture.get("stderr_utf8"), str):
+            raise InvocationError(f"capture streams invalid for {case.case_id}")
+        if capture.get("status") == "success":
+            if capture.get("failure") is not None or capture.get("exit_code") != 0 or capture.get("action") != _parse_response(capture["stdout_utf8"].encode("utf-8"), case.case_id):
+                raise InvocationError(f"capture response mismatch for {case.case_id}")
+        elif capture.get("status") != "failure" or capture.get("action") is not None or not isinstance(capture.get("failure"), str):
+            raise InvocationError(f"capture terminal status invalid for {case.case_id}")
+        latency = capture.get("runner_latency_ms")
+        if type(latency) not in (int, float) or not math.isfinite(latency) or latency < 0:
+            raise InvocationError(f"capture latency invalid for {case.case_id}")
+        captures.append(capture)
+    return manifest, captures, path
+
+
+def _metric(observations: list[float | None], *, lower: bool = False) -> dict[str, Any]:
+    valid = [value for value in observations if value is not None]
+    missing = len(observations) - len(valid)
+    return {
+        "value": sum(valid) / len(valid) if valid and missing == 0 else None,
+        "observed_value": sum(valid) / len(valid) if valid else None,
+        "eligible_count": len(observations), "observed_count": len(valid),
+        "missing_count": missing, "complete": bool(valid) and missing == 0,
+        "direction": "lower" if lower else "higher",
+    }
+
+
+def _percentile(sorted_values: list[float], fraction: float) -> float:
+    position = (len(sorted_values) - 1) * fraction
+    lower = math.floor(position)
+    upper = math.ceil(position)
+    return sorted_values[lower] + (sorted_values[upper] - sorted_values[lower]) * (position - lower)
+
+
+def _judgments(pack: InvocationPack, captures: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    rows: list[dict[str, Any]] = []
+    for case, capture in zip(pack.cases, captures):
+        gold = pack.gold[case.case_id]
+        action = capture["action"]
+        rows.append({
+            "case_id": case.case_id, "company_id": case.company_id,
+            "gold_action": gold, "candidate_action": action,
+            "correct": (action == gold) if action is not None else None,
+            "status": capture["status"], "runner_latency_ms": capture["runner_latency_ms"],
+        })
+    return rows
+
+
+def _score(pack: InvocationPack, run_manifest: dict[str, Any], judgments: list[dict[str, Any]], eval_id: str) -> dict[str, Any]:
+    metrics = {
+        "three_way_accuracy": _metric([float(row["correct"]) if row["correct"] is not None else None for row in judgments]),
+        "required_lookup_recall": _metric([float(row["candidate_action"] == "CALL_RAG") if row["candidate_action"] is not None else None for row in judgments if row["gold_action"] == "CALL_RAG"]),
+        "unnecessary_call_rate": _metric([float(row["candidate_action"] == "CALL_RAG") if row["candidate_action"] is not None else None for row in judgments if row["gold_action"] != "CALL_RAG"], lower=True),
+        "clarification_accuracy": _metric([float(row["candidate_action"] == "ASK_BUYER") if row["candidate_action"] is not None else None for row in judgments if row["gold_action"] == "ASK_BUYER"]),
+        "answer_from_context_accuracy": _metric([float(row["candidate_action"] == "ANSWER_FROM_CONTEXT") if row["candidate_action"] is not None else None for row in judgments if row["gold_action"] == "ANSWER_FROM_CONTEXT"]),
+    }
+    latencies = sorted(row["runner_latency_ms"] for row in judgments)
+    return {
+        "schema_version": "sphynx.rag.invoke.metrics.v1", "evaluation_id": eval_id,
+        "run_id": run_manifest["run_id"], "pack_id": PACK_ID, "metric_release": METRIC_RELEASE,
+        "grader_release": GRADER_RELEASE, "trust_tier": "local_unverified",
+        "development_only": True, "input_sha256": pack.input_sha256,
+        "gold_sha256": pack.gold_sha256, "candidate": run_manifest["candidate"],
+        "execution": run_manifest["execution"], "runner": run_manifest["runner"],
+        "case_count": len(pack.cases),
+        "success_count": sum(row["status"] == "success" for row in judgments),
+        "failure_count": sum(row["status"] != "success" for row in judgments),
+        "metrics": metrics,
+        "runner_latency_ms": {"p50": _percentile(latencies, 0.5), "p95": _percentile(latencies, 0.95)},
+        "customer_slices": {
+            company: _metric([float(row["correct"]) if row["correct"] is not None else None for row in judgments if row["company_id"] == company])
+            for company in sorted({case.company_id for case in pack.cases})
+        },
+        **(pack.identity() if "bank_id" in run_manifest else {}),
+    }
+
+
+def grade_invocation(
+    run_ref: InvocationRunRef | str | os.PathLike[str],
+    data_dir: str | os.PathLike[str],
+    *, bank_id: str | None = None,
+) -> InvocationEvaluationRef:
+    """Grade saved captures locally; labels never enter candidate requests."""
+
+    pack = load_invocation_pack(data_dir, bank_id=bank_id)
+    run_manifest, captures, run_path = _verified_run(run_ref, pack)
+    judgments = _judgments(pack, captures)
+    eval_id = "invoke-eval-" + uuid.uuid4().hex
+    eval_path = run_path / "evaluations" / eval_id
+    eval_path.mkdir(parents=True, exist_ok=False)
+    score = _score(pack, run_manifest, judgments, eval_id)
+    judgment_bytes = "".join(canonical_json(row) + "\n" for row in judgments).encode("utf-8")
+    with (eval_path / "judgments.jsonl").open("xb") as stream:
+        stream.write(judgment_bytes)
+    score_hash = _write_json(eval_path / "metrics.json", score)
+    _write_json(eval_path / "manifest.json", {
+        "schema_version": "sphynx.rag.invoke.evaluation.v1", "evaluation_id": eval_id,
+        "run_id": run_path.name, "pack_id": PACK_ID,
+        "judgments_sha256": _sha(judgment_bytes), "metrics_sha256": score_hash,
+        **(pack.identity() if "bank_id" in run_manifest else {}),
+    })
+    return InvocationEvaluationRef(eval_id, eval_path, run_path.name)
+
+
+def _verified_evaluation(
+    ref: InvocationEvaluationRef | str | os.PathLike[str],
+    data_dir: str | os.PathLike[str],
+    *, bank_id: str | None = None,
+) -> tuple[dict[str, Any], list[dict[str, Any]]]:
+    path = Path(ref.path if isinstance(ref, InvocationEvaluationRef) else ref).expanduser().resolve()
+    if isinstance(ref, InvocationEvaluationRef) and (ref.evaluation_id != path.name or ref.run_id != path.parent.parent.name):
+        raise InvocationError("evaluation reference differs from its directory")
+    manifest = _read_json(path / "manifest.json")
+    if (not _EVAL_ID.fullmatch(path.name) or manifest.get("evaluation_id") != path.name
+            or manifest.get("schema_version") != "sphynx.rag.invoke.evaluation.v1"
+            or manifest.get("pack_id") != PACK_ID or manifest.get("run_id") != path.parent.parent.name):
+        raise InvocationError("evaluation identity is invalid")
+    if {item.name for item in path.iterdir()} != {"manifest.json", "metrics.json", "judgments.jsonl"}:
+        raise InvocationError("evaluation file inventory is incomplete or has unindexed files")
+    try:
+        judgments_bytes = (path / "judgments.jsonl").read_bytes()
+        metrics_bytes = (path / "metrics.json").read_bytes()
+    except OSError as exc:
+        raise InvocationError("evaluation file is missing") from exc
+    if _sha(judgments_bytes) != manifest.get("judgments_sha256") or _sha(metrics_bytes) != manifest.get("metrics_sha256"):
+        raise InvocationError("evaluation hash mismatch")
+    pack = load_invocation_pack(data_dir, bank_id=bank_id)
+    identity = pack.identity()
+    if any(key in manifest for key in identity):
+        if any(manifest.get(key) != value for key, value in identity.items()):
+            raise InvocationError("evaluation bank identity differs from the pinned invocation cohort")
+    elif pack.bank_id != "excerpt-free-200":
+        raise InvocationError("evaluation is missing its bank identity")
+    score = _read_json(path / "metrics.json")
+    judgments = _read_jsonl_bytes(judgments_bytes, "judgments")
+    if (score.get("evaluation_id") != path.name or score.get("run_id") != manifest["run_id"]
+            or score.get("pack_id") != PACK_ID or score.get("metric_release") != METRIC_RELEASE
+            or score.get("grader_release") != GRADER_RELEASE or score.get("trust_tier") != "local_unverified"
+            or len(judgments) != len(pack.cases) or score.get("case_count") != len(pack.cases)):
+        raise InvocationError("evaluation release or count is invalid")
+    run_manifest, captures, _ = _verified_run(path.parent.parent, pack)
+    if ("bank_id" in manifest) != ("bank_id" in run_manifest):
+        raise InvocationError("evaluation and run bank identities differ")
+    expected_judgments = _judgments(pack, captures)
+    expected_score = _score(pack, run_manifest, expected_judgments, path.name)
+    if judgments != expected_judgments or score != expected_score:
+        raise InvocationError("evaluation differs from the pinned labels or verified source captures")
+    return score, judgments
+
+
+def report_invocation(
+    ref: InvocationEvaluationRef | str | os.PathLike[str],
+    data_dir: str | os.PathLike[str],
+    *, bank_id: str | None = None,
+) -> InvocationScorecard:
+    """Read a hash-verified private development scorecard."""
+
+    score, _ = _verified_evaluation(ref, data_dir, bank_id=bank_id)
+    return InvocationScorecard(score)
+
+
+def compare_invocation(
+    evaluation_a: InvocationEvaluationRef | str | os.PathLike[str],
+    evaluation_b: InvocationEvaluationRef | str | os.PathLike[str],
+    data_dir: str | os.PathLike[str],
+    *, bank_id: str | None = None,
+) -> InvocationComparison:
+    """Paired three-way comparison on exactly the same pinned local cohort."""
+
+    a, rows_a = _verified_evaluation(evaluation_a, data_dir, bank_id=bank_id)
+    b, rows_b = _verified_evaluation(evaluation_b, data_dir, bank_id=bank_id)
+    for key in ("pack_id", "input_sha256", "gold_sha256", "metric_release", "grader_release", "execution", "runner"):
+        if a.get(key) != b.get(key):
+            raise InvocationError(f"comparison identity mismatch: {key}")
+    if [(row["case_id"], row["gold_action"]) for row in rows_a] != [
+        (row["case_id"], row["gold_action"]) for row in rows_b
+    ]:
+        raise InvocationError("comparison case order or labels differ")
+    deltas = [float(right["correct"]) - float(left["correct"])
+              for left, right in zip(rows_a, rows_b)
+              if left["correct"] is not None and right["correct"] is not None]
+    return InvocationComparison({
+        "pack_id": PACK_ID, "evaluation_a": a["evaluation_id"],
+        "evaluation_b": b["evaluation_id"], "trust_tier": "local_unverified",
+        "bank_id": a.get("bank_id", "excerpt-free-200"),
+        "bank_release": a.get("bank_release", "v1-development-200"),
+        "population": a.get("population", "authored_development"),
+        "eligible_count": len(rows_a), "paired_count": len(deltas),
+        "missing_pair_count": len(rows_a) - len(deltas),
+        "mean_delta_b_minus_a": sum(deltas) / len(deltas) if len(deltas) == len(rows_a) else None,
+        "observed_mean_delta_b_minus_a": sum(deltas) / len(deltas) if deltas else None,
+    })