sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/fast_search.py

@@ -0,0 +1,285 @@
+"""Fast Search HTTP smoke adapter.
+
+Ordinary search responses contain ranked text but no stable passage identity or
+proof of corpus/label alignment. This adapter deliberately cannot produce a
+``rag.rerank.v1`` quality score or comparison group.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+import time
+import urllib.error
+import urllib.request
+from dataclasses import dataclass, field
+from typing import Callable, Mapping
+from urllib.parse import quote, urlsplit
+
+from .contracts import ContractError, _id, _text, canonical_json
+from .http_transport import open_no_redirect
+
+FAST_SEARCH_TRACK = "smoke_latency"
+MAX_RESPONSE_BYTES = 4 * 1024 * 1024
+
+
+@dataclass(frozen=True)
+class FastSearchConfig:
+    """Explicit endpoint and KB identity for one smoke request."""
+
+    base_url: str
+    org: str
+    kb: str
+    top_k: int = 5
+    timeout_seconds: float = 30.0
+    source_id: str | None = None
+    domain: str | None = None
+
+    def __post_init__(self) -> None:
+        parsed = urlsplit(self.base_url)
+        if (
+            parsed.scheme != "https"
+            or not parsed.hostname
+            or parsed.username is not None
+            or parsed.password is not None
+            or parsed.query
+            or parsed.fragment
+        ):
+            raise ContractError("base_url must be an HTTPS origin or path without credentials, query, or fragment")
+        _id(self.org, "org")
+        _id(self.kb, "kb")
+        if type(self.top_k) is not int or not 1 <= self.top_k <= 1000:
+            raise ContractError("top_k must be an integer from 1 through 1000")
+        if type(self.timeout_seconds) not in (int, float) or not math.isfinite(self.timeout_seconds) or not 0 < self.timeout_seconds <= 120:
+            raise ContractError("timeout_seconds must be finite, positive, and at most 120")
+        if self.source_id is not None:
+            _text(self.source_id, "source_id")
+        if self.domain is not None:
+            _text(self.domain, "domain")
+
+    @property
+    def search_url(self) -> str:
+        return (
+            f"{self.base_url.rstrip('/')}/{quote(self.org, safe='')}/kbs/"
+            f"{quote(self.kb, safe='')}/search"
+        )
+
+
+@dataclass(frozen=True)
+class TransportResponse:
+    status_code: int
+    body: bytes
+
+    def __post_init__(self) -> None:
+        if type(self.status_code) is not int or not 100 <= self.status_code <= 599:
+            raise ContractError("transport status_code must be a valid HTTP status")
+        if not isinstance(self.body, bytes):
+            raise ContractError("transport body must be bytes")
+
+
+Transport = Callable[[str, bytes, Mapping[str, str], float], TransportResponse]
+
+
+@dataclass(frozen=True)
+class FastSearchHit:
+    """A hit in server-supplied order, without an invented passage ID."""
+
+    rank: int
+    score: float
+    title: str
+    section_path: str
+    url: str
+    text: str
+    chunk_index: int
+    total_chunks: int
+
+    def to_dict(self) -> dict[str, object]:
+        return {
+            "rank": self.rank,
+            "score": self.score,
+            "title": self.title,
+            "section_path": self.section_path,
+            "url": self.url,
+            "text": self.text,
+            "chunk_index": self.chunk_index,
+            "total_chunks": self.total_chunks,
+        }
+
+
+@dataclass(frozen=True)
+class FastSearchSmokeResult:
+    """Explicitly ineligible for labeled frozen-pool quality comparisons."""
+
+    org: str
+    kb: str
+    query: str
+    top_k: int
+    status: str
+    http_status: int | None
+    latency_ms: float
+    hits: tuple[FastSearchHit, ...]
+    response_sha256: str | None
+    response_size_bytes: int | None
+    error: str | None
+    reported_latency_ms: float | None = None
+    track: str = field(default=FAST_SEARCH_TRACK, init=False)
+    trust_tier: str = field(default="local_unverified", init=False)
+    identity_capability: str = field(default="stable_source_and_chunk_ids_unavailable", init=False)
+    corpus_alignment: str = field(default="unverified", init=False)
+    quality_eligible: bool = field(default=False, init=False)
+    comparison_group: None = field(default=None, init=False)
+
+    def to_dict(self) -> dict[str, object]:
+        return {
+            "track": self.track,
+            "trust_tier": self.trust_tier,
+            "org": self.org,
+            "kb": self.kb,
+            "query": self.query,
+            "top_k": self.top_k,
+            "status": self.status,
+            "http_status": self.http_status,
+            "latency_ms": self.latency_ms,
+            "reported_latency_ms": self.reported_latency_ms,
+            "hits": [hit.to_dict() for hit in self.hits],
+            "response_sha256": self.response_sha256,
+            "response_size_bytes": self.response_size_bytes,
+            "error": self.error,
+            "identity_capability": self.identity_capability,
+            "corpus_alignment": self.corpus_alignment,
+            "quality_eligible": self.quality_eligible,
+            "comparison_group": self.comparison_group,
+        }
+
+
+def _urllib_transport(url: str, body: bytes, headers: Mapping[str, str], timeout_seconds: float) -> TransportResponse:
+    request = urllib.request.Request(url, data=body, headers=dict(headers), method="POST")
+    try:
+        with open_no_redirect(request, timeout=timeout_seconds) as response:
+            return TransportResponse(response.status, response.read(MAX_RESPONSE_BYTES + 1))
+    except urllib.error.HTTPError as exc:
+        return TransportResponse(exc.code, exc.read(MAX_RESPONSE_BYTES + 1))
+
+
+def _require_string(value: object, field: str) -> str:
+    if not isinstance(value, str):
+        raise ContractError(f"{field} must be a string")
+    return value
+
+
+def _require_int(value: object, field: str, minimum: int) -> int:
+    if type(value) is not int or value < minimum:
+        raise ContractError(f"{field} must be an integer >= {minimum}")
+    return value
+
+
+def _parse_hits(body: bytes, config: FastSearchConfig, query: str) -> tuple[tuple[FastSearchHit, ...], float | None]:
+    try:
+        decoded = json.loads(
+            body.decode("utf-8"),
+            parse_constant=lambda constant: (_ for _ in ()).throw(ValueError(f"nonfinite JSON value {constant}")),
+        )
+    except (UnicodeDecodeError, json.JSONDecodeError, ValueError) as exc:
+        raise ContractError(f"Fast Search returned invalid UTF-8 JSON: {exc}") from exc
+    if not isinstance(decoded, dict):
+        raise ContractError("Fast Search response must be a JSON object")
+    if decoded.get("org") != config.org or decoded.get("kb") != config.kb or decoded.get("query") != query:
+        raise ContractError("Fast Search response org, kb, or query does not match the request")
+    raw_hits = decoded.get("results")
+    if not isinstance(raw_hits, list):
+        raise ContractError("Fast Search response results must be an array")
+    hits: list[FastSearchHit] = []
+    for index, raw in enumerate(raw_hits):
+        if not isinstance(raw, dict):
+            raise ContractError(f"results[{index}] must be an object")
+        score = raw.get("score")
+        if type(score) not in (int, float) or not math.isfinite(score):
+            raise ContractError(f"results[{index}].score must be a finite number")
+        hits.append(
+            FastSearchHit(
+                rank=_require_int(raw.get("rank"), f"results[{index}].rank", 1),
+                score=float(score),
+                title=_require_string(raw.get("title"), f"results[{index}].title"),
+                section_path=_require_string(raw.get("section_path"), f"results[{index}].section_path"),
+                url=_require_string(raw.get("url"), f"results[{index}].url"),
+                text=_require_string(raw.get("text"), f"results[{index}].text"),
+                chunk_index=_require_int(raw.get("chunk_index"), f"results[{index}].chunk_index", 0),
+                total_chunks=_require_int(raw.get("total_chunks"), f"results[{index}].total_chunks", 1),
+            )
+        )
+    reported = decoded.get("latency_ms")
+    if reported is not None:
+        if type(reported) not in (int, float) or not math.isfinite(reported) or reported < 0:
+            raise ContractError("Fast Search reported latency_ms must be a finite nonnegative number")
+        reported = float(reported)
+    return tuple(hits), reported
+
+
+def search_fast_search(
+    config: FastSearchConfig,
+    query: str,
+    *,
+    transport: Transport | None = None,
+    headers: Mapping[str, str] | None = None,
+    clock_ns: Callable[[], int] = time.perf_counter_ns,
+) -> FastSearchSmokeResult:
+    """Call Fast Search once and retain smoke/latency evidence on every outcome.
+
+    ``transport`` receives URL, UTF-8 JSON body, headers, and timeout; the
+    default transport uses the Python standard library. Caller-supplied headers
+    (including credentials) are sent but never placed in the result.
+    """
+
+    if not isinstance(config, FastSearchConfig):
+        raise ContractError("config must be FastSearchConfig")
+    _text(query, "query")
+    payload: dict[str, object] = {"query": query, "top_k": config.top_k}
+    if config.source_id is not None:
+        payload["source_id"] = config.source_id
+    if config.domain is not None:
+        payload["domain"] = config.domain
+    request_headers = {"Content-Type": "application/json", "Accept": "application/json"}
+    if headers:
+        request_headers.update(headers)
+    start = clock_ns()
+    try:
+        response = (transport or _urllib_transport)(
+            config.search_url,
+            canonical_json(payload).encode("utf-8"),
+            request_headers,
+            float(config.timeout_seconds),
+        )
+        if not isinstance(response, TransportResponse):
+            raise ContractError("transport must return TransportResponse")
+    except Exception as exc:
+        latency_ms = max(0.0, (clock_ns() - start) / 1_000_000)
+        return FastSearchSmokeResult(
+            config.org, config.kb, query, config.top_k,
+            "transport_error", None, latency_ms, (), None, None,
+            f"transport failed ({type(exc).__name__})",
+        )
+    latency_ms = max(0.0, (clock_ns() - start) / 1_000_000)
+    digest = "sha256:" + hashlib.sha256(response.body).hexdigest()
+    size = len(response.body)
+    common = (config.org, config.kb, query, config.top_k)
+    if size > MAX_RESPONSE_BYTES:
+        return FastSearchSmokeResult(
+            *common, "invalid_response", response.status_code, latency_ms, (), digest, size,
+            f"response exceeds {MAX_RESPONSE_BYTES} bytes",
+        )
+    if not 200 <= response.status_code < 300:
+        return FastSearchSmokeResult(
+            *common, "http_error", response.status_code, latency_ms, (), digest, size,
+            f"Fast Search returned HTTP {response.status_code}",
+        )
+    try:
+        hits, reported_latency_ms = _parse_hits(response.body, config, query)
+    except ContractError as exc:
+        return FastSearchSmokeResult(
+            *common, "invalid_response", response.status_code, latency_ms, (), digest, size, str(exc),
+        )
+    return FastSearchSmokeResult(
+        *common, "ok", response.status_code, latency_ms, hits, digest, size, None,
+        reported_latency_ms=reported_latency_ms,
+    )

sphynx/full_rag.py

@@ -0,0 +1,341 @@
+"""Source-parity full-RAG traces, retained grading and fixed-generator diagnostics.
+
+Opaque subject traces and componentized diagnostic traces are separate scopes.
+Reference actions, qrels, facts and semantic review are never candidate inputs.
+Local adapters are trusted in-process Python; metadata is not runtime attestation.
+"""
+from __future__ import annotations
+from collections.abc import Mapping
+from copy import deepcopy
+from dataclasses import replace
+from pathlib import Path
+from time import perf_counter
+import re
+
+from .contracts import ContractError
+from .integration_contracts import CandidateConfiguration,ComparisonIdentity,TrackCaseCapture,value_digest,require_digest,require_comparable
+from .dataset_v3 import load_dataset_v3
+from ._track_capture import (strict_json,create_run,write_artifact,finish_run,read_artifact,
+    verify_track_run,write_evaluation,verify_track_evaluation,byte_digest)
+from .chunking import _policy_for_source,_select_tasks,_verify_approval
+from .pipeline import _detach,_require,_number
+from ._full_rag_trace import Level3Trace,validate_level3_trace,CONTEXT_CONDITIONS
+from ._full_rag_grading import ClaimVerdict,grade_level3,evaluate_counterfactuals,CounterfactualObservation
+
+PACK_ID="rag.full_rag.v1"
+INTERFACE="sphynx.rag.full_rag.v1"
+METRIC_RELEASE="rag.full_rag.source-level3.metric.v1"
+GRADER_RELEASE="rag.full_rag.source-level3.grader.v1"
+POLICY_VERSION="sphynx.rag.full_rag.policy.v1"
+JUDGMENTS_SCHEMA="sphynx.rag.full_rag.judgments.v1"
+_SEMANTIC=("answer_required_fact_coverage","answer_completeness","exact_value_unit_date_accuracy","faithfulness","unsupported_claim_rate","citation_support")
+
+
+def build_full_rag_request(task,policy,context_ids):
+    pool=[{key:row[key] for key in ("candidate_id","input","input_sha256","dense_rank","dense_score")}
+        for row in task["candidates"]]
+    by_id={row["candidate_id"]:row for row in pool}
+    _require(type(context_ids) in (tuple,list) and all(type(cid) is str for cid in context_ids)
+        and len(context_ids)==len(set(context_ids)) and set(context_ids)<=set(by_id),"full-RAG context inventory differs")
+    body={"schema_version":"sphynx.rag.full_rag.request.v1","case_id":task["task_id"],
+        "customer_id":task["customer_id"],"query":task["query"],"conversation_context":task["conversation_context"],
+        "candidates":pool,"context_condition":policy["context_condition"],"context_candidate_ids":list(context_ids),
+        "context_text":"\n\n".join(by_id[cid]["input"] for cid in context_ids),
+        "policy":{key:value for key,value in policy.items() if key!="contexts"}}
+    body["input_digest"]=value_digest(body)
+    return _detach(body)
+
+
+def validate_full_rag_response(response,request,*,expected_invocation=False):
+    fields={"schema_version","case_id","input_digest","trace"}
+    componentized=request["policy"]["execution_shape"]=="componentized"
+    if componentized:fields.add("stages")
+    _require(type(response) is dict and set(response)==fields,"full-RAG response fields differ")
+    _require(response["schema_version"]=="sphynx.rag.full_rag.response.v1" and response["case_id"]==request["case_id"]
+        and response["input_digest"]==request["input_digest"],"full-RAG response does not bind its request")
+    raw=response["trace"]
+    if componentized:
+        from ._componentized_full_rag import compose_full_rag
+        stages=response["stages"]
+        _require(type(stages) is dict and set(stages)=={"router","rewriter","generator"},"full-RAG component outputs differ")
+        derived=compose_full_rag(request,lambda kind,operation,payload:deepcopy(stages[kind]))
+        _require(_detach(derived)==response,"full-RAG trace differs from retained component outputs")
+    _require(type(raw) is dict and "expected_invocation" not in raw,"candidate trace cannot carry a reference action")
+    trace=deepcopy(raw)
+    _require(trace.get("task_id")==request["case_id"] and trace.get("as_uttered_query")==request["query"],"full-RAG trace request identity drift")
+    trace["expected_invocation"]=expected_invocation
+    unknown_latency=trace.get("latency_ms")=={}
+    if unknown_latency:trace["latency_ms"]={key:0.0 for key in ("routing","rewrite","retrieval","generation","total")}
+    else:_require(type(trace.get("latency_ms")) is dict and all(_number(v) and v>=0 for v in trace["latency_ms"].values()),"full-RAG latency values differ")
+    citations=trace.get("citations")
+    _require(type(citations) is list and all(type(row) is dict and type(row.get("start")) is int and type(row.get("end")) is int
+        and type(row.get("citation_id")) is str and row["citation_id"] and type(row.get("evidence_id")) is str and row["evidence_id"] for row in citations),"citation identities/offsets differ")
+    _require(len({row["citation_id"] for row in citations})==len(citations),"citation IDs must be unique")
+    parsed=validate_level3_trace(trace)
+    supplied=list(request["context_candidate_ids"])
+    _require(all(cid in supplied for cid in parsed.retrieved_candidate_ids)
+        and [cid for cid in supplied if cid in parsed.retrieved_candidate_ids]==list(parsed.retrieved_candidate_ids),
+        "trace retrieval is not an ordered subset of the supplied frozen contexts")
+    if request["policy"]["execution_shape"]=="componentized":
+        expected_context=tuple(supplied) if parsed.actual_rag_call_count else ()
+        _require(parsed.handed_context_ids==expected_context,"componentized context handoff differs from its fixed generator input")
+    if unknown_latency:parsed=replace(parsed,latency_ms={})
+    return parsed
+
+
+def _policy(dataset,digest,candidate,*,context_condition,contexts,execution_shape,**options):
+    _require(context_condition in CONTEXT_CONDITIONS and execution_shape in ("componentized","opaque"),"full-RAG execution boundary differs")
+    generator=candidate.parameters.get("generator_digest");prompt=candidate.parameters.get("prompt_digest")
+    require_digest(generator,"generator_digest");require_digest(prompt,"prompt_digest")
+    policy=_policy_for_source(dataset,digest,**options)
+    policy.update(version=POLICY_VERSION,context_condition=context_condition,contexts=_detach(contexts),execution_shape=execution_shape,
+        generator_digest=generator,prompt_digest=prompt,execution_mode=candidate.execution_mode,
+        timing_authority="component_reported_unattested",semantic_review_policy="capture-bound-owner-judgments.v1")
+    return policy
+
+
+def _identity(dataset,digest,tasks,policy):
+    fixed={k:v for k,v in policy.items() if k not in ("context_condition","contexts")}
+    semantic_refs=[{key:task[key] for key in ("task_id","should_call_rag","expected_behavior","evidence_groups")} for task in tasks]
+    condition="oracle" if policy["context_condition"]=="oracle" else "empty" if policy["context_condition"]=="empty" else "retrieved"
+    return ComparisonIdentity("rag_full_rag",PACK_ID,"v1","sable.rag.dataset.v3",digest,
+        value_digest([task["task_id"] for task in tasks]),value_digest(semantic_refs),value_digest(fixed),
+        METRIC_RELEASE,GRADER_RELEASE,policy["boundary_digest"] or value_digest({"boundary":policy["execution_shape"],"dataset":digest}),
+        context_condition=condition,generator_digest=policy["generator_digest"])
+
+
+def run_full_rag(candidate,adapter,dataset_path,output_dir,*,expected_sha256,synthetic=False,
+        owner_authorized=False,pack_manifest_path=None,trials=1,attempts=1,case_ids=None,boundary_digest=None,
+        execution_shape="opaque",context_condition="dense",contexts=None):
+    _require(isinstance(candidate,CandidateConfiguration) and candidate.interface_version==INTERFACE,"full-RAG candidate interface differs")
+    _require(candidate.execution_mode in ("trusted_local","recorded"),"full-RAG mode requires an explicit local adapter or replay")
+    dataset=load_dataset_v3(dataset_path,expected_sha256=expected_sha256);tasks=_select_tasks(dataset,case_ids)
+    if contexts is None:
+        _require(context_condition in ("dense","empty"),"oracle/A0/challenger diagnostics need explicit frozen contexts")
+        contexts={task["task_id"]:[] if context_condition=="empty" else [row["candidate_id"] for row in sorted(task["candidates"],key=lambda row:row["dense_rank"])[:5]] for task in tasks}
+    _require(isinstance(contexts,Mapping) and set(contexts)=={task["task_id"] for task in tasks},"full-RAG context cohort differs")
+    policy=_policy(dataset,expected_sha256,candidate,context_condition=context_condition,contexts=contexts,execution_shape=execution_shape,
+        synthetic=synthetic,owner_authorized=owner_authorized,pack_manifest_path=pack_manifest_path,
+        trials=trials,attempts=attempts,boundary_digest=boundary_digest)
+    requests={task["task_id"]:build_full_rag_request(task,policy,contexts[task["task_id"]]) for task in tasks}
+    if context_condition=="empty":_require(all(not values for values in contexts.values()),"empty context diagnostics must actually be empty")
+    if candidate.execution_mode=="recorded":
+        _require(isinstance(adapter,Mapping) and candidate.parameters.get("recordings_digest")==value_digest(adapter),"full-RAG replay recording is not pinned")
+        adapter=_detach(adapter)
+    else:_require(callable(adapter),"full-RAG adapter must be callable")
+    observed={"approved_manifest_source":None if synthetic else Path(pack_manifest_path).read_text(encoding="utf-8"),"frozen_policy":policy}
+    _verify_approval(dataset,expected_sha256,policy,{"approved_manifest_source":observed["approved_manifest_source"]})
+    identity=_identity(dataset,expected_sha256,tasks,policy);ids=[task["task_id"] for task in tasks]
+    directory=create_run(output_dir,identity,candidate,ids);captures=[]
+    for index,task in enumerate(tasks):
+        request=requests[task["task_id"]]
+        for trial in range(1,trials+1):
+            for attempt in range(1,attempts+1):
+                prefix=f"cases/{index}/{trial}/{attempt}/";req=write_artifact(directory,prefix+"request.json",request)
+                response=None;failure=None;begin=perf_counter()
+                try:
+                    raw=adapter[value_digest(request)] if candidate.execution_mode=="recorded" else adapter(_detach(request))
+                    response=write_artifact(directory,prefix+"response.json",raw);validate_full_rag_response(raw,request)
+                except Exception:failure="invalid_response" if response is not None else "candidate_failure"
+                observation=write_artifact(directory,prefix+"observed.json",observed)
+                captures.append(TrackCaseCapture(task["task_id"],trial,attempt,req,response,observation,failure,(perf_counter()-begin)*1000))
+    return finish_run(directory,identity,candidate,ids,captures)
+
+
+def _verified(run,dataset_path,digest):
+    dataset=load_dataset_v3(dataset_path,expected_sha256=digest);manifest=verify_track_run(run)
+    _require(manifest.candidate.interface_version==INTERFACE,"capture is not full-RAG")
+    tasks=_select_tasks(dataset,manifest.case_ids)
+    _require(manifest.captures[0].observed is not None,"full-RAG frozen control metadata is missing")
+    first_observed=read_artifact(run.path,manifest.captures[0].observed)
+    _require(type(first_observed) is dict and set(first_observed)=={"approved_manifest_source","frozen_policy"},"full-RAG control metadata fields differ")
+    policy=first_observed["frozen_policy"]
+    required={"version","synthetic","approved_manifest_digest","trials","attempts","selection","boundary_digest","context_condition",
+        "contexts","execution_shape","generator_digest","prompt_digest","execution_mode","timing_authority","semantic_review_policy"}
+    _require(type(policy) is dict and set(policy)==required and policy["version"]==POLICY_VERSION
+        and type(policy["synthetic"]) is bool and policy["selection"]=="first_success_per_trial"
+        and policy["context_condition"] in CONTEXT_CONDITIONS and policy["execution_shape"] in ("opaque","componentized")
+        and policy["execution_mode"]==manifest.candidate.execution_mode and policy["timing_authority"]=="component_reported_unattested"
+        and policy["semantic_review_policy"]=="capture-bound-owner-judgments.v1","full-RAG policy fields differ")
+    _require(all(type(policy[key]) is int and policy[key]>=1 for key in ("trials","attempts")),"full-RAG trial policy differs")
+    _require(policy["generator_digest"]==manifest.candidate.parameters["generator_digest"] and
+        policy["prompt_digest"]==manifest.candidate.parameters["prompt_digest"],"declared generator/prompt changed")
+    requests={task["task_id"]:build_full_rag_request(task,policy,policy["contexts"][task["task_id"]]) for task in tasks}
+    require_comparable(_identity(dataset,digest,tasks,policy),manifest.identity)
+    verify_track_run(run,expected_case_ids=manifest.case_ids,expected_requests=requests,trials=policy["trials"],attempts=policy["attempts"])
+    for capture in manifest.captures:
+        _require(capture.observed is not None,"full-RAG approval observation is missing")
+        observed=read_artifact(run.path,capture.observed)
+        _require(type(observed) is dict and set(observed)=={"approved_manifest_source","frozen_policy"}
+            and observed["frozen_policy"]==policy,"full-RAG frozen controls changed between cases")
+        _verify_approval(dataset,digest,policy,{"approved_manifest_source":observed["approved_manifest_source"]})
+    return manifest,tasks,policy,requests
+
+
+def create_full_rag_review_template(run,dataset_path,*,expected_sha256):
+    manifest,tasks,policy,requests=_verified(run,dataset_path,expected_sha256)
+    rows=[]
+    for capture in manifest.captures:
+        if capture.failure is None and capture.response is not None:
+            rows.append({"case_id":capture.case_id,"trial_index":capture.trial_index,"attempt_index":capture.attempt_index,
+                "response_digest":capture.response.sha256,"required_facts":{},"exact_values":[],"claims":[],
+                "claims_exhaustive":False,"review_status":"pending"})
+    return {"schema_version":JUDGMENTS_SCHEMA,"capture_manifest_digest":run.manifest_sha256,"dataset_digest":expected_sha256,
+        "reviewer_id":None,"reviewed_at_utc":None,"authority":"pending","rows":rows}
+
+
+def _reviews(value,run,digest,manifest,tasks):
+    if value is None:return {},None,None
+    raw=Path(value).read_bytes() if isinstance(value,(str,Path)) else None
+    review=strict_json(raw) if raw is not None else _detach(value)
+    _require(type(review) is dict and set(review)=={"schema_version","capture_manifest_digest","dataset_digest","reviewer_id","reviewed_at_utc","authority","rows"}
+        and review["schema_version"]==JUDGMENTS_SCHEMA and review["capture_manifest_digest"]==run.manifest_sha256
+        and review["dataset_digest"]==digest,"semantic judgments are not bound to this capture/dataset")
+    _require(type(review["reviewer_id"]) is str and review["reviewer_id"].strip() and type(review["reviewed_at_utc"]) is str
+        and re.fullmatch(r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z",review["reviewed_at_utc"])
+        and review["authority"] in ("owner_human_review","synthetic_fixture"),"semantic review identity/authority is incomplete")
+    _require(type(review["rows"]) is list,"semantic review rows differ")
+    first_policy=read_artifact(run.path,manifest.captures[0].request)["policy"]
+    _require(review["authority"]!="synthetic_fixture" or first_policy["synthetic"],
+        "synthetic judgments cannot approve a customer capture")
+    captures={(row.case_id,row.trial_index,row.attempt_index):row for row in manifest.captures};by_key={}
+    for row in review["rows"]:
+        fields={"case_id","trial_index","attempt_index","response_digest","required_facts","exact_values","claims","claims_exhaustive","review_status"}
+        _require(type(row) is dict and set(row)==fields and row["review_status"]=="reviewed" and type(row["claims_exhaustive"]) is bool,"semantic review row is incomplete")
+        key=(row["case_id"],row["trial_index"],row["attempt_index"]);capture=captures.get(key)
+        _require(key not in by_key and capture is not None and capture.failure is None and capture.response is not None
+            and row["response_digest"]==capture.response.sha256,"semantic review response membership/drift")
+        _require(type(row["required_facts"]) is dict and all(type(k) is str and k and type(v) is list and v
+            and all(type(phrase) is str and phrase.strip() for phrase in v) for k,v in row["required_facts"].items())
+            and type(row["exact_values"]) is list and all(type(v) is str and v.strip() for v in row["exact_values"])
+            and type(row["claims"]) is list,"semantic review facts/values/claims differ")
+        by_key[key]=row
+    truth={}
+    for row in review["rows"]:
+        criteria={"required_facts":row["required_facts"],"exact_values":row["exact_values"]}
+        previous=truth.get(row["case_id"])
+        _require(previous is None or previous==criteria,"semantic truth changed between trials/attempts")
+        truth[row["case_id"]]=criteria
+    policy_digest=value_digest({"schema_version":"sphynx.rag.full_rag.semantic-policy.v1",
+        "metric_release":METRIC_RELEASE,"authority":review["authority"],"truth":truth})
+    return by_key,value_digest(review),policy_digest
+
+
+def _grade_metrics(run,dataset_path,digest,judgments=None):
+    manifest,tasks,policy,requests=_verified(run,dataset_path,digest)
+    reviews,review_digest,review_policy=_reviews(judgments,run,digest,manifest,tasks);rows=[]
+    for task in tasks:
+        for trial in range(1,policy["trials"]+1):
+            score=None;selected=None;reviewed=False
+            for capture in sorted(manifest.captures,key=lambda row:row.attempt_index):
+                if capture.case_id!=task["task_id"] or capture.trial_index!=trial or capture.failure is not None:continue
+                _require(capture.response is not None,"successful full-RAG capture lacks response")
+                trace=validate_full_rag_response(read_artifact(run.path,capture.response),requests[task["task_id"]],expected_invocation=task["should_call_rag"])
+                review=reviews.get((capture.case_id,trial,capture.attempt_index));claims=[]
+                if review is not None:
+                    for raw in review["claims"]:
+                        _require(type(raw) is dict and set(raw)=={"start","end","supported_by_evidence_ids"}
+                            and type(raw["start"]) is int and type(raw["end"]) is int and type(raw["supported_by_evidence_ids"]) is list
+                            and all(type(cid) is str for cid in raw["supported_by_evidence_ids"])
+                            and len(set(raw["supported_by_evidence_ids"]))==len(raw["supported_by_evidence_ids"])
+                            and set(raw["supported_by_evidence_ids"])<=set(trace.handed_context_ids),"claim support leaves exact handed evidence")
+                        claims.append(ClaimVerdict(raw["start"],raw["end"],tuple(raw["supported_by_evidence_ids"])))
+                    ordered=sorted(claims,key=lambda claim:(claim.start,claim.end))
+                    _require(all(left.end<=right.start for left,right in zip(ordered,ordered[1:])),
+                        "reviewed claims overlap or repeat answer spans")
+                current=grade_level3(task,trace,required_facts={} if review is None else review["required_facts"],
+                    exact_values=[] if review is None else review["exact_values"],claims=claims)
+                if review is None:
+                    for key in _SEMANTIC:current[key]=None
+                elif not review["claims_exhaustive"]:
+                    for key in ("faithfulness","unsupported_claim_rate"):current[key]=None
+                if selected is None:score,selected,reviewed=current,capture,review is not None
+            rows.append({"case_id":task["task_id"],"trial_index":trial,"status":"failed" if selected is None else "measured",
+                "metrics":score,"semantic_reviewed":reviewed,"runner_wall_ms":None if selected is None else selected.runner_latency_ms,
+                "response_digest":None if selected is None else selected.response.sha256})
+    keys=("rag_call_true_positive","rag_call_false_positive","rag_call_false_negative","rewrite_preservation","evidence_handoff_completeness",
+        "answer_required_fact_coverage","exact_value_unit_date_accuracy","faithfulness","unsupported_claim_rate","citation_precision","citation_recall",
+        "citation_support","correct_abstention","answer_completeness","subject_error")
+    summary={}
+    for key in keys:
+        values=[row["metrics"][key] for row in rows if row["metrics"] is not None and row["metrics"][key] is not None]
+        summary[key]={"value":sum(values)/len(values) if len(values)==len(rows) else None,
+            "observed_mean":sum(values)/len(values) if values else None,"observed_count":len(values),
+            "eligible_count":len(rows),"missing_count":len(rows)-len(values)}
+    complete=all(row["metrics"] is not None for row in rows)
+    counts={key:sum(row["metrics"][key] for row in rows if row["metrics"] is not None)
+        for key in ("rag_call_true_positive","rag_call_false_positive","rag_call_false_negative")}
+    for key,denominator in (("rag_call_precision",counts["rag_call_true_positive"]+counts["rag_call_false_positive"]),
+        ("rag_call_recall",counts["rag_call_true_positive"]+counts["rag_call_false_negative"])):
+        summary[key]={"value":counts["rag_call_true_positive"]/denominator if complete and denominator else None,
+            "numerator":counts["rag_call_true_positive"],"denominator":denominator,
+            "complete":complete,"undefined":complete and not denominator,
+            "eligible_count":len(rows),"observed_count":sum(row["metrics"] is not None for row in rows),
+            "missing_count":sum(row["metrics"] is None for row in rows)}
+    return manifest,{"summary":summary,"cases":rows,"coverage":sum(row["metrics"] is not None for row in rows)/len(rows),
+        "execution_shape":policy["execution_shape"],"context_condition":policy["context_condition"],
+        "generator_digest":policy["generator_digest"],"prompt_digest":policy["prompt_digest"],
+        "execution_mode":policy["execution_mode"],"timing_authority":"component_reported_unattested",
+        "quality_scope":"synthetic_fixture" if policy["synthetic"] else "owner_approved",
+        "review_digest":review_digest,"judgment_policy_digest":review_policy},review_digest,review_policy
+
+
+def grade_full_rag(run,dataset_path,*,expected_sha256,judgments=None):
+    manifest,metrics,review_digest,review_policy=_grade_metrics(run,dataset_path,expected_sha256,judgments)
+    if review_digest is not None:
+        value=strict_json(Path(judgments).read_bytes()) if isinstance(judgments,(str,Path)) else _detach(judgments)
+        relative="semantic-reviews/"+review_digest.removeprefix("sha256:")+".json"
+        if not (run.path/relative).exists():write_artifact(run.path,relative,value)
+        metrics["review_artifact"]=relative
+    return write_evaluation(run,replace(manifest.identity,judgment_policy_digest=review_policy),metrics,
+        judgment_artifact_digest=review_digest)
+
+
+def report_full_rag(evaluation,dataset_path,*,expected_sha256):
+    report=verify_track_evaluation(evaluation).to_dict();review=None
+    if report["metrics"].get("review_artifact"):
+        from ._track_capture import safe_path
+        path=safe_path(evaluation.run.path,report["metrics"]["review_artifact"]);review=strict_json(path.read_bytes())
+        _require(value_digest(review)==report["judgment_artifact_digest"],"retained semantic review digest differs")
+    _,expected,review_digest,review_policy=_grade_metrics(evaluation.run,dataset_path,expected_sha256,review)
+    if review_digest is not None:expected["review_artifact"]=report["metrics"]["review_artifact"]
+    _require(report["metrics"]==expected,"full-RAG evaluation differs from retained trace/review")
+    _require(report["identity"]["judgment_policy_digest"]==review_policy,"semantic policy identity differs")
+    return report
+
+
+def compare_full_rag(baseline,candidate,dataset_path,*,expected_sha256):
+    a=report_full_rag(baseline,dataset_path,expected_sha256=expected_sha256)
+    b=report_full_rag(candidate,dataset_path,expected_sha256=expected_sha256)
+    require_comparable(ComparisonIdentity.from_dict(a["identity"]),ComparisonIdentity.from_dict(b["identity"]))
+    _require(a["metrics"]["context_condition"]==b["metrics"]["context_condition"],"full-RAG context diagnostic differs")
+    _require([(row["case_id"],row["trial_index"]) for row in a["metrics"]["cases"]]==
+        [(row["case_id"],row["trial_index"]) for row in b["metrics"]["cases"]],"full-RAG paired cohort differs")
+    return {"schema_version":"sphynx.rag.full_rag.comparison.v1","baseline":a["evaluation_id"],"candidate":b["evaluation_id"],
+        "delta":{key:None if value["value"] is None or b["metrics"]["summary"][key]["value"] is None
+            else b["metrics"]["summary"][key]["value"]-value["value"] for key,value in a["metrics"]["summary"].items()},
+        "timing_comparable":False,"notice":"Review scopes and opaque/componentized boundaries are held fixed; local timings are not attested."}
+
+
+def compare_full_rag_contexts(evaluations,dataset_path,*,expected_sha256):
+    _require(isinstance(evaluations,Mapping) and set(evaluations)==CONTEXT_CONDITIONS,"counterfactuals require exactly five fixed contexts")
+    reports={key:report_full_rag(value,dataset_path,expected_sha256=expected_sha256) for key,value in evaluations.items()}
+    base=reports["oracle"];identity=ComparisonIdentity.from_dict(base["identity"])
+    for condition,report in reports.items():
+        _require(report["metrics"]["context_condition"]==condition,"counterfactual condition alias changed")
+        require_comparable(identity,ComparisonIdentity.from_dict(report["identity"]),context_conditions=True)
+        _require(report["metrics"]["generator_digest"]==base["metrics"]["generator_digest"] and
+            report["metrics"]["prompt_digest"]==base["metrics"]["prompt_digest"],"counterfactual generator/prompt changed")
+    # This diagnostic uses retained coverage only. No generator/candidate is rerun.
+    rows={key:{(row["case_id"],row["trial_index"]):row for row in report["metrics"]["cases"]} for key,report in reports.items()}
+    _require(all(set(value)==set(rows["oracle"]) for value in rows.values()),"counterfactual task/trial cohorts differ")
+    result=[]
+    for case in rows["oracle"]:
+        coverage={key:rows[key][case]["metrics"]["answer_required_fact_coverage"] if rows[key][case]["metrics"] else None for key in CONTEXT_CONDITIONS}
+        def gap(a,b):return None if coverage[a] is None or coverage[b] is None else coverage[a]-coverage[b]
+        result.append({"case_id":case[0],"trial_index":case[1],"condition_coverage":coverage,
+            "retrieval_ranking_gap":gap("oracle","challenger"),"challenger_vs_a0":gap("challenger","a0"),
+            "a0_ranking_contribution":gap("a0","dense"),"rag_dependency":gap("challenger","empty")})
+    return {"schema_version":"sphynx.rag.full_rag.context-comparison.v1","cases":result,
+        "generator_digest":base["metrics"]["generator_digest"],"prompt_digest":base["metrics"]["prompt_digest"],
+        "notice":"Fixed-generator context diagnostic, separate from architecture selection; missing semantic truth remains unavailable."}

sphynx/hosted/__init__.py

@@ -0,0 +1,18 @@
+"""Hosted submission preparation. Remote execution is not connected yet."""
+
+from .contracts import (
+    CandidateConfig, CandidateFamily, HostedError, SelectedCandidateRepository,
+    parse_candidate_config, parse_candidate_family, parse_candidate_repository,
+    verify_candidate_repository,
+)
+from .registry import HostedRegistry, RepositoryRegistration, load_registry
+from .submission import FrozenSubmission, GitHubReader, prepare_submission
+
+__all__ = [
+    "CandidateConfig", "HostedError", "parse_candidate_config", "HostedRegistry",
+    "RepositoryRegistration", "load_registry", "FrozenSubmission", "GitHubReader",
+    "prepare_submission",
+]
+
+__all__ += ["CandidateFamily", "SelectedCandidateRepository", "parse_candidate_family",
+            "parse_candidate_repository", "verify_candidate_repository"]