sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/pipeline.py

@@ -0,0 +1,313 @@
+"""Capture and grade a frozen N11–N17 pipeline without exposing reference labels.
+
+The mathematical core is an imports-only port of sablebench's Level 2 grader.
+Execution is explicitly trusted local or pinned replay, never an OS sandbox.
+Missing labels/failed trials remain unavailable; recorded timings are replay.
+"""
+from __future__ import annotations
+
+from copy import deepcopy
+from collections.abc import Mapping
+import math
+from pathlib import Path
+from time import perf_counter
+
+from .contracts import ContractError, canonical_json
+from .integration_contracts import CandidateConfiguration, ComparisonIdentity, TrackCaseCapture, value_digest, require_comparable
+from .dataset_v3 import load_dataset_v3
+from ._track_capture import (byte_digest, strict_json, create_run, write_artifact, finish_run,
+    read_artifact, verify_track_run, write_evaluation, verify_track_evaluation)
+from .chunking import _policy_for_source, _select_tasks, _verify_approval
+from ._pipeline_core import STAGES, evaluate_level2_run, validate_controlled_change
+
+PACK_ID = "rag.pipeline.v1"
+INTERFACE = "sphynx.rag.pipeline.v1"
+METRIC_RELEASE = "rag.pipeline.source-level2.metric.v1"
+GRADER_RELEASE = "rag.pipeline.source-level2.grader.v1"
+POLICY_VERSION = "sphynx.rag.pipeline.policy.v1"
+_CANDIDATE_FIELDS = {"candidate_id","dense_rank","dense_score","reranker_logit","reranker_rank",
+    "fusion_score","fusion_rank","dedup_decision","post_dedup_rank","gate_decision","gate_threshold","final_rank"}
+
+
+def _require(ok, message):
+    if not ok: raise ContractError(message)
+
+
+def _detach(value):
+    if isinstance(value,Mapping):
+        _require(all(type(key) is str for key in value),"JSON keys must be strings")
+        value={key:_detach(child) for key,child in value.items()}
+    elif isinstance(value,(list,tuple)):
+        value=[_detach(child) for child in value]
+    return strict_json(canonical_json(value))
+
+
+def _number(value):
+    try:return type(value) in (int,float) and math.isfinite(value)
+    except OverflowError:return False
+
+
+def build_pipeline_request(task, policy):
+    """The request contains input texts/identities, never qrels or evidence truth."""
+    pool = [{key:row[key] for key in ("candidate_id","input","input_sha256","dense_rank","dense_score")}
+            for row in task["candidates"]]
+    body = {"schema_version":"sphynx.rag.pipeline.request.v1", "case_id":task["task_id"],
+        "customer_id":task["customer_id"], "query":task["query"],
+        "conversation_context":task["conversation_context"], "candidates":pool, "policy":policy}
+    if policy.get("boundary")=="composed_source_documents":
+        from .chunking import _documents
+        body["documents"]=_documents(task)
+    body["input_digest"] = value_digest(body)
+    return _detach(body)
+
+
+def validate_pipeline_response(response, request):
+    fields={"schema_version","case_id","input_digest","construction","embedding","candidates","latency_state","latency_ms"}
+    composed=request["policy"].get("boundary")=="composed_source_documents"
+    if composed:fields.add("stages")
+    _require(type(response) is dict and set(response)==fields, "pipeline response fields differ")
+    _require(response["schema_version"]==("sphynx.rag.pipeline.composed.response.v1" if composed else "sphynx.rag.pipeline.response.v1")
+        and response["case_id"]==request["case_id"] and response["input_digest"]==request["input_digest"],
+        "pipeline response does not bind its request")
+    _require(type(response["construction"]) is dict and type(response["embedding"]) is dict,
+        "pipeline construction/embedding traces are required")
+    candidates=response["candidates"]
+    _require(type(candidates) is list and all(type(row) is dict and set(row)==_CANDIDATE_FIELDS for row in candidates),
+        "pipeline must return the exact label-free candidate trace")
+    if composed:
+        from ._composed_pipeline import validate_chunks,compose_pipeline
+        expected=validate_chunks(response["construction"].get("chunks"),request["documents"])
+        _require(type(response["stages"]) is dict and set(response["stages"])=={"chunker","embedder","retriever","reranker","fusion","dedup","gate"},"composed stages are incomplete")
+        # Reconstruct only the retained transformation, never execute adapters.
+        derived=compose_pipeline(request,lambda kind,operation,payload:deepcopy(response["stages"][kind]))
+        derived["latency_state"]=response["latency_state"]
+        _require(_detach(derived)==response,"composed stage outputs differ from the retained final trace")
+    else:expected={row["candidate_id"]:row for row in request["candidates"]}
+    ids=[row["candidate_id"] for row in candidates]
+    _require(len(ids)==len(set(ids)) and set(ids)==set(expected),"pipeline candidate membership changed")
+    for row in candidates:
+        source=expected[row["candidate_id"]]
+        _require(composed or row["dense_rank"]==source["dense_rank"] and row["dense_score"]==source["dense_score"],
+            "pipeline N11 dense input changed")
+        for key in ("dense_score","reranker_logit","fusion_score","gate_threshold"):
+            _require(_number(row[key]),"pipeline score must be finite")
+        for key in ("dense_rank","reranker_rank","fusion_rank"):
+            _require(type(row[key]) is int and row[key]>=1,"pipeline ranks must be positive integers")
+        for key in ("post_dedup_rank","final_rank"):
+            _require(row[key] is None or type(row[key]) is int and row[key]>=1,"pipeline optional ranks differ")
+        _require(row["dedup_decision"] in ("keep","drop") and row["gate_decision"] in ("pass","reject"),
+            "pipeline decisions differ")
+        if row["dedup_decision"]=="drop":
+            _require(row["post_dedup_rank"] is None and row["gate_decision"]=="reject" and row["final_rank"] is None,
+                "deduplicated candidates cannot reenter later stages")
+        else:
+            _require(row["post_dedup_rank"] is not None,"kept candidates require post-dedup ranks")
+        _require(row["final_rank"] is None or row["gate_decision"]=="pass", "gated candidates cannot reenter N17")
+    _require(response["latency_state"] in ("cold","warm"),"pipeline latency state differs")
+    _require(type(response["latency_ms"]) is dict and set(response["latency_ms"])<=
+        {"retrieval","reranker","fusion","dedup","gate","total","chunking","embedding"},"pipeline timing fields differ")
+    _require(all(_number(value) and value>=0 for value in response["latency_ms"].values()),"pipeline timings must be finite/nonnegative")
+    # Structural core validation uses neutral labels only; real labels enter later.
+    neutral={**deepcopy(response),"query_id":request["case_id"],"answerable":False,
+        "input_contract":{"candidate_count":len(candidates),"identity_sha256":request["input_digest"]}}
+    neutral["candidates"]=[{**row,"qrel":0,"evidence_group_ids":[]} for row in candidates]
+    evaluate_level2_run({"controlled_config":request["policy"]["controlled_config"],"queries":[neutral]})
+    kept=sorted((row for row in candidates if row["dedup_decision"]=="keep"),key=lambda row:row["post_dedup_rank"])
+    _require([row["post_dedup_rank"] for row in kept]==list(range(1,len(kept)+1)),"post-dedup ranks are not contiguous")
+    return _detach(response)
+
+
+def _policy(dataset, source_digest, candidate, *, boundary="frozen_n11", **options):
+    _require(boundary in ("frozen_n11","composed_source_documents"),"pipeline execution boundary differs")
+    policy=_policy_for_source(dataset,source_digest,**options)
+    controlled=candidate.parameters.get("controlled_config")
+    _require(isinstance(controlled,Mapping) and type(controlled.get("top_k")) is int
+        and 1<=controlled["top_k"]<=50,"candidate needs a bounded controlled_config.top_k")
+    policy.update(version=POLICY_VERSION,controlled_config=_detach(controlled),
+        boundary=boundary,
+        execution_mode=candidate.execution_mode,timing_authority="component_reported_unattested",
+        missing_label_policy="unavailable_not_zero")
+    return policy
+
+
+def _identity(dataset, digest, tasks, policy):
+    # Controlled component values are captured separately. The comparison gate
+    # explicitly verifies every changed path rather than hiding them in one key.
+    fixed={key:value for key,value in policy.items() if key!="controlled_config"}
+    return ComparisonIdentity("rag_pipeline",PACK_ID,"v1","sable.rag.dataset.v3",digest,
+        value_digest([task["task_id"] for task in tasks]),
+        value_digest([{key:task[key] for key in ("task_id","answerability","candidates","evidence_groups")} for task in tasks]),
+        value_digest(fixed),METRIC_RELEASE,GRADER_RELEASE,
+        policy["boundary_digest"] or value_digest({"boundary":"N11-N17", "inputs":digest}))
+
+
+def run_pipeline(candidate, adapter, dataset_path, output_dir, *, expected_sha256,
+                 synthetic=False, owner_authorized=False, pack_manifest_path=None,
+                 trials=1, attempts=1, case_ids=None, boundary_digest=None,boundary="frozen_n11"):
+    _require(isinstance(candidate,CandidateConfiguration) and candidate.interface_version==INTERFACE,
+        "pipeline candidate interface differs")
+    _require(candidate.execution_mode in ("trusted_local","recorded"),"pipeline execution mode requires an explicit adapter")
+    dataset=load_dataset_v3(dataset_path,expected_sha256=expected_sha256)
+    policy=_policy(dataset,expected_sha256,candidate,synthetic=synthetic,owner_authorized=owner_authorized,
+        pack_manifest_path=pack_manifest_path,trials=trials,attempts=attempts,boundary_digest=boundary_digest,boundary=boundary)
+    tasks=_select_tasks(dataset,case_ids)
+    if candidate.execution_mode=="recorded":
+        _require(isinstance(adapter,Mapping) and candidate.parameters.get("recordings_digest")==value_digest(adapter),
+            "pipeline replay requires the complete pinned recording")
+        adapter=_detach(adapter)
+    else: _require(callable(adapter),"pipeline adapter must be callable")
+    observed={"approved_manifest_source":None if synthetic else Path(pack_manifest_path).read_text(encoding="utf-8")}
+    _verify_approval(dataset,expected_sha256,policy,observed)
+    identity=_identity(dataset,expected_sha256,tasks,policy)
+    ids=[task["task_id"] for task in tasks]
+    directory=create_run(output_dir,identity,candidate,ids)
+    captures=[]
+    for index,task in enumerate(tasks):
+        request=build_pipeline_request(task,policy)
+        for trial in range(1,trials+1):
+            for attempt in range(1,attempts+1):
+                prefix=f"cases/{index}/{trial}/{attempt}/"
+                req=write_artifact(directory,prefix+"request.json",request)
+                response=None;failure=None;begin=perf_counter()
+                try:
+                    raw=adapter[value_digest(request)] if candidate.execution_mode=="recorded" else adapter(_detach(request))
+                    response=write_artifact(directory,prefix+"response.json",raw)
+                    validate_pipeline_response(raw,request)
+                except Exception:
+                    failure="invalid_response" if response is not None else "candidate_failure"
+                observation=write_artifact(directory,prefix+"observed.json",observed)
+                captures.append(TrackCaseCapture(task["task_id"],trial,attempt,req,response,observation,failure,
+                    (perf_counter()-begin)*1000))
+    return finish_run(directory,identity,candidate,ids,captures)
+
+
+def _verified(run,dataset_path,digest):
+    dataset=load_dataset_v3(dataset_path,expected_sha256=digest)
+    manifest=verify_track_run(run)
+    _require(manifest.candidate.interface_version==INTERFACE,"capture is not a pipeline experiment")
+    tasks=_select_tasks(dataset,manifest.case_ids)
+    policy=read_artifact(run.path,manifest.captures[0].request).get("policy")
+    _require(type(policy) is dict and set(policy)=={"version","synthetic","approved_manifest_digest","trials","attempts","selection","boundary_digest",
+        "controlled_config","execution_mode","timing_authority","missing_label_policy","boundary"} and policy.get("version")==POLICY_VERSION and
+        policy.get("execution_mode")==manifest.candidate.execution_mode,"captured pipeline policy differs")
+    _require(type(policy["synthetic"]) is bool and policy["selection"]=="first_success_per_trial"
+        and policy["timing_authority"]=="component_reported_unattested" and policy["missing_label_policy"]=="unavailable_not_zero"
+        and policy["controlled_config"]==_detach(manifest.candidate.parameters["controlled_config"])
+        and policy["boundary"] in ("frozen_n11","composed_source_documents"),"pipeline policy is not the frozen declared policy")
+    _require(all(type(policy[key]) is int and policy[key]>=1 for key in ("trials","attempts")),"pipeline trial/attempt policy differs")
+    expected=_policy(dataset,digest,manifest.candidate,synthetic=policy.get("synthetic"),
+        owner_authorized=True,pack_manifest_path=None,trials=policy.get("trials"),attempts=policy.get("attempts"),
+        boundary_digest=policy.get("boundary_digest"),boundary=policy["boundary"]) if policy.get("synthetic") else None
+    if expected is not None:_require(policy==expected,"pipeline policy shape or values changed")
+    require_comparable(_identity(dataset,digest,tasks,policy),manifest.identity)
+    requests={task["task_id"]:build_pipeline_request(task,policy) for task in tasks}
+    verify_track_run(run,expected_case_ids=manifest.case_ids,expected_requests=requests,
+        trials=policy["trials"],attempts=policy["attempts"])
+    for capture in manifest.captures:
+        _require(capture.observed is not None,"pipeline approval observation is missing")
+        _verify_approval(dataset,digest,policy,read_artifact(run.path,capture.observed))
+    return manifest,tasks,policy,requests
+
+
+def _score(task,response,request):
+    response=validate_pipeline_response(response,request)
+    labels={row["candidate_id"]:row for row in task["candidates"]}
+    if request["policy"]["boundary"]=="composed_source_documents":
+        from ._composed_pipeline import validate_chunks,truth_for_chunks
+        chunks=validate_chunks(response["construction"]["chunks"],request["documents"])
+        labels=truth_for_chunks(task,chunks)
+        if labels is None:return None,len(task["candidates"])
+    missing=sum(row.get("qrel") is None or row.get("judgment_status")!="judged" for row in labels.values())
+    if missing:return None,missing
+    query={key:response[key] for key in ("construction","embedding","latency_state","latency_ms")}
+    query.update(query_id=task["task_id"],answerable=task["answerability"]=="answerable",
+        input_contract={"candidate_count":len(labels),"identity_sha256":request["input_digest"]},
+        candidates=[{**row,"qrel":labels[row["candidate_id"]]["qrel"],
+            "evidence_group_ids":labels[row["candidate_id"]]["evidence_group_ids"]} for row in response["candidates"]])
+    score=evaluate_level2_run({"controlled_config":request["policy"]["controlled_config"],"queries":[query]})
+    # Keep the canonical core's diagnostic output, but do not promote its legacy
+    # zero-on-empty aggregate into a defined ranking score for an unsupported pool.
+    metric_eligibility={"ndcg_at_5":any(row["qrel"]>0 for row in task["candidates"]),
+        "recall_at_5":any(row["qrel"]>=2 for row in task["candidates"]),
+        "mrr":any(row["qrel"]>=2 for row in task["candidates"])}
+    score["metric_eligibility"]=metric_eligibility
+    if not all(metric_eligibility.values()):
+        score["source_stage_metrics"]=_detach(score["stage_metrics"])
+        for stage in STAGES:
+            for key,eligible in metric_eligibility.items():
+                if not eligible:score["stage_metrics"][stage][key]=None
+        score["ranking_status"]="partially_eligible_no_strong_reference" if any(metric_eligibility.values()) else "undefined_no_positive_reference"
+    else:score["ranking_status"]="measured"
+    return score,0
+
+
+def _grade_metrics(run,dataset_path,expected_sha256):
+    manifest,tasks,policy,requests=_verified(run,dataset_path,expected_sha256)
+    rows=[]
+    for task in tasks:
+        for trial in range(1,policy["trials"]+1):
+            score=None;selected=None;missing=0
+            for capture in sorted(manifest.captures,key=lambda row:row.attempt_index):
+                if capture.case_id!=task["task_id"] or capture.trial_index!=trial or capture.failure is not None:continue
+                _require(capture.response is not None,"successful pipeline capture has no response")
+                value,absent=_score(task,read_artifact(run.path,capture.response),requests[task["task_id"]])
+                if selected is None:score,missing,selected=value,absent,capture
+            rows.append({"case_id":task["task_id"],"trial_index":trial,"metrics":score,
+                "status":"failed" if selected is None else "labels_missing" if missing else "measured",
+                "missing_label_count":missing,"runner_wall_ms":None if selected is None else selected.runner_latency_ms})
+    good=[row["metrics"] for row in rows if row["metrics"] is not None]
+    def aggregate(stage,key):
+        values=[row["stage_metrics"][stage][key] for row in good if row["stage_metrics"][stage][key] is not None]
+        return None if len(good)!=len(rows) or not values else sum(values)/len(values)
+    stage={stage:{key:aggregate(stage,key) for key in ("candidate_count","recall_at_5","ndcg_at_5","mrr")} for stage in STAGES}
+    undefined=sum(row["metrics"] is not None and row["metrics"]["ranking_status"]=="undefined_no_positive_reference" for row in rows)
+    undefined_by_metric={key:sum(not row["metric_eligibility"][key] for row in good) for key in ("recall_at_5","ndcg_at_5","mrr")}
+    accounting={stage:{key:{"eligible_count":len(rows)-undefined_by_metric.get(key,0),
+        "observed_count":sum(row["stage_metrics"][stage][key] is not None for row in good),
+        "missing_count":len(rows)-len(good),"undefined_count":undefined_by_metric.get(key,0)}
+        for key in ("candidate_count","recall_at_5","ndcg_at_5","mrr")} for stage in STAGES}
+    return manifest,{"stage_metrics":stage,"cases":rows,
+        "stage_metric_accounting":accounting,"ranking_eligible_count":len(rows)-undefined,
+        "ranking_fully_eligible_count":sum(all(row["metric_eligibility"].values()) for row in good),
+        "ranking_undefined_no_positive_count":undefined,"quality_complete_meaning":"all_capture_and_reference_rows_available",
+        "coverage":sum(row["status"]!="failed" for row in rows)/len(rows),"quality_complete":len(good)==len(rows),
+        "quality_observed_count":len(good),"quality_missing_count":len(rows)-len(good),
+        "quality_scope":"synthetic_fixture" if policy["synthetic"] else "owner_approved",
+        "execution_mode":policy["execution_mode"],"timing_authority":"component_reported_unattested",
+        "runner_timing_scope":"replay_adapter_wall" if policy["execution_mode"]=="recorded" else "trusted_local_adapter_wall",
+        "controlled_config":policy["controlled_config"],"boundary":policy["boundary"]}
+
+
+def grade_pipeline(run,dataset_path,*,expected_sha256):
+    manifest,metrics=_grade_metrics(run,dataset_path,expected_sha256)
+    return write_evaluation(run,manifest.identity,metrics)
+
+
+def report_pipeline(evaluation,dataset_path,*,expected_sha256):
+    _,expected=_grade_metrics(evaluation.run,dataset_path,expected_sha256)
+    report=verify_track_evaluation(evaluation).to_dict()
+    _require(report["metrics"]==expected,"pipeline evaluation differs from its retained captures/references")
+    return report
+
+
+def compare_pipeline(baseline,candidate,dataset_path,*,expected_sha256,declared_paths=()):
+    left=report_pipeline(baseline,dataset_path,expected_sha256=expected_sha256)
+    right=report_pipeline(candidate,dataset_path,expected_sha256=expected_sha256)
+    return _compare_report_data(left,right,declared_paths=declared_paths)
+
+
+def _compare_report_data(left,right,*,declared_paths=()):
+    """Native comparison arithmetic over already verified reports; no I/O/grant."""
+    require_comparable(ComparisonIdentity.from_dict(left["identity"]),ComparisonIdentity.from_dict(right["identity"]))
+    a,b=left["metrics"],right["metrics"]
+    changed=None
+    if a["controlled_config"]!=b["controlled_config"]:
+        changed=validate_controlled_change(a["controlled_config"],b["controlled_config"],declared_paths)
+    else:_require(not declared_paths,"declared pipeline change did not occur")
+    _require([(r["case_id"],r["trial_index"]) for r in a["cases"]]==[(r["case_id"],r["trial_index"]) for r in b["cases"]],"pipeline paired cohort differs")
+    delta={stage:{key:None if a["stage_metrics"][stage][key] is None or b["stage_metrics"][stage][key] is None
+        else b["stage_metrics"][stage][key]-a["stage_metrics"][stage][key] for key in a["stage_metrics"][stage]} for stage in STAGES}
+    return {"schema_version":"sphynx.rag.pipeline.comparison.v1","baseline":left["evaluation_id"],
+        "candidate":right["evaluation_id"],"changed_component":changed,"delta":delta,
+        "direction":"candidate_minus_baseline","timing_comparable":False,
+        "notice":"Component timings are caller-reported diagnostics; local execution is not attested."}