sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/_local_eval.py

@@ -0,0 +1,268 @@
+"""Shared, local-only immutable capture primitives for external RAG packs.
+
+Hashes detect changed records; they are not authentication or OS isolation.
+Owner-held inputs are deliberately not copied into installed package data.
+"""
+from __future__ import annotations
+
+import hashlib
+import math
+import platform
+import random
+import uuid
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Callable, Mapping
+
+from .contracts import ContractError, canonical_json
+from .retrieval import _strict_json
+
+
+def digest(raw: bytes) -> str:
+    return "sha256:" + hashlib.sha256(raw).hexdigest()
+
+
+def encoded(value: Any) -> bytes:
+    return canonical_json(value).encode("utf-8")
+
+
+def identity(value: Any) -> str:
+    return digest(encoded(value))
+
+
+def read_json(path: Path) -> Any:
+    return _strict_json(path.read_bytes())
+
+
+def require(condition: bool, message: str) -> None:
+    if not condition:
+        raise ContractError(message)
+
+
+def text(value: Any, field: str) -> str:
+    require(isinstance(value, str) and bool(value.strip()), field + " must be nonempty text")
+    return value
+
+
+def pinned_file(root: Path, relative: str, expected: Mapping[str, Any]) -> bytes:
+    path = Path(relative)
+    require(not path.is_absolute() and not path.drive and ".." not in path.parts
+            and "\\" not in relative and ":" not in relative, "unsafe manifest path")
+    target = (root / path).resolve(strict=True)
+    require(target.is_relative_to(root), "manifest path leaves dataset directory")
+    raw = target.read_bytes()
+    require(digest(raw) == expected.get("sha256") and len(raw) == expected.get("size_bytes"),
+            "manifest file bytes differ from pinned identity")
+    return raw
+
+
+def metric(values: list[float | None], *, missing_status: str = "incomplete") -> dict[str, Any]:
+    observed = [v for v in values if v is not None]
+    mean = sum(observed) / len(observed) if observed else None
+    return {"value": mean if len(observed) == len(values) else None,
+            "observed_mean": mean, "eligible_count": len(values),
+            "observed_count": len(observed), "missing_count": len(values) - len(observed),
+            "undefined_count": int(not values),
+            "status": "undefined" if not values else missing_status if len(observed) != len(values) else "measured"}
+
+
+def percentile(values: list[float], fraction: float) -> float | None:
+    if not values:
+        return None
+    rows = sorted(values)
+    position = (len(rows) - 1) * fraction
+    lo = math.floor(position)
+    hi = math.ceil(position)
+    return rows[lo] + (rows[hi] - rows[lo]) * (position - lo)
+
+
+def clustered_interval(rows: list[tuple[str, str, float]], samples: int = 2000) -> list[float] | None:
+    """Resample whole families within customer/domain; no utterance pseudoreplication."""
+    if not rows:
+        return None
+    strata: dict[str, dict[str, list[float]]] = {}
+    for stratum, cluster, value in rows:
+        strata.setdefault(stratum, {}).setdefault(cluster, []).append(value)
+    rng = random.Random(0)
+    means = []
+    for _ in range(samples):
+        draw = []
+        for clusters in strata.values():
+            groups = list(clusters.values())
+            for _ in groups:
+                draw.extend(rng.choice(groups))
+        means.append(sum(draw) / len(draw))
+    return [percentile(means, .025), percentile(means, .975)]
+
+
+@dataclass(frozen=True)
+class LocalRunRef:
+    path: Path
+    manifest_sha256: str
+
+    @property
+    def run_id(self) -> str:
+        return self.path.name
+
+
+@dataclass(frozen=True)
+class LocalEvaluationRef:
+    path: Path
+    scorecard_sha256: str
+    run: LocalRunRef
+
+    @property
+    def evaluation_id(self) -> str:
+        return self.path.name
+
+
+def open_local_run(path: str | Path, *, expected_sha256: str | None = None) -> LocalRunRef:
+    """Reopen retained evidence. Grading still validates the external pack and every file."""
+    root = Path(path).expanduser().resolve(strict=True)
+    raw = (root / "manifest.json").read_bytes()
+    actual = digest(raw)
+    if expected_sha256 is not None:
+        require(actual == "sha256:" + expected_sha256.removeprefix("sha256:"), "capture manifest SHA-256 differs")
+    value = _strict_json(raw)
+    require(value.get("schema_version") == "sphynx.local-capture.v1" and value.get("run_id") == root.name, "unsupported local capture")
+    return LocalRunRef(root, actual)
+
+
+def open_local_evaluation(path: str | Path) -> LocalEvaluationRef:
+    root = Path(path).expanduser().resolve(strict=True)
+    require(root.parent.name == "evaluations", "evaluation must be inside its original run")
+    raw = (root / "scorecard.json").read_bytes()
+    value = _strict_json(raw)
+    require(value.get("evaluation_id") == root.name, "evaluation path identity differs")
+    run = open_local_run(root.parent.parent, expected_sha256=value.get("capture_manifest_sha256"))
+    return LocalEvaluationRef(root, digest(raw), run)
+
+
+@dataclass(frozen=True)
+class LocalScorecard:
+    data: Mapping[str, Any]
+
+    @property
+    def metrics(self) -> Mapping[str, Any]:
+        return self.data["metrics"]
+
+    def to_dict(self) -> dict[str, Any]:
+        return _strict_json(encoded(dict(self.data)))
+
+    def to_json(self) -> str:
+        return canonical_json(self.to_dict())
+
+    def to_text(self) -> str:
+        from .metric_views import summarize_metrics
+        return summarize_metrics(self.data).to_text()
+
+    def to_markdown(self) -> str:
+        from .metric_views import summarize_metrics
+        return summarize_metrics(self.data).to_markdown()
+
+
+def capture(pack_identity: dict[str, Any], candidate: dict[str, Any], policy: dict[str, Any],
+            requests: list[dict[str, Any]], execute: Callable, output_dir: str | Path) -> LocalRunRef:
+    root = Path(output_dir).expanduser().resolve() / "runs" / ("run-" + uuid.uuid4().hex)
+    root.mkdir(parents=True, exist_ok=False)
+    records = []
+    for index, request in enumerate(requests):
+        raw_request = encoded(request)
+        stdout, stderr, exit_code, failure, latency_ms, extra = execute(raw_request, request)
+        files = {}
+        for suffix, raw in (("request.json", raw_request), ("stdout.bin", stdout),
+                            ("stderr.bin", stderr), ("observed.json", encoded(extra))):
+            relative = f"{index:06d}.{suffix}"
+            (root / relative).write_bytes(raw)
+            files[suffix] = {"path": relative, "sha256": digest(raw), "size_bytes": len(raw)}
+        records.append({"case_id": request["case_id"], "files": files, "exit_code": exit_code,
+                        "failure": failure, "runner_latency_ms": latency_ms})
+    value = {"schema_version": "sphynx.local-capture.v1", "run_id": root.name,
+             "pack": pack_identity, "candidate": candidate, "execution_policy": policy,
+             "runtime": {"python": platform.python_version(), "platform": platform.platform()},
+             "trust_tier": "local_unverified", "captures": records}
+    raw = encoded(value)
+    (root / "manifest.json").write_bytes(raw)
+    return LocalRunRef(root, digest(raw))
+
+
+def verified_capture(run: LocalRunRef, pack_identity: dict[str, Any],
+                     requests: list[dict[str, Any]], policy: dict[str, Any]) -> tuple[dict[str, Any], list[dict[str, Any]]]:
+    require(isinstance(run, LocalRunRef), "run must be a LocalRunRef from this SDK")
+    root = run.path.resolve(strict=True)
+    raw = (root / "manifest.json").read_bytes()
+    require(digest(raw) == run.manifest_sha256, "capture manifest changed")
+    manifest = _strict_json(raw)
+    require(manifest.get("schema_version") == "sphynx.local-capture.v1"
+            and manifest.get("run_id") == run.run_id and manifest.get("pack") == pack_identity
+            and manifest.get("execution_policy") == policy, "capture pack or policy identity differs")
+    captures = manifest.get("captures")
+    require(isinstance(captures, list) and len(captures) == len(requests), "capture cohort differs")
+    rows = []
+    for index, (row, request) in enumerate(zip(captures, requests)):
+        require(row.get("case_id") == request["case_id"], "capture case order differs")
+        require(set(row.get("files", {})) == {"request.json", "stdout.bin", "stderr.bin", "observed.json"},
+                "capture file inventory differs")
+        payloads = {}
+        for suffix, entry in row["files"].items():
+            require(entry.get("path") == f"{index:06d}.{suffix}", "capture file path differs")
+            payloads[suffix] = pinned_file(root, entry["path"], entry)
+        require(payloads["request.json"] == encoded(request), "capture request differs from pinned case")
+        span = row.get("runner_latency_ms")
+        require(type(span) in (int, float) and math.isfinite(span) and span >= 0, "invalid runner span")
+        failure = row.get("failure")
+        require(failure is None or isinstance(failure, str), "invalid failure record")
+        require(type(row.get("exit_code")) is int or row.get("exit_code") is None, "invalid exit record")
+        response = None
+        if failure is None:
+            try:
+                response = _strict_json(payloads["stdout.bin"])
+            except (ValueError, ContractError, UnicodeError, RecursionError):
+                failure = "invalid_json"
+        rows.append({**row, "failure": failure, "response": response,
+                     "observed": _strict_json(payloads["observed.json"])})
+    return manifest, rows
+
+
+def save_evaluation(run: LocalRunRef, data: dict[str, Any], annotations: dict[str, Any] | None = None) -> LocalEvaluationRef:
+    payload = {**data, "run_id": run.run_id, "capture_manifest_sha256": run.manifest_sha256,
+               "annotations": annotations, "annotations_sha256": identity(annotations)}
+    evaluation_id = "eval-" + identity(payload).split(":")[1]
+    raw = encoded({**payload, "evaluation_id": evaluation_id})
+    root = run.path / "evaluations" / evaluation_id
+    if root.exists():
+        require((root / "scorecard.json").read_bytes() == raw, "existing evaluation bytes changed")
+    else:
+        root.mkdir(parents=True, exist_ok=False)
+        (root / "scorecard.json").write_bytes(raw)
+    return LocalEvaluationRef(root, digest(raw), run)
+
+
+def report_evaluation(evaluation: LocalEvaluationRef, recompute: Callable) -> LocalScorecard:
+    require(isinstance(evaluation, LocalEvaluationRef), "evaluation must be a LocalEvaluationRef")
+    raw = (evaluation.path / "scorecard.json").read_bytes()
+    require(digest(raw) == evaluation.scorecard_sha256, "evaluation bytes changed")
+    saved = _strict_json(raw)
+    base = recompute(saved["annotations"])
+    expected = {**base, "run_id": evaluation.run.run_id,
+                "capture_manifest_sha256": evaluation.run.manifest_sha256,
+                "annotations": saved["annotations"], "annotations_sha256": identity(saved["annotations"])}
+    evaluation_id = "eval-" + identity(expected).split(":")[1]
+    require(evaluation_id == evaluation.evaluation_id
+            and saved == {**expected, "evaluation_id": evaluation_id}, "evaluation does not match regraded captures")
+    # Raw annotations and references stay in the owner bundle, not the report.
+    return LocalScorecard({k: v for k, v in saved.items() if k != "annotations"})
+
+
+def compare_scorecards(a: LocalScorecard, b: LocalScorecard) -> dict[str, Any]:
+    for field in ("pack", "execution_policy", "metric_release", "grader_release", "runtime"):
+        require(a.data[field] == b.data[field], "comparison requires matching " + field)
+    deltas = {}
+    for key, left in a.metrics.items():
+        right = b.metrics[key]
+        av, bv = left.get("value"), right.get("value")
+        deltas[key] = {"a": av, "b": bv, "delta_b_minus_a": bv - av if av is not None and bv is not None else None,
+                       "status": "measured" if av is not None and bv is not None else "incomplete"}
+    return {"schema_version": "sphynx.local-comparison.v1", "trust_tier": "local_unverified",
+            "pack": a.data["pack"], "candidate_a": a.data["candidate"], "candidate_b": b.data["candidate"],
+            "metrics": deltas}

sphynx/_pipeline_core.py

@@ -0,0 +1,243 @@
+"""Pinned local fast-search trace validation and stage attribution."""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Mapping, Sequence
+from typing import cast
+
+from ._rag_ranking import ndcg, recall_at, reciprocal_rank
+from typing import Any
+JsonValue = Any
+
+STAGES = ("N11", "N12", "N13", "N14", "N15", "N16", "N17")
+CHANGE_COMPONENTS = frozenset({"chunking", "embedding", "reranker", "fusion", "gating", "top_k"})
+
+
+class Level2Error(ValueError):
+    """Raised for incomplete traces or uncontrolled pipeline comparisons."""
+
+
+def _flatten(value: Mapping[str, object], prefix: str = "") -> dict[str, object]:
+    result: dict[str, object] = {}
+    for key, item in value.items():
+        path = f"{prefix}.{key}" if prefix else key
+        if isinstance(item, Mapping):
+            result.update(_flatten(cast("Mapping[str, object]", item), path))
+        else:
+            result[path] = item
+    return result
+
+
+def validate_controlled_change(
+    baseline: Mapping[str, object],
+    challenger: Mapping[str, object],
+    declared_paths: Sequence[str],
+) -> str:
+    """Require changes to exactly one declared component and no hidden field."""
+
+    left, right = _flatten(baseline), _flatten(challenger)
+    changed = {path for path in left.keys() | right.keys() if left.get(path) != right.get(path)}
+    if changed != set(declared_paths) or not changed:
+        raise Level2Error(f"controlled changes differ; actual={sorted(changed)}")
+    components = {path.split(".", 1)[0] for path in changed}
+    if len(components) != 1 or next(iter(components)) not in CHANGE_COMPONENTS:
+        raise Level2Error("comparison changes more than one approved component")
+    return next(iter(components))
+
+
+def validate_fast_search_pin(
+    pin: Mapping[str, object], actual_revision: str, available_paths: Sequence[str]
+) -> None:
+    if pin.get("repository") != "sable-inc/fast-search":
+        raise Level2Error("fast-search repository identity is invalid")
+    if pin.get("revision") != actual_revision:
+        raise Level2Error("local fast-search revision does not match the pin")
+    required = cast("list[str]", pin.get("required_paths"))
+    missing = set(required) - set(available_paths)
+    if missing:
+        raise Level2Error(f"pinned checkout lacks required paths: {sorted(missing)}")
+
+
+def _ordered(candidates: Sequence[Mapping[str, object]], stage: str) -> list[Mapping[str, object]]:
+    if stage == "N11":
+        return sorted(candidates, key=lambda value: cast("int", value["dense_rank"]))
+    if stage in {"N12", "N13"}:
+        return sorted(
+            candidates,
+            key=lambda value: (
+                -cast("float", value["reranker_logit"]),
+                cast("str", value["candidate_id"]),
+            ),
+        )
+    if stage == "N14":
+        return sorted(candidates, key=lambda value: cast("int", value["fusion_rank"]))
+    if stage == "N15":
+        selected = [value for value in candidates if value["dedup_decision"] == "keep"]
+        return sorted(selected, key=lambda value: cast("int", value["post_dedup_rank"]))
+    if stage == "N16":
+        selected = [
+            value
+            for value in candidates
+            if value["dedup_decision"] == "keep" and value["gate_decision"] == "pass"
+        ]
+        return sorted(selected, key=lambda value: cast("int", value["post_dedup_rank"]))
+    selected = [value for value in candidates if value["final_rank"] is not None]
+    return sorted(selected, key=lambda value: cast("int", value["final_rank"]))
+
+
+def _percentile(values: Sequence[float], fraction: float) -> float | None:
+    if not values:
+        return None
+    ordered = sorted(values)
+    index = min(math.ceil(fraction * len(ordered)) - 1, len(ordered) - 1)
+    return ordered[max(index, 0)]
+
+
+def evaluate_level2_run(run: Mapping[str, object]) -> dict[str, JsonValue]:
+    """Validate N11-N17 identity/order and report causal stage/latency metrics."""
+
+    queries = cast("list[Mapping[str, object]]", run.get("queries"))
+    if not queries:
+        raise Level2Error("level2 run requires queries")
+    stage_values: dict[str, list[dict[str, float | int | None]]] = {stage: [] for stage in STAGES}
+    zero_result_answerable = 0
+    unsupported_final = 0
+    unsupported_query_count = 0
+    final_count = 0
+    first_degradation: dict[str, str | None] = {}
+    transition_rescue = transition_harm = 0
+    latency: dict[str, dict[str, list[float]]] = {
+        state: {name: [] for name in ("retrieval", "reranker", "fusion", "dedup", "gate", "total")}
+        for state in ("cold", "warm")
+    }
+    for query_index, query in enumerate(queries):
+        for field in ("construction", "embedding", "input_contract"):
+            if not isinstance(query.get(field), Mapping):
+                raise Level2Error(f"query {query_index} lacks {field} trace")
+        candidates = cast("list[Mapping[str, object]]", query.get("candidates"))
+        identities = [cast("str", candidate["candidate_id"]) for candidate in candidates]
+        if len(identities) != len(set(identities)):
+            raise Level2Error("N11 candidate identities must be unique")
+        for candidate in candidates:
+            required = {
+                "candidate_id",
+                "dense_rank",
+                "dense_score",
+                "reranker_logit",
+                "reranker_rank",
+                "fusion_score",
+                "fusion_rank",
+                "dedup_decision",
+                "post_dedup_rank",
+                "gate_decision",
+                "gate_threshold",
+                "final_rank",
+                "qrel",
+                "evidence_group_ids",
+            }
+            if not required <= candidate.keys():
+                raise Level2Error("candidate trace does not cover N11 through N17")
+        reranker = _ordered(candidates, "N13")
+        if [candidate["reranker_rank"] for candidate in reranker] != list(
+            range(1, len(reranker) + 1)
+        ):
+            raise Level2Error("N13 ranks do not follow reranker logits")
+        fusion = _ordered(candidates, "N14")
+        if [candidate["fusion_rank"] for candidate in fusion] != list(range(1, len(fusion) + 1)):
+            raise Level2Error("N14 fusion ranks are not contiguous")
+        threshold = {cast("float", candidate["gate_threshold"]) for candidate in candidates}
+        if len(threshold) > 1:
+            raise Level2Error("N16 thresholds differ inside one query")
+        top_k = cast("int", cast("Mapping[str, object]", run["controlled_config"])["top_k"])
+        final = _ordered(candidates, "N17")
+        if len(final) > top_k:
+            raise Level2Error("N17 exceeds configured top_k")
+        if [candidate["final_rank"] for candidate in final] != list(range(1, len(final) + 1)):
+            raise Level2Error("N17 final ranks are not contiguous")
+        n16_ids = {
+            cast("str", candidate["candidate_id"]) for candidate in _ordered(candidates, "N16")
+        }
+        if not {cast("str", candidate["candidate_id"]) for candidate in final} <= n16_ids:
+            raise Level2Error("N17 contains a candidate removed before final top-k")
+
+        prior_recall: float | None = None
+        degraded: str | None = None
+        prior_top: set[str] = set()
+        for stage in STAGES:
+            ordered = _ordered(candidates, stage)
+            grades = [cast("int", candidate["qrel"]) for candidate in ordered]
+            current_recall = recall_at(grades, 5, 2)
+            stage_values[stage].append(
+                {
+                    "candidate_count": len(ordered),
+                    "recall_at_5": current_recall,
+                    "ndcg_at_5": ndcg(grades, 5),
+                    "mrr": reciprocal_rank(grades, 2),
+                }
+            )
+            current_top = {
+                cast("str", candidate["candidate_id"])
+                for candidate in ordered[:5]
+                if cast("int", candidate["qrel"]) >= 2
+            }
+            if prior_recall is not None and current_recall is not None:
+                transition_rescue += len(current_top - prior_top)
+                transition_harm += len(prior_top - current_top)
+                if degraded is None and current_recall < prior_recall:
+                    degraded = stage
+            prior_recall, prior_top = current_recall, current_top
+        first_degradation[cast("str", query["query_id"])] = degraded
+        answerable = bool(query["answerable"])
+        zero_result_answerable += int(answerable and not final)
+        unsupported_query_count += int(not answerable)
+        unsupported_final += int(not answerable and bool(final))
+        final_count += len(final)
+        state = cast("str", query.get("latency_state", "cold" if query_index == 0 else "warm"))
+        if state not in latency:
+            raise Level2Error("latency_state must be cold or warm")
+        for name, value in cast("Mapping[str, object]", query["latency_ms"]).items():
+            number = float(cast("int | float", value))
+            if name in latency[state] and math.isfinite(number) and number >= 0:
+                latency[state][name].append(number)
+
+    summary: dict[str, JsonValue] = {}
+    for stage, observations in stage_values.items():
+        summary[stage] = {
+            metric: sum(
+                cast("float", observation[metric])
+                for observation in observations
+                if observation[metric] is not None
+            )
+            / max(sum(observation[metric] is not None for observation in observations), 1)
+            for metric in ("candidate_count", "recall_at_5", "ndcg_at_5", "mrr")
+        }
+    return {
+        "stage_metrics": summary,
+        "first_degradation_stage": cast("dict[str, JsonValue]", first_degradation),
+        "transition_rescue_count": transition_rescue,
+        "transition_harm_count": transition_harm,
+        "zero_result_answerable_count": zero_result_answerable,
+        "unsupported_leakage_rate": (
+            None if unsupported_query_count == 0 else unsupported_final / unsupported_query_count
+        ),
+        "context_precision": sum(
+            cast("int", candidate["qrel"]) >= 2
+            for query in queries
+            for candidate in _ordered(
+                cast("list[Mapping[str, object]]", query["candidates"]), "N17"
+            )
+        )
+        / max(final_count, 1),
+        "latency_ms": {
+            state: {
+                name: {
+                    "p50": _percentile(values, 0.5),
+                    "p90": _percentile(values, 0.9),
+                    "p95": _percentile(values, 0.95),
+                }
+                for name, values in components.items()
+            }
+            for state, components in latency.items()
+        },
+    }