sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/public_bundle.py

@@ -0,0 +1,173 @@
+"""Prepare and verify a separate public-only reranking data bundle.
+
+The SDK wheel contains no benchmark texts. A release owner must authorize data
+distribution separately. Hashes bind bytes; they are not a signature or proof
+that an arbitrary supplied source has redistribution permission.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+import hashlib
+import json
+from pathlib import Path
+import re
+
+from ._local_eval import read_json, require
+from .public_rerank import DOMAINS, PUBLIC_POOL_SHA256, load_public_rerank_pack
+
+SCHEMA = "sphynx.public-rerank.bundle.v1"
+FILES = ("frozen-pools.json", "DATA_SOURCES.md")
+ROW_FIELDS = ("dataset", "split", "query_id", "task_id", "query")
+PASSAGE_FIELDS = ("candidate_id", "document_id", "input", "first_stage_rank", "first_stage_score", "grade")
+
+
+def _hash(raw: bytes) -> str:
+    return hashlib.sha256(raw).hexdigest()
+
+
+def _json(value: object) -> bytes:
+    return (json.dumps(value, sort_keys=True, indent=2, ensure_ascii=False, allow_nan=False) + "\n").encode("utf-8")
+
+
+@dataclass(frozen=True)
+class PublicRerankBundle:
+    path: Path
+    manifest_sha256: str
+    dataset_sha256: str
+
+    @property
+    def dataset_path(self) -> Path:
+        return self.path / "frozen-pools.json"
+
+
+def prepare_public_rerank_bundle(source: str | Path, output_dir: str | Path, *,
+                                 expected_source_sha256: str = PUBLIC_POOL_SHA256) -> PublicRerankBundle:
+    """Export only six public domains and explicit grading fields to a new folder.
+
+    Default input is the pinned research source. An explicit different hash is
+    a caller-approved custom source, not an official Sphynx benchmark release.
+    No network calls, cloud uploads or model execution occur.
+    """
+    source = Path(source).expanduser().resolve(strict=True)
+    source_raw = source.read_bytes()
+    expected = expected_source_sha256.removeprefix("sha256:")
+    require(_hash(source_raw) == expected, "public source SHA-256 differs")
+    packs = [load_public_rerank_pack(source, split=split, expected_sha256=expected)
+             for split in ("development", "confirmation")]
+    rows = []
+    for pack in packs:
+        for row in pack.rows:
+            rows.append({**{key: row[key] for key in ROW_FIELDS},
+                         "candidates": [{key: c[key] for key in PASSAGE_FIELDS} for c in row["candidates"]]})
+    payload = {"schema_version": "expanded-reranker.frozen-pools.v1", "candidate_depth": 15,
+               "first_stage": "bm25_okapi_public_screen", "queries": rows}
+    notices = ["# Public reranking data sources", "",
+               "This is a frozen excerpt of six BEIR datasets, not the full BEIR benchmark.",
+               "Keep this notice with the data. The SDK license does not license third-party data.",
+               "BEIR's dataset disclaimer: https://github.com/beir-cellar/beir#-disclaimer", "",
+               "Each BEIR-maintained card below declares CC BY-SA 4.0. Check the original",
+               "dataset terms and attribution before use or redistribution. This notice records",
+               "the published declaration; it is not a legal clearance of source content.",
+               "License text: https://creativecommons.org/licenses/by-sa/4.0/", ""]
+    notices.extend(f"- {domain}: https://huggingface.co/datasets/BeIR/{domain}" for domain in DOMAINS)
+    notices.extend(["", "Changes: select 30 queries per domain (15 per split), preserve the frozen",
+                    "15-passage BM25 pools and native qrels, remove duplicate title/text fields",
+                    "and unrelated metadata. Seven internal illustrative cases are excluded.",
+                    "Both splits were previously opened and support exploratory comparisons.",
+                    "Candidate inputs omit ground truths. Local owners can read public qrels.", ""])
+    contents = {"frozen-pools.json": _json(payload), "DATA_SOURCES.md": "\n".join(notices).encode("utf-8")}
+    manifest = {"schema_version": SCHEMA, "pack_id": "rag.rerank.beir.v1", "query_count": 180,
+                "split_counts": {"development": 90, "confirmation": 90}, "domains": list(DOMAINS),
+                "candidate_depth": 15, "source_sha256": expected,
+                "source_authority": "pinned_research_source" if expected == PUBLIC_POOL_SHA256 else "caller_approved_custom",
+                "data_distribution": "separate_owner_authorization_required",
+                "files": {name: {"sha256": _hash(raw), "size_bytes": len(raw)} for name, raw in contents.items()}}
+    root = Path(output_dir).expanduser().absolute()
+    require(not root.exists() and not root.is_symlink(), "bundle output must be a new folder")
+    root.mkdir(parents=True, exist_ok=False)
+    for name, raw in contents.items():
+        with (root / name).open("xb") as stream:
+            stream.write(raw)
+    raw = _json(manifest)
+    with (root / "manifest.json").open("xb") as stream:
+        stream.write(raw)
+    return load_public_rerank_bundle(root, expected_manifest_sha256=_hash(raw))
+
+
+def load_public_rerank_bundle(data_dir: str | Path, *, expected_manifest_sha256: str) -> PublicRerankBundle:
+    """Verify an owner-supplied manifest pin before loading any experiment data.
+
+    Get the pin from your release owner through a trusted channel. Do not replace
+    it with a freshly computed hash when verification fails.
+    """
+    expected = expected_manifest_sha256.removeprefix("sha256:")
+    require(re.fullmatch(r"[0-9a-f]{64}", expected) is not None, "manifest pin must be a lowercase SHA-256")
+    root = Path(data_dir).expanduser().resolve(strict=True)
+    require(root.is_dir(), "bundle must be a directory")
+    paths = {path.name for path in root.iterdir()}
+    require(paths == {*FILES, "manifest.json"}, "bundle inventory must contain exactly the approved three files")
+    for name in paths:
+        path = root / name
+        require(path.is_file() and not path.is_symlink() and path.resolve().parent == root, "unsafe bundle file")
+    raw = (root / "manifest.json").read_bytes()
+    require(_hash(raw) == expected, "bundle manifest SHA-256 differs")
+    manifest = read_json(root / "manifest.json")
+    require(manifest.get("schema_version") == SCHEMA and manifest.get("pack_id") == "rag.rerank.beir.v1"
+            and manifest.get("query_count") == 180 and manifest.get("domains") == list(DOMAINS)
+            and manifest.get("split_counts") == {"development": 90, "confirmation": 90}
+            and manifest.get("candidate_depth") == 15, "unsupported bundle identity")
+    require(isinstance(manifest.get("files"), dict) and set(manifest["files"]) == set(FILES), "invalid bundle inventory")
+    for name in FILES:
+        raw = (root / name).read_bytes()
+        record = manifest["files"][name]
+        require(isinstance(record, dict) and set(record) == {"sha256", "size_bytes"}
+                and _hash(raw) == record["sha256"] and len(raw) == record["size_bytes"], "bundle file differs: " + name)
+    dataset = root / "frozen-pools.json"
+    data = read_json(dataset)
+    require(set(data) == {"schema_version", "candidate_depth", "first_stage", "queries"}, "non-public bundle metadata")
+    require(isinstance(data["queries"], list) and len(data["queries"]) == 180, "bundle must contain 180 public cases")
+    for row in data["queries"]:
+        require(isinstance(row, dict) and set(row) == {*ROW_FIELDS, "candidates"}
+                and row.get("dataset") in DOMAINS, "non-public bundle row")
+        require(isinstance(row["candidates"], list) and all(isinstance(c, dict) and set(c) == set(PASSAGE_FIELDS)
+                for c in row["candidates"]), "non-public passage metadata")
+    pin = manifest["files"]["frozen-pools.json"]["sha256"]
+    for split in ("development", "confirmation"):
+        load_public_rerank_pack(dataset, split=split, expected_sha256=pin)
+    return PublicRerankBundle(root, expected, pin)
+
+RELEASE_BUNDLE_SCHEMA = "sphynx.public-rerank.release-bundle.v1"
+
+
+def load_public_release_bundle(data_dir: str | Path, *, release_id: str,
+                               expected_manifest_sha256: str) -> PublicRerankBundle:
+    """Verify a separate revised pool bundle against a trusted manifest pin.
+
+    An owner supplies the exact pinned source bytes and its attribution notice.
+    The SDK does not distribute the source dataset or infer distribution rights.
+    """
+    from .public_rerank import load_public_pool_release
+    expected = expected_manifest_sha256.removeprefix("sha256:")
+    require(re.fullmatch(r"[0-9a-f]{64}", expected) is not None, "manifest pin must be a lowercase SHA-256")
+    root = Path(data_dir).expanduser().resolve(strict=True)
+    require(root.is_dir() and {p.name for p in root.iterdir()} == {*FILES, "manifest.json"},
+            "release bundle inventory must contain exactly three approved files")
+    for name in (*FILES, "manifest.json"):
+        path = root / name
+        require(path.is_file() and not path.is_symlink() and path.resolve().parent == root, "unsafe release bundle file")
+    raw = (root / "manifest.json").read_bytes()
+    require(_hash(raw) == expected, "release bundle manifest SHA-256 differs")
+    manifest = read_json(root / "manifest.json")
+    require(set(manifest) == {"schema_version", "release_id", "dataset_id", "files"}
+            and manifest.get("schema_version") == RELEASE_BUNDLE_SCHEMA
+            and manifest.get("release_id") == release_id, "release bundle identity differs")
+    records = manifest.get("files")
+    require(isinstance(records, dict) and set(records) == set(FILES), "invalid release bundle inventory")
+    for name in FILES:
+        raw = (root / name).read_bytes(); record = records[name]
+        require(isinstance(record, dict) and set(record) == {"sha256", "size_bytes"}
+                and type(record["size_bytes"]) is int and record["size_bytes"] == len(raw)
+                and record["sha256"] == _hash(raw), "release bundle file differs: " + name)
+    pack = load_public_pool_release(root / "frozen-pools.json", release_id=release_id)
+    require(manifest["dataset_id"] == pack.identity["dataset_id"], "release bundle dataset identity differs")
+    return PublicRerankBundle(root, expected, pack.identity["dataset_digest"].removeprefix("sha256:"))

sphynx/public_eval.py

@@ -0,0 +1,62 @@
+"""Public hosted evaluation metadata; no corpus, labels, runner or cloud SDK."""
+from __future__ import annotations
+from .evals import ArtifactBinding, ArtifactReference, EvaluationDefinition
+
+EVALUATION_ID = "rag.rerank.public.v1"
+
+
+BENCHMARK_REF = "beir.scifact.bm25-top15.dev15@1.0.0"
+
+
+LABEL_REF = "beir.scifact.native-qrels.dev15@1.0.0"
+
+
+MANIFEST_DIGEST = "sha256:9335ce0aaa3e53942c8cba2deff126af5ad93319d383cb4eb70fceb443990835"
+
+
+BENCHMARK_DIGEST = "sha256:fda436c6e006233a4f53eaadf8633b05111f0c1c21b2b099972c518e02aa9971"
+
+
+LABEL_DIGEST = "sha256:18b60b759c05ac9a8d5ac14477ff509e67ec9c38ffd760c20b0b0f54313ad198"
+
+
+GRADER_RELEASE = "rag.rerank.scifact.dev15.grader.v1"
+
+
+METRIC_RELEASE = "rag.rerank.beir.metric.v1"
+
+
+INPUT_DIGEST = "sha256:3a80c8b09cb26c96c7551d6b194d8195fc848ea7a0491c8f014a30e505bf5140"
+
+
+def _object(properties, *, required=(), default=None):
+    value = {"type": "object", "properties": properties, "required": list(required), "additionalProperties": False}
+    if default is not None:
+        value["default"] = default
+    return value
+
+
+def public_definition():
+    """Explicit benchmark selection is required; protocol choices are fixed."""
+    schema = _object({
+        "benchmark": {"type": "string", "enum": [BENCHMARK_REF]},
+        "label_release": {"type": "string", "enum": [LABEL_REF], "default": LABEL_REF},
+        "split": {"type": "string", "enum": ["development"], "default": "development"},
+        "top_k": {"type": "integer", "enum": [5], "default": 5},
+        "relevance_threshold": {"type": "integer", "enum": [1], "default": 1},
+        "runtime": _object({"trials": {"type": "integer", "enum": [1], "default": 1},
+            "per_case_timeout_seconds": {"type": "integer", "enum": [30], "default": 30}}, default={}),
+        "reporting": _object({"format": {"type": "string", "enum": ["markdown", "json"], "default": "markdown"}}, default={}),
+    }, required=("benchmark",))
+    return EvaluationDefinition(EVALUATION_ID, "v1", "SciFact development reranking on fixed passage pools", "runnable",
+        "sphynx.rag.rerank.v1", {"adapter": "hosted-public-rerank.v1", "grader": GRADER_RELEASE, "metrics": METRIC_RELEASE, "capture_policy": "public-rerank.streams.v1"},
+        schema, ("/benchmark", "/label_release", "/split", "/top_k", "/relevance_threshold", "/runtime"),
+        ("/runtime",), (ArtifactBinding("/benchmark", "benchmark_release"), ArtifactBinding("/label_release", "label_release")),
+        ("ndcg_at_5", "hit_at_5", "mrr_at_5", "pool_oracle_coverage", "runner_latency_ms", "valid_response_coverage", "candidate_failure_rate"))
+
+
+def public_artifacts():
+    return (ArtifactReference(BENCHMARK_REF, "benchmark_release", "1.0.0", BENCHMARK_DIGEST, public=True),
+            ArtifactReference(LABEL_REF, "label_release", "1.0.0", LABEL_DIGEST, public=True))
+
+

sphynx/public_rerank.py

@@ -0,0 +1,355 @@
+"""Versioned six-domain public BEIR frozen-pool reranking development pack."""
+from __future__ import annotations
+
+from dataclasses import dataclass
+import math
+from pathlib import Path
+from typing import Any
+
+from ._local_eval import (LocalEvaluationRef, LocalRunRef, LocalScorecard, capture, compare_scorecards,
+                         digest, identity, metric, percentile, read_json, report_evaluation, require,
+                         save_evaluation, text, verified_capture)
+from .contracts import (CandidateSpec, INTERFACE_VERSION, REQUEST_SCHEMA_VERSION,
+                        validate_candidate_request, validate_candidate_response, ranked_candidate_ids)
+from .invocation import _execute_candidate
+
+PUBLIC_RERANK_PACK_ID = "rag.rerank.beir.v1"
+PUBLIC_POOL_SHA256 = "89b713acfd071cafd65fc40581d721399aeee79a132c87e20b03a2b6c401941e"
+DOMAINS = ("scifact", "nfcorpus", "fiqa", "arguana", "trec-covid", "scidocs")
+METRIC_RELEASE = "rag.rerank.beir.metric.v1"
+GRADER_RELEASE = "rag.rerank.beir.grader.v1"
+
+
+class PublicRerankPack:
+    """Owner-authorized external data; native public qrels remain grader-only."""
+
+    def __init__(self, source_digest: str, split: str, rows: list[dict[str, Any]]):
+        self.rows = rows
+        self.split = split
+        self.identity = {"pack_id": PUBLIC_RERANK_PACK_ID, "pack_release": "v1.external",
+                         "dataset_digest": source_digest, "cohort_digest": identity(rows), "split": split,
+                         "domains": list(DOMAINS), "candidate_depth": 15,
+                         "first_stage": "bm25_okapi_public_screen", "relevance_threshold": 1,
+                         "label_status": "public_qrels", "development_only": True,
+                         "confirmation_status": "previously_opened_exploratory"}
+
+
+def load_public_rerank_pack(dataset_path: str | Path, *, split: str = "development",
+                            expected_sha256: str = PUBLIC_POOL_SHA256) -> PublicRerankPack:
+    """Select exactly one original split; exclude every Sable illustrative fixture."""
+    require(split in ("development", "confirmation"), "split must be development or confirmation")
+    source = Path(dataset_path).expanduser().resolve(strict=True)
+    source_digest = digest(source.read_bytes())
+    require(source_digest == "sha256:" + expected_sha256.removeprefix("sha256:"), "public dataset SHA-256 differs")
+    data = read_json(source)
+    require(data.get("schema_version") == "expanded-reranker.frozen-pools.v1"
+            and data.get("candidate_depth") == 15 and data.get("first_stage") == "bm25_okapi_public_screen",
+            "unsupported public pool source or first-stage policy")
+    require(isinstance(data.get("queries"), list), "public queries must be a list")
+    selected = []
+    ids = set()
+    for item in data["queries"]:
+        if item.get("dataset") not in DOMAINS:
+            continue
+        require(item.get("split") in ("development", "confirmation"), "unexpected public split")
+        key = (item["dataset"], text(item.get("query_id"), "query_id"))
+        require(key not in ids, "public query must appear in only one split")
+        ids.add(key)
+        text(item.get("query"), "query")
+        text(item.get("task_id"), "task_id")
+        candidates = item.get("candidates")
+        require(isinstance(candidates, list) and len(candidates) == 15, "public pools must have depth 15")
+        seen = set()
+        documents = set()
+        for index, candidate in enumerate(candidates):
+            cid = text(candidate.get("candidate_id"), "candidate_id")
+            document = text(candidate.get("document_id"), "document_id")
+            require(cid not in seen and document not in documents, "duplicate public pool candidate or document")
+            seen.add(cid)
+            documents.add(document)
+            text(candidate.get("input"), "candidate.input")
+            require(type(candidate.get("first_stage_rank")) is int and candidate["first_stage_rank"] == index + 1,
+                    "public first-stage ranks must equal frozen pool order")
+            score = candidate.get("first_stage_score")
+            require(type(score) in (int, float) and math.isfinite(score), "invalid public first-stage score")
+            require(type(candidate.get("grade")) is int and 0 <= candidate["grade"] <= 3,
+                    "public relevance grades must retain native nonnegative qrels")
+        if item["split"] == split:
+            selected.append(item)
+    require(all(sum(row["dataset"] == domain for row in selected) == 15 for domain in DOMAINS),
+            "public split must contain the original 15 queries in each of six domains")
+    require(len({row["task_id"] for row in selected}) == len(selected), "public task IDs must be unique")
+    return PublicRerankPack(source_digest, split, selected)
+
+
+def build_public_rerank_request(pack: PublicRerankPack, case_id: str) -> dict[str, Any]:
+    row = next((row for row in pack.rows if row["task_id"] == case_id), None)
+    require(row is not None, "case does not belong to public reranking split")
+    pool = row["candidates"]
+    if "dataset_id" in pack.identity:
+        # The legacy request protocol requires dense_rank to equal request position.
+        pool = sorted(pool, key=lambda candidate: candidate["first_stage_rank"])
+    candidates = [{"candidate_id": c["candidate_id"], "input": c["input"],
+                   "dense_rank": c["first_stage_rank"], "dense_score": c["first_stage_score"]}
+                  for c in pool]
+    # dense_* are legacy names; the pack identity declares BM25 or the Snowflake proxy.
+    value = {"schema_version": REQUEST_SCHEMA_VERSION, "pack_id": PUBLIC_RERANK_PACK_ID,
+             "case_id": row["task_id"], "query_id": row["query_id"], "query": row["query"],
+             "versions": {"corpus": pack.identity["dataset_digest"], "index": pack.identity["first_stage"],
+                          "chunking": "public-document-v1",
+                          "embedding": identity(pack.identity["first_stage_identity"])
+                              if "first_stage_identity" in pack.identity else "none-bm25"},
+             "pool_digest": identity({"query": row["query"], "candidates": candidates}), "candidates": candidates}
+    return validate_candidate_request(value).to_dict()
+
+
+def _policy(pack: PublicRerankPack | None = None) -> dict[str, Any]:
+    revised = pack is not None and "dataset_id" in pack.identity
+    return {"policy_version": "rag.rerank.beir.policy.v2" if revised else "rag.rerank.beir.policy.v1", "timeout_seconds": 30.0, "trials": 1,
+            "candidate_depth": 15, "ranking_depth": 5, "isolation": "local_unsandboxed",
+            "cache_policy": "caller_managed_not_observed"}
+
+
+def _rank_public_response(pack, row, response, request):
+    """Retain the original source-pool tie order on revised releases only."""
+    if "dataset_id" not in pack.identity:
+        return ranked_candidate_ids(response, request)
+    require(pack.identity.get("ranking_tie_policy") == "frozen_candidate_source_rank",
+            "revised public ranking tie policy differs")
+    source_rank = {candidate["candidate_id"]: candidate["candidate_source_rank"]
+                   for candidate in row["candidates"]}
+    values = {score.candidate_id: score.ranking_score for score in response.scores}
+    return tuple(sorted(values, key=lambda cid: (-values[cid], source_rank[cid])))
+
+
+def run_public_rerank(candidate: CandidateSpec, dataset_path: str | Path, output_dir: str | Path, *,
+                      split: str = "development", expected_sha256: str = PUBLIC_POOL_SHA256,
+                      release_id: str | None = None) -> LocalRunRef:
+    require(isinstance(candidate, CandidateSpec) and candidate.interface_version == INTERFACE_VERSION,
+            "use a reranking CandidateSpec with sphynx.rag.rerank.v1")
+    pack = _load_selected(dataset_path, split, expected_sha256, release_id)
+    requests = [build_public_rerank_request(pack, row["task_id"]) for row in pack.rows]
+
+    def execute(raw: bytes, request: dict[str, Any]):
+        stdout, stderr, exit_code, failure, latency = _execute_candidate(candidate, raw, 30.0)
+        return stdout, stderr, exit_code, failure, latency, {}
+
+    return capture(pack.identity, {"name": candidate.name, "version": candidate.version,
+                   "interface_version": candidate.interface_version, "argv": list(candidate.argv),
+                   "identity_authority": "caller_declared_not_code_attestation"}, _policy(pack), requests, execute, output_dir)
+
+
+def _compute(run: LocalRunRef, pack: PublicRerankPack) -> dict[str, Any]:
+    requests = [build_public_rerank_request(pack, row["task_id"]) for row in pack.rows]
+    revised = "dataset_id" in pack.identity
+    domain_ids = tuple(pack.identity["domains"])
+    manifest, captures = verified_capture(run, pack.identity, requests, _policy(pack))
+    cases = []
+    for row, request, record in zip(pack.rows, requests, captures):
+        failure = record["failure"]
+        ranking = None
+        if failure is None:
+            try:
+                parsed_request = validate_candidate_request(request)
+                response = validate_candidate_response(record["response"], parsed_request)
+                ranking = _rank_public_response(pack, row, response, parsed_request)
+            except (ValueError, TypeError, KeyError):
+                failure = "invalid_response"
+        grades = {c["candidate_id"]: c["grade"] for c in row["candidates"]}
+        labels_complete = all(grade is not None for grade in grades.values())
+        oracle = any(grade is not None and grade > 0 for grade in grades.values())
+        if revised and not oracle and not labels_complete:
+            oracle = None
+        scores = {key: None for key in ("ndcg_at_5", "hit_at_5", "mrr_at_5")}
+        if failure is None and oracle and labels_complete:
+            relevance = [grades[cid] for cid in ranking[:5]]
+            def dcg(values):
+                return sum((2 ** grade - 1) / math.log2(i + 2) for i, grade in enumerate(values))
+            scores["ndcg_at_5"] = dcg(relevance) / dcg(sorted(grades.values(), reverse=True)[:5])
+            scores["hit_at_5"] = float(any(grade > 0 for grade in relevance))
+            scores["mrr_at_5"] = next((1 / (i + 1) for i, grade in enumerate(relevance) if grade > 0), 0.0)
+        case = {"case_id": row["task_id"], "domain": row["dataset"], "failure": failure,
+                "pool_oracle": oracle, "metrics": scores, "runner_latency_ms": record["runner_latency_ms"]}
+        if revised:
+            case.update(labels_complete=labels_complete, quality_eligible=oracle is True and labels_complete)
+        cases.append(case)
+    eligible = lambda row: row["pool_oracle"] and row.get("labels_complete", True)
+    metrics = {key: metric([row["metrics"][key] for row in cases if eligible(row)])
+               for key in ("ndcg_at_5", "hit_at_5", "mrr_at_5")}
+    for entry in metrics.values():
+        entry["undefined_count"] = sum(row["pool_oracle"] is False and row.get("labels_complete", True) for row in cases)
+        if revised:
+            entry["blocked_reference_count"] = sum(not row["labels_complete"] for row in cases)
+    metrics["pool_oracle_coverage"] = metric([None if row["pool_oracle"] is None else float(row["pool_oracle"]) for row in cases])
+    metrics["valid_response_coverage"] = metric([float(row["failure"] is None) for row in cases])
+    metrics["candidate_failure_rate"] = metric([float(row["failure"] is not None) for row in cases])
+    spans = [row["runner_latency_ms"] for row in cases]
+    observed_spans = [value for value in spans if value is not None]
+    metrics["runner_latency_ms"] = {**metric(spans),
+        "value": percentile(observed_spans, .5) if observed_spans else None,
+        "p95": percentile(observed_spans, .95) if observed_spans else None}
+    domains = {}
+    for domain in domain_ids:
+        selected = [row for row in cases if row["domain"] == domain]
+        domains[domain] = {key: metric([row["metrics"][key] for row in selected if eligible(row)])
+                           for key in ("ndcg_at_5", "hit_at_5", "mrr_at_5")}
+        domains[domain]["pool_oracle_coverage"] = metric([None if row["pool_oracle"] is None else float(row["pool_oracle"]) for row in selected])
+    macro = {key: metric([domains[domain][key]["value"] for domain in domain_ids])
+             for key in ("ndcg_at_5", "hit_at_5", "mrr_at_5")}
+    return {"schema_version": "sphynx.public-rerank.scorecard.v1", "pack_id": PUBLIC_RERANK_PACK_ID,
+            "track": "rag.rerank.public", "pack": pack.identity, "candidate": manifest["candidate"],
+            "execution_policy": _policy(pack), "runtime": manifest["runtime"], "trust_tier": "local_unverified", "development_only": True,
+            "label_status": "public_qrels", "metric_semantics": (
+                "Native qrels >0; conditional on frozen-pool oracle and complete qrels. Declared release domains. Previously opened splits."
+                if revised else "Native qrels >0; conditional on BM25 pool oracle. Previously opened splits."),
+            "metric_release": "rag.rerank.beir.metric.v2" if revised else METRIC_RELEASE,
+            "grader_release": "rag.rerank.beir.grader.v2" if revised else GRADER_RELEASE,
+            "metrics": metrics, "domain_slices": domains, "equal_domain_macro": macro, "cases": cases,
+            "counts": {"attempted_count": len(cases), "success_count": sum(row["failure"] is None for row in cases),
+                       "failure_count": sum(row["failure"] is not None for row in cases)}}
+
+
+def grade_public_rerank(run: LocalRunRef, dataset_path: str | Path, *, split: str = "development",
+                        expected_sha256: str = PUBLIC_POOL_SHA256, release_id: str | None = None) -> LocalEvaluationRef:
+    pack = _load_selected(dataset_path, split, expected_sha256, release_id)
+    return save_evaluation(run, _compute(run, pack))
+
+
+def report_public_rerank(evaluation: LocalEvaluationRef, dataset_path: str | Path, *, split: str = "development",
+                         expected_sha256: str = PUBLIC_POOL_SHA256, release_id: str | None = None) -> LocalScorecard:
+    pack = _load_selected(dataset_path, split, expected_sha256, release_id)
+    return report_evaluation(evaluation, lambda annotations: _compute(evaluation.run, pack))
+
+
+def compare_public_rerank(a: LocalEvaluationRef, b: LocalEvaluationRef, dataset_path: str | Path, *,
+                          split: str = "development", expected_sha256: str = PUBLIC_POOL_SHA256,
+                          release_id: str | None = None) -> dict[str, Any]:
+    options = {"split": split, "expected_sha256": expected_sha256, "release_id": release_id}
+    return compare_scorecards(report_public_rerank(a, dataset_path, **options), report_public_rerank(b, dataset_path, **options))
+
+# Available revised pools have separate immutable identities and loader policy.
+
+def _load_selected(dataset_path, split, expected_sha256, release_id):
+    if release_id is None:
+        return load_public_rerank_pack(dataset_path, split=split, expected_sha256=expected_sha256)
+    require(split == "confirmation", "revised releases require split='confirmation'")
+    pack = load_public_pool_release(dataset_path, release_id=release_id)
+    require(expected_sha256 == PUBLIC_POOL_SHA256 or
+            "sha256:" + expected_sha256.removeprefix("sha256:") == pack.identity["dataset_digest"],
+            "explicit public release SHA-256 differs")
+    return pack
+
+
+@dataclass(frozen=True)
+class PublicPoolRelease:
+    release_id: str
+    dataset_id: str
+    source_sha256: str
+    domains: tuple[str, ...]
+    config_sha256: str
+    candidate_identity_sha256: str
+    previous_pool_sha256: str | tuple[tuple[str, str], ...]
+    previous_field: str
+    schema_version: str = "expanded-reranker.fresh-snowflake-pools.v1"
+    queries_per_domain: int = 20
+    candidate_depth: int = 15
+
+
+PUBLIC_POOL_RELEASES = (
+    PublicPoolRelease("fresh-e2", "beir.fresh-e2.confirmation.v1",
+        "1168a3f2079ae192bc54ed681976004ad375882fbc7800b611c5bb50fbf7b8b4", DOMAINS,
+        "809cc4988a9faba34b460576968ebf1c04a2723d7adefbc878c4e20f54351218",
+        "34c1da7ccc31aef04a9a88992334944b3ca07a06e9aa9f35ac84f385a54fe5ce",
+        PUBLIC_POOL_SHA256, "previous_pools_sha256"),
+    PublicPoolRelease("guarded-jina", "beir.guarded-jina.confirmation.v1",
+        "8a519db2b0e62622705c7e05d71aac3c526d4a3dc97308e248839c5558f9ce49",
+        ("scifact", "nfcorpus", "fiqa", "arguana", "scidocs", "webis-touche2020"),
+        "edf1b7a2fd789f7dc11e9d8d264e6868fa88bdfaef694278f687b55cdd397c36",
+        "4aa6c61bf60a4692bc577c7b4372ef8d71de1a08fbf8c7160894bf2035409612",
+        (("expanded-reranker\\data\\frozen-pools.json", PUBLIC_POOL_SHA256),
+         ("expanded-reranker\\fresh-e2\\data\\snowflake-pools.json",
+          "1168a3f2079ae192bc54ed681976004ad375882fbc7800b611c5bb50fbf7b8b4")), "previous_pool_sha256"),
+)
+SNOWFLAKE_FIRST_STAGE = {
+    "model_id": "Snowflake/snowflake-arctic-embed-m-v1.5",
+    "revision": "e58a8f756156a1293d763f17e3aae643474e9b8a",
+    "query_prompt": "query", "similarity": "dot_product",
+    "role": "public Snowflake dense proxy; not a reproduction of the unavailable production Snowflake service",
+}
+
+
+def list_public_pool_releases() -> tuple[PublicPoolRelease, ...]:
+    """List revised content-free identities without opening data or models."""
+    return PUBLIC_POOL_RELEASES
+
+
+def load_public_pool_release(dataset_path: str | Path, *, release_id: str) -> PublicRerankPack:
+    """Load one pinned 120-query release; preserve null/missing qrels as unknown.
+
+    These sources previously defaulted absent native qrels to zero. Retained
+    integer zero is a source judgment policy, not proof of negative review.
+    This loader performs no scoring and does not execute Snowflake or Jina.
+    """
+    release = next((r for r in PUBLIC_POOL_RELEASES if r.release_id == release_id), None)
+    require(release is not None, "unknown revised public release")
+    source = Path(dataset_path).expanduser().resolve(strict=True)
+    source_digest = digest(source.read_bytes())
+    require(source_digest == "sha256:" + release.source_sha256, "public release SHA-256 differs")
+    data = read_json(source)
+    require(isinstance(data, dict), "public release must be a JSON object")
+    require(data.get("schema_version") == release.schema_version
+            and data.get("candidate_depth") == release.candidate_depth
+            and data.get("config_sha256") == release.config_sha256
+            and data.get("candidate_identity_pools_sha256") == release.candidate_identity_sha256
+            and data.get(release.previous_field) == (dict(release.previous_pool_sha256)
+                if isinstance(release.previous_pool_sha256, tuple) else release.previous_pool_sha256)
+            and data.get("first_stage") == SNOWFLAKE_FIRST_STAGE,
+            "public release schema, provenance or first-stage identity differs")
+    rows = data.get("queries")
+    require(isinstance(rows, list) and len(rows) == len(release.domains) * release.queries_per_domain,
+            "revised public release must contain exactly its 120 queries")
+    ids, tasks = set(), set()
+    unknown = 0
+    for row in rows:
+        require(isinstance(row, dict) and row.get("dataset") in release.domains
+                and row.get("split") == "confirmation", "revised public domain or split differs")
+        key = (row["dataset"], text(row.get("query_id"), "query_id"))
+        task = text(row.get("task_id"), "task_id")
+        require(key not in ids and task not in tasks, "duplicate revised public query or task")
+        ids.add(key); tasks.add(task)
+        text(row.get("query"), "query")
+        candidates = row.get("candidates")
+        require(isinstance(candidates, list) and len(candidates) == release.candidate_depth,
+                "revised public candidate depth differs")
+        cids, docs, ranks, source_ranks = set(), set(), set(), set()
+        for index, candidate in enumerate(candidates):
+            require(isinstance(candidate, dict), "invalid revised public candidate")
+            cid = text(candidate.get("candidate_id"), "candidate_id")
+            doc = text(candidate.get("document_id"), "document_id")
+            require(cid not in cids and doc not in docs, "duplicate revised public candidate or document")
+            cids.add(cid); docs.add(doc)
+            text(candidate.get("input"), "candidate.input")
+            for field, seen in (("first_stage_rank", ranks), ("candidate_source_rank", source_ranks)):
+                rank = candidate.get(field)
+                require(type(rank) is int and 1 <= rank <= release.candidate_depth and rank not in seen,
+                        "revised public ranks must be complete permutations")
+                seen.add(rank)
+            require(candidate["candidate_source_rank"] == index + 1, "BM25 source ranks differ from pool order")
+            for field in ("first_stage_score", "candidate_source_score"):
+                score = candidate.get(field)
+                require(type(score) in (int, float) and math.isfinite(score), "invalid revised public score")
+            grade = candidate.get("grade")
+            require(grade is None or (type(grade) is int and 0 <= grade <= 3), "invalid revised public qrel")
+            unknown += int(grade is None)
+        # Keep the source order, dense ranks and unknown grades intact.
+    require(all(sum(row["dataset"] == domain for row in rows) == release.queries_per_domain
+                for domain in release.domains), "revised public domain population differs")
+    pack = PublicRerankPack(source_digest, "confirmation", rows)
+    pack.identity.update(dataset_id=release.dataset_id, pack_release=release.dataset_id,
+        domains=list(release.domains), first_stage="snowflake_arctic_embed_m_v1_5_public_proxy",
+        ranking_tie_policy="frozen_candidate_source_rank",
+        first_stage_identity=dict(SNOWFLAKE_FIRST_STAGE), candidate_source="bm25_top15_frozen_identities",
+        judgment_policy="source_absent_native_qrels_default_zero_explicit_null_unknown",
+        unknown_qrel_count=unknown, reference_digest=source_digest,
+        confirmation_status="previously_opened_exploratory")
+    return pack