sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/sable_rerank.py

@@ -0,0 +1,489 @@
+"""Owner-held Sable reranker development packs.
+
+The dataset path is supplied by the owner at runtime. Neither passages nor
+reference labels are installed in the wheel. Local execution is untrusted:
+the candidate process can read files available to its OS identity, so genuinely
+blind holdouts require a separate platform runner.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass, replace
+from pathlib import Path
+from typing import Any, Mapping
+
+from .contracts import INTERFACE_VERSION, SABLE_RERANK_DEV_PACK_ID, CandidateSpec, ContractError, build_candidate_request, canonical_json
+from .evidence import RunRef, read_capture_artifact, read_capture_manifest, read_run_manifest, verify_run
+from .metrics import METRIC_RELEASE, aggregate_case_metrics, grade_case
+from .packs import ExecutionPolicy, PackDescriptor
+from .rag import RagRerankPack, validate_dataset_v2
+from .runner import ENV_POLICY_ID, MAX_STREAM_BYTES, _case_snapshot, _parse_response, _run_with_cases
+
+SABLE_PACK_ID = SABLE_RERANK_DEV_PACK_ID
+_SCHEMA = "sphynx.sable.rerank.private.v1"
+_ORIGINS = frozenset({"reconstructed_development", "exact_pre_rerank"})
+
+
+@dataclass(frozen=True)
+class PrivatePack:
+    pack: RagRerankPack
+    origin: str
+    label_status: str
+    source_revision: str
+
+    @property
+    def dataset_digest(self) -> str:
+        return self.pack.dataset_digest
+
+
+@dataclass(frozen=True)
+class PrivateEvaluation:
+    run_id: str
+    pack_id: str
+    dataset_digest: str
+    origin: str
+    label_status: str
+    candidate: Mapping[str, str]
+    execution_policy: Mapping[str, Any]
+    runner: Mapping[str, str]
+    status: str
+    cases: tuple[Mapping[str, Any], ...]
+    metrics: Mapping[str, Mapping[str, Any]]
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schema_version": "sphynx.sable.rerank.evaluation.v1",
+            "run_id": self.run_id,
+            "pack_id": self.pack_id,
+            "dataset_digest": self.dataset_digest,
+            "origin": self.origin,
+            "label_status": self.label_status,
+            "trust_tier": "local_unverified",
+            "candidate": dict(self.candidate),
+            "execution_policy": dict(self.execution_policy),
+            "runner": dict(self.runner),
+            "status": self.status,
+            "cases": [dict(case) for case in self.cases],
+            "metrics": {key: dict(value) for key, value in self.metrics.items()},
+        }
+
+    def to_json(self) -> str:
+        return canonical_json(self.to_dict())
+
+
+def load_private_pack(dataset_path: str | Path, *, expected_sha256: str | None = None) -> PrivatePack:
+    """Validate an external dataset and bind its complete bytes to a release.
+
+    ``expected_sha256`` is useful when loading a separately approved release;
+    a changed label or passage byte then fails before candidate execution.
+    """
+
+    path = Path(dataset_path).expanduser().resolve(strict=True)
+    payload = path.read_bytes()
+    digest = hashlib.sha256(payload).hexdigest()
+    if expected_sha256 is not None and expected_sha256.removeprefix("sha256:") != digest:
+        raise ContractError("private pack bytes differ from expected SHA-256")
+    try:
+        value = json.loads(payload)
+    except (UnicodeError, json.JSONDecodeError) as exc:
+        raise ContractError("private pack must be UTF-8 JSON") from exc
+    if not isinstance(value, dict):
+        raise ContractError("private pack must be a JSON object")
+    meta = value.get("sphynx_private_pack")
+    if not isinstance(meta, dict) or meta.get("schema_version") != _SCHEMA:
+        raise ContractError("private pack metadata/schema is missing")
+    if meta.get("pack_id") != SABLE_PACK_ID or meta.get("visibility") != "private":
+        raise ContractError("private Sable pack identity or visibility is invalid")
+    origin = meta.get("origin")
+    label_status = meta.get("label_status")
+    if origin not in _ORIGINS or label_status not in {"provisional", "human_adjudicated"}:
+        raise ContractError("private pack origin or label status is invalid")
+    if origin == "reconstructed_development" and label_status != "provisional":
+        raise ContractError("reconstructed pools cannot claim adjudicated benchmark labels")
+    if label_status == "human_adjudicated" and meta.get("privacy_review") != "approved":
+        raise ContractError("adjudicated private pack requires approved privacy review")
+    source_revision = meta.get("source_revision")
+    if not isinstance(source_revision, str) or not source_revision.strip():
+        raise ContractError("private pack needs source_revision")
+    cases, provisional_refs, threshold = validate_dataset_v2(value)
+    for case, reference in zip(cases, provisional_refs):
+        has_required_group = any(group.required for group in reference.evidence_groups)
+        has_relevant_passage = any(item.relevance_grade >= threshold for item in reference.candidates)
+        if has_required_group and not has_relevant_passage:
+            raise ContractError(f"{case.case_id}: answerable reranker pool has no relevant passage; track retrieval coverage separately")
+        if not has_required_group and has_relevant_passage:
+            raise ContractError(f"{case.case_id}: unsupported reranker pool contains a relevant passage")
+    if label_status == "human_adjudicated":
+        for task in value["tasks"]:
+            if not all(isinstance(item.get("adjudication"), dict) for item in task["candidates"]):
+                raise ContractError("human_adjudicated status requires every candidate adjudication")
+    references = tuple(replace(ref, label_authority=label_status) for ref in provisional_refs)
+    descriptor = PackDescriptor(
+        pack_id=SABLE_PACK_ID,
+        pack_release="v1." + digest,
+        title="Owner-held Sable passage reranking development pack",
+        track="rag_rerank",
+        interface_version=INTERFACE_VERSION,
+        case_schema_version=value["schema_version"],
+        request_schema_version="sphynx.rag.rerank.request.v1",
+        response_schema_version="sphynx.rag.rerank.response.v1",
+        status="runnable",
+        description="Owner-held Sable data; local runs are development results only.",
+        execution_policy=ExecutionPolicy("v1", 1, 30.0, "local-unbounded.v1"),
+        case_count=len(cases),
+        metric_release=METRIC_RELEASE,
+    )
+    pack = RagRerankPack(descriptor, cases, references, METRIC_RELEASE, "sha256:" + digest, threshold)
+    return PrivatePack(pack, origin, label_status, source_revision)
+
+
+def run_private_rerank(
+    candidate: CandidateSpec, dataset_path: str | Path, output_dir: str | Path,
+    *, expected_sha256: str | None = None,
+) -> RunRef:
+    """Run a candidate on the complete frozen pool without sending labels."""
+
+    private = load_private_pack(dataset_path, expected_sha256=expected_sha256)
+    return _run_with_cases(candidate, private.pack.descriptor, private.pack.cases, output_dir)
+
+
+def grade_private_rerank(
+    run_ref: RunRef | str | Path, dataset_path: str | Path,
+    *, expected_sha256: str | None = None,
+) -> PrivateEvaluation:
+    """Replay verified raw captures against a matching owner-held dataset."""
+
+    private = load_private_pack(dataset_path, expected_sha256=expected_sha256)
+    pack = private.pack
+    run_path = (run_ref.path if isinstance(run_ref, RunRef) else Path(run_ref)).expanduser().resolve()
+    manifest = read_run_manifest(run_path)
+    ref = RunRef(manifest["run_id"], run_path, manifest["pack_id"])
+    verify_run(ref)
+    if manifest.get("pack_id") != SABLE_PACK_ID or manifest.get("pack_release") != pack.descriptor.pack_release:
+        raise ContractError("run and private pack release differ")
+    if manifest.get("case_count") != len(pack.cases) or manifest.get("capture_count") != len(pack.cases):
+        raise ContractError("run does not cover every private pack case exactly once")
+    policy = pack.descriptor.execution_policy
+    expected_policy = {
+        "policy_version": policy.policy_version, "trials": policy.trials,
+        "per_case_timeout_seconds": policy.per_case_timeout_seconds,
+        "resource_policy_id": policy.resource_policy_id,
+        "environment_policy_id": ENV_POLICY_ID,
+        "stdout_stderr_retention_limit_bytes_each": MAX_STREAM_BYTES,
+    }
+    if manifest.get("execution_policy") != expected_policy:
+        raise ContractError("run execution policy differs from private pack")
+    candidate_identity = manifest.get("candidate")
+    if not isinstance(candidate_identity, dict) or candidate_identity.get("interface_version") != INTERFACE_VERSION:
+        raise ContractError("run candidate interface differs from private pack")
+    runner_identity = manifest.get("runner")
+    if not isinstance(runner_identity, dict) or runner_identity.get("release") != "v1" or runner_identity.get("clock_domain") != "python.monotonic_ns.process_local":
+        raise ContractError("run runner identity is unsupported")
+    expected_cases = [
+        {
+            "dataset_id": case.dataset_id, "dataset_version": case.dataset_version,
+            "case_id": case.case_id, "query_id": case.query_id, "pool_digest": case.pool_digest,
+            "split": case.split, "split_group_id": case.split_group_id, "cluster_id": case.cluster_id,
+            "suite_type": case.suite_type, "versions": case.versions.to_dict(),
+        }
+        for case in pack.cases
+    ]
+    if manifest.get("cases") != expected_cases or manifest.get("track") != "rag_rerank" or manifest.get("trust_tier") != "local_unverified" or manifest.get("provenance") != "runner_observed":
+        raise ContractError("run pool, cohort, order, or trust identity differs from private pack")
+    ids = manifest.get("capture_ids")
+    if not isinstance(ids, list) or len(ids) != len(pack.cases):
+        raise ContractError("run capture order is invalid")
+    results = []
+    case_rows: list[Mapping[str, Any]] = []
+    for case, capture_id in zip(pack.cases, ids):
+        capture = read_capture_manifest(ref, capture_id)
+        identity = capture.get("identity", {})
+        if not isinstance(identity, dict) or any((
+            identity.get("case_id") != case.case_id,
+            identity.get("pool_digest") != case.pool_digest,
+            identity.get("trial_index") != 1,
+            identity.get("run_id") != ref.run_id,
+        )):
+            raise ContractError(f"capture {capture_id} does not match private case")
+        request = build_candidate_request(case, SABLE_PACK_ID)
+        if read_capture_artifact(ref, capture_id, "candidate_request") != (request.to_json() + "\n").encode("utf-8"):
+            raise ContractError(f"candidate request differs from private case {case.case_id}")
+        if read_capture_artifact(ref, capture_id, "case_snapshot") != (canonical_json(_case_snapshot(case)) + "\n").encode("utf-8"):
+            raise ContractError(f"case snapshot differs from private case {case.case_id}")
+        status = capture.get("status")
+        ranked = None
+        if status == "success":
+            ranked, error_code, _ = _parse_response(read_capture_artifact(ref, capture_id, "candidate_stdout"), request)
+            if error_code or ranked is None or capture.get("ranked_candidate_ids") != list(ranked):
+                raise ContractError(f"saved ranking differs from raw response for {case.case_id}")
+        elif capture.get("ranked_candidate_ids") is not None:
+            raise ContractError(f"failed capture has a ranking for {case.case_id}")
+        timing = capture.get("timing")
+        latency_ns = timing.get("candidate_latency_ns") if isinstance(timing, dict) else None
+        if latency_ns is not None and (type(latency_ns) is not int or latency_ns < 0):
+            raise ContractError(f"invalid timing for {case.case_id}")
+        result = grade_case(
+            case, pack.reference_for(case.case_id), ranked,
+            latency_ns / 1_000_000 if latency_ns is not None else None,
+            terminal_status="ok" if status == "success" else str(status),
+            relevance_threshold=pack.relevance_threshold, pack=pack,
+        )
+        results.append(result)
+        case_rows.append({
+            "case_id": case.case_id, "pool_digest": case.pool_digest,
+            "status": status, "values": dict(result.values),
+            "undefined_reasons": dict(result.undefined_reasons),
+        })
+    metrics = {name: value.to_dict() for name, value in aggregate_case_metrics(results, pack=pack).items()}
+    return PrivateEvaluation(
+        ref.run_id, SABLE_PACK_ID, private.dataset_digest, private.origin,
+        private.label_status, manifest["candidate"], expected_policy, runner_identity,
+        "completed" if all(row["status"] == "success" for row in case_rows) else "incomplete",
+        tuple(case_rows), metrics,
+    )
+
+
+def compare_private_rerank(a: PrivateEvaluation, b: PrivateEvaluation) -> dict[str, Any]:
+    """Compare two fully matched frozen-pool runs with paired case deltas."""
+
+    if a.pack_id != b.pack_id or a.dataset_digest != b.dataset_digest:
+        raise ContractError("private reranker comparisons require the exact same pack bytes")
+    if [row["pool_digest"] for row in a.cases] != [row["pool_digest"] for row in b.cases]:
+        raise ContractError("private reranker comparisons require identical ordered pools")
+    if a.execution_policy != b.execution_policy or a.runner != b.runner:
+        raise ContractError("private reranker comparisons require matching runner and execution policy")
+    metrics: dict[str, Any] = {}
+    for metric_id in a.metrics:
+        av, bv = a.metrics[metric_id], b.metrics[metric_id]
+        pairs = [
+            {"case_id": left["case_id"], "a": left["values"][metric_id], "b": right["values"][metric_id],
+             "delta_b_minus_a": right["values"][metric_id] - left["values"][metric_id]}
+            for left, right in zip(a.cases, b.cases)
+            if left["values"][metric_id] is not None and right["values"][metric_id] is not None
+        ]
+        metrics[metric_id] = {
+            "a": av["value"], "b": bv["value"],
+            "delta_b_minus_a": bv["value"] - av["value"] if av["value"] is not None and bv["value"] is not None else None,
+            "paired_cases": pairs,
+        }
+    return {
+        "pack_id": a.pack_id, "dataset_digest": a.dataset_digest,
+        "origin": a.origin, "trust_tier": "local_unverified", "metrics": metrics,
+    }
+
+
+def build_cherry_development_dataset(pool_path: str | Path, labels_path: str | Path) -> dict[str, Any]:
+    """Adapt the existing reconstructed Cherry study without bundling its data.
+
+    The result is a development dataset only. The study did not capture the
+    actual production Pinecone pool, contextualized passages, or rewritten
+    query, and its labels have not been Sphynx-adjudicated.
+    """
+
+    pool_bytes = Path(pool_path).read_bytes()
+    label_bytes = Path(labels_path).read_bytes()
+    pool = json.loads(pool_bytes)
+    labels = json.loads(label_bytes)
+    if pool.get("schema_version") != "cherry-pre-rerank-dense-pools.v1":
+        raise ContractError("unsupported Cherry frozen-pool source schema")
+    if pool.get("status") != "frozen_before_reranker_scoring":
+        raise ContractError("Cherry pool was not frozen before reranker scoring")
+    if labels.get("schema_version") != "cherry-reranker-labels.v1":
+        raise ContractError("unsupported Cherry label source schema")
+    frozen_body = {key: value for key, value in pool.items() if key not in {"runtime", "pool_sha256"}}
+    expected_pool_sha = hashlib.sha256(canonical_json(frozen_body).encode("utf-8")).hexdigest()
+    if pool.get("pool_sha256") != expected_pool_sha or labels.get("pool_sha256") != expected_pool_sha:
+        raise ContractError("Cherry pool and labels do not share the frozen source digest")
+    queries = pool.get("queries")
+    label_queries = labels.get("queries")
+    if not isinstance(queries, list) or not isinstance(label_queries, dict) or len(queries) != len(label_queries):
+        raise ContractError("Cherry questions and labels disagree")
+    source_sha = hashlib.sha256(pool_bytes).hexdigest()
+    label_sha = hashlib.sha256(label_bytes).hexdigest()
+    tasks: list[dict[str, Any]] = []
+    for index, item in enumerate(queries):
+        case_id = item["task_id"]
+        if case_id not in label_queries or item.get("candidate_count") != len(item.get("candidates", [])):
+            raise ContractError(f"Cherry pool or labels missing for {case_id}")
+        qrels = label_queries[case_id].get("relevance")
+        if not isinstance(qrels, dict):
+            raise ContractError(f"Cherry qrels missing for {case_id}")
+        candidates_raw = item["candidates"]
+        candidate_ids = {candidate["candidate_id"] for candidate in candidates_raw}
+        if len(candidate_ids) != len(candidates_raw) or not set(qrels) <= candidate_ids:
+            raise ContractError(f"Cherry qrels do not match frozen pool for {case_id}")
+        answerable = any(int(value.get("grade", 0)) >= 2 for value in qrels.values())
+        group_id = f"evidence:{index:02d}"
+        acceptable = [candidate["candidate_id"] for candidate in candidates_raw
+                      if int(qrels.get(candidate["candidate_id"], {}).get("grade", 0)) >= 2]
+        candidates = []
+        for rank, candidate in enumerate(candidates_raw, start=1):
+            label = qrels.get(candidate["candidate_id"], {})
+            source_id = str(candidate["source_id"])
+            chunk_index = int(candidate["chunk_index"])
+            candidates.append({
+                "candidate_id": candidate["candidate_id"],
+                "document_id": "source:" + source_id,
+                "source_id": source_id,
+                "chunk_id": f"{source_id}:chunk:{chunk_index}",
+                "url": f"urn:sable:cherry:{source_id}:{chunk_index}",
+                "input": candidate["input"],
+                "dense_rank": rank,
+                "dense_score": candidate["first_stage_score"],
+                "relevance_grade": int(label.get("grade", 0)),
+                "evidence_group_ids": [group_id] if candidate["candidate_id"] in acceptable else [],
+                "known_hard_negative": bool(label.get("hard_negative", False)),
+            })
+        task = {
+            "task_id": case_id, "query_id": case_id, "cluster_id": case_id,
+            "split_group_id": case_id, "domain": "cherry", "query": item["query"],
+            "answerability": "answerable" if answerable else "unsupported",
+            "business_severity": "medium",
+            "query_categories": ["straightforward_factual" if answerable else "unsupported_not_in_kb"],
+            "customer": "cherry", "split": "dev", "suite_type": "customer_acceptance",
+            "versions": {
+                "corpus": "cherry-reconstruction:" + str(pool["source_corpus_sha256"]),
+                "index": "snowflake-local:" + source_sha,
+                "chunking": "public-api-chunks",
+                "embedding": "snowflake-arctic-m-v2:" + str(pool["first_stage"]["revision"]),
+            },
+            "candidates": candidates,
+            "evidence_groups": [{
+                "evidence_group_id": group_id,
+                "description": "Answer-bearing passages with Cherry study relevance grade at least two",
+                "required": True,
+                "acceptable_candidate_ids": acceptable,
+            }] if answerable else [],
+        }
+        if not answerable:
+            task["expected_unsupported_behavior"] = "abstain"
+        tasks.append(task)
+    dataset = {
+        "schema_version": "rag-evals.dataset.v2",
+        "dataset_id": "sable.cherry.reconstructed.v1",
+        "version": "v1",
+        "default_relevance_threshold": 2,
+        "sphynx_private_pack": {
+            "schema_version": _SCHEMA, "pack_id": SABLE_PACK_ID,
+            "visibility": "private", "origin": "reconstructed_development",
+            "label_status": "provisional",
+            "source_revision": str(pool["first_stage"]["fast_search_source_revision"]),
+            "source_pool_sha256": source_sha, "source_label_sha256": label_sha,
+        },
+        "tasks": tasks,
+    }
+    validate_dataset_v2(dataset)
+    return dataset
+
+
+def build_exact_sable_dataset(
+    records: list[Mapping[str, Any]], *, dataset_id: str, dataset_version: str,
+    source_revision: str, label_status: str = "provisional",
+    privacy_review: str = "pending",
+) -> dict[str, Any]:
+    """Build a private pack from reviewed Fast Search pre-rerank exports.
+
+    Every record must provide ``case_id``, ``snapshot`` (the opt-in capture),
+    ``versions`` (corpus/index/chunking/embedding), ``answerability``, and a
+    complete ``labels`` mapping keyed by vector ID. Label groups and cohort
+    fields are owner-supplied; no relevance is inferred from model output.
+    """
+
+    if not records or label_status not in {"provisional", "human_adjudicated"}:
+        raise ContractError("exact Sable dataset needs records and valid label status")
+    if label_status == "human_adjudicated" and privacy_review != "approved":
+        raise ContractError("adjudicated dataset requires approved privacy review")
+    tasks = []
+    snapshots = []
+    for record in records:
+        if not isinstance(record, Mapping):
+            raise ContractError("each exact Sable case must be an object")
+        case_id = record.get("case_id")
+        snapshot = record.get("snapshot")
+        labels = record.get("labels")
+        versions = record.get("versions")
+        if not isinstance(snapshot, Mapping) or snapshot.get("schema_version") != "sphynx.fast-search.pre-rerank.v1":
+            raise ContractError(f"{case_id}: invalid Fast Search capture schema")
+        if snapshot.get("boundary") != "after_pinecone_before_reranker":
+            raise ContractError(f"{case_id}: capture is not at the exact pre-rerank boundary")
+        pool = snapshot.get("candidates")
+        if not isinstance(pool, list) or not pool or snapshot.get("candidate_count") != len(pool):
+            raise ContractError(f"{case_id}: capture has no complete candidate pool")
+        if not isinstance(labels, Mapping) or set(labels) != {item.get("candidate_id") for item in pool}:
+            raise ContractError(f"{case_id}: every captured vector needs exactly one label")
+        if not isinstance(versions, Mapping) or set(versions) != {"corpus", "index", "chunking", "embedding"}:
+            raise ContractError(f"{case_id}: explicit corpus/index/chunking/embedding versions required")
+        candidates = []
+        for rank, item in enumerate(pool, 1):
+            if not isinstance(item, Mapping) or item.get("dense_rank") != rank:
+                raise ContractError(f"{case_id}: captured dense rank/order is invalid")
+            candidate_id = item["candidate_id"]
+            label = labels[candidate_id]
+            if not isinstance(label, Mapping) or "relevance_grade" not in label or "evidence_group_ids" not in label:
+                raise ContractError(f"{case_id}: incomplete label for {candidate_id}")
+            candidate = {
+                "candidate_id": candidate_id,
+                "document_id": item["document_id"],
+                "source_id": item["source_id"],
+                "chunk_id": candidate_id,
+                "url": item.get("url") or f"urn:fast-search:{candidate_id}",
+                "input": item["reranker_input"],
+                "dense_rank": rank,
+                "dense_score": item["dense_score"],
+                "relevance_grade": label["relevance_grade"],
+                "evidence_group_ids": label["evidence_group_ids"],
+                "known_hard_negative": bool(label.get("known_hard_negative", False)),
+                "raw_text": item.get("raw_text"),
+                "source_url": item.get("url"),
+                "title": item.get("title"),
+                "section_path": item.get("section_path"),
+                "pipeline_id": item.get("pipeline_id"),
+            }
+            if "adjudication" in label:
+                candidate["adjudication"] = label["adjudication"]
+            candidates.append(candidate)
+        split = record.get("split", "dev")
+        task = {
+            "task_id": case_id, "query_id": case_id,
+            "cluster_id": record.get("cluster_id", case_id),
+            "split_group_id": record.get("split_group_id", case_id),
+            "domain": record.get("domain", "sable"),
+            "query": snapshot["query"],
+            "answerability": record.get("answerability"),
+            "business_severity": record.get("business_severity", "medium"),
+            "query_categories": record.get("query_categories", ["straightforward_factual"]),
+            "customer": snapshot["org"],
+            "split": split,
+            "suite_type": record.get("suite_type", "untouched_holdout" if split == "holdout" else "customer_acceptance"),
+            "versions": dict(versions),
+            "candidates": candidates,
+            "evidence_groups": record.get("evidence_groups", []),
+            "capture_scope": {
+                "org": snapshot["org"], "kbs": snapshot["kbs"],
+                "filters": snapshot["filters"],
+                "requested_top_k": snapshot["requested_top_k"],
+                "retrieval_depth": snapshot["retrieval_depth"],
+            },
+        }
+        if task["answerability"] == "unsupported":
+            task["expected_unsupported_behavior"] = "abstain"
+        tasks.append(task)
+        snapshots.append(snapshot)
+    dataset = {
+        "schema_version": "rag-evals.dataset.v2",
+        "dataset_id": dataset_id,
+        "version": dataset_version,
+        "default_relevance_threshold": 2,
+        "sphynx_private_pack": {
+            "schema_version": _SCHEMA, "pack_id": SABLE_PACK_ID,
+            "visibility": "private", "origin": "exact_pre_rerank",
+            "label_status": label_status, "privacy_review": privacy_review,
+            "source_revision": source_revision,
+            "capture_sha256": hashlib.sha256(canonical_json(snapshots).encode("utf-8")).hexdigest(),
+        },
+        "tasks": tasks,
+    }
+    validate_dataset_v2(dataset)
+    return dataset