sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/customer_rerank.py

@@ -0,0 +1,299 @@
+"""Build owner-held development reranker pools from existing customer KBs.
+
+This adapter measures recovery of machine-generated source anchors. It does
+not turn unmatched passages into reviewed semantic negatives. The explicit
+``unjudged_policy`` prevents accidentally treating this as a benchmark qrel set.
+No customer material is bundled with Sphynx and no network calls are made.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+import re
+from collections import Counter
+from pathlib import Path
+from typing import Any, Iterable
+
+from .contracts import ContractError, canonical_json
+from .rag import validate_dataset_v2
+from .sable_rerank import SABLE_PACK_ID, PrivateEvaluation, load_private_pack
+
+LABEL_BASIS = "machine_generated_source_anchor_recovery"
+UNJUDGED_POLICY = "assume_zero_development"
+DEFAULT_CUSTOMERS = ("figma", "monday", "notion", "tipalti", "vanta")
+_TOKENS = re.compile(r"[a-z0-9]+")
+_STOP = frozenset({"a", "an", "and", "are", "can", "do", "how", "i", "in", "is", "of", "the", "to", "what", "about", "should", "customer", "know"})
+
+
+def _sha(payload: bytes) -> str:
+    return hashlib.sha256(payload).hexdigest()
+
+
+def _read_json(path: Path) -> dict[str, Any]:
+    value = json.loads(path.read_text(encoding="utf-8"))
+    if not isinstance(value, dict):
+        raise ContractError(f"{path.name}: expected JSON object")
+    return value
+
+
+def _manifest(path: Path, schema: str | None = None) -> dict[str, Any]:
+    value = _read_json(path)
+    digest = _sha(canonical_json({key: item for key, item in value.items() if key != "manifest_sha256"}).encode("utf-8"))
+    if value.get("manifest_sha256") != digest:
+        raise ContractError(f"{path.name}: source manifest digest mismatch")
+    if schema is not None and value.get("schema_version") != schema:
+        raise ContractError(f"{path.name}: unsupported source manifest schema")
+    return value
+
+
+def _read_jsonl(path: Path) -> list[dict[str, Any]]:
+    rows = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
+    if not rows or any(not isinstance(row, dict) for row in rows):
+        raise ContractError(f"{path.name}: expected nonempty JSONL objects")
+    return rows
+
+
+def _unique(rows: Iterable[dict[str, Any]], field: str) -> dict[str, dict[str, Any]]:
+    result: dict[str, dict[str, Any]] = {}
+    for row in rows:
+        identity = row.get(field)
+        if not isinstance(identity, str) or not identity or identity in result:
+            raise ContractError(f"missing or duplicated {field}")
+        result[identity] = row
+    return result
+
+
+def _tokens(text: str) -> list[str]:
+    return [token for token in _TOKENS.findall(text.lower()) if token not in _STOP]
+
+
+def _rank(query: str, chunks: list[dict[str, Any]]) -> list[tuple[float, dict[str, Any]]]:
+    """Deterministic BM25 with k1=1.2, b=0.75, sorted chunk-ID ties."""
+
+    documents = [Counter(_tokens(chunk["text"])) for chunk in chunks]
+    lengths = [sum(document.values()) for document in documents]
+    average = sum(lengths) / len(lengths) if sum(lengths) else 1.0
+    terms = set(_tokens(query))
+    frequency = {term: sum(term in document for document in documents) for term in terms}
+    scores = []
+    for chunk, document, length in zip(chunks, documents, lengths):
+        score = 0.0
+        for term in terms:
+            tf = document[term]
+            if tf:
+                idf = math.log(1.0 + (len(chunks) - frequency[term] + 0.5) / (frequency[term] + 0.5))
+                score += idf * tf * 2.2 / (tf + 1.2 * (0.25 + 0.75 * length / average))
+        scores.append((score, chunk))
+    return sorted(scores, key=lambda item: (-item[0], item[1]["chunk_id"]))
+
+
+def build_customer_development_dataset(
+    question_bank_root: str | Path,
+    knowledge_base_root: str | Path,
+    *,
+    unjudged_policy: str,
+    customers: Iterable[str] = DEFAULT_CUSTOMERS,
+    candidate_depth: int = 15,
+) -> dict[str, Any]:
+    """Freeze actual local KB pools for every anchored ``call_rag`` question.
+
+    Only development call cases are used; clarification/context controls are
+    excluded by their action. Gold anchors are NEVER injected into retrieval.
+    Cases with no retrieved anchor remain in ``retrieval_coverage.excluded``.
+    An anchor gets grade 3; every other passage gets grade 0 meaning "not this
+    known anchor", NOT "reviewed irrelevant". These are anchor-recovery proxy
+    metrics and must be displayed with that qualification.
+    """
+
+    if unjudged_policy != UNJUDGED_POLICY:
+        raise ContractError("customer development pools require explicit assume_zero_development policy")
+    if type(candidate_depth) is not int or not 5 <= candidate_depth <= 100:
+        raise ContractError("candidate_depth must be an integer from 5 to 100")
+    customer_ids = tuple(customers)
+    if not customer_ids or len(set(customer_ids)) != len(customer_ids) or any(
+        not isinstance(company, str) or not re.fullmatch(r"[a-z0-9-]+", company) for company in customer_ids
+    ):
+        raise ContractError("customers must be unique simple identifiers")
+    bank = Path(question_bank_root).expanduser().resolve(strict=True)
+    kb = Path(knowledge_base_root).expanduser().resolve(strict=True)
+    question_manifest = _manifest(bank / "manifest.json", "sable.invocation-question-corpus.v1")
+    corpus_manifest = _manifest(kb / "manifest.json")
+    if question_manifest.get("knowledge_base_manifest_sha256") != corpus_manifest["manifest_sha256"]:
+        raise ContractError("question bank and source knowledge base are not the same pinned release")
+    if not question_manifest.get("development_only") or question_manifest.get("protected_test_split"):
+        raise ContractError("this builder accepts development question banks only")
+    input_path, gold_path = bank / "model-input.jsonl", bank / "gold.jsonl"
+    inputs = _unique(_read_jsonl(input_path), "question_id")
+    gold = _unique(_read_jsonl(gold_path), "question_id")
+    if set(inputs) != set(gold) or len(inputs) != question_manifest.get("question_count"):
+        raise ContractError("question bank input/gold membership or declared count mismatch")
+    manifest_companies = {row["company_id"]: row for row in corpus_manifest["companies"]}
+    tasks: list[dict[str, Any]] = []
+    sources, coverage = [], {}
+    for company in sorted(customer_ids):
+        if company not in manifest_companies:
+            raise ContractError(f"source KB has no company {company}")
+        company_root = kb / "companies" / company
+        company_manifest = _manifest(company_root / "manifest.json", "sable.company-kb.v1")
+        if company_manifest["manifest_sha256"] != manifest_companies[company]["manifest_sha256"]:
+            raise ContractError(f"{company}: company/global manifest mismatch")
+        chunks_path = (company_root / company_manifest["retrieval_chunks"]).resolve(strict=True)
+        if not chunks_path.is_relative_to(company_root.resolve()):
+            raise ContractError(f"{company}: chunk path leaves company source directory")
+        chunks = _read_jsonl(chunks_path)
+        by_chunk = _unique(chunks, "chunk_id")
+        if len(chunks) != company_manifest.get("chunk_count") or len(chunks) < candidate_depth:
+            raise ContractError(f"{company}: KB chunk count disagrees or is too small")
+        for chunk in chunks:
+            if chunk.get("company_id") != company or not isinstance(chunk.get("text"), str) or not chunk["text"]:
+                raise ContractError(f"{company}: malformed or wrong-customer chunk")
+            if _sha(chunk["text"].encode("utf-8")) != chunk.get("text_sha256"):
+                raise ContractError(f"{company}: chunk text digest mismatch")
+        corpus_sha = _sha(chunks_path.read_bytes())
+        sources.append({
+            "customer": company, "chunk_count": len(chunks), "chunks_sha256": corpus_sha,
+            "manifest_sha256": company_manifest["manifest_sha256"],
+            "source_classes": dict(sorted(Counter(chunk["source_class"] for chunk in chunks).items())),
+        })
+        eligible = retrieved = anchor_count = retrieved_anchor_count = 0
+        excluded = []
+        for identity, label in sorted(gold.items()):
+            row = inputs[identity]
+            if row.get("company_id") != label.get("company_id") or row.get("user_turn") != label.get("user_turn") or row.get("conversation_context") != label.get("conversation_context"):
+                raise ContractError(f"{identity}: input and gold question/context differ")
+            if label["company_id"] != company or label["gold"].get("invocation_action") != "call_rag":
+                continue
+            if label.get("split") != "development":
+                raise ContractError(f"{identity}: selected source case is not development")
+            anchors = set(label["gold"].get("evidence_chunk_ids", []))
+            documents = set(label["gold"].get("evidence_document_ids", []))
+            if not anchors or not anchors <= set(by_chunk) or any(by_chunk[cid]["document_id"] not in documents for cid in anchors):
+                raise ContractError(f"{identity}: source anchor/document is missing or mismatched")
+            eligible += 1
+            anchor_count += len(anchors)
+            pool = _rank(row["user_turn"], chunks)[:candidate_depth]
+            present = anchors & {chunk["chunk_id"] for _, chunk in pool}
+            retrieved_anchor_count += len(present)
+            if not present:
+                excluded.append({"case_id": identity, "reason": "no_source_anchor_in_retrieved_pool", "anchor_count": len(anchors)})
+                continue
+            retrieved += 1
+            group_id = "anchor:" + identity
+            candidates = []
+            for rank, (score, chunk) in enumerate(pool, 1):
+                matched = chunk["chunk_id"] in anchors
+                candidates.append({
+                    "candidate_id": chunk["chunk_id"], "document_id": chunk["document_id"],
+                    "source_id": chunk["document_id"], "chunk_id": chunk["chunk_id"],
+                    "url": chunk["source_uri"], "input": chunk["text"],
+                    "dense_rank": rank, "dense_score": score,
+                    "relevance_grade": 3 if matched else 0,
+                    "evidence_group_ids": [group_id] if matched else [],
+                    "known_hard_negative": False,
+                    "label_basis": "source_anchor_match" if matched else "unjudged_not_known_anchor",
+                    "source_class": chunk["source_class"], "text_sha256": chunk["text_sha256"],
+                })
+            tasks.append({
+                "task_id": identity, "query_id": identity,
+                "cluster_id": label["family_id"], "split_group_id": label["family_id"],
+                "domain": company, "customer": company, "query": row["user_turn"],
+                "answerability": "answerable", "business_severity": "medium",
+                "query_categories": ["straightforward_factual"], "topic": label["topic"],
+                "split": "dev", "suite_type": "customer_acceptance",
+                "versions": {"corpus": "source-kb:" + corpus_sha,
+                             "index": "bm25-reconstructed.v1:" + corpus_sha,
+                             "chunking": company_manifest["parser_version"],
+                             "embedding": "none.lexical-bm25.v1"},
+                "candidates": candidates,
+                "evidence_groups": [{"evidence_group_id": group_id,
+                                     "description": "Recover a machine-generated source anchor; semantic sufficiency is not adjudicated",
+                                     "required": True, "acceptable_candidate_ids": sorted(present)}],
+                "source_anchor_count": len(anchors), "retrieved_source_anchor_count": len(present),
+                "source_question_provenance": label["source_provenance"],
+                "source_review_status": label["review_status"],
+            })
+        if eligible == 0:
+            raise ContractError(f"{company}: no anchored development call questions")
+        coverage[company] = {
+            "eligible_query_count": eligible, "scored_query_count": retrieved,
+            "retrieval_missing_query_count": eligible - retrieved,
+            "anchor_pool_coverage": retrieved / eligible,
+            "source_anchor_count": anchor_count, "retrieved_anchor_count": retrieved_anchor_count,
+            "anchor_recall_at_pool_depth": retrieved_anchor_count / anchor_count,
+            "excluded": excluded,
+        }
+    if not tasks:
+        raise ContractError("no retrieved source anchors available for reranker grading")
+    dataset = {
+        "schema_version": "rag-evals.dataset.v2", "dataset_id": "sable.customer-source-anchor.dev.v1",
+        "version": "v1", "default_relevance_threshold": 2,
+        "sphynx_private_pack": {
+            "schema_version": "sphynx.sable.rerank.private.v1", "pack_id": SABLE_PACK_ID,
+            "visibility": "private", "origin": "reconstructed_development", "label_status": "provisional",
+            "source_revision": "source-kb:" + corpus_manifest["manifest_sha256"],
+            "label_basis": LABEL_BASIS, "unjudged_policy": UNJUDGED_POLICY,
+            "metric_semantics": "source-anchor recovery proxy; unmatched chunks are unjudged, not human negatives",
+            "oracle_anchor_injection": False, "candidate_source": "deterministic local BM25 top-k",
+            "candidate_depth": candidate_depth,
+            "question_manifest_sha256": question_manifest["manifest_sha256"],
+            "source_input_sha256": _sha(input_path.read_bytes()), "source_label_sha256": _sha(gold_path.read_bytes()),
+            "knowledge_base_manifest_sha256": corpus_manifest["manifest_sha256"],
+            "sources": sources, "retrieval_coverage": coverage,
+        },
+        "tasks": tasks,
+    }
+    validate_dataset_v2(dataset)
+    return dataset
+
+
+def summarize_customer_rerank(evaluation: PrivateEvaluation, dataset_path: str | Path) -> dict[str, Any]:
+    """Return qualified aggregate and customer metrics without passage/label text."""
+
+    private = load_private_pack(dataset_path)
+    if evaluation.dataset_digest != private.dataset_digest:
+        raise ContractError("customer summary requires the exact evaluated dataset bytes")
+    dataset = _read_json(Path(dataset_path))
+    meta = dataset["sphynx_private_pack"]
+    if meta.get("label_basis") != LABEL_BASIS:
+        raise ContractError("customer summary requires source-anchor recovery labels")
+    customers = {task["task_id"]: task["customer"] for task in dataset["tasks"]}
+    per_customer = {}
+    for company in sorted(meta["retrieval_coverage"]):
+        rows = [row for row in evaluation.cases if customers.get(row["case_id"]) == company]
+        metrics = {}
+        for metric_id in evaluation.metrics:
+            values = sorted(row["values"][metric_id] for row in rows if row["values"].get(metric_id) is not None)
+            complete = bool(rows) and len(values) == len(rows)
+            if metric_id == "runner_latency_ms":
+                def percentile(fraction: float) -> float | None:
+                    if not complete or not values:
+                        return None
+                    position = (len(values) - 1) * fraction
+                    left = int(position)
+                    return values[left] + (values[min(left + 1, len(values) - 1)] - values[left]) * (position - left)
+                value, p95 = percentile(0.5), percentile(0.95)
+            else:
+                value = sum(values) / len(values) if complete and values else None
+                p95 = None
+            metrics[metric_id] = {"value": value, "p95": p95, "eligible_count": len(rows), "observed_count": len(values), "complete": complete}
+        per_customer[company] = {"case_count": len(rows), "retrieval_coverage": meta["retrieval_coverage"][company], "metrics": metrics}
+    # Excluded-case IDs belong in the owner-held dataset, not this scorecard.
+    for item in per_customer.values():
+        item["retrieval_coverage"] = {key: value for key, value in item["retrieval_coverage"].items() if key != "excluded"}
+    return {
+        "schema_version": "sphynx.customer.rerank.summary.v1", "dataset_digest": private.dataset_digest,
+        "pack_id": evaluation.pack_id,
+        "run_id": evaluation.run_id, "candidate": dict(evaluation.candidate), "status": evaluation.status,
+        "origin": evaluation.origin, "development_only": True,
+        "trust_tier": "local_unverified", "label_status": "provisional", "label_basis": LABEL_BASIS,
+        "metric_semantics": meta["metric_semantics"], "oracle_anchor_injection": False,
+        "candidate_depth": meta["candidate_depth"], "customer_count": len(per_customer),
+        "case_count": len(evaluation.cases), "candidate_count": sum(len(task["candidates"]) for task in dataset["tasks"]),
+        "success_count": sum(row["status"] == "success" for row in evaluation.cases),
+        "failure_count": sum(row["status"] != "success" for row in evaluation.cases),
+        "eligible_query_count": sum(item["eligible_query_count"] for item in meta["retrieval_coverage"].values()),
+        "retrieval_missing_query_count": sum(item["retrieval_missing_query_count"] for item in meta["retrieval_coverage"].values()),
+        "metrics": {key: dict(value) for key, value in evaluation.metrics.items()}, "customers": per_customer,
+    }

sphynx/customer_retrieval.py

@@ -0,0 +1,146 @@
+"""Freeze full owner-held customer corpora for retrieval architecture evals.
+
+Source anchors are provisional recovery proxies, not human relevance labels.
+Unlike the reranker builder, no question is excluded for a retrieval miss and
+no candidate pool is constructed here. Each architecture searches the corpus.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import re
+from collections import Counter
+from pathlib import Path
+from typing import Any, Iterable
+
+from .contracts import ContractError
+from .customer_rerank import (
+    DEFAULT_CUSTOMERS, LABEL_BASIS, UNJUDGED_POLICY, _manifest, _read_jsonl, _unique,
+)
+
+
+def build_customer_retrieval_dataset(
+    question_bank_root: str | Path,
+    knowledge_base_root: str | Path,
+    *,
+    unjudged_policy: str,
+    customers: Iterable[str] = DEFAULT_CUSTOMERS,
+) -> dict[str, Any]:
+    """Return every anchored development CALL_RAG case and its full corpus.
+
+The returned object includes private references. Store it outside Git and do
+not pass it to a candidate. ``build_retrieval_request`` strips references and
+filters corpus chunks to the case's customer. A required group means recover
+any accepted source anchor for this case; semantic answer sufficiency remains
+unreviewed. Neither anchors nor alternative passages are injected into pools.
+"""
+    if unjudged_policy != UNJUDGED_POLICY:
+        raise ContractError("explicit assume_zero_development policy is required")
+    customer_ids = tuple(customers)
+    if (not customer_ids or len(set(customer_ids)) != len(customer_ids)
+            or any(not isinstance(c, str) or not re.fullmatch(r"[a-z0-9-]+", c)
+                   for c in customer_ids)):
+        raise ContractError("customers must be unique simple identifiers")
+    bank = Path(question_bank_root).expanduser().resolve(strict=True)
+    kb = Path(knowledge_base_root).expanduser().resolve(strict=True)
+    question_manifest = _manifest(bank / "manifest.json", "sable.invocation-question-corpus.v1")
+    corpus_manifest = _manifest(kb / "manifest.json")
+    if question_manifest.get("knowledge_base_manifest_sha256") != corpus_manifest["manifest_sha256"]:
+        raise ContractError("question bank and corpus are not the same pinned release")
+    if not question_manifest.get("development_only") or question_manifest.get("protected_test_split"):
+        raise ContractError("only development question banks are accepted")
+    input_path, gold_path = bank / "model-input.jsonl", bank / "gold.jsonl"
+    inputs = _unique(_read_jsonl(input_path), "question_id")
+    gold = _unique(_read_jsonl(gold_path), "question_id")
+    if set(inputs) != set(gold) or len(inputs) != question_manifest.get("question_count"):
+        raise ContractError("question input/gold identities or declared counts differ")
+    for identity, label in gold.items():
+        row = inputs[identity]
+        if any(row.get(key) != label.get(key) for key in ("company_id", "user_turn", "conversation_context")):
+            raise ContractError(f"{identity}: input and gold differ")
+    companies = _unique(corpus_manifest["companies"], "company_id")
+    corpus: list[dict[str, Any]] = []
+    cases: list[dict[str, Any]] = []
+    sources: list[dict[str, Any]] = []
+    counts: dict[str, int] = {}
+    for customer in sorted(customer_ids):
+        if customer not in companies:
+            raise ContractError(f"missing source company {customer}")
+        directory = (kb / "companies" / customer).resolve(strict=True)
+        if not directory.is_relative_to(kb):
+            raise ContractError("customer source leaves knowledge base directory")
+        manifest = _manifest(directory / "manifest.json", "sable.company-kb.v1")
+        if manifest["manifest_sha256"] != companies[customer]["manifest_sha256"]:
+            raise ContractError(f"{customer}: company/global source manifest differs")
+        path = (directory / manifest["retrieval_chunks"]).resolve(strict=True)
+        if not path.is_relative_to(directory):
+            raise ContractError("corpus path leaves customer directory")
+        chunks = _read_jsonl(path)
+        by_id = _unique(chunks, "chunk_id")
+        if len(chunks) != manifest.get("chunk_count"):
+            raise ContractError(f"{customer}: corpus count differs")
+        for chunk in chunks:
+            text = chunk.get("text")
+            if chunk.get("company_id") != customer or not isinstance(text, str) or not text:
+                raise ContractError(f"{customer}: invalid corpus chunk")
+            if hashlib.sha256(text.encode("utf-8")).hexdigest() != chunk.get("text_sha256"):
+                raise ContractError(f"{customer}: corpus text digest differs")
+            corpus.append({
+                "chunk_id": chunk["chunk_id"], "document_id": chunk["document_id"],
+                "customer": customer, "text": text, "text_sha256": chunk["text_sha256"],
+                "source_class": chunk["source_class"], "source_uri": chunk["source_uri"],
+            })
+        sources.append({
+            "customer": customer, "chunk_count": len(chunks),
+            "chunks_sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
+            "manifest_sha256": manifest["manifest_sha256"],
+            "chunking_version": manifest["parser_version"],
+            "source_classes": dict(sorted(Counter(c["source_class"] for c in chunks).items())),
+        })
+        counts[customer] = 0
+        for identity, label in sorted(gold.items()):
+            if label.get("company_id") != customer or label.get("gold", {}).get("invocation_action") != "call_rag":
+                continue
+            if label.get("split") != "development":
+                raise ContractError(f"{identity}: only development cases are accepted")
+            anchors = label["gold"].get("evidence_chunk_ids", [])
+            documents = label["gold"].get("evidence_document_ids", [])
+            if (not isinstance(anchors, list) or not anchors or len(set(anchors)) != len(anchors)
+                    or not set(anchors) <= set(by_id)
+                    or any(by_id[a]["document_id"] not in documents for a in anchors)):
+                raise ContractError(f"{identity}: source anchors are absent or inconsistent")
+            query = inputs[identity]["user_turn"]
+            family = label.get("family_id")
+            if not isinstance(query, str) or not query.strip() or not isinstance(family, str) or not family:
+                raise ContractError(f"{identity}: query or independent family identity is missing")
+            cases.append({
+                "query_id": identity, "query": query, "customer": customer,
+                "cluster_id": family, "conversation_context": inputs[identity].get("conversation_context", []),
+                "references": {
+                    "evidence_groups": [{"evidence_group_id": "anchor:" + identity,
+                                         "required": True, "acceptable_chunk_ids": sorted(anchors)}],
+                    "graded_relevance": {anchor: 3 for anchor in sorted(anchors)},
+                },
+            })
+            counts[customer] += 1
+        if not counts[customer]:
+            raise ContractError(f"{customer}: no anchored retrieval questions")
+    _unique(corpus, "chunk_id")
+    _unique(cases, "query_id")
+    return {
+        "schema_version": "sphynx.rag.retrieve.dataset.v1",
+        "metadata": {
+            "pack_id": "rag.retrieve.v1", "origin": "reconstructed_development",
+            "label_status": "provisional", "label_basis": LABEL_BASIS,
+            "unjudged_policy": UNJUDGED_POLICY, "development_only": True,
+            "metric_semantics": "source-anchor recovery proxy; complete-evidence means complete known anchor groups, not reviewed answer sufficiency",
+            "oracle_anchor_injection": False, "retrieval_population": "all anchored development CALL_RAG questions",
+            "knowledge_base_manifest_sha256": corpus_manifest["manifest_sha256"],
+            "question_manifest_sha256": question_manifest["manifest_sha256"],
+            "source_input_sha256": hashlib.sha256(input_path.read_bytes()).hexdigest(),
+            "source_label_sha256": hashlib.sha256(gold_path.read_bytes()).hexdigest(),
+            "case_count": len(cases), "customer_case_counts": counts, "sources": sources,
+        },
+        "corpus": sorted(corpus, key=lambda c: c["chunk_id"]),
+        "cases": sorted(cases, key=lambda c: (c["customer"], c["query_id"])),
+    }

sphynx/data/benchmark-releases.v1.json

@@ -0,0 +1,164 @@
+{
+  "benchmarks": [
+    {
+      "cases": [
+        {
+          "case_id": "core-onboarding-security",
+          "company": "synthetic",
+          "form_receipt": null,
+          "form_status": "unclassified",
+          "input_digest": "sha256:de96e83e201c629361c57d5831eee6f4e91ce22c9aeea5c1334b6f4cd235ed70",
+          "input_form": "unclassified",
+          "length_words": 8,
+          "split": "dev"
+        },
+        {
+          "case_id": "core-onboarding-security-paraphrase",
+          "company": "synthetic",
+          "form_receipt": null,
+          "form_status": "unclassified",
+          "input_digest": "sha256:7428d18761ac9c9f0df1f0de1a7cf319bc4461009fcb4d4932115e784fddc483",
+          "input_form": "unclassified",
+          "length_words": 7,
+          "split": "dev"
+        },
+        {
+          "case_id": "customer-seat-transfer",
+          "company": "synthetic",
+          "form_receipt": null,
+          "form_status": "unclassified",
+          "input_digest": "sha256:0f2fd15f28a8f15e3c3b4eaf09e8b79bcdf5740ffffdd61ef48d742851fb7de9",
+          "input_form": "unclassified",
+          "length_words": 9,
+          "split": "test"
+        },
+        {
+          "case_id": "production-passage-not-url",
+          "company": "synthetic",
+          "form_receipt": null,
+          "form_status": "unclassified",
+          "input_digest": "sha256:efb18047d05174c26aafc52938e0374b37ff00a142a996154502523ab9179ac0",
+          "input_form": "unclassified",
+          "length_words": 7,
+          "split": "test"
+        },
+        {
+          "case_id": "adversarial-unsupported-sso",
+          "company": "synthetic",
+          "form_receipt": null,
+          "form_status": "unclassified",
+          "input_digest": "sha256:0acddc94d4ae4bd8f35663d863711ee7c1e9d4192333be1d8aae35efbfe5e404",
+          "input_form": "unclassified",
+          "length_words": 7,
+          "split": "test"
+        },
+        {
+          "case_id": "adversarial-duplicate-guides",
+          "company": "synthetic",
+          "form_receipt": null,
+          "form_status": "unclassified",
+          "input_digest": "sha256:81c3019143d778315ff90b630a40af9bf8ffeb65dd51f81c0ab162b1c6201ef0",
+          "input_form": "unclassified",
+          "length_words": 6,
+          "split": "test"
+        }
+      ],
+      "corpus": {
+        "digest": "sha256:fa979226c3ead51d5b153a3343b6ef107161b1bb62947f483b10acfa3d86a50d",
+        "ref": "synthetic.rerank.corpus-projection@2.0.0",
+        "version": "2.0.0"
+      },
+      "evaluation_id": "rag.rerank.v1",
+      "input": {
+        "digest": "sha256:5b78e9060f9dcfd963b91820ae2e93306afad9b69b8f9275dfc77ed2a1a796b7",
+        "ref": "synthetic.rerank.input-projection@2.0.0",
+        "version": "2.0.0"
+      },
+      "intended_use": "illustrative",
+      "interface_version": "sphynx.rag.rerank.v1",
+      "label_set": {
+        "digest": "sha256:951cd8806fe47a0c52cc630aecb044e726af68835776a8f5b136a3a6c57efa68",
+        "ref": "synthetic.rerank.labels@2.0.0",
+        "version": "2.0.0"
+      },
+      "metric_profile": "rag.rerank.metric.v1",
+      "protocol_revision": "rag.rerank.v1",
+      "ref": "synthetic.rerank.fixture@2.0.0",
+      "required_input_form": "any",
+      "required_label_profile": "passage_qrels_and_evidence_groups",
+      "schema_version": "sphynx.benchmark-release.v1",
+      "scope": "component",
+      "version": "2.0.0"
+    }
+  ],
+  "independent_assets_archived": false,
+  "label_sets": [
+    {
+      "authority": "synthetic_fixture",
+      "cases": [
+        {
+          "case_id": "core-onboarding-security",
+          "input_digest": "sha256:de96e83e201c629361c57d5831eee6f4e91ce22c9aeea5c1334b6f4cd235ed70",
+          "status": "judged"
+        },
+        {
+          "case_id": "core-onboarding-security-paraphrase",
+          "input_digest": "sha256:7428d18761ac9c9f0df1f0de1a7cf319bc4461009fcb4d4932115e784fddc483",
+          "status": "judged"
+        },
+        {
+          "case_id": "customer-seat-transfer",
+          "input_digest": "sha256:0f2fd15f28a8f15e3c3b4eaf09e8b79bcdf5740ffffdd61ef48d742851fb7de9",
+          "status": "judged"
+        },
+        {
+          "case_id": "production-passage-not-url",
+          "input_digest": "sha256:efb18047d05174c26aafc52938e0374b37ff00a142a996154502523ab9179ac0",
+          "status": "judged"
+        },
+        {
+          "case_id": "adversarial-unsupported-sso",
+          "input_digest": "sha256:0acddc94d4ae4bd8f35663d863711ee7c1e9d4192333be1d8aae35efbfe5e404",
+          "status": "judged"
+        },
+        {
+          "case_id": "adversarial-duplicate-guides",
+          "input_digest": "sha256:81c3019143d778315ff90b630a40af9bf8ffeb65dd51f81c0ab162b1c6201ef0",
+          "status": "judged"
+        }
+      ],
+      "corpus": {
+        "digest": "sha256:fa979226c3ead51d5b153a3343b6ef107161b1bb62947f483b10acfa3d86a50d",
+        "ref": "synthetic.rerank.corpus-projection@2.0.0",
+        "version": "2.0.0"
+      },
+      "input": {
+        "digest": "sha256:5b78e9060f9dcfd963b91820ae2e93306afad9b69b8f9275dfc77ed2a1a796b7",
+        "ref": "synthetic.rerank.input-projection@2.0.0",
+        "version": "2.0.0"
+      },
+      "label_profile": "passage_qrels_and_evidence_groups",
+      "lineage": [],
+      "missingness": "complete_required",
+      "policy_revision": "rag.rerank.synthetic-label-policy.v1",
+      "ref": "synthetic.rerank.labels@2.0.0",
+      "reference": {
+        "digest": "sha256:eef5634b7bb182c18241f146c5dc22d20b44fd65131c0de7344cd070028b6f24",
+        "ref": "synthetic.rerank.reference-projection@2.0.0",
+        "version": "2.0.0"
+      },
+      "review_receipts": [],
+      "rubric_revision": "rag.rerank.metric.v1",
+      "schema_version": "sphynx.label-set-release.v1",
+      "version": "2.0.0"
+    }
+  ],
+  "metadata_only": true,
+  "projection_version": "sphynx.synthetic-rerank.metadata-projections.v1",
+  "schema_version": "sphynx.benchmark-catalog.v1",
+  "source": {
+    "digest": "sha256:74884a0176738649d85cc8b681323103961939aaa0f3d5f1dd1565ed2e53d86b",
+    "ref": "help-center-evidence-mini@2.0.0",
+    "version": "2.0.0"
+  }
+}