sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/official_benchmarks.py

@@ -0,0 +1,385 @@
+"""Explicit official-source intake, normalized once rather than repeated per case.
+
+Downloads are opt-in. Importing this module never downloads data or runs a model.
+Benchmark labels are retained as dataset annotations, never semantic judge truth.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+import copy
+from datetime import datetime, timezone
+import hashlib
+import io
+import json
+from pathlib import Path, PurePosixPath
+import ssl
+import urllib.request
+import urllib.error
+from zipfile import ZipFile
+
+from ._rag_diagnostics_contracts import ContractError
+from ._rag_diagnostics_importers import _json
+
+MAX_SOURCE_BYTES = 100_000_000
+MAX_UNPACKED_BYTES = 200_000_000
+REGISTRY = {
+    'beir.scifact.test': {
+        'name': 'SciFact, BEIR retrieval test split',
+        'source_url': 'https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/scifact.zip',
+        'published_md5': '5f7d1de60b170fc8027bb7898e2efca1',
+        'published_sha256': None,
+        'format': 'beir-zip', 'split': 'test',
+        'upstream': 'https://github.com/beir-cellar/beir/wiki/Datasets-available',
+        'license_source': 'https://github.com/allenai/scifact/blob/master/LICENSE.md',
+        'licenses': ['CC-BY-4.0 (claims/evidence annotations)', 'ODC-By-1.0 (abstract corpus)'],
+        'scope': 'BEIR retrieval split; not the hidden original SciFact verification test set',
+    },
+    'hotpotqa.distractor.dev': {
+        'name': 'HotpotQA distractor development set',
+        'source_url': 'https://curtis.ml.cmu.edu/datasets/hotpot/hotpot_dev_distractor_v1.json',
+        'published_md5': None, 'format': 'hotpot-json', 'split': 'dev',
+        'published_sha256': '4e9ecb5c8d3b719f624d66b60f8d56bf227f03914f5f0753d6fa1b359d7104ea',
+        'checksum_source': 'https://huggingface.co/datasets/hotpotqa/hotpot_qa/raw/bf65d2739d97a2681f26ef950b2fc400575dbba3/dataset_infos.json',
+        'fallback_url': 'https://huggingface.co/datasets/namlh2004/hotpotqa/resolve/7e54db4656209750ff487f6fdf8e39a66dba136b/hotpot_dev_distractor_v1.json',
+        'fallback_sha256': 'e3da074df24e8369009918aa5cdbdd254dadcde4c63f7569d36afd6f2268caa8',
+        'upstream': 'https://hotpotqa.github.io/',
+        'license_source': 'https://hotpotqa.github.io/',
+        'licenses': ['CC-BY-SA-4.0'],
+        'scope': 'Per-question supplied distractor context; not full-Wikipedia open retrieval',
+    },
+}
+
+
+def _need(condition, message):
+    if not condition:
+        raise ContractError(message)
+
+
+def _sha(raw):
+    return hashlib.sha256(raw).hexdigest()
+
+
+def _raw(path):
+    path = Path(path)
+    _need(path.is_file() and not path.is_symlink() and 0 < path.stat().st_size <= MAX_SOURCE_BYTES,
+          'Expected a regular source file no larger than 100MB')
+    raw = path.read_bytes()
+    _need(0 < len(raw) <= MAX_SOURCE_BYTES, 'Empty or oversized benchmark source')
+    return raw
+
+
+def _id(value):
+    _need(type(value) is str and 0 < len(value) <= 180 and not any(c.isspace() for c in value), 'Invalid benchmark ID')
+    return value
+
+
+def list_official_benchmarks() -> list[dict]:
+    """Describe the two supported upstream formats without accessing any files."""
+    return [dict(dataset_id=key, **copy.deepcopy(value)) for key, value in REGISTRY.items()]
+
+
+def fetch_official_benchmark_source(dataset_id: str, destination: str | Path) -> dict:
+    """Download one allowlisted public source to a new file, with bounded I/O.
+
+    The returned SHA256 freezes the received bytes. A published MD5, when present,
+    is an upstream compatibility check, not a cryptographic signature.
+    """
+    _need(dataset_id in REGISTRY, 'Unsupported official dataset')
+    info = REGISTRY[dataset_id]
+    destination = Path(destination)
+    _need(not destination.exists(), 'Source destination already exists')
+    url = info['source_url']
+    request = urllib.request.Request(url, headers={'User-Agent': 'Sphynx-benchmark-intake/1'})
+    fallback = False
+    try:
+        with urllib.request.urlopen(request, timeout=15, context=ssl.create_default_context()) as response:
+            _need(response.url == url, 'Unexpected upstream redirect; review the source URL')
+            raw = response.read(MAX_SOURCE_BYTES + 1)
+    except (urllib.error.URLError, TimeoutError):
+        _need('fallback_url' in info, 'Official source unavailable and no pinned fallback exists')
+        fallback = True
+        with urllib.request.urlopen(info['fallback_url'], timeout=30, context=ssl.create_default_context()) as response:
+            mirror = response.read(MAX_SOURCE_BYTES + 1)
+        _need(len(mirror) <= MAX_SOURCE_BYTES and _sha(mirror) == info['fallback_sha256'], 'Pinned mirror bytes differ')
+        # The mirror changed JSON whitespace only. Reconstruct the upstream
+        # ujson serialization, and require the independently recorded ORIGINAL
+        # byte hash. This never accepts semantic similarity in place of identity.
+        raw = json.dumps(_json(mirror), ensure_ascii=True, allow_nan=False,
+                         separators=(',', ':')).replace('/', '\\/').encode('utf-8')
+    _need(0 < len(raw) <= MAX_SOURCE_BYTES, 'Empty or oversized official download')
+    md5 = hashlib.md5(raw, usedforsecurity=False).hexdigest()
+    _need(info['published_md5'] is None or md5 == info['published_md5'], 'Published upstream checksum differs')
+    _need(info['published_sha256'] is None or _sha(raw) == info['published_sha256'], 'Published upstream SHA256 differs')
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    with destination.open('xb') as handle:
+        handle.write(raw)
+    return {'dataset_id': dataset_id, 'source_url': url, 'path': str(destination.resolve()),
+            'sha256': _sha(raw), 'bytes': len(raw), 'md5': md5,
+            'upstream_md5_matched': info['published_md5'] is not None,
+            'upstream_sha256_matched': info['published_sha256'] is not None,
+            'fallback_used': fallback,
+            'transfer_url': info['fallback_url'] if fallback else url,
+            'fallback_sha256': info.get('fallback_sha256') if fallback else None,
+            'reconstruction': 'compact ASCII JSON with escaped forward slashes; exact original SHA256 matched' if fallback else None}
+
+
+def _jsonl(raw):
+    rows = []
+    for line in raw.splitlines():
+        if line.strip():
+            row = _json(line)
+            _need(type(row) is dict, 'Expected JSONL objects')
+            rows.append(row)
+    _need(rows, 'Expected nonempty JSONL input')
+    return rows
+
+
+def _beir(raw):
+    with ZipFile(io.BytesIO(raw)) as archive:
+        entries = archive.infolist()
+        _need(len(entries) <= 1000 and len({i.filename for i in entries}) == len(entries), 'Duplicate or excessive archive members')
+        _need(sum(i.file_size for i in entries) <= MAX_UNPACKED_BYTES, 'Archive exceeds uncompressed limit')
+        for item in entries:
+            path = PurePosixPath(item.filename)
+            _need(not path.is_absolute() and '..' not in path.parts and '\\' not in item.filename
+                  and ':' not in item.filename and (item.external_attr >> 16) & 0o170000 != 0o120000,
+                  'Unsafe benchmark archive member')
+        def member(name):
+            info = archive.getinfo('scifact/' + name)
+            _need(info.file_size <= MAX_SOURCE_BYTES, 'Oversized archive member')
+            raw = archive.read(info)
+            _need(len(raw) <= MAX_SOURCE_BYTES, 'Oversized archive member')
+            return raw
+        corpus, queries = _jsonl(member('corpus.jsonl')), _jsonl(member('queries.jsonl'))
+        text = member('qrels/test.tsv').decode('utf-8')
+    corp = {}
+    for row in corpus:
+        did = _id(row.get('_id'))
+        _need(did not in corp and type(row.get('text')) is str and type(row.get('title', '')) is str, 'Invalid corpus row')
+        corp[did] = row
+    all_queries = {}
+    for row in queries:
+        qid = _id(row.get('_id'))
+        _need(qid not in all_queries and type(row.get('text')) is str and row['text'].strip(), 'Invalid query row')
+        all_queries[qid] = row
+    qrels, pairs = [], set()
+    lines = [line for line in text.splitlines() if line.strip()]
+    _need(lines and lines[0].split('\t') == ['query-id', 'corpus-id', 'score'], 'Expected BEIR qrel header')
+    for line in lines[1:]:
+        parts = line.split('\t')
+        _need(len(parts) == 3 and parts[2].isdigit(), 'Invalid qrel row')
+        qid, did, score = parts[0], parts[1], int(parts[2])
+        _need(qid in all_queries and did in corp and (qid, did) not in pairs and 0 <= score <= 100, 'Invalid qrel population')
+        pairs.add((qid, did))
+        qrels.append((qid, did, score))
+    selected = {qid for qid, _, _ in qrels}
+    _need(selected, 'Empty evaluation qrel split')
+    return list(corp.values()), [row for qid, row in all_queries.items() if qid in selected], qrels, []
+
+
+def _hotpot(raw):
+    data = _json(raw)
+    _need(type(data) is list and 0 < len(data) <= 100000, 'Expected HotpotQA records')
+    corpus, queries, qrels, annotations, ids = {}, [], [], [], set()
+    for row in data:
+        _need(type(row) is dict, 'Invalid HotpotQA case')
+        qid = _id(row.get('_id'))
+        _need(qid not in ids and type(row.get('question')) is str and row['question'].strip()
+              and type(row.get('answer')) is str, 'Invalid or duplicate HotpotQA question')
+        ids.add(qid)
+        context = row.get('context')
+        _need(type(context) is list and 0 < len(context) <= 100, 'Invalid distractor context')
+        by_title, context_ids = {}, []
+        for item in context:
+            _need(type(item) is list and len(item) == 2 and type(item[0]) is str
+                  and type(item[1]) is list and all(type(s) is str for s in item[1]), 'Invalid context sentences')
+            title, sentences = item
+            _need(title not in by_title, 'Duplicate context title')
+            did = 'hp-' + _sha(json.dumps(item, ensure_ascii=False, separators=(',', ':')).encode())
+            corpus[did] = {'_id': did, 'title': title, 'text': ''.join(sentences), 'sentences': sentences}
+            by_title[title] = (did, len(sentences))
+            context_ids.append(did)
+        supporting = row.get('supporting_facts')
+        _need(type(supporting) is list and supporting, 'Supporting-fact annotations required')
+        evidence, seen_facts, issues = set(), set(), []
+        for item in supporting:
+            _need(type(item) is list and len(item) == 2 and type(item[0]) is str and type(item[1]) is int
+                  and item[1] >= 0, 'Malformed supporting-fact annotation')
+            fact = tuple(item)
+            if fact in seen_facts:
+                issues.append({'kind': 'duplicate_supporting_fact', 'fact': item})
+            seen_facts.add(fact)
+            if item[0] not in by_title:
+                issues.append({'kind': 'missing_supporting_title', 'fact': item})
+            elif item[1] >= by_title[item[0]][1]:
+                issues.append({'kind': 'supporting_sentence_out_of_range', 'fact': item})
+            else:
+                evidence.add(by_title[item[0]][0])
+        queries.append({'_id': qid, 'text': row['question'], 'context_ids': context_ids,
+                        'question_type': row.get('type'), 'difficulty': row.get('level')})
+        if not issues:
+            qrels.extend((qid, did, 1) for did in sorted(evidence))
+        annotations.append({'query_id': qid, 'answer': row['answer'], 'supporting_facts': supporting,
+                            'authority': 'upstream_dataset_annotation', 'independent_trace_review': False,
+                            'support_reference_status': 'unavailable_upstream_annotation_issue' if issues else 'available',
+                            'validation_issues': issues})
+    return list(corpus.values()), queries, qrels, annotations
+
+
+def prepare_official_benchmark(dataset_id: str, source_path: str | Path, output_dir: str | Path,
+                               *, expected_sha256: str) -> dict:
+    """Normalize pinned upstream bytes into a new checksummed, private bundle.
+
+    Source shape and checksum are validated; caller-supplied bytes do not prove
+    their origin. Preserve the download receipt alongside this bundle.
+    """
+    _need(dataset_id in REGISTRY, 'Unsupported official dataset')
+    raw = _raw(source_path)
+    _need(type(expected_sha256) is str and _sha(raw) == expected_sha256, 'Source SHA256 differs from explicit pin')
+    info = REGISTRY[dataset_id]
+    if info['published_md5'] is not None:
+        _need(hashlib.md5(raw, usedforsecurity=False).hexdigest() == info['published_md5'],
+              'Source differs from published upstream checksum')
+    if info['published_sha256'] is not None:
+        _need(_sha(raw) == info['published_sha256'], 'Source differs from published upstream SHA256')
+    corpus, queries, qrels, annotations = _beir(raw) if info['format'] == 'beir-zip' else _hotpot(raw)
+    output = Path(output_dir)
+    _need(not output.exists(), 'Benchmark bundle already exists; choose a new release directory')
+    output.mkdir(parents=True, exist_ok=False)
+    files = {}
+    def write(name, content):
+        _need(len(content) <= MAX_UNPACKED_BYTES, 'Normalized file exceeds bundle limit')
+        with (output / name).open('xb') as handle:
+            handle.write(content)
+        files[name] = {'sha256': _sha(content), 'bytes': len(content)}
+    def rows(items):
+        return b''.join((json.dumps(row, ensure_ascii=False, allow_nan=False, separators=(',', ':'))+'\n').encode('utf-8') for row in items)
+    write('corpus.jsonl', rows(corpus))
+    write('queries.jsonl', rows(queries))
+    write('qrels.tsv', ('query-id\tcorpus-id\tscore\n' + ''.join(f'{q}\t{d}\t{s}\n' for q,d,s in qrels)).encode('utf-8'))
+    write('annotations.jsonl', rows(annotations))
+    manifest = {'schema_version': 'sphynx.official-benchmark.v1', 'dataset_id': dataset_id,
+        'prepared_at_utc': datetime.now(timezone.utc).isoformat(), 'upstream': info,
+        'source_sha256': expected_sha256, 'source_bytes': len(raw), 'files': files,
+        'counts': {'documents': len(corpus), 'queries': len(queries), 'qrels': len(qrels), 'answer_annotations': len(annotations)},
+        'reference_validation': {'queries_with_issues': sum(bool(a['validation_issues']) for a in annotations),
+                                 'policy': 'retain every query and raw annotation; affected support references are unavailable, never repaired or counted as zero'},
+        'intake_source_sha256': _sha(Path(__file__).read_bytes()),
+        'semantic_review': 'KIV; upstream annotations do not review candidate outputs',
+        'population': 'the complete selected public split; not representative Sable customer traffic',
+        'sampling': {'automatic_subsampling': False, 'independent_clusters_verified': False},
+        'execution': {'models_run': False, 'hosted_admission': False},
+        'origin_basis': 'exact caller-pinned bytes; preserve the independent upstream download receipt'}
+    with (output / 'manifest.json').open('x', encoding='utf-8', newline='\n') as handle:
+        json.dump(manifest, handle, indent=2, ensure_ascii=False, allow_nan=False)
+        handle.write('\n')
+    return {'path': str(output.resolve()), 'manifest_sha256': _sha((output/'manifest.json').read_bytes()), **manifest['counts']}
+
+
+@dataclass(frozen=True)
+class OfficialBenchmark:
+    """Verified normalized data with labels separate from candidate-safe inputs."""
+    manifest: dict
+    corpus: dict
+    queries: dict
+    qrels: dict
+    annotations: dict
+
+    def candidate_case(self, query_id: str) -> dict:
+        query = self.queries[query_id]
+        ids = query['context_ids'] if 'context_ids' in query else self.corpus
+        return {'id': query_id, 'question': query['text'], 'documents': [
+            {'id': did, 'title': self.corpus[did].get('title', ''), 'text': self.corpus[did]['text']}
+            for did in ids]}
+
+
+def load_official_benchmark(path: str | Path, *, expected_manifest_sha256: str) -> OfficialBenchmark:
+    """Verify every normalized file against an explicitly pinned final manifest."""
+    root = Path(path)
+    raw = _raw(root / 'manifest.json')
+    _need(_sha(raw) == expected_manifest_sha256, 'Benchmark manifest checksum differs')
+    manifest = _json(raw)
+    _need(type(manifest) is dict and manifest.get('schema_version') == 'sphynx.official-benchmark.v1'
+          and manifest.get('dataset_id') in REGISTRY, 'Unsupported official benchmark manifest')
+    names = {'corpus.jsonl', 'queries.jsonl', 'qrels.tsv', 'annotations.jsonl'}
+    _need(set(manifest.get('files', {})) == names, 'Invalid bundle file closure')
+    contents = {}
+    for name in sorted(names):
+        target = root / name
+        _need(target.is_file() and not target.is_symlink() and target.stat().st_size <= MAX_UNPACKED_BYTES, 'Missing or oversized bundle file')
+        data = target.read_bytes()
+        _need(_sha(data) == manifest['files'][name]['sha256'] and len(data) == manifest['files'][name]['bytes'], 'Normalized benchmark bytes changed')
+        contents[name] = data
+    def keyed(rows, key):
+        result = {}
+        for row in rows:
+            identity = _id(row.get(key))
+            _need(identity not in result, 'Duplicate normalized identity')
+            result[identity] = row
+        return result
+    corpus = keyed(_jsonl(contents['corpus.jsonl']), '_id')
+    queries = keyed(_jsonl(contents['queries.jsonl']), '_id')
+    annotations = keyed(_jsonl(contents['annotations.jsonl']), 'query_id') if contents['annotations.jsonl'] else {}
+    _need(set(annotations) <= set(queries), 'Unknown annotated question')
+    for row in corpus.values():
+        _need(type(row.get('text')) is str and type(row.get('title', '')) is str, 'Invalid normalized passage')
+    corpus_ids = set(corpus)
+    for row in queries.values():
+        _need(type(row.get('text')) is str and row['text'].strip(), 'Invalid normalized query')
+        if 'context_ids' in row:
+            ids = row['context_ids']
+            _need(type(ids) is list and all(type(i) is str for i in ids)
+                  and len(ids) == len(set(ids)) and set(ids) <= corpus_ids, 'Invalid normalized context')
+    qrels = {qid: {} for qid in queries}
+    for qid, annotation in annotations.items():
+        _need(type(annotation.get('validation_issues')) is list and annotation.get('support_reference_status')
+              in ('available', 'unavailable_upstream_annotation_issue'), 'Missing reference validation status')
+        if annotation['validation_issues']:
+            _need(annotation['support_reference_status'] == 'unavailable_upstream_annotation_issue', 'Inconsistent reference status')
+            qrels[qid] = None
+    lines = contents['qrels.tsv'].decode('utf-8').splitlines()
+    _need(lines and lines[0] == 'query-id\tcorpus-id\tscore', 'Invalid normalized qrel header')
+    for line in lines[1:]:
+        parts = line.split('\t')
+        _need(len(parts) == 3 and parts[2].isdigit(), 'Invalid normalized qrel')
+        qid, did, value = parts
+        _need(qid in queries and did in corpus and qrels[qid] is not None and did not in qrels[qid] and 0 <= int(value) <= 100,
+              'Invalid normalized qrel identity or grade')
+        _need('context_ids' not in queries[qid] or did in queries[qid]['context_ids'], 'Qrel outside question context')
+        qrels[qid][did] = int(value)
+    _need(manifest.get('counts') == {'documents': len(corpus), 'queries': len(queries),
+          'qrels': sum(len(values) for values in qrels.values() if values is not None),
+          'answer_annotations': len(annotations)}, 'Normalized population differs')
+    return OfficialBenchmark(manifest, corpus, queries, qrels, annotations)
+
+
+def main(argv=None):
+    """Explicit source-download and offline import commands; no inference."""
+    import argparse
+    parser = argparse.ArgumentParser(description=__doc__)
+    commands = parser.add_subparsers(dest='command', required=True)
+    commands.add_parser('list')
+    fetch = commands.add_parser('fetch')
+    fetch.add_argument('dataset_id', choices=REGISTRY)
+    fetch.add_argument('--output', required=True)
+    prepare = commands.add_parser('prepare')
+    prepare.add_argument('dataset_id', choices=REGISTRY)
+    prepare.add_argument('--source', required=True)
+    prepare.add_argument('--sha256', required=True)
+    prepare.add_argument('--output', required=True)
+    args = parser.parse_args(argv)
+    try:
+        if args.command == 'list':
+            result = list_official_benchmarks()
+        elif args.command == 'fetch':
+            result = fetch_official_benchmark_source(args.dataset_id, args.output)
+        else:
+            result = prepare_official_benchmark(args.dataset_id, args.source, args.output, expected_sha256=args.sha256)
+    except (ContractError, OSError, ValueError) as exc:
+        parser.error(str(exc))
+    print(json.dumps(result, indent=2, ensure_ascii=False, allow_nan=False))
+    return 0
+
+
+if __name__ == '__main__':
+    raise SystemExit(main())

sphynx/packs.py

@@ -0,0 +1,258 @@
+"""Installed pack descriptors and candidate protocol compatibility checks."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .contracts import (
+    CASE_SCHEMA_VERSION,
+    INTERFACE_VERSION,
+    PACK_ID,
+    SABLE_RERANK_DEV_PACK_ID,
+    REQUEST_SCHEMA_VERSION,
+    RESPONSE_SCHEMA_VERSION,
+    CandidateSpec,
+)
+
+
+@dataclass(frozen=True)
+class ExecutionPolicy:
+    """Pack-owned local trial and timeout policy; no CPU/memory isolation."""
+
+    policy_version: str
+    trials: int
+    per_case_timeout_seconds: float
+    resource_policy_id: str
+
+
+@dataclass(frozen=True)
+class PackDescriptor:
+    pack_id: str
+    pack_release: str
+    title: str
+    track: str
+    interface_version: str
+    case_schema_version: str
+    request_schema_version: str
+    response_schema_version: str
+    status: str
+    description: str
+    execution_policy: ExecutionPolicy
+    case_count: int | None = None
+    metric_release: str | None = None
+    required_inputs: tuple[str, ...] = ()
+    setup_instructions: tuple[str, ...] = ()
+    run_api: str | None = None
+
+    @property
+    def runnable(self) -> bool:
+        return self.status == "runnable"
+
+
+@dataclass(frozen=True)
+class CandidateCheck:
+    candidate_name: str
+    pack_id: str
+    compatible: bool
+    pack_runnable: bool
+    reasons: tuple[str, ...]
+    pack: PackDescriptor | None
+
+
+_PACKS: tuple[PackDescriptor, ...] = (
+    PackDescriptor(
+        pack_id=PACK_ID,
+        pack_release="v1",
+        title="Illustrative frozen-pool RAG reranking",
+        track="rag_rerank",
+        interface_version=INTERFACE_VERSION,
+        case_schema_version=CASE_SCHEMA_VERSION,
+        request_schema_version=REQUEST_SCHEMA_VERSION,
+        response_schema_version=RESPONSE_SCHEMA_VERSION,
+        status="runnable",
+        description="Six synthetic illustrative frozen-pool cases; local scores are not a validated benchmark.",
+        execution_policy=ExecutionPolicy(
+            policy_version="v1",
+            trials=1,
+            per_case_timeout_seconds=30.0,
+            resource_policy_id="local-unbounded.v1",
+        ),
+        case_count=6,
+        metric_release="rag.rerank.metric.v1",
+        required_inputs=("Compatible reranking CandidateSpec and a writable output directory.",),
+        setup_instructions=(
+            "The six synthetic cases are bundled with the SDK; no external dataset, model, provider, or cloud is required.",
+            "Call run(candidate, 'rag.rerank.v1', output_dir), or run_demo(output_dir) for both packaged examples.",
+            "Local labels are owner-readable; results remain local_unverified and are not a validated benchmark.",
+        ),
+        run_api="sphynx.run",
+    ),
+    PackDescriptor(
+        pack_id=SABLE_RERANK_DEV_PACK_ID,
+        pack_release="v1.external",
+        title="Owner-held Sable passage reranking development",
+        track="rag_rerank",
+        interface_version=INTERFACE_VERSION,
+        case_schema_version=CASE_SCHEMA_VERSION,
+        request_schema_version=REQUEST_SCHEMA_VERSION,
+        response_schema_version=RESPONSE_SCHEMA_VERSION,
+        status="requires_local_data",
+        description="Pass the owner-held dataset to run_private_rerank; release identity binds its complete SHA-256.",
+        execution_policy=ExecutionPolicy(
+            policy_version="v1", trials=1, per_case_timeout_seconds=30.0,
+            resource_policy_id="local-unbounded.v1",
+        ),
+        metric_release="rag.rerank.metric.v1",
+        required_inputs=(
+            "Authorized owner-held frozen-pool dataset JSON and its expected SHA-256.",
+            "Compatible reranking CandidateSpec and a writable output directory.",
+        ),
+        setup_instructions=(
+            "Obtain permission to use the local customer bank; no private dataset is bundled or downloaded.",
+            "Call run_private_rerank(candidate, dataset_path, output_dir, expected_sha256=expected_sha256).",
+            "Retain the same dataset and expected SHA-256 for grade_private_rerank; check its label status before interpreting scores.",
+            "Install any model or provider dependencies required by your candidate separately.",
+        ),
+        run_api="sphynx.run_private_rerank",
+    ),
+    PackDescriptor(
+        pack_id="rag.retrieve.v1",
+        pack_release="v1.external",
+        title="Owner-held full retrieval architecture development",
+        track="rag_retrieve",
+        interface_version="sphynx.rag.retrieve.v1",
+        case_schema_version="sphynx.rag.retrieve.dataset.v1",
+        request_schema_version="sphynx.rag.retrieve.request.v1",
+        response_schema_version="sphynx.rag.retrieve.response.v1",
+        status="requires_local_data",
+        description="Full corpus retrieval, fusion and context selection; use run_retrieval or capture_retrieval with owner-held data. Candidate pools may differ.",
+        execution_policy=ExecutionPolicy(
+            policy_version="rag.retrieve.policy.v1", trials=1,
+            per_case_timeout_seconds=0.0, resource_policy_id="local-in-process-unbounded.v1",
+        ),
+        metric_release="rag.retrieve.metric.v1",
+        required_inputs=(
+            "Authorized owner-held retrieval dataset JSON containing the corpus, question cohort, and reference labels.",
+            "Candidate identity plus a trusted adapter callable or captured batch responses; a writable output directory.",
+        ),
+        setup_instructions=(
+            "Prepare authorized question-bank and corpus assets; no customer data, index, embedding model, or provider access is bundled.",
+            "Call run_retrieval(candidate, adapter, dataset_path, output_dir); use capture_retrieval(candidate, dataset_path, responses, output_dir) for batch captures.",
+            "Use expected_sha256 to check an approved dataset digest; retain the dataset for grade_retrieval.",
+            "The callable runs in process without an OS sandbox or timeout; planned answer metrics are not computed.",
+        ),
+        run_api="sphynx.run_retrieval",
+    ),
+    PackDescriptor(
+        pack_id="rag.invoke.v1",
+        pack_release="v1-external-banks",
+        title="Five-customer RAG invocation development",
+        track="rag_invoke",
+        interface_version="sphynx.rag.invoke.v1",
+        case_schema_version="sphynx.rag.invoke.request.v1",
+        request_schema_version="sphynx.rag.invoke.request.v1",
+        response_schema_version="sphynx.rag.invoke.response.v1",
+        status="requires_local_data",
+        description="Pinned 200/300 development and 90/80 regression banks; use run_invocation with owner-held local data.",
+        execution_policy=ExecutionPolicy(
+            policy_version="v1", trials=1, per_case_timeout_seconds=30.0,
+            resource_policy_id="local-unbounded.v1",
+        ),
+        case_count=None,
+        metric_release="rag.invoke.metric.v1",
+        required_inputs=(
+            "Authorized local bank directory with pinned manifest.json, model-input.jsonl, and gold.jsonl for a release listed by list_invocation_banks().",
+            "Invocation CandidateSpec using sphynx.rag.invoke.v1 and a writable output directory.",
+        ),
+        setup_instructions=(
+            "Use list_invocation_banks() to choose excerpt-free-200, development-300, narrow-policy-90, or decision-boundary-80; files are not bundled or downloaded.",
+            "Call run_invocation(candidate, data_dir, output_dir, bank_id=bank_id); the loader checks the manifest, pinned hashes, case counts, and labels.",
+            "Retain the same bank for grade_invocation; install candidate-specific provider or model dependencies separately.",
+            "Local labels are owner-readable; development routing scores remain local_unverified.",
+            "Development-300 includes supplied source-excerpt controls; regression banks are previously exercised, not blind holdouts.",
+        ),
+        run_api="sphynx.run_invocation",
+    ),
+    PackDescriptor(
+        pack_id="rag.question_robustness.v1", pack_release="v1.external",
+        title="Paired question wording robustness development", track="rag_question_robustness",
+        interface_version="sphynx.rag.question_robustness.v1",
+        case_schema_version="sphynx.rag.question_robustness.pilot.draft.v1",
+        request_schema_version="sphynx.rag.question_robustness.request.v1",
+        response_schema_version="sphynx.rag.question_robustness.response.v1",
+        status="requires_local_data",
+        description="Pinned owner-held paired pilot; pending human review remains provisional. Observed search and owner query judgments support separate grades.",
+        execution_policy=ExecutionPolicy("rag.question_robustness.policy.v1", 1, 0.0, "trusted-local-in-process.v1"),
+        case_count=60, metric_release="rag.question_robustness.metric.v1",
+        required_inputs=("Owner-held pilot directory, approved manifest SHA-256, trusted adapter and pinned candidate metadata.",),
+        setup_instructions=(
+            "Call run_question_robustness(candidate, adapter, data_dir, output_dir, mode='routing_only' or 'observed_pipeline').",
+            "Use ObservedSearch.search in pipeline adapters; candidate assertions alone do not establish search or evidence.",
+            "Call grade_question_robustness; supply separate capture-bound owner query judgments for semantic metrics.",
+            "Original labels remain provisional; adapters run in process without a timeout or OS sandbox. No cloud projection is installed.",
+        ), run_api="sphynx.run_question_robustness",
+    ),
+    PackDescriptor(
+        pack_id="rag.rerank.beir.v1", pack_release="v1.external",
+        title="Six-domain public BEIR reranking development", track="rag_rerank_public",
+        interface_version=INTERFACE_VERSION, case_schema_version="expanded-reranker.frozen-pools.v1",
+        request_schema_version=REQUEST_SCHEMA_VERSION, response_schema_version=RESPONSE_SCHEMA_VERSION,
+        status="requires_local_data",
+        description="180 public queries with frozen BM25 pools; select one 90-query original split. Seven Sable illustrations are excluded.",
+        execution_policy=ExecutionPolicy("rag.rerank.beir.policy.v1", 1, 30.0, "local-unbounded.v1"),
+        case_count=90, metric_release="rag.rerank.beir.metric.v1",
+        required_inputs=("Authorized external frozen-pools.json, expected SHA-256 and compatible reranking CandidateSpec.",),
+        setup_instructions=(
+            "Call run_public_rerank(candidate, dataset_path, output_dir, split='development' or 'confirmation').",
+            "Use grade_public_rerank, report_public_rerank and compare_public_rerank with the same dataset and split.",
+            "Native positive qrels use grade >0. Ranking quality is conditional on relevant passages being present in the BM25 pool.",
+            "Both original splits were previously opened and are exploratory; raw public datasets are not redistributed with the SDK.",
+        ), run_api="sphynx.run_public_rerank",
+    ),
+
+)
+
+
+def list_packs() -> tuple[PackDescriptor, ...]:
+    """Return descriptors installed with the package, independent of CWD."""
+
+    return _PACKS
+
+
+def get_pack(pack_id: str) -> PackDescriptor:
+    """Return an installed descriptor or raise an actionable lookup error."""
+
+    for pack in _PACKS:
+        if pack.pack_id == pack_id:
+            return pack
+    available = ", ".join(pack.pack_id for pack in _PACKS) or "none"
+    raise ValueError(f"Unknown pack {pack_id!r}. Installed packs: {available}")
+
+
+def check_candidate(candidate: CandidateSpec, pack_id: str) -> CandidateCheck:
+    """Check static interface compatibility; does not execute the candidate.
+
+    ``pack_runnable`` is separate because a valid candidate protocol can exist
+    before the pack's cases and grader are shipped.
+    """
+
+    if not isinstance(candidate, CandidateSpec):
+        raise TypeError("candidate must be CandidateSpec")
+    try:
+        pack = get_pack(pack_id)
+    except ValueError as exc:
+        return CandidateCheck(candidate.name, pack_id, False, False, (str(exc),), None)
+    reasons: list[str] = []
+    if candidate.interface_version != pack.interface_version:
+        reasons.append(
+            f"Candidate interface {candidate.interface_version!r} is incompatible with {pack.pack_id}; "
+            f"use {pack.interface_version!r}."
+        )
+    return CandidateCheck(
+        candidate_name=candidate.name,
+        pack_id=pack.pack_id,
+        compatible=not reasons,
+        pack_runnable=pack.runnable,
+        reasons=tuple(reasons),
+        pack=pack,
+    )