if-split 0.1.0__py3-none-any.whl

ifsplit/dataset.py

@@ -0,0 +1,112 @@
+"""Stage 8 - Thin loader over a manifest, with cluster-balanced sampling.
+
+Reads ``manifest.json`` and exposes train/val/test views (entry ids, ligand
+classes, and the entry->cluster map). Coordinate/featurization loading is
+intentionally *not* here - it's the optional downstream concern (PLAN.md §1.5);
+a model repo plugs its own featurizer onto these entry lists.
+
+The PDB is heavily redundant (thousands of near-identical lysozyme / kinase
+co-crystals). Training by sampling entries uniformly drowns the model in
+over-represented folds. ``sample_by_cluster`` draws one entry per sequence
+cluster per epoch, which is the bigger training-quality lever than ligand tiering
+- and it is free here because the clusters are already computed. Sampling is
+deterministic given a seed (no global RNG), so an epoch is reproducible.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass
+from pathlib import Path
+
+from .manifest import read_classes, read_clusters, read_id_list, read_manifest
+
+SPLITS = ("train", "val", "test")
+
+
+def _stable_rank(key: str, seed: int) -> int:
+    """Deterministic pseudo-random rank for ``key`` under ``seed`` (no global RNG)."""
+    digest = hashlib.blake2b(f"{seed}:{key}".encode(), digest_size=8).digest()
+    return int.from_bytes(digest, "big")
+
+
+@dataclass
+class SplitView:
+    name: str
+    entry_ids: list[str]
+    ligand_classes: dict[str, list[str]]  # entry_id -> classes
+    entry_clusters: dict[str, str]  # entry_id -> cluster key
+
+    def __len__(self) -> int:
+        return len(self.entry_ids)
+
+    def with_class(self, cls: str) -> list[str]:
+        """Entry ids in this split tagged with ligand class ``cls``."""
+        return [e for e in self.entry_ids if cls in self.ligand_classes.get(e, [])]
+
+    def cluster_groups(self) -> dict[str, list[str]]:
+        """Map cluster key -> sorted entry ids within this split."""
+        groups: dict[str, list[str]] = {}
+        for e in self.entry_ids:
+            key = self.entry_clusters.get(e, e)
+            groups.setdefault(key, []).append(e)
+        return {k: sorted(v) for k, v in sorted(groups.items())}
+
+    def sample_by_cluster(self, seed: int = 0) -> list[str]:
+        """One entry per cluster, chosen deterministically by ``seed``.
+
+        De-redundifies the split: each sequence cluster contributes exactly one
+        representative, so over-represented folds don't dominate. Vary ``seed``
+        across epochs to rotate which member of each cluster is drawn. Returns a
+        deterministically ordered list (sorted by the same stable rank).
+        """
+        chosen: list[tuple[int, str]] = []
+        for key, members in self.cluster_groups().items():
+            rep = min(members, key=lambda e: (_stable_rank(e, seed), e))
+            chosen.append((_stable_rank(key, seed), rep))
+        return [e for _, e in sorted(chosen)]
+
+
+class IFSplitDataset:
+    """Read-only view over a built manifest's train/val/test partition."""
+
+    def __init__(self, manifest_path: str | Path) -> None:
+        self._m = read_manifest(manifest_path)
+        self._dir = Path(manifest_path).parent
+        self.dataset_version: str = self._m["dataset_version"]
+        self.config_hash: str = self._m["config_hash"]
+        files = self._m.get("files", {})
+        self._split_files: dict[str, str] = files.get("splits", {})
+        # Supporting maps live in sidecar files referenced by the manifest.
+        self._classes = read_classes(
+            self._dir / files.get("ligand_classes", "ligands.classes.json")
+        )
+        self._entry_clusters = read_clusters(self._dir / files.get("clusters", "clusters.json"))
+
+    def split(self, name: str) -> SplitView:
+        if name not in SPLITS:
+            raise KeyError(f"unknown split {name!r}; expected one of {SPLITS}")
+        fname = self._split_files.get(name, f"{name}.json")
+        ids = read_id_list(self._dir / fname)
+        return SplitView(
+            name=name,
+            entry_ids=ids,
+            ligand_classes={e: self._classes.get(e, []) for e in ids},
+            entry_clusters={e: self._entry_clusters.get(e, e) for e in ids},
+        )
+
+    @property
+    def train(self) -> SplitView:
+        return self.split("train")
+
+    @property
+    def val(self) -> SplitView:
+        return self.split("val")
+
+    @property
+    def test(self) -> SplitView:
+        return self.split("test")
+
+
+def load_dataset(manifest_path: str | Path) -> IFSplitDataset:
+    return IFSplitDataset(manifest_path)

ifsplit/download.py

@@ -0,0 +1,229 @@
+"""Stage 2 - Optional structure hydration (the only stage that touches coordinates).
+
+`build` never calls this. `fetch` materializes the mmCIF files for a *built*
+manifest into an MLOps-friendly tree that anyone can pick up and train on:
+
+    <root>/
+      structures/
+        train/  hh/4hhb-assembly1.cif.gz        # split-partitioned (browsable),
+        val/    ...                              # sharded by the PDB middle-two
+        test/   ab/1abc-assembly1.cif.gz         # chars (PDB "divided" scheme)
+      index.jsonl                                # one row/structure (zero-dep)
+      index.parquet                              # same, columnar (if pyarrow present)
+      manifest.json                              # copy of the source split manifest
+      DATASET_CARD.md                            # provenance + how-to-load
+
+Design choices that make it "pristine":
+- **Content-addressed integrity:** every file's SHA-256 is recorded in the index,
+  so a re-fetch / transfer can be verified and the pull is resumable (existing,
+  hash-matching files are skipped).
+- **Deterministic paths:** path is a pure function of (split, entry_id, assembly),
+  so two people fetching the same manifest get byte-identical trees.
+- **Explicit scope:** the caller must choose --split / --all; large pulls require
+  --yes. No accidental terabyte (the lightweight-by-default contract).
+- **No coordinates in the split itself:** this is downstream of `build`; the
+  manifest + lock remain tiny and coordinate-free.
+"""
+
+from __future__ import annotations
+
+import gzip
+import hashlib
+import time
+from collections.abc import Callable, Iterable
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import httpx
+
+from . import __version__
+
+FILE_BASE = "https://files.rcsb.org/download"
+SPLITS = ("train", "val", "test")
+ProgressFn = Callable[[str], None]
+
+# RCSB "divided" sharding: middle two characters of the 4-char core id.
+# 4HHB -> "hh"; for extended ids (pdb_0000XXXX) we shard on the core's chars 2-3.
+_EXTENDED_PREFIX = "pdb_0000"
+
+
+def core_id(entry_id: str) -> str:
+    """Lowercase 'core' id used for filenames/sharding (legacy or extended)."""
+    e = entry_id.lower()
+    if e.startswith(_EXTENDED_PREFIX) and len(e) > len(_EXTENDED_PREFIX):
+        return e[len(_EXTENDED_PREFIX) :]
+    if e.startswith("pdb_"):
+        return e[len("pdb_") :]
+    return e
+
+
+def shard_for(entry_id: str) -> str:
+    """Two-char shard (PDB divided scheme). Falls back to a stable 2-char hash."""
+    c = core_id(entry_id)
+    if len(c) >= 3:
+        return c[1:3]
+    return hashlib.blake2b(c.encode(), digest_size=1).hexdigest()  # 2 hex chars
+
+
+def filename_for(entry_id: str, *, assembly: bool) -> str:
+    suffix = "-assembly1.cif.gz" if assembly else ".cif.gz"
+    return f"{core_id(entry_id)}{suffix}"
+
+
+def url_for(entry_id: str, *, assembly: bool) -> str:
+    return f"{FILE_BASE}/{filename_for(entry_id, assembly=assembly)}"
+
+
+def rel_path_for(entry_id: str, split: str, *, assembly: bool) -> Path:
+    return (
+        Path("structures") / split / shard_for(entry_id) / filename_for(entry_id, assembly=assembly)
+    )
+
+
+@dataclass
+class FetchResult:
+    fetched: list[str] = field(default_factory=list)
+    skipped: list[str] = field(default_factory=list)  # already present + hash-ok
+    failed: list[tuple[str, str]] = field(default_factory=list)  # (entry_id, reason)
+    index_rows: list[dict] = field(default_factory=list)
+
+
+def _sha256_file(path: Path) -> str:
+    h = hashlib.sha256()
+    with path.open("rb") as fh:
+        for chunk in iter(lambda: fh.read(1 << 20), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+class StructureFetcher:
+    """Polite, resumable mmCIF downloader (assembly 1 or asymmetric unit)."""
+
+    def __init__(
+        self,
+        *,
+        assembly: bool = True,
+        workers: int = 8,
+        timeout: float = 120.0,
+        max_retries: int = 4,
+        backoff_base: float = 1.5,
+        sleep=time.sleep,
+    ) -> None:
+        self.assembly = assembly
+        self.workers = workers
+        self._timeout = timeout
+        self._max_retries = max_retries
+        self._backoff_base = backoff_base
+        self._sleep = sleep
+        self._client = httpx.Client(
+            timeout=timeout,
+            headers={"User-Agent": f"IF-Split/{__version__} (structure fetch)"},
+            follow_redirects=True,
+        )
+
+    def __enter__(self) -> StructureFetcher:
+        return self
+
+    def __exit__(self, *exc) -> None:
+        self._client.close()
+
+    def close(self) -> None:
+        self._client.close()
+
+    def _get(self, url: str) -> bytes:
+        last: Exception | None = None
+        for attempt in range(self._max_retries + 1):
+            try:
+                resp = self._client.get(url)
+            except httpx.HTTPError as exc:
+                last = exc
+            else:
+                if resp.status_code == 200:
+                    return resp.content
+                if resp.status_code == 404:
+                    raise FileNotFoundError(f"not on RCSB (404): {url}")
+                last = RuntimeError(f"HTTP {resp.status_code}: {url}")
+            if attempt < self._max_retries:
+                self._sleep(self._backoff_base**attempt)
+        raise RuntimeError(f"download failed after retries: {url} ({last})")
+
+    def estimate_bytes(self, entry_ids: list[str], sample: int = 12) -> int | None:
+        """Rough total-size estimate from HEAD on a sample (None if unavailable)."""
+        sizes: list[int] = []
+        for eid in entry_ids[:sample]:
+            try:
+                r = self._client.head(url_for(eid, assembly=self.assembly))
+                cl = r.headers.get("content-length")
+                if r.status_code == 200 and cl:
+                    sizes.append(int(cl))
+            except httpx.HTTPError:
+                continue
+        if not sizes:
+            return None
+        avg = sum(sizes) / len(sizes)
+        return int(avg * len(entry_ids))
+
+    def _fetch_one(self, entry_id: str, split: str, root: Path) -> dict:
+        rel = rel_path_for(entry_id, split, assembly=self.assembly)
+        dest = root / rel
+        if dest.exists():  # resume: trust an existing, readable file
+            return {
+                "entry_id": entry_id,
+                "split": split,
+                "path": str(rel),
+                "sha256": _sha256_file(dest),
+                "status": "skipped",
+            }
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        data = self._get(url_for(entry_id, assembly=self.assembly))
+        gzip.decompress(data)  # integrity check: must be valid gzip
+        tmp = dest.with_suffix(dest.suffix + ".part")
+        tmp.write_bytes(data)
+        tmp.replace(dest)
+        return {
+            "entry_id": entry_id,
+            "split": split,
+            "path": str(rel),
+            "sha256": hashlib.sha256(data).hexdigest(),
+            "status": "fetched",
+        }
+
+    def fetch(
+        self,
+        targets: Iterable[tuple[str, str]],  # (entry_id, split)
+        root: Path,
+        *,
+        progress: ProgressFn | None = None,
+    ) -> FetchResult:
+        targets = list(targets)
+        result = FetchResult()
+        done = 0
+        total = len(targets)
+
+        def say(msg: str) -> None:
+            if progress:
+                progress(msg)
+
+        with ThreadPoolExecutor(max_workers=self.workers) as pool:
+            futs = {
+                pool.submit(self._fetch_one, eid, split, root): (eid, split)
+                for eid, split in targets
+            }
+            for fut in as_completed(futs):
+                eid, _split = futs[fut]
+                done += 1
+                try:
+                    row = fut.result()
+                except Exception as exc:
+                    result.failed.append((eid, str(exc)))
+                else:
+                    result.index_rows.append(row)
+                    (result.skipped if row["status"] == "skipped" else result.fetched).append(eid)
+                if done % 100 == 0 or done == total:
+                    say(
+                        f"{done}/{total} ({len(result.fetched)} new, "
+                        f"{len(result.skipped)} cached, {len(result.failed)} failed)"
+                    )
+        result.index_rows.sort(key=lambda r: (r["split"], r["entry_id"]))
+        return result

ifsplit/enumerate.py

@@ -0,0 +1,111 @@
+"""Stage 1 - Enumerate candidates from RCSB (Search v2 + Data API).
+
+Selects entries by ``release_date <= snapshot_date`` plus the method/resolution
+filters, enriches each via the Data API (sequences, ligand comps, residue
+counts, assemblies), and writes the byte-stable ``candidates.jsonl`` -- the
+snapshot definition. No coordinates are downloaded (PLAN.md §1.5).
+"""
+
+from __future__ import annotations
+
+import sys
+import time
+from collections.abc import Callable
+from pathlib import Path
+
+from .config import Config
+from .rcsb import RcsbClient
+from .schema import CandidateRecord, canonical_jsonl_bytes, sha256_hex
+
+ProgressFn = Callable[[str], None]
+
+# How often (in enriched records) to emit a progress line.
+_REPORT_EVERY = 1000
+
+
+def fmt_duration(seconds: float) -> str:
+    """Human-friendly duration: ``45s`` / ``3m29s`` / ``1h02m``."""
+    s = int(max(0, seconds))
+    if s < 60:
+        return f"{s}s"
+    m, s = divmod(s, 60)
+    if m < 60:
+        return f"{m}m{s:02d}s"
+    h, m = divmod(m, 60)
+    return f"{h}h{m:02d}m"
+
+
+def progress_line(label: str, n: int, total: int, t0: float) -> str:
+    """A ``label: n/total (pct)  rate/s  eta ...`` line from a monotonic start."""
+    elapsed = time.monotonic() - t0
+    pct = (100 * n / total) if total else 0.0
+    msg = f"{label}: {n}/{total} ({pct:.0f}%)"
+    if elapsed > 0 and n > 0:
+        rate = n / elapsed
+        eta = (total - n) / rate if rate > 0 else 0.0
+        msg += f"  {rate:.0f}/s  eta {fmt_duration(eta)}"
+    return msg
+
+
+def make_console_progress(stream=None) -> ProgressFn:
+    """A timestamped, line-flushed progress printer for long CLI runs.
+
+    The flush is the important part: when stdout is redirected to a file, Python
+    block-buffers it, so unflushed progress lines stay invisible until the process
+    exits. This forces each line out immediately.
+    """
+    out = stream or sys.stdout
+
+    def say(msg: str) -> None:
+        print(f"  [{time.strftime('%H:%M:%S')}] {msg}", file=out, flush=True)
+
+    return say
+
+
+def enumerate_candidates(
+    cfg: Config,
+    out_dir: str | Path,
+    *,
+    limit: int | None = None,
+    client: RcsbClient | None = None,
+    progress: ProgressFn | None = None,
+) -> tuple[list[CandidateRecord], Path, str]:
+    """Run Stage 1.
+
+    Returns ``(records, candidates_path, sha256)``. ``candidates.jsonl`` is
+    written to ``out_dir`` in canonical (byte-stable) form.
+    """
+    out_dir = Path(out_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    def say(msg: str) -> None:
+        if progress:
+            progress(msg)
+
+    owns_client = client is None
+    client = client or RcsbClient()
+    try:
+        ids = client.search_entry_ids(cfg, limit=limit, progress=progress)
+        say(f"search: {len(ids)} entries match the snapshot")
+
+        records: list[CandidateRecord] = []
+        total = len(ids)
+        t0 = time.monotonic()
+        for raw in client.fetch_entries(ids):
+            records.append(CandidateRecord.from_data_api(raw))
+            if len(records) % _REPORT_EVERY == 0:
+                say(progress_line("enriched", len(records), total, t0))
+        say(progress_line("enriched", len(records), total, t0) + " (done)")
+    finally:
+        if owns_client:
+            client.close()
+
+    data = canonical_jsonl_bytes(records)
+    sha = sha256_hex(data)
+    candidates_path = out_dir / "candidates.jsonl"
+    candidates_path.write_bytes(data)
+    say(f"wrote {candidates_path} ({len(records)} records, sha256={sha[:12]}...)")
+
+    # Return in canonical (entry_id-sorted) order so callers see what was written.
+    records.sort(key=lambda r: r.entry_id)
+    return records, candidates_path, sha

ifsplit/hydrate.py

@@ -0,0 +1,216 @@
+"""Stage 2 orchestration: turn a built manifest into a hydrated, ML-ready dataset.
+
+Ties together split selection, the structure fetcher, the dual index
+(JSONL + optional Parquet), a copy of the manifest, and a generated dataset card.
+Kept separate from ``download.py`` (the transport layer) so the I/O policy and
+the MLOps packaging are easy to read and test independently.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any
+
+from . import __version__
+from .download import SPLITS, StructureFetcher
+from .manifest import (
+    CLASSES_FILENAME,
+    CLUSTERS_FILENAME,
+    SPLIT_FILES,
+    TIERS_FILENAME,
+    read_classes,
+    read_clusters,
+    read_id_list,
+    read_manifest,
+    read_tiers,
+)
+
+ProgressFn = Callable[[str], None]
+
+
+def select_targets(
+    manifest: dict[str, Any], splits: list[str], base_dir: Path | None = None
+) -> list[tuple[str, str]]:
+    """(entry_id, split) pairs for the requested splits, deterministically ordered.
+
+    Reads each split's id list from its file (train.json etc.) next to the
+    manifest. ``base_dir`` defaults to the manifest's directory.
+    """
+    base = Path(base_dir) if base_dir is not None else Path(".")
+    split_files = manifest.get("files", {}).get("splits", SPLIT_FILES)
+    targets: list[tuple[str, str]] = []
+    for split in splits:
+        fname = split_files.get(split, f"{split}.json")
+        for eid in sorted(read_id_list(base / fname)):
+            targets.append((eid, split))
+    return targets
+
+
+def _index_rows(
+    fetch_rows: list[dict],
+    classes: dict[str, list[str]],
+    entry_clusters: dict[str, str],
+    tiers: dict[str, dict] | None = None,
+) -> list[dict]:
+    """Enrich each fetched file with split metadata for the ML index."""
+    tiers = tiers or {}
+    rows: list[dict] = []
+    for r in fetch_rows:
+        eid = r["entry_id"]
+        rows.append(
+            {
+                "entry_id": eid,
+                "split": r["split"],
+                "path": r["path"],
+                "sha256": r["sha256"],
+                "cluster": entry_clusters.get(eid),
+                "ligand_classes": classes.get(eid, []),
+                "ligand_tiers": tiers.get(eid, {}),
+            }
+        )
+    rows.sort(key=lambda r: (r["split"], r["entry_id"]))
+    return rows
+
+
+def write_index(rows: list[dict], root: Path) -> dict[str, Path]:
+    """Write index.jsonl (always) and index.parquet (if pyarrow present)."""
+    out: dict[str, Path] = {}
+    jsonl = root / "index.jsonl"
+    with jsonl.open("w", encoding="utf-8") as fh:
+        for r in rows:
+            fh.write(json.dumps(r, sort_keys=True, separators=(",", ":")) + "\n")
+    out["jsonl"] = jsonl
+
+    try:
+        import pyarrow as pa
+        import pyarrow.parquet as pq
+    except ImportError:
+        return out
+
+    # Stringify nested fields for a clean, portable columnar schema.
+    table = pa.table(
+        {
+            "entry_id": [r["entry_id"] for r in rows],
+            "split": [r["split"] for r in rows],
+            "path": [r["path"] for r in rows],
+            "sha256": [r["sha256"] for r in rows],
+            "cluster": [r["cluster"] for r in rows],
+            "ligand_classes": [",".join(r["ligand_classes"]) for r in rows],
+            "ligand_tiers_json": [json.dumps(r["ligand_tiers"], sort_keys=True) for r in rows],
+        }
+    )
+    parquet = root / "index.parquet"
+    pq.write_table(table, parquet)
+    out["parquet"] = parquet
+    return out
+
+
+def _dataset_card(manifest: dict[str, Any], rows: list[dict], *, assembly: bool) -> str:
+    counts = {s: sum(1 for r in rows if r["split"] == s) for s in SPLITS}
+    cfg = manifest.get("config", {})
+    unit = "biological assembly 1" if assembly else "asymmetric unit"
+    cl = manifest["clustering"]
+    clustering = f"{cl['backend']} @ {cl['identity']}% identity"
+    return f"""# {manifest["dataset_version"]} — structures
+
+Generated by IF-Split {__version__}. Coordinates fetched from RCSB
+({unit}, gzipped mmCIF). Reproducible from the source split.
+
+## Provenance
+- **config hash:** `{manifest["config_hash"]}`
+- **snapshot date:** `{cfg.get("snapshot_date", "?")}` (entries with release_date <= this)
+- **clustering:** {clustering}
+- **candidates.sha256:** `{manifest["candidates"]["sha256"]}`
+
+## Splits (hydrated)
+| split | structures |
+|-------|-----------:|
+| train | {counts["train"]:>10,} |
+| val   | {counts["val"]:>10,} |
+| test  | {counts["test"]:>10,} |
+
+## Layout
+```
+structures/<split>/<shard>/<id>{"-assembly1" if assembly else ""}.cif.gz
+```
+`<shard>` is the PDB "divided" scheme — the middle two characters of the entry id
+(e.g. `4hhb` -> `hh/`) — so no single split directory holds an unwieldy number of
+files. Browsable by hand, scalable to the whole PDB.
+
+## Files
+- `index.jsonl` / `index.parquet` — one row per structure: entry_id, split, path,
+  sha256, cluster, ligand_classes, ligand_tiers. The `sha256` lets you verify
+  integrity and de-duplicate; `cluster` lets you sample cluster-balanced batches.
+- `manifest.json` — the full source split manifest (config, drop log, per-class
+  counts, component stats).
+
+## Load (Python)
+```python
+import pandas as pd
+df = pd.read_parquet("index.parquet")          # or pd.read_json("index.jsonl", lines=True)
+train = df[df.split == "train"]
+# one representative per cluster (de-redundified):
+epoch = train.sort_values("entry_id").groupby("cluster").head(1)
+```
+
+## Integrity
+```bash
+# every file's sha256 is in the index; re-fetch is resumable and verifiable.
+```
+"""
+
+
+def hydrate(
+    manifest_path: str | Path,
+    root: str | Path,
+    *,
+    splits: list[str] | None = None,
+    assembly: bool = True,
+    workers: int = 8,
+    fetcher: StructureFetcher | None = None,
+    progress: ProgressFn | None = None,
+) -> dict[str, Any]:
+    """Download structures for a manifest and write the ML-ready package.
+
+    Returns a summary dict. Idempotent: re-running skips files already present.
+    """
+    manifest = read_manifest(manifest_path)
+    src = Path(manifest_path).parent
+    files = manifest.get("files", {})
+    # Split membership + supporting maps live in sidecar files next to the manifest.
+    classes = read_classes(src / files.get("ligand_classes", CLASSES_FILENAME))
+    entry_clusters = read_clusters(src / files.get("clusters", CLUSTERS_FILENAME))
+    tiers = read_tiers(src / files.get("ligand_tiers", TIERS_FILENAME))
+    root = Path(root)
+    root.mkdir(parents=True, exist_ok=True)
+    splits = splits or list(SPLITS)
+
+    targets = select_targets(manifest, splits, base_dir=src)
+
+    owns = fetcher is None
+    fetcher = fetcher or StructureFetcher(assembly=assembly, workers=workers)
+    try:
+        res = fetcher.fetch(targets, root, progress=progress)
+    finally:
+        if owns:
+            fetcher.close()
+
+    rows = _index_rows(res.index_rows, classes, entry_clusters, tiers)
+    index_paths = write_index(rows, root)
+    (root / "manifest.json").write_text(
+        json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+    )
+    (root / "DATASET_CARD.md").write_text(
+        _dataset_card(manifest, rows, assembly=assembly), encoding="utf-8"
+    )
+
+    return {
+        "root": str(root),
+        "requested": len(targets),
+        "fetched": len(res.fetched),
+        "skipped": len(res.skipped),
+        "failed": res.failed,
+        "index": {k: str(v) for k, v in index_paths.items()},
+    }