spatialscape 0.3.0__py3-none-any.whl

spatialscape/site.py

@@ -0,0 +1,274 @@
+"""The gallery: `site.yaml` -> a folder you can upload (the packaged viewer, datasets.json and the bundles).
+
+`site.yaml` is the one place that describes the gallery: its title and intro, an optional navigation bar,
+where the bundles are hosted, and one card per dataset. A card is either a dataset yaml to build (`config:`),
+an already built bundle (`bundle:`), or a placeholder (`id:` + `name:`). `datasets.json` is written from it
+on every `site build` and is never edited by hand.
+"""
+from __future__ import annotations
+
+import json
+import os
+import shutil
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any, Literal
+
+import yaml
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+from .registry import build_registry, bundle_entry, find_bundles, packaged_app_dir, placeholder_entry
+
+Log = Callable[[str], None]
+
+
+class _Model(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+
+class PaperSpec(_Model):
+    title: str
+    url: str | None = None
+
+
+class NavLink(_Model):
+    label: str
+    url: str
+
+
+class NavSpec(_Model):
+    """The slim bar above the viewer and the gallery (the registry's `site` block)."""
+
+    title: str | None = None
+    logo: str | None = None
+    links: list[NavLink] = Field(default_factory=list)
+
+
+class SiteDataset(_Model):
+    """One gallery card: a dataset yaml to build, an already built bundle, or a placeholder."""
+
+    config: Path | None = None  # a dataset.yaml; built (incrementally) into bundles/<id> next to it
+    bundle: Path | None = None  # a built bundle directory
+    id: str | None = None  # a placeholder card ("coming soon"); needs `name`
+    name: str | None = None  # card title; default: the dataset's name
+    description: str | None = None  # card text; default: the dataset's description
+    platforms: list[str] = Field(default_factory=list)  # placeholder cards only; bundles report their own
+    status: Literal["live", "local", "coming soon"] | None = None
+    paper: PaperSpec | None = None
+    tags: list[str] = Field(default_factory=list)
+    accent: str | None = None  # color of the generated card art when there is no thumbnail
+    thumbnail: str | None = None  # image URL; default: <bundle>/thumbnail.png
+
+    @model_validator(mode="after")
+    def _one_source(self):
+        n = sum(x is not None for x in (self.config, self.bundle, self.id))
+        if n != 1:
+            raise ValueError("each dataset needs exactly one of `config:` (a dataset.yaml), `bundle:` (a built bundle) or `id:` (a placeholder card)")
+        if self.id is not None and not self.name:
+            raise ValueError(f"placeholder {self.id!r}: set `name:`")
+        return self
+
+
+class SiteConfig(_Model):
+    title: str = "spatialscape"
+    intro: str | None = None
+    nav: NavSpec | None = None
+    data_url: str | None = None  # bundles hosted elsewhere at <data_url>/<id>; nothing is copied into the site
+    datasets: list[SiteDataset]
+
+
+def load_site_config(path: str | Path) -> SiteConfig:
+    path = Path(path).expanduser().resolve()
+    raw = yaml.safe_load(path.read_text())
+    if not isinstance(raw, dict):
+        raise ValueError(f"{path}: expected a mapping (title, datasets, ...) at the top level")
+    site = SiteConfig.model_validate(raw)
+    base = path.parent
+    for d in site.datasets:
+        for attr in ("config", "bundle"):
+            p = getattr(d, attr)
+            if p is not None:
+                p = Path(os.path.expandvars(str(p))).expanduser()
+                setattr(d, attr, p if p.is_absolute() else (base / p).resolve())
+    return site
+
+
+def copy_app(app_dir: Path, out: Path) -> None:
+    out.mkdir(parents=True, exist_ok=True)
+    shutil.copy2(app_dir / "index.html", out / "index.html")
+    (out / ".nojekyll").write_text("")  # GitHub Pages: serve the folder as is, no Jekyll pass over a bundle's thousands of files
+    if (app_dir / "favicon.svg").exists():
+        shutil.copy2(app_dir / "favicon.svg", out / "favicon.svg")
+    assets_out = out / "assets"
+    if assets_out.exists():
+        shutil.rmtree(assets_out)
+    shutil.copytree(app_dir / "assets", assets_out)
+
+
+def _viewer(app_dir: Path | None) -> Path:
+    app_dir = app_dir or packaged_app_dir()
+    if app_dir is None:
+        raise FileNotFoundError(
+            "this install has no packaged viewer. Install spatialscape from PyPI, or in a source checkout run "
+            "`npm run build` then `python python/scripts/bundle_app.py`"
+        )
+    return app_dir
+
+
+def _place(bundle: Path, dest: Path, link: bool, log: Log) -> None:
+    if dest.resolve() == bundle.resolve():
+        return
+    if dest.is_symlink() or dest.is_file():
+        dest.unlink()
+    elif dest.exists():
+        shutil.rmtree(dest)
+    if link:
+        os.symlink(bundle.resolve(), dest, target_is_directory=True)
+    else:
+        # the per-sample build.json fingerprints and the config path hold paths from the building machine
+        shutil.copytree(bundle, dest, ignore=shutil.ignore_patterns("build.json"))
+        info = dest / "build-info.json"
+        if info.exists():
+            try:
+                d = json.loads(info.read_text())
+                d.pop("config", None)
+                info.write_text(json.dumps(d, indent=1))
+            except ValueError:
+                pass
+    log(f"{'linked' if link else 'copied'} {bundle} -> {dest}")
+
+
+def _overlay(entry: dict[str, Any], d: SiteDataset) -> dict[str, Any]:
+    """What site.yaml says about a card, on top of what its manifest says."""
+    if d.name:
+        entry["name"] = d.name
+    if d.description is not None:
+        entry["description"] = d.description
+    if d.status:
+        entry["status"] = d.status
+    if d.paper:
+        entry["paper"] = d.paper.model_dump(exclude_none=True)
+    if d.tags:
+        entry["tags"] = list(d.tags)
+    if d.accent:
+        entry["accent"] = d.accent
+    if d.thumbnail:
+        entry["thumbnail"] = d.thumbnail
+    return entry
+
+
+def build_site_from_config(site_yaml: Path, out: Path | None = None, *, link: bool = False, force: bool = False, app_dir: Path | None = None, log: Log = print) -> Path:
+    """Build every `config:` dataset that changed, place the bundles, write the viewer and `datasets.json`."""
+    from .build import build_bundle
+    from .config import load_config
+
+    site_yaml = Path(site_yaml).expanduser().resolve()
+    site = load_site_config(site_yaml)
+    out = Path(out).expanduser().resolve() if out is not None else site_yaml.parent / "site"
+    app = _viewer(app_dir)
+    entries: list[dict[str, Any]] = []
+    placed = 0
+    for d in site.datasets:
+        if d.id is not None:
+            entries.append(_overlay(placeholder_entry(d.id, d.name or d.id, d.description, d.platforms), d))
+            continue
+        if d.config is not None:
+            cfg = load_config(d.config)
+            log(f"{cfg.id}: {d.config}")
+            res = build_bundle(cfg, force=force, log=log)
+            for p in res.problems:
+                log(f"  ! {p}")
+            if res.problems:
+                raise ValueError(f"{cfg.id}: the bundle did not validate; fix the problems above and rerun")
+            bundle = res.out
+        else:
+            bundle = d.bundle
+            if not (bundle / "manifest.json").exists():
+                raise FileNotFoundError(f"{bundle}: no manifest.json here (not a built bundle)")
+        bid = json.loads((bundle / "manifest.json").read_text()).get("id") or bundle.name
+        if site.data_url:
+            url = f"{site.data_url.rstrip('/')}/{bid}"
+        else:
+            url = bid
+            if placed == 0:
+                copy_app(app, out)
+                log(f"viewer -> {out}")
+            _place(bundle, out / bid, link, log)
+        placed += 1
+        entries.append(_overlay(bundle_entry(bundle, url), d))
+    if placed == 0 or site.data_url:
+        copy_app(app, out)
+        log(f"viewer -> {out}")
+    doc: dict[str, Any] = {"title": site.title}
+    if site.intro:
+        doc["intro"] = site.intro
+    if site.nav:
+        doc["site"] = site.nav.model_dump(exclude_none=True)
+    doc["datasets"] = entries
+    reg = out / "datasets.json"
+    reg.write_text(json.dumps(doc, indent=2) + "\n")
+    log(f"registry -> {reg} ({len(entries)} dataset(s))")
+    return out
+
+
+def build_site(out: Path, bundles: list[Path], *, data_url: str | None = None, link: bool = False, app_dir: Path | None = None, log: Log = print) -> Path:
+    """A site from built bundles alone (no site.yaml): the viewer plus a registry generated from their manifests."""
+    out = Path(out).expanduser().resolve()
+    app = _viewer(app_dir)
+    expanded: list[Path] = []
+    for b in bundles:
+        found = find_bundles(Path(b))
+        if not found:
+            raise FileNotFoundError(f"{b}: no bundle here (no manifest.json in it or directly under it)")
+        expanded += found
+    copy_app(app, out)
+    log(f"viewer -> {out}")
+    if data_url is None:
+        for b in expanded:
+            bid = json.loads((b / "manifest.json").read_text()).get("id") or b.name
+            _place(b, out / bid, link, log)
+    doc = build_registry(expanded, url_for=(lambda b, bid: f"{data_url.rstrip('/')}/{bid}") if data_url else (lambda b, bid: bid))
+    reg = out / "datasets.json"
+    reg.write_text(json.dumps(doc, indent=2) + "\n")
+    log(f"registry -> {reg} ({len(doc['datasets'])} dataset(s))")
+    return out
+
+
+def _pad(text: str, comment: str, width: int = 44) -> str:
+    return f"{text:<{width}} # {comment}" if len(text) < width else f"{text}   # {comment}"
+
+
+def write_site_template(items: list[Path], out: Path, *, title: str | None = None) -> Path:
+    """A commented `site.yaml` listing the given dataset yamls and bundle folders."""
+    out = Path(out).expanduser()
+    out.parent.mkdir(parents=True, exist_ok=True)
+    base = out.parent.resolve()
+    L = [
+        "# site.yaml for spatialscape: the gallery and the viewer around it. Lines starting with # are optional.",
+        "# `spatialscape site build site.yaml` builds every `config:` dataset that changed, then writes a folder",
+        "# with the viewer, datasets.json and the bundles, ready to upload to any static host.",
+        f"title: {json.dumps(title) if title and (':' in title or '#' in title) else (title or 'My datasets')}",
+        "# intro: One or two sentences shown under the title.",
+        _pad("# nav:", "a slim bar above the viewer and the gallery"),
+        "#   title: Our lab",
+        "#   links: [{ label: Lab site, url: https://lab.example.org }]",
+        _pad("# data_url: https://data.example.org", "bundles hosted elsewhere at <data_url>/<id>; nothing is copied into the site"),
+        "datasets:",
+    ]
+    for item in items:
+        item = Path(item).expanduser().resolve()
+        rel = os.path.relpath(item, base)
+        if item.is_dir():
+            L.append(_pad(f"  - bundle: {rel}", "an already built bundle"))
+        else:
+            L.append(_pad(f"  - config: {rel}", "built into bundles/<id> next to it when the data or the yaml changed"))
+        L.append("    # paper: { title: Someone et al. 2026, url: https://doi.org/... }")
+        L.append("    # tags: [cortex, visium]")
+    L += [
+        _pad("  # - id: hippocampus", 'a "coming soon" card'),
+        "  #   name: Human hippocampus",
+        "  #   description: Later this year.",
+        "  #   platforms: [visium, snrnaseq]",
+    ]
+    out.write_text("\n".join(L) + "\n")
+    return out

spatialscape/synthetic.py

@@ -0,0 +1,142 @@
+"""Generate a tiny synthetic dataset (AnnData + PNG image + dataset.yaml) for tests and demos."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import anndata as ad
+import numpy as np
+import pandas as pd
+import scipy.sparse as sp
+import yaml
+
+CLUSTERS = ["Astro", "Excit_L2", "Excit_L5", "Inhib_PV", "Oligo", "Micro"]
+
+
+def _tissue(n: int, rng: np.random.Generator, w: float, h: float) -> np.ndarray:
+    # points inside an ellipse with a notch, in pixel units of a (h, w) image
+    pts = []
+    while len(pts) < n:
+        x, y = rng.uniform(0, w), rng.uniform(0, h)
+        nx, ny = (x - w / 2) / (w / 2), (y - h / 2) / (h / 2)
+        inside = nx * nx + ny * ny < 0.85 and not (nx > 0.3 and abs(ny) < 0.15)
+        if inside:
+            pts.append((x, y))
+    return np.array(pts)
+
+
+def make_sample(path: Path, n: int, n_genes: int, seed: int, w: int = 512, h: int = 384, counts: bool = False):
+    rng = np.random.default_rng(seed)
+    xy = _tissue(n, rng, w, h)
+    ang = np.arctan2(xy[:, 1] - h / 2, xy[:, 0] - w / 2)
+    cl = ((ang + np.pi) / (2 * np.pi) * len(CLUSTERS)).astype(int) % len(CLUSTERS)
+    genes = [f"G{i:03d}" for i in range(n_genes)]
+    centers = rng.uniform([0, 0], [w, h], size=(n_genes, 2))
+    X = np.zeros((n, n_genes), dtype=np.float32)
+    for g in range(n_genes):
+        d2 = ((xy - centers[g]) ** 2).sum(1)
+        X[:, g] = np.exp(-d2 / (2 * (0.2 * w) ** 2)) * rng.uniform(1, 4)
+    X += (cl[:, None] == (np.arange(n_genes)[None, :] % len(CLUSTERS))) * 1.5
+    X *= rng.random(X.shape) > 0.35  # sparsity
+    if counts:
+        X = rng.poisson(X * 3).astype(np.float32)
+    obs = pd.DataFrame(
+        {
+            "cluster": pd.Categorical([CLUSTERS[i] for i in cl]),
+            "total_counts": X.sum(1) * 100 + rng.normal(0, 5, n),
+            "sample_id": path.stem,
+        },
+        index=[f"cell_{i}" for i in range(n)],
+    )
+    obs.loc[obs.index[:7], "cluster"] = np.nan  # a few missing labels
+    a = ad.AnnData(X=sp.csr_matrix(X), obs=obs, var=pd.DataFrame(index=genes))
+    a.obsm["spatial"] = xy
+    a.obsm["X_umap"] = np.stack([np.cos(ang) * 3 + rng.normal(0, 0.3, n), np.sin(ang) * 3 + rng.normal(0, 0.3, n)], 1)
+    a.uns["spot_nn_spacing_level0_px"] = 10.0  # => 10 um per px
+    # fake cell polygons: wobbly hexagons around each point (pixel units, same frame as coords),
+    # stored both as obsm/segmentations (n, 6, 2) and as a long-format parquet file
+    ang_v = np.linspace(0, 2 * np.pi, 7)[:-1]
+    r = rng.uniform(2.5, 4.0, size=(n, 1)) * (1 + 0.15 * rng.standard_normal((n, 6)))
+    vx = xy[:, 0:1] + r * np.cos(ang_v)
+    vy = xy[:, 1:2] + r * np.sin(ang_v)
+    a.obsm["segmentations"] = np.stack([vx, vy], -1)
+    a.write_h5ad(path)
+    # image: gradient background + darker tissue ellipse, (h, w, 3) uint8
+    yy, xx = np.mgrid[0:h, 0:w]
+    nx, ny = (xx - w / 2) / (w / 2), (yy - h / 2) / (h / 2)
+    tissue = (nx * nx + ny * ny < 0.85).astype(np.float32)
+    img = np.stack(
+        [
+            235 - 90 * tissue + 10 * nx,
+            225 - 120 * tissue,
+            230 - 80 * tissue + 10 * ny,
+        ],
+        -1,
+    )
+    img = np.clip(img + rng.normal(0, 4, img.shape), 0, 255).astype(np.uint8)
+    import imageio.v3 as iio
+
+    iio.imwrite(path.with_suffix(".png"), img)
+    import pyarrow as pa
+    import pyarrow.parquet as pq
+
+    cid = np.repeat(np.asarray(obs.index), 6)
+    pq.write_table(
+        pa.table({"cell_id": cid, "vertex_x": vx.ravel(), "vertex_y": vy.ravel()}),
+        path.with_suffix(".polygons.parquet"),
+    )
+
+
+def make_synthetic(out: Path, n_cells: int = 2000, n_genes: int = 50) -> Path:
+    out = Path(out)
+    out.mkdir(parents=True, exist_ok=True)
+    make_sample(out / "sampleA.h5ad", n_cells, n_genes, seed=1)
+    make_sample(out / "sampleB.h5ad", int(n_cells * 1.4), n_genes, seed=2, counts=True)
+    cfg = {
+        "id": "synthetic",
+        "name": "Synthetic demo",
+        "description": "Two fake tissue sections plus an embedding, for tests and the hosted demo.",
+        "default_gene": "G001",
+        "default_color": {"field": "cluster"},
+        "layout": {"mode": "grid"},
+        "fields": [
+            {"id": "cluster", "name": "Cluster", "type": "categorical", "categories": CLUSTERS},
+            {"id": "total_counts", "name": "Total counts", "type": "continuous"},
+        ],
+        "samples": [
+            {
+                "id": "sampleA",
+                "name": "Sample A (lognorm)",
+                "platform": "visium",
+                "group": "donor1",
+                "path": "sampleA.h5ad",
+                "fields": {"cluster": "cluster", "total_counts": "total_counts"},
+                "images": [{"id": "he", "name": "H&E", "path": "sampleA.png"}],
+                "polygons": {"path": "sampleA.polygons.parquet"},
+            },
+            {
+                "id": "sampleB",
+                "name": "Sample B (counts, flipped)",
+                "platform": "visium",
+                "group": "donor2",
+                "path": "sampleB.h5ad",
+                "transform": {"flip": "x", "rotate": 90},
+                "fields": {"cluster": "cluster", "total_counts": "total_counts"},
+                "images": [{"id": "he", "name": "H&E", "path": "sampleB.png"}],
+                "polygons": {"path": "sampleB.polygons.parquet"},
+            },
+            {
+                "id": "umapA",
+                "name": "Sample A UMAP",
+                "platform": "snrnaseq",
+                "kind": "embedding",
+                "group": "donor1",
+                "path": "sampleA.h5ad",
+                "coords": "obsm/X_umap",
+                "fields": {"cluster": "cluster", "total_counts": "total_counts"},
+            },
+        ],
+    }
+    with open(out / "dataset.yaml", "w") as fh:
+        yaml.safe_dump(cfg, fh, sort_keys=False)
+    return out / "dataset.yaml"

spatialscape/thumbnail.py

@@ -0,0 +1,95 @@
+"""Thumbnails: one spatial section colored by a categorical field, rendered from the bundle's own arrays."""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import zarr
+from PIL import Image, ImageColor, ImageDraw
+
+from .manifest import Manifest
+
+BG = (17, 21, 27)
+
+
+def _hex(c: str) -> tuple[int, int, int]:
+    try:
+        r, g, b = ImageColor.getrgb(c)[:3]  # #rrggbb, #rgb and CSS names alike
+        return (r, g, b)
+    except ValueError:
+        return (136, 136, 136)
+
+
+def pick_field(m: Manifest, sample) -> str | None:
+    """Default color field if categorical and present on the sample, else the first categorical field it carries."""
+    cats = [f.id for f in m.fields if f.type == "categorical"]
+    if m.defaultColor.kind == "field" and m.defaultColor.field in cats and m.defaultColor.field in sample.fields:
+        return m.defaultColor.field
+    for fid in cats:
+        if fid in sample.fields:
+            return fid
+    return None
+
+
+def render_sample(root: Path, m: Manifest, sample, *, width: int = 512, height: int = 512, max_points: int = 80_000, pad: int = 16) -> Image.Image:
+    d = root / "samples" / sample.id
+    og = zarr.open_group(str(d / "obs.zarr"), mode="r")
+    xy = og["xy"][:]
+    n = len(xy)
+    step = max(1, n // max_points)
+    xy = xy[::step]
+    fid = pick_field(m, sample)
+    if fid:
+        f = m.field_by_id(fid)
+        codes = og["cat"][fid][::step]
+        colors = [_hex(c) for c in m.vocabularies[f.vocabulary].colors]
+        na = m.vocabularies[f.vocabulary].categories.index("NA") if "NA" in m.vocabularies[f.vocabulary].categories else -1
+    else:
+        codes = None
+        colors = [(200, 200, 200)]
+        na = -1
+    x0, y0 = xy.min(axis=0)
+    x1, y1 = xy.max(axis=0)
+    sw, sh = max(x1 - x0, 1e-9), max(y1 - y0, 1e-9)
+    scale = min((width - 2 * pad) / sw, (height - 2 * pad) / sh)
+    ox = (width - sw * scale) / 2
+    oy = (height - sh * scale) / 2
+    img = Image.new("RGB", (width, height), BG)
+    draw = ImageDraw.Draw(img)
+    r = max(0.6, min(3.0, 0.45 * sample.pointRadius * scale)) if sample.kind == "spatial" else 1.2
+    px = (xy[:, 0] - x0) * scale + ox
+    py = (xy[:, 1] - y0) * scale + oy
+    for i in range(len(xy)):
+        c = colors[codes[i]] if codes is not None and codes[i] < len(colors) else (120, 120, 120)
+        if codes is not None and codes[i] == na:
+            c = (70, 74, 80)
+        draw.ellipse([px[i] - r, py[i] - r, px[i] + r, py[i] + r], fill=c)
+    return img
+
+
+def write_thumbnails(root: Path, *, per_sample: bool = True, hero_id: str | None = None, log=lambda *_: None) -> list[Path]:
+    root = Path(root)
+    m = Manifest.model_validate_json((root / "manifest.json").read_text())
+    out: list[Path] = []
+    spatial = [s for s in m.samples if s.kind == "spatial"] or m.samples
+    if not spatial:
+        return out
+    hero_id = hero_id or m.thumbnailSample
+    hero = next((s for s in m.samples if s.id == hero_id), None) if hero_id else None
+    if hero_id and hero is None:
+        raise KeyError(f"thumbnail sample {hero_id!r} not in bundle")
+    hero = hero or next((s for s in spatial if s.id == (m.layout.order[0] if m.layout.order else None)), spatial[0])
+    img = render_sample(root, m, hero)
+    p = root / "thumbnail.png"
+    img.save(p, optimize=True)
+    out.append(p)
+    log(f"thumbnail: {hero.id} -> {p.name}")
+    if per_sample:
+        for s in m.samples:
+            img = render_sample(root, m, s, width=256, height=256, max_points=20_000, pad=8)
+            sp = root / "samples" / s.id / "thumbnail.png"
+            img.save(sp, optimize=True)
+            out.append(sp)
+    meta = {"hero": hero.id, "field": pick_field(m, hero)}
+    (root / "thumbnail.json").write_text(json.dumps(meta))
+    return out