spatialscape 0.3.0__py3-none-any.whl

spatialscape/readers.py

@@ -0,0 +1,195 @@
+"""Read AnnData (h5ad / zarr v2 / zarr v3) and SpatialData tables into a uniform object."""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import anndata as ad
+import numpy as np
+import pandas as pd
+
+from .config import SampleSpec
+
+
+@dataclass
+class SampleInput:
+    adata: ad.AnnData
+    coords: np.ndarray  # (n_obs, 2) float64 in source units
+    gene_names: np.ndarray  # (n_vars,) str
+    name_column: str | None = None  # var column the names came from (symbols for an Ensembl-indexed object)
+    uns: dict[str, Any] = field(default_factory=dict)
+
+
+MEMORY: dict[str, ad.AnnData] = {}  # objects handed to the build without a file (`spatialscape.show(adata)`); path "memory:<key>"
+
+
+def read_anndata(path: Path, table: str | None = None, *, layer: str | None = None, obsm_keys: list[str] | None = None) -> ad.AnnData:
+    """An AnnData from an h5ad, an AnnData Zarr store or a SpatialData table. With `layer` given, only that matrix,
+    obs, var, uns and the `obsm_keys` are read: a processed object's other layers, graphs and embeddings stay on
+    disk instead of multiplying the build's memory."""
+    if str(path).startswith("memory:"):
+        key = str(path)[len("memory:") :]
+        if key not in MEMORY:
+            raise KeyError(f"in-memory object {key!r} is no longer registered; call spatialscape.show() again")
+        return MEMORY[key]
+    path = Path(path)
+    if path.is_file():
+        if path.suffix != ".h5ad":
+            raise ValueError(f"unsupported file type: {path}")
+        if layer is None:
+            return ad.read_h5ad(path)
+        import h5py
+
+        with h5py.File(path, "r") as f:
+            return _read_parts(f, layer, obsm_keys or [])
+    if not path.is_dir():
+        raise FileNotFoundError(path)
+    target = _table_dir(path, table) if (path / "tables").is_dir() else path  # SpatialData store or AnnData Zarr
+    if layer is None:
+        return ad.read_zarr(target)
+    import zarr
+
+    return _read_parts(zarr.open_group(str(target), mode="r"), layer, obsm_keys or [])
+
+
+def _read_parts(g, layer: str, obsm_keys: list[str]) -> ad.AnnData:
+    from anndata.io import read_elem
+
+    if layer == "X":
+        if "X" not in g:
+            raise KeyError("the object has no X")
+        X = read_elem(g["X"])
+    else:
+        if "layers" not in g or layer not in g["layers"]:
+            raise KeyError(f"layer {layer!r} not found; layers: {list(g['layers'].keys()) if 'layers' in g else []}")
+        X = read_elem(g["layers"][layer])
+    obs = read_elem(g["obs"])
+    var = read_elem(g["var"]) if "var" in g else pd.DataFrame(index=[str(i) for i in range(X.shape[1])])
+    keys = list(obsm_keys)
+    if "obsm" in g:  # polygon stacks (n_obs, n_vertices, 2) are auto-detected by the build, so they come along
+        for k in g["obsm"]:
+            shape = getattr(g["obsm"][k], "shape", None)
+            if k not in keys and shape is not None and len(shape) == 3 and shape[-1] == 2:
+                keys.append(k)
+    obsm = {k: read_elem(g["obsm"][k]) for k in keys if "obsm" in g and k in g["obsm"]}
+    try:
+        uns = read_elem(g["uns"]) if "uns" in g else {}
+    except Exception:  # an uns entry anndata cannot read must not stop a build; nothing here needs it but scale factors
+        uns = {}
+    return ad.AnnData(X=X, obs=obs, var=var, obsm=obsm, uns=uns)
+
+
+def get_coords(adata: ad.AnnData, spec: str) -> np.ndarray:
+    if spec.startswith("obsm/"):
+        key = spec[len("obsm/") :]
+        if key not in adata.obsm:
+            raise KeyError(f"obsm['{key}'] not found; available: {list(adata.obsm.keys())}")
+        arr = adata.obsm[key]
+        arr = arr.to_numpy() if isinstance(arr, pd.DataFrame) else np.asarray(arr)
+        return np.asarray(arr[:, :2], dtype=np.float64)
+    if "," in spec:
+        xs, ys = (s.strip() for s in spec.split(",", 1))
+        cols = []
+        for s in (xs, ys):
+            if not s.startswith("obs/"):
+                raise ValueError(f"coords column spec must be obs/<col>, got {s}")
+            cols.append(np.asarray(adata.obs[s[4:]], dtype=np.float64))
+        return np.stack(cols, axis=1)
+    raise ValueError(f"unrecognised coords spec: {spec!r}")
+
+
+SYMBOL_COLUMNS = ("gene_name", "gene_symbol", "symbol", "Symbol", "gene_symbols", "feature_name", "features")
+
+
+def symbol_column(adata: ad.AnnData) -> str | None:
+    """The var column to name genes by when var_names are Ensembl ids (what people search for is the symbol)."""
+    idx = pd.Index(adata.var_names.astype(str))
+    head = idx[:2000]
+    if len(head) == 0 or head.str.match(r"^ENS[A-Z]*G\d+").mean() < 0.9:
+        return None
+    return next((c for c in SYMBOL_COLUMNS if c in adata.var.columns), None)
+
+
+def gene_names(adata: ad.AnnData, column: str | None) -> np.ndarray:
+    """Feature names in var order: `column` when given, else the symbol column for Ensembl-indexed objects, else
+    var_names. A missing symbol keeps its id; duplicates get .1, .2 suffixes."""
+    column = column or symbol_column(adata)
+    ids = pd.Index(adata.var_names.astype(str))
+    if column:
+        col = adata.var[column].astype(object)
+        names = pd.Index(np.where(col.isna() | (col.astype(str).str.strip() == ""), ids, col.astype(str)).astype(str))
+    else:
+        names = ids
+    if names.has_duplicates:
+        names = ad.utils.make_index_unique(names, join=".")
+    return np.asarray(names, dtype=str)
+
+
+def load_sample(spec: SampleSpec) -> SampleInput:
+    wanted = [spec.coords[len("obsm/") :]] if spec.coords.startswith("obsm/") else []
+    if spec.polygons is not None and spec.polygons.obsm:
+        wanted.append(spec.polygons.obsm)
+    adata = read_anndata(spec.path, spec.table, layer=spec.expression.layer, obsm_keys=wanted)
+    if spec.subset is not None:
+        col = spec.subset.column
+        if col not in adata.obs:
+            raise KeyError(f"sample {spec.id}: subset column {col!r} not found; obs columns: {', '.join(map(str, adata.obs.columns))}")
+        keep = adata.obs[col].astype(str).isin([str(v) for v in spec.subset.values]).to_numpy()
+        if not keep.any():
+            raise ValueError(f"sample {spec.id}: subset {col} in {spec.subset.values} matches no observations")
+        adata = adata[keep].copy()
+    coords = get_coords(adata, spec.coords)
+    if coords.shape[0] != adata.n_obs:
+        raise ValueError("coords length does not match n_obs")
+    uns = dict(adata.uns) if adata.uns is not None else {}
+    name_col = spec.gene_column or symbol_column(adata)
+    return SampleInput(adata=adata, coords=coords, gene_names=gene_names(adata, name_col), uns=uns, name_column=name_col)
+
+
+@dataclass
+class Peek:
+    """What `init` and `inspect` need from a sample file without reading its matrix."""
+
+    obs: pd.DataFrame
+    obsm: list[str]
+    layers: list[str]
+    n_vars: int
+
+
+def _table_dir(path: Path, table: str | None) -> Path:
+    tables = path / "tables"
+    if table is None:
+        keys = sorted(p.name for p in tables.iterdir() if p.is_dir())
+        if len(keys) != 1:
+            raise ValueError(f"SpatialData store has tables {keys}; set `table:`")
+        table = keys[0]
+    return tables / table
+
+
+def _peek_group(g, read_elem) -> Peek:
+    obs = read_elem(g["obs"])
+    obsm = list(g["obsm"].keys()) if "obsm" in g else []
+    layers = [k for k in (g["layers"].keys() if "layers" in g else []) if k]
+    n_vars = len(read_elem(g["var"])) if "var" in g else 0
+    return Peek(obs=obs, obsm=obsm, layers=layers, n_vars=n_vars)
+
+
+def peek(path: Path, table: str | None = None) -> Peek:
+    """Read `obs` plus the `obsm` and layer keys of an h5ad / AnnData Zarr / SpatialData table; the matrix stays on disk."""
+    from anndata.io import read_elem
+
+    path = Path(path)
+    if path.is_file():
+        if path.suffix != ".h5ad":
+            raise ValueError(f"unsupported file type: {path}")
+        import h5py
+
+        with h5py.File(path, "r") as f:
+            return _peek_group(f, read_elem)
+    if not path.is_dir():
+        raise FileNotFoundError(path)
+    import zarr
+
+    target = _table_dir(path, table) if (path / "tables").is_dir() else path
+    return _peek_group(zarr.open_group(str(target), mode="r"), read_elem)

spatialscape/registry.py

@@ -0,0 +1,83 @@
+"""Registry (datasets.json) entries derived from bundles on disk.
+
+The viewer's landing gallery reads `datasets.json` next to its `index.html`. `spatialscape serve` generates
+one on the fly for every bundle in the served folder; `spatialscape site build` writes one from `site.yaml`
+(or, for a plain folder of bundles, from their manifests alone).
+"""
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+from typing import Any
+
+from .manifest import Manifest
+
+
+def find_bundles(root: Path) -> list[Path]:
+    """Bundle directories directly under `root` (those holding a manifest.json), plus `root` itself if it is one."""
+    root = Path(root)
+    if (root / "manifest.json").exists():
+        return [root]
+    return sorted(p for p in root.iterdir() if p.is_dir() and (p / "manifest.json").exists())
+
+
+def bundle_entry(bundle: Path, url: str) -> dict[str, Any]:
+    """Registry entry for one bundle, read from its manifest and gene list."""
+    m = Manifest.model_validate_json((bundle / "manifest.json").read_text())
+    genes_path = bundle / "genes.json"
+    n_genes = len(json.loads(genes_path.read_text())) if genes_path.exists() else None
+    platforms: list[str] = []
+    for s in m.samples:
+        p = "snrnaseq" if s.kind == "embedding" else s.platform
+        if p not in platforms:
+            platforms.append(p)
+    entry: dict[str, Any] = {
+        "id": m.id,
+        "name": m.name,
+        "description": m.description or "",
+        "platforms": platforms,
+        "samples": len(m.samples),
+        "cells": sum(s.nObs for s in m.samples),
+        "url": url,
+        "status": "live",
+    }
+    if n_genes is not None:
+        entry["genes"] = n_genes
+    return entry
+
+
+def placeholder_entry(dataset_id: str, name: str, description: str | None = None, platforms: list[str] | None = None) -> dict[str, Any]:
+    """A complete "coming soon" card: every key the gallery reads is present, so the viewer never sees a half entry."""
+    return {
+        "id": dataset_id,
+        "name": name,
+        "description": description or "",
+        "platforms": list(platforms or []),
+        "samples": 0,
+        "cells": 0,
+        "url": "",
+        "status": "coming soon",
+    }
+
+
+def build_registry(bundles: list[Path], *, url_for: Any, title: str | None = None, intro: str | None = None) -> dict[str, Any]:
+    """A datasets.json document from bundles alone. `url_for(bundle_dir, manifest_id) -> str` decides where each is served."""
+    datasets = []
+    for b in bundles:
+        try:
+            m_id = json.loads((b / "manifest.json").read_text()).get("id") or b.name
+            datasets.append(bundle_entry(b, url_for(b, m_id)))
+        except Exception as e:  # one stale or half-written bundle must not take the gallery down with it
+            print(f"registry: skipping {b}: {str(e).splitlines()[0][:160]}", file=sys.stderr)
+    doc: dict[str, Any] = {"title": title or "spatialscape"}
+    if intro is not None:
+        doc["intro"] = intro
+    doc["datasets"] = datasets
+    return doc
+
+
+def packaged_app_dir() -> Path | None:
+    """The viewer build shipped inside the package, or None for a source checkout without one."""
+    d = Path(__file__).resolve().parent / "_app"
+    return d if (d / "index.html").exists() else None

spatialscape/serve.py

@@ -0,0 +1,262 @@
+"""Development server: the viewer app plus data bundles from one origin, with CORS and HTTP Range.
+
+Layout served at http://host:port/
+  /                      the viewer (packaged with the CLI); if the served folder is itself a built site
+                         (has index.html) it is served as-is instead
+  /datasets.json         the folder's own registry if present, else one generated from the bundles found
+  /<bundle>/...          each bundle directory in the served folder (range requests supported)
+  /assets/...            the viewer's static assets
+"""
+from __future__ import annotations
+
+import contextlib
+import json
+import os
+import posixpath
+import re
+import threading
+import time
+import webbrowser
+from functools import partial
+from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+
+from .registry import build_registry, find_bundles, packaged_app_dir
+
+RANGE_RE = re.compile(r"bytes=(\d*)-(\d*)")
+
+
+class RangeCORSHandler(SimpleHTTPRequestHandler):
+    app_dir: Path | None = None
+    data_dir: Path
+    session: bool = False  # `spatialscape.show()`: answers /_api/ping and stores the viewer's lasso selections and regions
+    _selections: dict[str, dict] = {}
+    _regions: dict[str, dict] = {}
+    _registry_cache: tuple[float, bytes] | None = None
+    _registry_lock = threading.Lock()
+
+    def end_headers(self):
+        self.send_header("Access-Control-Allow-Origin", "*")
+        self.send_header("Access-Control-Allow-Methods", "GET, HEAD, POST, OPTIONS")
+        self.send_header("Access-Control-Allow-Headers", "Range, Content-Type")
+        self.send_header("Access-Control-Allow-Private-Network", "true")
+        self.send_header("Access-Control-Expose-Headers", "Content-Range, Content-Length, Accept-Ranges, ETag")
+        self.send_header("Accept-Ranges", "bytes")
+        self.send_header("Timing-Allow-Origin", "*")
+        self.send_header("Cache-Control", "no-cache")
+        super().end_headers()
+
+    def do_OPTIONS(self):
+        self.send_response(204)
+        self.end_headers()
+
+    def log_message(self, fmt, *args):  # quieter
+        if os.environ.get("SSCAPE_SERVE_VERBOSE"):
+            super().log_message(fmt, *args)
+
+    # ---- routing: data folder first, then the packaged viewer
+    def list_directory(self, path):
+        """Only the root of a folder served without a viewer lists its bundles (there is nothing else to show
+        there); inside bundles nothing is browsed, they are addressed through manifest.json."""
+        if self.app_dir is None and Path(path).resolve() == self.data_dir.resolve():
+            return super().list_directory(path)
+        self.send_error(404, "no directory listings")
+        return None
+
+    def translate_path(self, path: str) -> str:
+        rel = posixpath.normpath(path.split("?", 1)[0].split("#", 1)[0])
+        data = Path(super().translate_path(rel))  # inside data_dir (SimpleHTTPRequestHandler blocks traversal)
+        is_root = rel.strip("/") == ""
+        if data.is_file() or (data.is_dir() and not is_root):
+            return str(data)  # a bundle file, or a bundle folder listing
+        if self.app_dir is not None:
+            rel_clean = rel.lstrip("/") or "index.html"
+            candidate = (self.app_dir / rel_clean).resolve()
+            if candidate.is_relative_to(self.app_dir) and candidate.is_file():
+                return str(candidate)  # the viewer at / and its assets
+        return str(data)
+
+    def do_GET(self):
+        if self._is_registry_request():
+            self._send_registry(head=False)
+            return
+        if self.session and self.path.startswith("/_api/"):
+            self._api_get()
+            return
+        super().do_GET()
+
+    def do_POST(self):
+        route = self.path.split("?", 1)[0]
+        if not (self.session and route in ("/_api/selection", "/_api/regions")):
+            self.send_error(405)
+            return
+        try:
+            n = int(self.headers.get("Content-Length") or 0)
+            doc = json.loads(self.rfile.read(n) or b"{}")
+            if not isinstance(doc, dict):
+                raise ValueError("body must be a JSON object")
+            key = str(doc.get("dataset") or "")
+        except (ValueError, TypeError):
+            self.send_error(400)
+            return
+        if route == "/_api/selection":
+            type(self)._selections[key] = {"version": doc.get("version", 0), "samples": doc.get("samples") or {}, "time": time.time()}
+        else:
+            type(self)._regions[key] = {"version": doc.get("version", 0), "groups": doc.get("groups") or [], "codes": doc.get("codes") or {}, "time": time.time()}
+        self.send_response(204)
+        self.end_headers()
+
+    def _api_get(self) -> None:
+        route, _, query = self.path.partition("?")
+        if route == "/_api/ping":
+            body = b'{"session": true}'
+        elif route in ("/_api/selection", "/_api/regions"):
+            key = dict(p.split("=", 1) for p in query.split("&") if "=" in p).get("d", "")
+            if route == "/_api/selection":
+                body = json.dumps(type(self)._selections.get(key) or {"version": 0, "samples": {}}).encode()
+            else:
+                body = json.dumps(type(self)._regions.get(key) or {"version": 0, "groups": [], "codes": {}}).encode()
+        else:
+            self.send_error(404)
+            return
+        self.send_response(200)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        self.wfile.write(body)
+
+    def do_HEAD(self):
+        if self._is_registry_request():
+            self._send_registry(head=True)
+            return
+        super().do_HEAD()
+
+    def _is_registry_request(self) -> bool:
+        return self.path.split("?", 1)[0] == "/datasets.json" and not (self.data_dir / "datasets.json").exists()
+
+    def _send_registry(self, *, head: bool) -> None:
+        body = self._registry_bytes()
+        self.send_response(200)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        if not head:
+            self.wfile.write(body)
+
+    def _registry_bytes(self) -> bytes:
+        cls = type(self)
+        with cls._registry_lock:
+            now = time.time()
+            if cls._registry_cache and now - cls._registry_cache[0] < 2.0:
+                return cls._registry_cache[1]
+            bundles = find_bundles(self.data_dir)
+            doc = build_registry(
+                bundles,
+                url_for=lambda b, _id: b.name if b != self.data_dir else ".",
+                title="spatialscape",
+                intro=f"Bundles in {self.data_dir.name or self.data_dir}/ on this machine. Open any of them, or paste another bundle URL below.",
+            )
+            if self.session:
+                doc["api"] = "/_api"  # the viewer posts lasso selections and regions here (spatialscape.show)
+            body = json.dumps(doc).encode()
+            cls._registry_cache = (now, body)
+            return body
+
+    # ---- byte ranges
+    def send_head(self):
+        rng = self.headers.get("Range")
+        path = self.translate_path(self.path)
+        if not rng or os.path.isdir(path) or not os.path.exists(path):
+            return super().send_head()
+        m = RANGE_RE.match(rng)
+        if not m or "," in rng:  # multiple ranges: answer with the whole file (200), which every client accepts
+            return super().send_head()
+        size = os.path.getsize(path)
+        start_s, end_s = m.groups()
+        if start_s == "" and end_s == "":
+            return super().send_head()
+        if start_s == "":  # suffix range: last N bytes
+            n = int(end_s)
+            start, end = max(0, size - n), size - 1
+        else:
+            start = int(start_s)
+            end = int(end_s) if end_s else size - 1
+        end = min(end, size - 1)
+        if start >= size or end < start:
+            self.send_response(416)
+            self.send_header("Content-Range", f"bytes */{size}")
+            self.end_headers()
+            return None
+        f = open(path, "rb")  # noqa: SIM115 - streamed and closed by the handler
+        f.seek(start)
+        self.send_response(206)
+        self.send_header("Content-Type", self.guess_type(path))
+        self.send_header("Content-Range", f"bytes {start}-{end}/{size}")
+        self.send_header("Content-Length", str(end - start + 1))
+        self.end_headers()
+        return _LimitedFile(f, end - start + 1)
+
+
+class _LimitedFile:
+    def __init__(self, f, n):
+        self.f, self.n = f, n
+
+    def read(self, k=-1):
+        if self.n <= 0:
+            return b""
+        k = self.n if k < 0 else min(k, self.n)
+        data = self.f.read(k)
+        self.n -= len(data)
+        return data
+
+    def close(self):
+        self.f.close()
+
+
+def make_server(directory: Path, port: int = 8787, host: str = "127.0.0.1", app_dir: Path | None = None, session: bool = False) -> ThreadingHTTPServer:
+    """A ready-to-run server. `app_dir=None` means: use the packaged viewer unless `directory` is itself a built site.
+    `session=True` adds the /_api routes `spatialscape.show()` uses to read lasso selections back."""
+    directory = Path(directory).resolve()
+    if (directory / "index.html").exists():
+        app_dir = None  # a built site: serve it as-is
+    elif app_dir is None:
+        app_dir = packaged_app_dir()
+    extensions = {**SimpleHTTPRequestHandler.extensions_map, ".json": "application/json", ".bw": "application/octet-stream", ".bed": "text/plain", "": "application/octet-stream"}
+    handler = type("Handler", (RangeCORSHandler,), {"app_dir": app_dir, "data_dir": directory, "session": session, "_selections": {}, "_regions": {}, "extensions_map": extensions})
+    factory = partial(handler, directory=str(directory))
+    return ThreadingHTTPServer((host, port), factory)
+
+
+def _port_free(host: str, port: int) -> bool:
+    import socket
+
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+        return sock.connect_ex((host, port)) != 0
+
+
+def serve(directory: Path, port: int = 8787, host: str = "127.0.0.1", open_browser: bool = False, port_chosen: bool = False) -> None:
+    """Start the server. With the default port, a busy port is skipped to the next free one; an explicit `--port` is honoured or refused."""
+    if not _port_free(host, port):
+        if port_chosen:
+            raise OSError(f"port {port} is already in use; stop that server or pass another --port")
+        wanted = port
+        while not _port_free(host, port) and port < wanted + 20:
+            port += 1
+        if not _port_free(host, port):
+            raise OSError(f"ports {wanted}-{port} are all in use; pass --port")
+        print(f"port {wanted} is in use; using {port}", flush=True)
+    httpd = make_server(directory, port=port, host=host)
+    handler_cls = httpd.RequestHandlerClass.func  # type: ignore[attr-defined]
+    url = f"http://{host}:{port}/"
+    directory = Path(directory).resolve()
+    n = len(find_bundles(directory))
+    if handler_cls.app_dir is not None:
+        print(f"viewer + {n} bundle(s) from {directory}\n  open {url}   (Ctrl-C stops the server)", flush=True)
+    elif (directory / "index.html").exists():
+        print(f"serving site {directory}\n  open {url}   (Ctrl-C stops the server)", flush=True)
+    else:
+        print(f"serving {n} bundle(s) from {directory} at {url} (no viewer in this install; open them in a hosted viewer with ?d=<url>)", flush=True)
+    if open_browser:
+        threading.Timer(0.6, lambda: webbrowser.open(url)).start()
+    with contextlib.suppress(KeyboardInterrupt):
+        httpd.serve_forever()