plexora 0.0.1__py3-none-any.whl

plexora/server/models/adapters/csv_adapter.py

@@ -0,0 +1,105 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import polars as pl
+
+from .base import NormalizedDatasource
+
+
+class CsvAdapter:
+    """Adapter for the flat-CSV feature-table workflow.
+
+    Takes the project's DataSpec (server/models/project.py): `src` says where
+    the file is, `roles` say what its columns mean. A role the project never
+    recorded is None here, which is why the coordinate columns are optional --
+    a CSV imported but not yet fully described still loads, it just has no
+    usable coordinates until something asks the user for them.
+    """
+
+    def __init__(self, spec):
+        self.csv_path = Path(spec.src)
+        self.x_column = spec.roles.x
+        self.y_column = spec.roles.y
+        self.id_field = spec.roles.cell_id
+        self.celltype_column = spec.roles.celltype
+        # The marker/metadata split the user confirmed at import. A CSV header
+        # does not draw that line itself, which is the whole reason the
+        # classification screen exists -- so this is the only thing that knows
+        # Area and Eccentricity are measurements rather than stains.
+        self.marker_columns = list(spec.columns.markers)
+        # Explicit opt-in only -- no heuristic guessing at whether the values
+        # "look" already transformed. Same contract as AnnDataAdapter, which is
+        # where this used to be honoured and only there: a CSV kept the flag
+        # and read straight past it, so a user who asked for the transform got
+        # a project that said it was transformed and was not.
+        self.apply_log_transform = bool(spec.is_transformed)
+
+    def read_obs_column(self, name: str):
+        """Always None: a CSV's table IS the file.
+
+        There is no second place to look -- `load_table()` reads every column,
+        so anything a caller could ask for is already in `frame()`. Returning
+        None says "nothing to add here", which is what lets the caller treat a
+        column that is missing from both as genuinely unknown rather than as a
+        format it forgot to handle.
+        """
+        return None
+
+    def load_table(self) -> NormalizedDatasource:
+        df = pl.read_csv(self.csv_path)
+        # Manufacture a stable positional 'id' column, mirroring pandas'
+        # implicit RangeIndex usage in the code this replaced -- must happen
+        # immediately after read_csv, before any other transform, since
+        # downstream code treats 'id' as a stable per-row identity.
+        df = df.with_row_index("id").with_columns(pl.col("id").cast(pl.Int64))
+        numeric_cols = [c for c, dt in df.schema.items() if dt in (pl.Float32, pl.Float64)]
+        df = df.with_columns([
+            pl.when(pl.col(c) == float("-inf")).then(0).otherwise(pl.col(c)).alias(c)
+            for c in numeric_cols
+        ])
+
+        if self.id_field and self.id_field in df.columns:
+            source_obs_ids = df[self.id_field].cast(pl.Utf8).to_list()
+        else:
+            source_obs_ids = df["id"].cast(pl.Utf8).to_list()
+
+        feature_columns = self._feature_columns(df)
+        if self.apply_log_transform:
+            # After the -inf guard above and over the markers only, matching
+            # AnnDataAdapter: transforming a coordinate or a cell id would move
+            # every cell on the image. Numeric ones at that -- the split is the
+            # user's to correct, so a text column can end up in the marker box,
+            # and log1p on it is an error rather than a bad number.
+            transform = [c for c in feature_columns if df.schema[c].is_numeric()]
+            df = df.with_columns([pl.col(c).log1p().alias(c) for c in transform])
+
+        return NormalizedDatasource(
+            table=df,
+            id_column="id",
+            source_obs_ids=source_obs_ids,
+            x_column=self.x_column,
+            y_column=self.y_column,
+            feature_columns=feature_columns,
+            celltype_column=self.celltype_column,
+        )
+
+    def _feature_columns(self, df) -> list[str]:
+        """Which columns hold marker intensities.
+
+        The recorded split when there is one, narrowed to columns this file
+        actually has: the answer can outlive the file it was given for, and a
+        marker naming a column that is gone would be log-transformed into a
+        polars error and offered as a gate with no data behind it.
+
+        Everything-but-the-roles is the fallback, for a project registered
+        before the classification screen ran. It cannot tell a stain from a
+        measurement -- that is what the screen is for -- so it is a last resort
+        and not the answer.
+        """
+        excluded = {"id", self.x_column, self.y_column, self.id_field, self.celltype_column}
+        recorded = [c for c in self.marker_columns
+                    if c in df.columns and c not in excluded]
+        if recorded:
+            return recorded
+        return [c for c in df.columns if c not in excluded]

plexora/server/models/adapters/inspection.py

@@ -0,0 +1,265 @@
+from __future__ import annotations
+
+import numpy as np
+
+from . import classify
+from .anndata_adapter import describe_obsm, is_likely_image_identifier_name
+
+# Read-only structural inspection of not-yet-registered source files -- used
+# by the standalone import wizard (a future Stage 3 endpoint) to populate a
+# progressive-disclosure config UI, and reusable directly from Jupyter/tests.
+# Deliberately separate from data_model.py's module-global-based caching:
+# these operate on files that aren't loaded (or registered) yet.
+
+_MAX_CANDIDATE_VALUES = 500
+
+
+def _is_subset_candidate(series) -> bool:
+    """A column is worth offering as an image/sample subset choice if it's
+    non-numeric (categorical/string-like) with more than one, but not an
+    unreasonably large number of, distinct values."""
+    try:
+        if np.issubdtype(series.dtype, np.number):
+            return False
+    except TypeError:
+        pass
+    n_unique = series.nunique(dropna=True)
+    return 1 < n_unique <= _MAX_CANDIDATE_VALUES
+
+
+def _inspect_adata(adata) -> dict:
+    """The format-independent half of inspection: everything the config UI
+    needs about an already-opened AnnData. Shared by inspect_anndata() and
+    inspect_spatialdata_table(), since a SpatialData table *is* an AnnData
+    and its config page offers exactly the same choices."""
+    obs_columns = []
+    for column in adata.obs.columns:
+        series = adata.obs[column]
+        is_candidate = _is_subset_candidate(series)
+        n_unique = series.nunique(dropna=True)
+        # Same name heuristic the adapter's own ambiguity guard enforces
+        # (adapters/anndata_adapter.py) -- lets the UI reserve its
+        # "this file may span multiple images" warning for genuine
+        # identifier-shaped columns instead of every categorical column
+        # (cell_type/cluster/condition columns are common and would
+        # otherwise trigger a misleading warning on ordinary single-image data).
+        likely_identifier = is_candidate and is_likely_image_identifier_name(column) and n_unique > 1
+        # Every column gets its actual values so the subset picker can
+        # always offer a dropdown of real categories instead of asking the
+        # user to type a value freehand -- is_subset_candidate above only
+        # gates the ambiguity-warning heuristic, not whether a column's
+        # values are shown at all. But a column with more distinct values
+        # than _MAX_CANDIDATE_VALUES (e.g. a per-cell numeric ID or
+        # centroid coordinate -- effectively unique per row) is never a
+        # usable subset choice, and for a real hundreds-of-thousands-of-rows
+        # table, listing it anyway inflates this inspection payload to tens
+        # of MB. A config page embeds that payload as a literal
+        # JavaScript object ({{data|tojson}}, not JSON.parse'd), so a
+        # bloated payload freezes the importing browser tab outright --
+        # capping here is what keeps that page loadable for large
+        # single-cell tables (confirmed against a real 686k-cell exemplar,
+        # whose id/centroid columns alone produced a 30MB payload).
+        values = (
+            sorted(str(v) for v in series.dropna().unique().tolist())
+            if n_unique <= _MAX_CANDIDATE_VALUES
+            else []
+        )
+        entry = {
+            "name": column,
+            "dtype": str(series.dtype),
+            "is_subset_candidate": is_candidate,
+            "likely_multi_image_identifier": likely_identifier,
+            "values": values,
+        }
+        obs_columns.append(entry)
+
+    return {
+        "obs_count": int(adata.n_obs),
+        # anndata's Layers/AxisArrays mappings can report a spurious
+        # `None` key (observed with anndata 0.13.2) even when no real
+        # layer/obsm entry of that name exists -- filter it out.
+        "obsm_keys": [k for k in adata.obsm.keys() if k],
+        # The same arrays with their shapes, which is what a user picking
+        # between them actually needs: `spatial` and `X_umap` are routinely both
+        # (n, 2) float32, so the name is the only thing separating a cell's
+        # position from its embedding -- and the name is exactly what cannot be
+        # trusted. Kept beside `obsm_keys` rather than replacing it so existing
+        # callers of that key are untouched.
+        "obsm": describe_obsm(adata),
+        "layers": [k for k in adata.layers.keys() if k],
+        "obs_columns": obs_columns,
+        "var_names": [str(v) for v in adata.var_names],
+        "n_var": int(adata.n_vars),
+    }
+
+
+def inspect_csv(path) -> dict:
+    """Structural inspection of a flat CSV: its columns and their dtypes.
+
+    Reads a small number of rows rather than the file: a real quantification
+    table runs to millions of rows, and nothing here needs values -- only
+    names, and enough of each column to tell a number from a label. Polars
+    infers from the rows it is given, so a column that is empty at the top of
+    the file can be misread; that is acceptable because the user confirms the
+    split on the classification screen, which is the whole reason it exists.
+    """
+    import polars as pl
+
+    frame = pl.read_csv(path, n_rows=_DTYPE_SAMPLE_ROWS)
+    columns = [{"name": name, "dtype": str(dtype)} for name, dtype in frame.schema.items()]
+    return {
+        "data_type": "csv",
+        "columns": columns,
+        **classify.classify_columns(columns),
+    }
+
+
+#: Rows read to infer CSV dtypes. Enough to get past a sparse first row,
+#: cheap enough to run on every keystroke-triggered inspection.
+_DTYPE_SAMPLE_ROWS = 200
+
+#: obsm keys that conventionally hold cell centroids, in preference order.
+_SPATIAL_OBSM_KEYS = ("spatial", "X_spatial", "spatial_centroid")
+
+
+def propose_read_spec(inspection: dict) -> dict:
+    """How to read this file, worked out from its own structure.
+
+    The import page asks for a path and nothing else, so everything the old
+    two-step config form made the user choose has to be answered from the file
+    itself. Almost all of it can be: `obsm['spatial']` is near-universal for
+    centroids, `X` is the feature matrix by definition, and a positional cell
+    id is the right default regardless (see the uint32-packing note in
+    datasource.py).
+
+    Returns the read spec plus `ambiguous`: the columns that make the file span
+    several images, which is the one thing that genuinely cannot be guessed --
+    picking an image for the user would silently load the wrong cells. The
+    caller turns that into a subset choice on the form.
+
+    An unresolved coordinate source is NOT an error here. It leaves the x/y
+    roles unset, and whatever first needs them asks for them.
+    """
+    obsm_keys = inspection.get("obsm_keys") or []
+    obs_columns = inspection.get("obs_columns") or []
+    names = [c.get("name") for c in obs_columns if c.get("name")]
+
+    coordinates = {}
+    obsm_key = next((k for k in _SPATIAL_OBSM_KEYS if k in obsm_keys), None)
+    if obsm_key:
+        coordinates = {"source": "obsm", "obsm_key": obsm_key}
+    else:
+        guessed = classify.guess_roles(names)
+        if guessed.get("x") and guessed.get("y"):
+            coordinates = {"source": "obs", "x_column": guessed["x"], "y_column": guessed["y"]}
+
+    return {
+        "coordinates": coordinates,
+        "features": {"source": "X"},
+        "ambiguous": [
+            c["name"] for c in obs_columns if c.get("likely_multi_image_identifier")
+        ],
+        **classify.classify_from_inspection(inspection),
+    }
+
+
+def inspect_anndata(path) -> dict:
+    """Structural inspection of an .h5ad file: obsm keys, layers, obs
+    columns (flagging which look like image/sample subset candidates and,
+    for those, their available values), var names, and observation count.
+    """
+    import anndata as ad
+
+    adata = ad.read_h5ad(path, backed='r')
+    try:
+        return {"data_type": "anndata", **_inspect_adata(adata)}
+    finally:
+        if adata.isbacked:
+            adata.file.close()
+
+
+def source_layers(spec) -> list[str]:
+    """The extra expression matrices a project's source file carries.
+
+    Recorded at import (DataSpec.layers), so this is only reached for a project
+    imported before that was recorded -- but it has to be reached, because the
+    one control that can fix a project reading the wrong matrix would otherwise
+    be the one control it never offers.
+
+    Cheaper than a full inspection on purpose: a zarr group listing, or a backed
+    h5ad open, so nothing materializes a second matrix just to learn its name.
+    Best-effort -- an unreadable file means no picker, which is what every
+    surface here did before there was one.
+    """
+    if spec is None or spec.type == "csv":
+        return []
+    if spec.layers:
+        return list(spec.layers)
+    try:
+        if spec.type == "spatialdata":
+            from .spatialdata_adapter import list_table_layers
+
+            return list_table_layers(spec.src, spec.table)
+        import anndata as ad
+
+        adata = ad.read_h5ad(spec.src, backed="r")
+        try:
+            return [str(name) for name in adata.layers.keys() if name]
+        finally:
+            if adata.isbacked:
+                adata.file.close()
+    except Exception:
+        return []
+
+
+def source_obsm(spec) -> list[dict]:
+    """The obsm arrays a project's source file carries, with their shapes.
+
+    The coordinate question's counterpart to `source_layers`, and reached for
+    the same reason: a project imported before obsm was recorded would
+    otherwise be offered no candidates at all, and that is precisely the
+    project whose coordinate source was picked by a name heuristic nobody was
+    shown.
+
+    Best-effort in the same way -- an unreadable file means no obsm choices,
+    which is what every surface offered before this existed.
+    """
+    if spec is None or spec.type == "csv":
+        return []
+    if spec.obsm:
+        return [dict(entry) for entry in spec.obsm]
+    try:
+        if spec.type == "spatialdata":
+            from .spatialdata_adapter import read_spatialdata_table
+
+            return describe_obsm(read_spatialdata_table(spec.src, spec.table))
+        import anndata as ad
+
+        adata = ad.read_h5ad(spec.src, backed="r")
+        try:
+            return describe_obsm(adata)
+        finally:
+            if adata.isbacked:
+                adata.file.close()
+    except Exception:
+        return []
+
+
+def inspect_spatialdata_table(store, table) -> dict:
+    """Same inspection, for one table inside a SpatialData (.zarr) store.
+
+    Reads only the named table (see spatialdata_adapter.read_spatialdata_table
+    for why the whole store is deliberately not opened). There's no backed
+    mode for a zarr-backed AnnData, so unlike inspect_anndata this
+    materializes the table -- acceptable because the import form has the
+    user pick a single table first, and its shape is shown there.
+    """
+    from .spatialdata_adapter import read_spatialdata_table
+
+    adata = read_spatialdata_table(store, table)
+    return {
+        "data_type": "spatialdata",
+        "store": str(store),
+        "table": str(table),
+        **_inspect_adata(adata),
+    }

plexora/server/models/adapters/spatialdata_adapter.py

@@ -0,0 +1,248 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from .anndata_adapter import AnnDataAdapter
+
+# A SpatialData store keeps every annotation table as a plain AnnData group
+# under <store>.zarr/tables/<name>, so once the user has picked *which* table
+# to use it is an ordinary AnnData and every downstream config field,
+# inspection result and adapter behavior is identical to the .h5ad path.
+# That's why SpatialDataAdapter subclasses AnnDataAdapter and overrides only
+# the read.
+TABLES_GROUP = "tables"
+
+# Zarr's own "this directory is a group" markers -- v3 writes zarr.json, v2
+# writes .zgroup. Both appear in a single real store: spatialdata 0.7 wrote
+# the exemplar LSP20209.zarr with a v3 root but v2 table groups, so anything
+# sniffing the layout has to accept either.
+_GROUP_MARKERS = ("zarr.json", ".zgroup")
+
+
+def table_path(store, table) -> Path:
+    """Resolve <store>/tables/<table>, rejecting anything that isn't a plain
+    table name. Without this an empty name resolves to the tables/ group
+    itself -- which *is* a valid zarr group, so the read gets that far and
+    then fails deep inside anndata with a confusing
+    "AnnData.__init__() got an unexpected keyword argument '<some table>'".
+    Separators are refused for the same reason a table name arriving from a
+    form post shouldn't be able to address anything outside tables/.
+    """
+    name = str(table or '').strip()
+    if not name or name in ('.', '..') or '/' in name or '\\' in name:
+        raise ValueError(f"Invalid SpatialData table name: {str(table)!r}")
+    return Path(store) / TABLES_GROUP / name
+
+
+def _is_group(path: Path) -> bool:
+    return path.is_dir() and any((path / marker).exists() for marker in _GROUP_MARKERS)
+
+
+def read_spatialdata_table(store, table):
+    """Read a single table out of a SpatialData store as an AnnData.
+
+    Deliberately reads *only* the named table rather than going through
+    spatialdata.read_zarr(store, selection=('tables',)), which eagerly
+    materializes every table's X: on the exemplar LSP20209.zarr that is
+    0.55s and +846MB (it loads two 1536-dim embedding tables nobody asked
+    for) versus 0.04s and +13MB for the one table actually being imported.
+    A future plugin that genuinely needs the other elements can open the
+    full store itself -- the config records the store path alongside the
+    table name for exactly that reason.
+
+    spatialdata._io.io_table._read_table is the same per-table reader
+    sd.read_zarr calls, so this keeps spatialdata's own TableModel version
+    parsing rather than reimplementing it. It's a private symbol though, so
+    a spatialdata upgrade that moves it falls back to anndata's reader
+    (_read_table is a thin wrapper over exactly that, plus attrs
+    normalization we don't consume) instead of breaking a packaged build.
+    A test pins the import so the move is noticed rather than silently
+    absorbed.
+    """
+    path = table_path(store, table)
+    if not _is_group(path):
+        raise ValueError(
+            f"SpatialData store {str(store)!r} has no table named {str(table)!r} "
+            f"(expected a zarr group at {path})"
+        )
+
+    try:
+        from spatialdata._io.io_table import _read_table
+    except ImportError:
+        import anndata as ad
+
+        return ad.read_zarr(path)
+    return _read_table(path)
+
+
+def _node_length(node) -> int | None:
+    """Length of an on-disk AnnData node along its first axis, from zarr
+    metadata only -- never reads values.
+
+    A column is a plain array under zarr v2 but frequently a *group* under
+    v3, where anndata encodes strings as 'nullable-string-array'
+    (values/mask) and categoricals as 'categorical' (codes/categories);
+    sparse X is a group too, carrying its shape in an attr. Both encodings
+    occur in one real store -- spatialdata 0.7 wrote the exemplar
+    LSP20209.zarr with a v3 root but v2 table groups -- so this handles all
+    of them rather than assuming either.
+    """
+    shape = getattr(node, "shape", None)
+    if shape is not None:
+        return int(shape[0])
+    attr_shape = node.attrs.get("shape")
+    if attr_shape:
+        return int(attr_shape[0])
+    # A group standing in for one column: every encoding keeps a full-length
+    # array under one of these names ('categories' is deliberately absent --
+    # it holds the distinct levels, not one entry per observation).
+    for child in ("codes", "values", "data"):
+        try:
+            child_shape = getattr(node[child], "shape", None)
+        except KeyError:
+            continue
+        if child_shape is not None:
+            return int(child_shape[0])
+    return None
+
+
+def _frame_length(group, key) -> int | None:
+    """Row count of an on-disk AnnData dataframe group (obs/var). Prefers the
+    index anndata records under the '_index' attr, falling back to any column
+    -- they all share the axis length."""
+    try:
+        frame = group[key]
+    except KeyError:
+        return None
+    index_name = frame.attrs.get("_index")
+    candidates = ([index_name] if index_name else []) + [
+        k for k in frame.keys() if k != index_name
+    ]
+    for candidate in candidates:
+        try:
+            length = _node_length(frame[candidate])
+        except (KeyError, AttributeError, IndexError, TypeError):
+            continue
+        if length is not None:
+            return length
+    return None
+
+
+def _table_shape(group) -> tuple[int | None, int | None]:
+    """(n_obs, n_var) for a table group. X carries both in one place when it
+    is readable (a dense array's shape, or a sparse group's 'shape' attr);
+    obs/var are the fallback for a table whose X is absent."""
+    try:
+        x_shape = getattr(group["X"], "shape", None) or group["X"].attrs.get("shape")
+        if x_shape is not None and len(x_shape) >= 2:
+            return int(x_shape[0]), int(x_shape[1])
+    except (KeyError, AttributeError, IndexError, TypeError):
+        pass
+    return _frame_length(group, "obs"), _frame_length(group, "var")
+
+
+def list_spatialdata_tables(store) -> list[dict]:
+    """Every table in a SpatialData store, with its shape, for the import
+    form's table picker. Metadata-only: shapes come from zarr's array
+    headers, so listing all three tables of the exemplar store (one of them
+    243348x1536) costs ~5ms and ~1MB rather than loading any values.
+
+    Raises ValueError if `store` isn't a readable SpatialData store, so the
+    form can tell "not a store" apart from "a store with no tables" (a real
+    state -- a store may hold only images/labels/shapes).
+    """
+    import zarr
+
+    store_path = Path(store)
+    if not _is_group(store_path):
+        raise ValueError(f"{str(store)!r} is not a zarr store.")
+    tables_dir = store_path / TABLES_GROUP
+    if not tables_dir.is_dir():
+        return []
+
+    tables = []
+    for entry in sorted(tables_dir.iterdir()):
+        if not _is_group(entry):
+            continue
+        try:
+            # Path (not str) matches spatialdata's own reader: zarr v3 parses
+            # a string store as a URL, so table names containing characters
+            # like '#' would be truncated.
+            group = zarr.open_group(entry, mode="r")
+        except Exception:
+            continue
+        n_obs, n_var = _table_shape(group)
+        tables.append({"name": entry.name, "n_obs": n_obs, "n_var": n_var})
+    return tables
+
+
+def list_table_layers(store, table) -> list[str]:
+    """The extra expression matrices one table carries alongside its X.
+
+    Metadata-only, like list_spatialdata_tables above: zarr's group listing
+    names the children without reading any of them, so this costs a directory
+    walk rather than the hundreds of MB materializing a second matrix would.
+
+    Best-effort by design -- an unreadable or layer-less table is an ordinary
+    empty answer, not an error. The caller is asking "is there anything to
+    choose between here?", and "no" is a fine reply.
+    """
+    import zarr
+
+    path = table_path(store, table)
+    if not _is_group(path):
+        return []
+    try:
+        group = zarr.open_group(path, mode="r")
+        return sorted(str(name) for name in group["layers"].keys() if name)
+    except Exception:
+        return []
+
+
+class SpatialDataAdapter(AnnDataAdapter):
+    """Adapter for a single table inside a SpatialData (.zarr) store.
+
+    `spec.src` is the store root and `spec.table` names the table within it;
+    every other field means exactly what it means for AnnData (see
+    anndata_adapter.py), because the resolved table *is* an AnnData.
+
+    Table coordinates are used as-is, in the pixel space of the registered
+    image -- no SpatialData coordinate-system transform is applied. That
+    holds for tables written alongside the image they annotate (the
+    exemplar store's obsm['spatial'] is literally its X_centroid/Y_centroid
+    columns); a store needing a transform would have to grow explicit
+    support here.
+    """
+
+    def __init__(self, spec):
+        super().__init__(spec)
+        self.table = spec.table
+        if not self.table:
+            raise ValueError(
+                "A SpatialData datasource needs a table -- it names which "
+                "table inside the .zarr store to load."
+            )
+
+    def _read_adata(self):
+        return read_spatialdata_table(self.path, self.table)
+
+    def _read_obs(self):
+        """The table's obs group, without the rest of the table.
+
+        The same saving as the .h5ad path and for the same reason, only larger:
+        `read_spatialdata_table` already avoids the store's *other* tables, but
+        it still materializes this one's X -- and the exemplar store keeps a
+        1536-dimensional embedding table whose X is most of its 846 MB. An
+        annotation column is a directory read next to that.
+        """
+        import zarr
+
+        try:
+            from anndata.io import read_elem  # anndata >= 0.10, public API
+        except ImportError:  # pragma: no cover - older anndata
+            from anndata._io.specs import read_elem
+
+        # Path, not str: zarr v3 parses a string store as a URL, so a table name
+        # containing '#' would be truncated (same reason as list_spatialdata_tables).
+        group = zarr.open_group(table_path(self.path, self.table), mode="r")
+        return read_elem(group["obs"])