plexora 0.0.1__py3-none-any.whl

plexora/plugins/cell_explorer/server/routes.py

@@ -0,0 +1,167 @@
+"""Cell Explorer's HTTP surface.
+
+Thin on purpose: parse, hand to the right module, translate the outcome. What a
+variable is lives in `variables`, how values are packed lives in `values`, and
+who may overwrite whom lives in `state`, so there is one place to read each and
+no route can quietly disagree with another.
+
+Status codes carry meaning, because the client acts on them differently:
+
+    400  the request is wrong -- an unknown datasource, a column this project
+         does not have. Retrying it unchanged will fail the same way.
+    409  the request was fine but somebody else saved first. The client has
+         preferences worth keeping, so this opens a prompt rather than
+         discarding either side.
+    422  the stored settings are from a newer Plexora. Nothing is written and
+         the panel says so, rather than overwriting what it cannot read.
+"""
+
+from flask import Blueprint, Response, jsonify, request
+
+from plexora import api
+from plexora.plugins.cell_explorer.server import values as value_encoder
+from plexora.plugins.cell_explorer.server import variables
+from plexora.plugins.cell_explorer.server.state import (
+    CellExplorerRepository,
+    ConflictError,
+    UnreadableState,
+)
+
+# template_folder/static_folder make this plugin self-contained: Flask's
+# DispatchingJinjaLoader already searches every blueprint's template folder, and
+# the blueprint serves its own assets.
+cell_explorer_bp = Blueprint(
+    'cell_explorer', __name__,
+    template_folder='../templates',
+    static_folder='../static',
+    static_url_path='/static',
+)
+
+
+def _dataset(datasource):
+    """A dataset for a datasource that exists, or a KeyError."""
+    if not datasource:
+        raise KeyError(datasource)
+    return api.dataset(datasource)
+
+
+@cell_explorer_bp.route('/api/variables', methods=['GET'])
+def get_variables():
+    """Every column that can be coloured by, described well enough to render
+    the panel without asking for any values."""
+    try:
+        dataset = _dataset(request.args.get('datasource'))
+    except KeyError:
+        return jsonify(success=False, error="Unknown datasource"), 400
+
+    if not dataset.table.available:
+        # Not an error: a project can legitimately be an image and nothing else,
+        # and the panel has an empty state that says what to do about it.
+        return api.json_response({"success": True, "variables": [],
+                                  "reason": "no_table"})
+
+    return api.json_response({
+        "success": True,
+        "variables": variables.describe_all(dataset),
+        # What core can draw cells with, so the panel can say "there is nothing
+        # to draw these on" before it fetches a single value.
+        "can_draw": {
+            "segmentation": dataset.segmentation.available,
+            "segmentation_pending": dataset.segmentation.pending,
+            "centroids": bool(dataset.schema and dataset.schema.x and dataset.schema.y),
+        },
+    })
+
+
+@cell_explorer_bp.route('/api/values', methods=['GET'])
+def get_values():
+    """One column's values as a packed binary buffer.
+
+    Not JSON: a million cells is ordinary here, and the difference is ~30 MB of
+    objects to parse against ~6 MB of typed array to read. See values.py for the
+    layouts, which the client mirrors in cellExplorerApi.js.
+    """
+    column = request.args.get('column') or ''
+    try:
+        dataset = _dataset(request.args.get('datasource'))
+    except KeyError:
+        return jsonify(success=False, error="Unknown datasource"), 400
+
+    descriptor = variables.find(dataset, column) if column else None
+    if descriptor is None:
+        return jsonify(success=False,
+                       error=f"No such metadata column: {column!r}"), 400
+
+    kind = request.args.get('kind')
+    if kind in ("categorical", "continuous") and kind != descriptor["kind"]:
+        # The user overrode the inference. Honoured only where the descriptor
+        # carries both halves (see variables.describe), so an override cannot
+        # ask for an encoding whose dictionary was never computed.
+        if not descriptor.get("ambiguous"):
+            return jsonify(success=False,
+                           error=f"{column!r} cannot be read as {kind}"), 400
+        descriptor = {**descriptor, "kind": kind}
+
+    try:
+        payload, resolved_kind, count = value_encoder.encode(dataset, column, descriptor)
+    except (KeyError, ValueError) as exc:
+        return jsonify(success=False, error=str(exc)), 400
+
+    response = Response(payload, mimetype="application/octet-stream")
+    response.headers["Content-Encoding"] = "gzip"
+    response.headers["X-Cell-Count"] = str(count)
+    response.headers["X-Value-Kind"] = resolved_kind
+    # So a proxy or a browser cache cannot serve one column's buffer for
+    # another's -- the URL differs, but the headers above are what the client
+    # reads to decide how to decode it.
+    response.headers["Cache-Control"] = "no-store"
+    return response
+
+
+@cell_explorer_bp.route('/api/state', methods=['GET'])
+def get_state():
+    """The stored display preferences, plus the revision to write against."""
+    datasource = request.args.get('datasource')
+    try:
+        _dataset(datasource)
+    except KeyError:
+        return jsonify(success=False, error="Unknown datasource"), 400
+
+    try:
+        state = CellExplorerRepository(datasource).load()
+    except UnreadableState as exc:
+        return jsonify(success=False, error=str(exc),
+                       schema_version=exc.schema_version), 422
+    except ValueError as exc:
+        return jsonify(success=False, error=str(exc)), 400
+
+    return api.json_response({"success": True, **state})
+
+
+@cell_explorer_bp.route('/api/state', methods=['POST'])
+def post_state():
+    """Store display preferences, if nobody else has written since the caller
+    last read."""
+    body = request.get_json(silent=True)
+    if not isinstance(body, dict):
+        return jsonify(success=False, error="expected a JSON object"), 400
+
+    datasource = body.get('datasource') or request.args.get('datasource')
+    try:
+        _dataset(datasource)
+    except KeyError:
+        return jsonify(success=False, error="Unknown datasource"), 400
+
+    try:
+        state = CellExplorerRepository(datasource).save(
+            body.get('revision'), body.get('settings'))
+    except ConflictError as exc:
+        return jsonify(success=False, error="conflict",
+                       revision=exc.current_revision), 409
+    except UnreadableState as exc:
+        return jsonify(success=False, error=str(exc),
+                       schema_version=exc.schema_version), 422
+    except ValueError as exc:
+        return jsonify(success=False, error=str(exc)), 400
+
+    return api.json_response({"success": True, **state})

plexora/plugins/cell_explorer/server/state.py

@@ -0,0 +1,281 @@
+"""What Cell Explorer remembers between sessions, and who may overwrite whom.
+
+Only display preferences: which column was showing, what colour each category
+got, which were hidden, the palette and range for a numeric one, the opacity.
+Never values. The table is the source of truth, and caching a copy of it here
+would mean a project whose data changed underneath still painted the old
+picture -- silently, and looking entirely plausible.
+
+The store writes one opaque blob per plugin per datasource: no partial updates,
+no revisions of its own. So the revision lives INSIDE the document and is
+checked here, which is also why routes never touch the store directly.
+
+Why a revision at all for a single-user desktop app: two tabs on the same
+project is not exotic, and neither is a tab left open from yesterday. Both hold
+a full copy of these preferences and both autosave. Last-writer-wins means the
+stale tab quietly reinstates its whole world -- every recoloured category
+reverted, with no error and nothing to notice. The check costs an integer
+comparison.
+
+Unknown keys are preserved rather than dropped (see `normalize`). A newer
+Plexora writing a field this one does not model must not have it stripped by the
+first save from an older tab.
+"""
+
+from __future__ import annotations
+
+import json
+import threading
+
+from plexora import api
+
+PLUGIN_NAME = "cell_explorer"
+
+#: Bumped only for a change this version cannot read. A field that is merely
+#: new does not need it -- `normalize` fills defaults and keeps what it does not
+#: recognise, so additive changes are already safe both ways.
+SCHEMA_VERSION = 1
+
+#: Matches ImageViewer.DEFAULT_CELL_LAYER_OPACITY. Filled masks hide the tissue
+#: at full strength and outlines read poorly when faint, and one control serves
+#: both.
+DEFAULT_OPACITY = 0.7
+
+DEFAULT_PALETTE = "viridis"
+PALETTES = ("viridis", "magma", "cividis", "coolwarm", "custom")
+MODES = ("none", "centroids", "outlines", "filled")
+
+
+class ConflictError(Exception):
+    """Somebody else wrote since the caller last read.
+
+    Carries the current revision so the client can say what happened rather
+    than just failing.
+    """
+
+    def __init__(self, current_revision):
+        super().__init__("Cell Explorer settings changed in another session")
+        self.current_revision = current_revision
+
+
+class UnreadableState(Exception):
+    """The stored document is from a version this one cannot read.
+
+    Deliberately not "start fresh". Overwriting a newer tab's settings because
+    this build does not understand them destroys work to avoid showing a
+    banner.
+    """
+
+    def __init__(self, schema_version):
+        super().__init__(
+            f"stored settings use schema version {schema_version}, "
+            f"which this version of Plexora cannot read"
+        )
+        self.schema_version = schema_version
+
+
+#: One lock per datasource, held across the read-modify-write in `save`.
+#: Plexora serves on waitress, which is multi-threaded, so two tabs saving at
+#: the same moment really do land in two threads -- and without this they can
+#: both read revision 41, both write revision 42, and one of them vanishes
+#: without ever being told it conflicted.
+_LOCKS: dict[str, threading.Lock] = {}
+_LOCKS_GUARD = threading.Lock()
+
+
+def _lock_for(datasource):
+    with _LOCKS_GUARD:
+        lock = _LOCKS.get(datasource)
+        if lock is None:
+            lock = _LOCKS[datasource] = threading.Lock()
+        return lock
+
+
+def default_state() -> dict:
+    return {
+        "schema_version": SCHEMA_VERSION,
+        "revision": 0,
+        # Which column is showing. None means "decide for me" -- the panel then
+        # picks the best candidate rather than opening on nothing.
+        "selected": None,
+        "display": {"mode": None, "opacity": DEFAULT_OPACITY},
+        # column -> "categorical" | "continuous", where the user disagreed with
+        # the inference. Per column, because the answer is about that column.
+        "overrides": {},
+        # column -> {"colors": {category: "#rrggbb"}, "hidden": [category, ...]}
+        "categorical": {},
+        # column -> {"palette", "custom": {"low", "high"}, "range": {...},
+        #            "hidden": bool}
+        "continuous": {},
+    }
+
+
+class CellExplorerRepository:
+    """One project's display preferences."""
+
+    def __init__(self, datasource):
+        self.datasource = datasource
+        self._store = api.store(datasource, PLUGIN_NAME)
+
+    def load(self) -> dict:
+        """The stored document, or the default for a project nobody has opened.
+
+        The default is not persisted: opening the panel on a fresh project
+        should leave the project exactly as it found it.
+        """
+        blob = self._store.get_state()
+        if not blob:
+            return default_state()
+        try:
+            raw = json.loads(blob.decode("utf-8"))
+        except (UnicodeDecodeError, ValueError) as exc:
+            # Deliberately loud. Quietly handing back an empty document presents
+            # "your settings are gone" as "this project has no settings", and
+            # the next autosave makes it true.
+            raise ValueError(f"stored Cell Explorer settings could not be read: {exc}") from exc
+        return normalize(raw)
+
+    def save(self, base_revision, settings) -> dict:
+        """Store the document if the caller was up to date, and bump the revision.
+
+        Returns the stored document. Raises ConflictError when somebody else
+        wrote first (nothing is stored), ValueError when the payload is not a
+        document.
+        """
+        if not isinstance(base_revision, int) or isinstance(base_revision, bool):
+            raise ValueError("base_revision must be an integer")
+        if not isinstance(settings, dict):
+            raise ValueError("settings must be an object")
+
+        with _lock_for(self.datasource):
+            current = self.load()
+            if current["revision"] != base_revision:
+                raise ConflictError(current["revision"])
+            updated = normalize(settings)
+            updated["revision"] = current["revision"] + 1
+            self._store.put_state(
+                json.dumps(updated, separators=(",", ":")).encode("utf-8"))
+            return updated
+
+
+def normalize(raw) -> dict:
+    """A stored document, cleaned into the shape the panel expects.
+
+    Everything is bounds-checked rather than trusted: this blob has been on disk
+    across upgrades, and a palette name or an opacity that no longer means
+    anything should fall back to a default rather than reach the renderer.
+
+    Per-column entries are NOT filtered against the current table. A column that
+    is temporarily absent -- a data file swapped, a different image loaded --
+    comes back, and dropping its colours the one time it was missing loses work
+    a user did deliberately. The panel ignores entries it cannot match.
+    """
+    if not isinstance(raw, dict):
+        raise ValueError("stored settings are not an object")
+
+    version = raw.get("schema_version", SCHEMA_VERSION)
+    if not isinstance(version, int) or version > SCHEMA_VERSION:
+        raise UnreadableState(version)
+
+    state = dict(raw)
+    state["schema_version"] = SCHEMA_VERSION
+    state["revision"] = _int(raw.get("revision"), 0)
+    state["selected"] = _text_or_none(raw.get("selected"))
+    state["display"] = _display(raw.get("display"))
+    state["overrides"] = {
+        str(column): kind
+        for column, kind in _mapping(raw.get("overrides")).items()
+        if kind in ("categorical", "continuous")
+    }
+    state["categorical"] = {
+        str(column): _categorical_entry(entry)
+        for column, entry in _mapping(raw.get("categorical")).items()
+    }
+    state["continuous"] = {
+        str(column): _continuous_entry(entry)
+        for column, entry in _mapping(raw.get("continuous")).items()
+    }
+    return state
+
+
+def _display(raw) -> dict:
+    raw = _mapping(raw)
+    mode = raw.get("mode")
+    return {
+        "mode": mode if mode in MODES else None,
+        "opacity": _clamped(raw.get("opacity"), DEFAULT_OPACITY),
+    }
+
+
+def _categorical_entry(raw) -> dict:
+    raw = _mapping(raw)
+    colors = {
+        str(category): color
+        for category, color in _mapping(raw.get("colors")).items()
+        if _is_hex(color)
+    }
+    hidden = sorted({str(category) for category in raw.get("hidden") or ()
+                     if isinstance(category, (str, int, float, bool))})
+    return {"colors": colors, "hidden": hidden}
+
+
+def _continuous_entry(raw) -> dict:
+    raw = _mapping(raw)
+    palette = raw.get("palette")
+    custom = _mapping(raw.get("custom"))
+    range_raw = _mapping(raw.get("range"))
+    low = _number_or_none(range_raw.get("min"))
+    high = _number_or_none(range_raw.get("max"))
+    manual = (range_raw.get("mode") == "manual"
+              and low is not None and high is not None and low < high)
+    return {
+        "palette": palette if palette in PALETTES else DEFAULT_PALETTE,
+        "custom": {
+            "low": custom.get("low") if _is_hex(custom.get("low")) else None,
+            "high": custom.get("high") if _is_hex(custom.get("high")) else None,
+        },
+        # Whether the overlay is drawn at all. A boolean here and a list of
+        # labels on a categorical entry: a ramp has no rows to hide one of, so
+        # the whole column is the unit. Listed explicitly because these entries
+        # are rebuilt key by key rather than passed through -- an unmodelled
+        # field on one of them is dropped by the first save.
+        "hidden": bool(raw.get("hidden")),
+        # A manual range that no longer makes sense (min >= max after the source
+        # changed) reverts to auto rather than being kept and refusing to draw.
+        "range": ({"mode": "manual", "min": low, "max": high} if manual
+                  else {"mode": "auto"}),
+    }
+
+
+def _mapping(value) -> dict:
+    return value if isinstance(value, dict) else {}
+
+
+def _int(value, fallback) -> int:
+    return value if isinstance(value, int) and not isinstance(value, bool) else fallback
+
+
+def _text_or_none(value):
+    return value if isinstance(value, str) and value else None
+
+
+def _number_or_none(value):
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        return None
+    return float(value) if value == value and abs(value) != float("inf") else None
+
+
+def _clamped(value, fallback) -> float:
+    number = _number_or_none(value)
+    if number is None:
+        return fallback
+    return max(0.0, min(1.0, number))
+
+
+def _is_hex(value) -> bool:
+    """#rrggbb only. Anything else reaches a canvas fillStyle, where an
+    unrecognised string is silently ignored and the cell keeps whatever colour
+    was set last -- a wrong picture rather than a missing one."""
+    if not isinstance(value, str) or len(value) != 7 or not value.startswith("#"):
+        return False
+    return all(character in "0123456789abcdefABCDEF" for character in value[1:])

plexora/plugins/cell_explorer/server/values.py

@@ -0,0 +1,145 @@
+"""One column's values, on the wire.
+
+A million cells is an ordinary image here, and the browser needs a value per
+cell to build a colour lookup table. As JSON objects that is roughly 30 MB of
+`{"cell_id": 1, "value": 0.52}` to serialize, send and parse; as a packed
+structured array it is 6 MB, gzipped to less, and arrives as typed arrays the
+LUT builder can read directly with no per-row work at all.
+
+Two layouts, one per kind:
+
+    categorical   id: uint32, code: uint16      -- an index into the descriptor's
+                                                   category list
+    continuous    id: uint32, value: float32
+
+Categorical is dictionary-encoded rather than sending repeated strings, which is
+where most of the saving is: "Macrophage" is eleven bytes per cell and two as a
+code. The dictionary is the descriptor's `categories`, already fetched, so
+nothing is sent twice.
+
+float32 rather than float64 because this drives a colour ramp with 256 stops.
+The stored data is untouched -- this is a display copy.
+
+**The ids are the contract.** They are the same values the segmentation mask's
+labels carry and the same ones the centroid cache packs, derived by the same
+rule (`centroid_tiles.build_cache`): the cell-id column, cast to float, rows
+with a non-finite id dropped, the rest to uint32. A different rule here would
+produce a payload that is individually correct and lines up with nothing.
+"""
+
+from __future__ import annotations
+
+import gzip
+
+import numpy as np
+import polars as pl
+
+from plexora.plugins.cell_explorer.server import variables
+
+#: Code for "this cell has no value". uint16 max, so it can never collide with a
+#: real category index -- a column with 65535 categories is refused long before
+#: this by the identifier check.
+MISSING_CODE = 0xFFFF
+
+CATEGORICAL_DTYPE = np.dtype([("id", "<u4"), ("code", "<u2")])
+CONTINUOUS_DTYPE = np.dtype([("id", "<u4"), ("value", "<f4")])
+
+
+def cell_ids(dataset):
+    """(ids, keep) for the loaded table.
+
+    `keep` is the row mask, returned alongside rather than applied, because
+    every caller has a values array that has to lose exactly the same rows. A
+    cell whose id will not survive the round trip to uint32 has no way to be
+    matched against a mask label, so it is dropped rather than drawn somewhere
+    arbitrary.
+    """
+    frame = dataset.table.frame()
+    if frame is None:
+        return np.empty(0, dtype=np.uint32), np.empty(0, dtype=bool)
+    column = (dataset.schema.cell_id if dataset.schema else None) or "id"
+    if column not in frame.columns:
+        # The role names a column of the SOURCE for the structural formats; the
+        # table the adapter emits carries its own positional "id". Falling back
+        # keeps a project whose role was answered against obs working.
+        column = "id"
+    raw = (frame[column].cast(pl.Float64, strict=False)
+           .fill_null(float("nan")).to_numpy())
+    keep = np.isfinite(raw)
+    return raw[keep].astype(np.uint32, copy=False), keep
+
+
+def encode(dataset, column: str, descriptor: dict) -> tuple[bytes, str, int]:
+    """(gzipped bytes, kind, cell count) for one column.
+
+    `descriptor` is passed in rather than looked up so the codes below are
+    indexed against the very category list the client was given. Recomputing it
+    here would work right up until the two orders diverged, at which point every
+    cell is labelled as some other category and nothing reports it.
+    """
+    ids, keep = cell_ids(dataset)
+    values = np.asarray(dataset.table.metadata_values(column).values)
+    if values.size == keep.size:
+        values = values[keep]
+    elif values.size != ids.size:
+        raise ValueError(
+            f"column {column!r} has {values.size} values but the table has "
+            f"{keep.size} rows"
+        )
+
+    kind = descriptor.get("kind", "categorical")
+    if kind == "continuous":
+        payload = _continuous(ids, values)
+    else:
+        payload = _categorical(ids, values, descriptor.get("categories") or [])
+    return gzip.compress(payload.tobytes(), 5), kind, int(ids.size)
+
+
+def _continuous(ids, values):
+    numeric = _as_float(values)
+    # Infinities join NaN as "no value". They are not the top of the scale --
+    # clipping them there would paint a handful of cells the extreme colour and
+    # say nothing about why.
+    numeric = np.where(np.isfinite(numeric), numeric, np.nan)
+    record = np.empty(ids.size, dtype=CONTINUOUS_DTYPE)
+    record["id"] = ids
+    record["value"] = numeric.astype(np.float32, copy=False)
+    return record
+
+
+def _as_float(values):
+    if values.dtype.kind in "iuf":
+        return values.astype(np.float64, copy=False)
+    if values.dtype.kind == "b":
+        return values.astype(np.float64)
+    # A text column the user overrode to continuous. Anything unparseable
+    # becomes NaN, which is the same "no value" every other route takes.
+    return np.array([_to_float(value) for value in values], dtype=np.float64)
+
+
+def _to_float(value):
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return float("nan")
+
+
+def _categorical(ids, values, categories):
+    text, missing = variables.as_text(values)
+    labels = np.array([entry["value"] for entry in categories], dtype=str)
+
+    codes = np.full(ids.size, MISSING_CODE, dtype=np.uint16)
+    if labels.size and text.size:
+        # Vectorized lookup: sort the dictionary, binary-search every value into
+        # it, then map back through the sort permutation. A dict comprehension
+        # over the values would be a Python loop over millions of rows.
+        order = np.argsort(labels)
+        ordered = labels[order]
+        position = np.clip(np.searchsorted(ordered, text), 0, ordered.size - 1)
+        matched = (ordered[position] == text) & ~missing
+        codes[matched] = order[position][matched].astype(np.uint16)
+
+    record = np.empty(ids.size, dtype=CATEGORICAL_DTYPE)
+    record["id"] = ids
+    record["code"] = codes
+    return record