plexora 0.0.1__py3-none-any.whl

plexora/plugins/gating/server/model.py

@@ -0,0 +1,320 @@
+"""Gating's query and persistence logic.
+
+Reaches the host only through `plexora.api`. Everything it needs -- the
+feature table, the role->column map, per-plugin storage -- arrives as handles,
+so this module names no core internals and holds no core state.
+"""
+
+import pickle
+
+import numpy as np
+import polars as pl
+from scipy.stats import norm
+from sklearn.mixture import GaussianMixture
+
+from plexora import api
+from plexora.plugins.gating.server.database import LEGACY_STATE_TABLE
+
+#: Identifies this plugin's storage namespace. Must stay 'gating' -- it is what
+#: `plugin_gating_state` is keyed on, and changing it would strand saved gates.
+PLUGIN_NAME = "gating"
+
+
+def _store(datasource_name):
+    """Gating's slice of this datasource's database.
+
+    `legacy_state_table` points at the un-namespaced `gatinglist` table earlier
+    builds wrote directly. Reads fall back to it when the namespaced table is
+    empty, so upgrading the host does not lose a user's saved gates; writes
+    always go to the namespaced table, so the old one is left frozen rather
+    than kept in sync.
+    """
+    return api.store(datasource_name, PLUGIN_NAME, legacy_state_table=LEGACY_STATE_TABLE)
+
+
+def get_gated_cells(datasource_name, gates, start_keys):
+    table = api.dataset(datasource_name).table
+    if not gates:
+        return []
+    id_key = start_keys[0]
+    values = table.frame()[id_key].to_numpy()[table.range_mask(gates)].tolist()
+    return [{id_key: v} for v in values]
+
+
+def download_gating_csv(datasource_name, gates, channels, selection_ids, encoding):
+    dataset = api.dataset(datasource_name)
+    df = dataset.table.frame()
+
+    csv = df
+    # The role, not a literal column name -- schema.cell_id is whatever the
+    # project recorded. No fallback: this plugin declares cell_id in its
+    # Requires, so core collects it before the tool can open at all, and a
+    # literal default here would only mask a project that slipped through.
+    idField = dataset.schema.cell_id
+
+    if selection_ids:
+        datasource_filter = df.filter(pl.col(idField).is_in(selection_ids))
+    else:
+        datasource_filter = df
+
+    expr = None
+    for key, value in gates.items():
+        cond = (pl.col(key) > value[0]) & (pl.col(key) < value[1])
+        expr = cond if expr is None else (expr & cond)
+    if expr is not None:
+        ids = datasource_filter.filter(expr)['id'].to_numpy()
+    else:
+        # No gates set: no filter, nothing gated in. (pandas' .query('')
+        # used to raise ValueError here -- fixed rather than preserved.)
+        ids = np.array([], dtype=np.int64)
+
+    if 'Area' in channels:
+        del channels['Area']
+    is_in_ids = pl.col('id').is_in(ids)
+    for channel in channels:
+        if channel in gates:
+            # Cast to the original column's dtype for CSV-text parity with
+            # the pandas version: csv.loc[mask, channel] = 1 silently
+            # upcast an int literal into what's typically a float64 marker
+            # column (rendering "1.0"), whereas a bare Polars int literal
+            # would render "1" -- a real text diff in the exported CSV.
+            dtype = csv.schema[channel]
+            if encoding == 'binary':
+                value_expr = pl.when(is_in_ids).then(pl.lit(1)).otherwise(pl.lit(0)).cast(dtype)
+            else:
+                value_expr = pl.when(is_in_ids).then(pl.col(channel)).otherwise(pl.lit(0).cast(dtype))
+            csv = csv.with_columns(value_expr.alias(channel))
+        else:
+            csv = csv.with_columns(pl.lit(0).alias(channel))
+
+    return csv
+
+
+def download_gates(datasource_name, gates, channels):
+    rows = []
+    for key, value in channels.items():
+        rows.append([key, value[0], value[1]])
+    csv = pl.DataFrame(rows, schema=['channel', 'gate_start', 'gate_end'], orient='row')
+    csv = csv.with_columns(pl.lit(False).alias('gate_active'))
+
+    schema = csv.schema
+    for channel in gates:
+        is_channel = pl.col('channel') == channel
+        csv = csv.with_columns([
+            pl.when(is_channel).then(pl.lit(True)).otherwise(pl.col('gate_active')).alias('gate_active'),
+            pl.when(is_channel).then(pl.lit(gates[channel][0]).cast(schema['gate_start']))
+              .otherwise(pl.col('gate_start')).alias('gate_start'),
+            pl.when(is_channel).then(pl.lit(gates[channel][1]).cast(schema['gate_end']))
+              .otherwise(pl.col('gate_end')).alias('gate_end'),
+        ])
+
+    return csv
+
+
+def save_gating_list(datasource_name, gates, channels):
+    rows = []
+    for key, value in channels.items():
+        rows.append([key, value[0], value[1]])
+    csv = pl.DataFrame(rows, schema=['channel', 'gate_start', 'gate_end'], orient='row')
+    csv = csv.with_columns(pl.lit(False).alias('gate_active'))
+
+    schema = csv.schema
+    for channel in gates:
+        is_channel = pl.col('channel') == channel
+        csv = csv.with_columns([
+            pl.when(is_channel).then(pl.lit(True)).otherwise(pl.col('gate_active')).alias('gate_active'),
+            pl.when(is_channel).then(pl.lit(gates[channel][0]).cast(schema['gate_start']))
+              .otherwise(pl.col('gate_start')).alias('gate_start'),
+            pl.when(is_channel).then(pl.lit(gates[channel][1]).cast(schema['gate_end']))
+              .otherwise(pl.col('gate_end')).alias('gate_end'),
+        ])
+
+    temp = csv.to_dicts()
+    f = pickle.dumps(temp, protocol=4)
+    _store(datasource_name).put_state(f)
+
+
+def get_saved_gating_list(datasource_name):
+    cells = _store(datasource_name).get_state()
+    if cells is None:
+        return None
+    return pickle.loads(cells)
+
+
+#: Components in the auto-gate fit.
+#:
+#: Three, not two. A marker's negative population is not a point -- it is a
+#: broad distribution of background and autofluorescence, and on a log scale it
+#: is close to symmetric. A two-component fit given that has an easier job
+#: splitting the *background* down the middle than separating background from
+#: positives, and the resulting gate sits inside the negative population: every
+#: cell above the middle of the background reads as positive.
+#:
+#: With a third component the background can occupy two and the bright tail
+#: gets its own, so the boundary between the top two is a real one. Same count,
+#: and the same reasoning, as core's `get_channel_gmm` for image channels.
+_GATE_COMPONENTS = 3
+
+#: EM cost scales with N per iteration, and a 1-D mixture's fitted parameters
+#: barely move between 100k rows and millions. Fixed seed, so a given column
+#: gates the same way twice.
+_GMM_FIT_SAMPLE_CAP = 100_000
+
+
+def _fit_mixture(values, components):
+    """(means, sds, weights), ascending by mean, or None if it cannot fit.
+
+    A column with fewer distinct values than components -- a flag, a constant,
+    a nearly empty channel -- has no mixture to find, and sklearn either raises
+    or returns degenerate components. Saying so is better than a gate derived
+    from noise.
+    """
+    values = values[np.isfinite(values)]
+    if values.size < components or np.unique(values).size < components:
+        return None
+
+    sample = values
+    if sample.shape[0] > _GMM_FIT_SAMPLE_CAP:
+        rng = np.random.default_rng(0)
+        sample = sample[rng.choice(sample.shape[0], size=_GMM_FIT_SAMPLE_CAP,
+                                   replace=False)]
+
+    gmm = GaussianMixture(components, max_iter=1000, tol=1e-6, random_state=0)
+    gmm.fit(sample.reshape(-1, 1))
+    order = np.argsort(gmm.means_[:, 0])
+    return (gmm.means_[order, 0],
+            np.sqrt(gmm.covariances_[order, 0, 0]),
+            gmm.weights_[order])
+
+
+def _populations(fitted, x):
+    """(background, positive) weighted densities at `x`.
+
+    Two populations out of three components: the brightest one is the positive
+    population, and everything below it pooled is the background. Pooling
+    rather than taking the second component alone is what makes the pair add up
+    to the whole distribution, so the two curves drawn over the histogram cover
+    it instead of leaving the largest peak unexplained.
+    """
+    means, sds, weights = fitted
+    positive = norm(means[-1], sds[-1]).pdf(x) * weights[-1]
+    background = sum(norm(means[i], sds[i]).pdf(x) * weights[i]
+                     for i in range(len(means) - 1))
+    return background, positive
+
+
+def _crossover(fitted):
+    """Where the positive population overtakes the background.
+
+    The threshold, rather than `mean(means)` -- which is what this used to take
+    and which ignores both how wide each component is and how much of the data
+    it holds. A narrow background beside a broad positive tail has its midpoint
+    far below the point where the two actually change places, and that
+    difference is the gate being wrong by tens of percent of the cells.
+    """
+    means = fitted[0]
+    x = np.linspace(means[0], means[-1], 2000)
+    background, positive = _populations(fitted, x)
+    above = np.flatnonzero(positive > background)
+    # Never overtakes: nothing here is separable, so the top component's own
+    # centre is as honest an answer as there is.
+    return float(x[above[0]]) if above.size else float(means[-1])
+
+
+def auto_gate(values, log_transformed, at=None):
+    """Where the positive population starts, in the values' own units.
+
+    Fitted on a log scale whichever scale the table is stored on: marker
+    intensities are log-normal, and a mixture of *normals* fitted to raw counts
+    is fitting the wrong shape -- the components chase the skew instead of the
+    populations. So raw values are log1p'd for the fit and the answer mapped
+    back with expm1, while values the project already log1p'd are fitted as
+    they stand. The same underlying data then gives the same gate either way,
+    which it did not before: turning the log switch on used to move the gate
+    from roughly the right place to the middle of the background.
+
+    That is also why the project's own flag decides this rather than a guess at
+    the numbers. Logging twice is its own failure -- it compresses the
+    separation until a marker with 3% positives gates at 28% -- so this cannot
+    be "always log1p", and nothing in the values themselves tells the two
+    apart.
+
+    `at` asks for the fitted curves as well, evaluated at those points and in
+    the values' own units. Returns (gate, background, positive), any of which
+    is None when the column has no mixture to find -- a constant, a flag, a
+    nearly empty channel. The caller ships nothing rather than a number derived
+    from noise.
+    """
+    values = values[np.isfinite(values)]
+    # log1p rather than log: it is the transform Plexora itself applies, expm1
+    # inverts it exactly, and it is defined at zero -- which is where a large
+    # part of a quantification column sits. Negative values (arcsinh, z-scored)
+    # have no log to take, so those are fitted as they stand.
+    to_log = not log_transformed and values.size > 0 and values.min() >= 0
+    fitted = _fit_mixture(np.log1p(values) if to_log else values, _GATE_COMPONENTS)
+    if fitted is None:
+        return None, None, None
+
+    gate = _crossover(fitted)
+    gate = float(np.expm1(gate)) if to_log else gate
+    if at is None:
+        return gate, None, None
+
+    # Back into the values' own units, which is what the histogram underneath
+    # these curves is binned in. Densities do not survive a change of variable
+    # unchanged -- dividing by (1 + x) is the log1p Jacobian, and without it
+    # the curves would be the right shape at the wrong height.
+    background, positive = _populations(fitted, np.log1p(at) if to_log else at)
+    if to_log:
+        background, positive = background / (1 + at), positive / (1 + at)
+    return gate, background, positive
+
+
+def _curve(x, y):
+    """A fitted density as the client plots it. Empty when there was no fit."""
+    if y is None:
+        return []
+    return [{'x': float(x[i]), 'y': float(y[i])} for i in range(len(x))]
+
+
+def get_gating_gmm(channel_name, datasource_name, selection_ids):
+    dataset = api.dataset(datasource_name)
+    df = dataset.table.frame()
+
+    selection_key = tuple(sorted(selection_ids)) if selection_ids else None
+    cache_key = (channel_name, selection_key)
+
+    def _compute():
+        packet_gmm = {}
+
+        idField = dataset.schema.cell_id
+        if selection_ids:
+            datasource_filter = df.filter(pl.col(idField).is_in(selection_ids))
+        else:
+            # No selection to filter by (the only case current callers use,
+            # since lasso/spatial-selection was removed) -- avoid a full
+            # 2M-row copy that's immediately discarded.
+            datasource_filter = df
+
+        column_data = df[channel_name].to_numpy()
+        # The histogram the curves below are laid over -- binned on the whole
+        # column, in its own units, and deliberately not subsampled.
+        bin_edges = np.histogram_bin_edges(column_data[~np.isnan(column_data)], bins=50)
+        midpoints = (bin_edges[1:] + bin_edges[:-1]) / 2
+
+        column_data_filtered = datasource_filter[channel_name].to_numpy()
+
+        # One fit answers both: where to put the gate, and the two curves that
+        # show why it went there. They used to be able to disagree -- the
+        # curves were the fit, the gate was a summary of it that ignored their
+        # widths -- so the auto button landed somewhere the picture did not
+        # explain.
+        gate, background, positive = auto_gate(
+            column_data_filtered, dataset.table.log_transformed, at=midpoints)
+        if gate is not None:
+            packet_gmm['gate'] = gate
+        packet_gmm['gmm_1'] = _curve(midpoints, background)
+        packet_gmm['gmm_2'] = _curve(midpoints, positive)
+        return packet_gmm
+
+    return dataset.cached(cache_key, _compute)

plexora/plugins/gating/server/routes.py

@@ -0,0 +1,235 @@
+import json
+
+from flask import Blueprint, Response, abort, jsonify, request, stream_with_context
+import polars as pl
+
+from plexora import api
+from plexora.plugins.gating.server import anndata_gates
+from plexora.plugins.gating.server import model as gating_model
+from plexora.plugins.gating.server.model import PLUGIN_NAME
+
+# template_folder/static_folder make this plugin self-contained: Flask's
+# DispatchingJinjaLoader already searches every blueprint's template folder, and
+# the blueprint serves its own assets. Both were available all along and unused
+# -- core previously had to know where a module's files lived.
+gating_bp = Blueprint(
+    'gating', __name__,
+    template_folder='../templates',
+    static_folder='../static',
+    static_url_path='/static',
+)
+
+
+def _files(datasource):
+    """This plugin's own file storage for a datasource -- where the uploaded
+    gates CSV lands. Scoped per plugin rather than written straight into the
+    datasource directory, so two plugins cannot overwrite each other's uploads
+    and an uninstall knows what belongs to whom."""
+    return api.store(datasource, PLUGIN_NAME).directory()
+
+
+@gating_bp.route('/get_gated_cell_ids', methods=['GET'])
+def get_gated_cell_ids():
+    datasource = request.args.get('datasource')
+    filter = json.loads(request.args.get('filter'))
+    start_keys = list(request.args.get('start_keys').split(','))
+    resp = gating_model.get_gated_cells(datasource, filter, start_keys)
+    return api.json_response(resp)
+
+
+@gating_bp.route('/get_gating_gmm', methods=['POST'])
+def get_gating_gmm():
+    post_data = json.loads(request.data)
+    channel = post_data['channel']
+    datasource = post_data['datasource']
+    selection_ids = post_data['selection_ids']
+    resp = gating_model.get_gating_gmm(channel, datasource, selection_ids)
+    return api.json_response(resp)
+
+
+@gating_bp.route('/upload_gates', methods=['POST'])
+def upload_gates():
+    file = request.files['file']
+    if file.filename.endswith('.csv') == False:
+        abort(422)
+    datasource = request.form['datasource']
+    file.save(_files(datasource) / 'uploaded_gates.csv')
+    resp = jsonify(success=True)
+    return resp
+
+
+def _stream_csv(df, chunksize=100_000):
+    """Yield a large DataFrame as CSV in row chunks instead of materializing
+    the full serialized string (and holding it alongside the DataFrame) in
+    memory at once, as df.write_csv() would for a multi-million-row gating
+    export. Polars has no built-in chunked-string-generator, so this slices
+    and writes each chunk by hand."""
+    header = True
+    for start in range(0, df.height, chunksize):
+        yield df.slice(start, chunksize).write_csv(include_header=header)
+        header = False
+
+
+@gating_bp.route('/download_gating_csv', methods=['POST'])
+def download_gating_csv():
+    datasource = request.form['datasource']
+    filename = request.form['filename']
+
+    filter = json.loads(request.form['filter'])
+    channels = json.loads(request.form['channels'])
+    selection_ids = json.loads(request.form['selection_ids'])
+    fullCsv = json.loads(request.form['fullCsv'])
+    encoding = request.form['encoding']
+    if fullCsv:
+        csv = gating_model.download_gating_csv(datasource, filter, channels, selection_ids, encoding)
+        return Response(
+            stream_with_context(_stream_csv(csv)),
+            mimetype="text/csv",
+            headers={"Content-disposition":
+                         "attachment; filename=" + filename + ".csv"})
+    else:
+        csv = gating_model.download_gates(datasource, filter, channels)
+        return Response(
+            csv.write_csv(),
+            mimetype="text/csv",
+            headers={"Content-disposition":
+                         "attachment; filename=" + filename + ".csv"})
+
+
+@gating_bp.route('/save_gating_list', methods=['POST'])
+def save_gating_list():
+    post_data = json.loads(request.data)
+
+    datasource = post_data['datasource']
+    filter = post_data['filter']
+    channels = post_data['channels']
+
+    # DB-only on every save -- the .h5ad file is only ever written to
+    # explicitly, via the "Save Gates to AnnData" button (save_gates_to_anndata()
+    # below). Writing to the source file on every debounced slider edit was
+    # tried and reverted: the user wants edits to stay local/undo-able in the
+    # DB until they deliberately commit them to the file.
+    gating_model.save_gating_list(datasource, filter, channels)
+
+    resp = jsonify(success=True)
+    return resp
+
+
+@gating_bp.route('/get_saved_gating_list', methods=['GET'])
+def get_saved_gating_list():
+    datasource = request.args.get('datasource')
+    resp = gating_model.get_saved_gating_list(datasource)
+    return api.json_response(resp)
+
+
+@gating_bp.route('/save_gates_to_anndata', methods=['POST'])
+def save_gates_to_anndata():
+    """Writes only adata.uns[table_name] (lower gate bound per marker, one
+    column per image) back into the source .h5ad -- never a full anndata
+    rewrite. Gates are derived from the persisted GatingList row (the DB),
+    not from anything the client sends -- the DB is the single source of
+    truth here, same as CSV download/restore-on-reload.
+
+    Important: gate_active is NOT the right signal for "was this channel
+    customized" -- it only ever reflects whichever single marker is
+    currently displayed (gatingList.selections is reset on every marker
+    switch, by design, for the live single-marker slider/segmentation-
+    outline view). The gate_start/gate_end *values* for every OTHER
+    previously-gated channel are still correctly persisted though --
+    save_gating_list writes them from gating_channels, which is never
+    wiped -- so a channel counts as "has a gate" here by comparing its
+    stored gate_start/gate_end against its own true full data range
+    (get_datasource_description), exactly like the marker dropdown's
+    green-dot indicator does client-side (viewerSidebar.js's
+    hasCustomGate), not by trusting gate_active."""
+    post_data = json.loads(request.data)
+    datasource = post_data['datasource']
+    table_name = post_data.get('table_name') or 'gates'
+
+    try:
+        dataset = api.dataset(datasource)
+    except KeyError:
+        return jsonify(success=False, error="Unknown datasource"), 400
+    # A SpatialData datasource's gates go into the uns of the one table it
+    # was imported from, using the same codec -- see anndata_gates._open_group.
+    if dataset.source_kind not in ('anndata', 'spatialdata'):
+        return jsonify(success=False, error="Not an AnnData or SpatialData datasource"), 400
+    # The role, from the project record. This used to be a free-text box on
+    # the panel defaulting to the literal 'imageid' -- the plugin asking the
+    # user for something core already had a place to store. It is declared in
+    # Requires as an optional role now, so the host collects it once and every
+    # plugin sees the same answer.
+    imageid_column = dataset.schema.image_id
+    if not imageid_column:
+        return jsonify(success=False, error="No image ID column recorded for this project",
+                       needs="role:image_id"), 400
+
+    saved_rows = gating_model.get_saved_gating_list(datasource) or []
+    description = dataset.table.describe()
+    active_gates = {}
+    for row in saved_rows:
+        channel = row.get('channel')
+        if not channel:
+            continue
+        gate_start = row.get('gate_start')
+        gate_end = row.get('gate_end')
+        if gate_start is None or gate_end is None:
+            continue
+        desc = description.get(channel) or {}
+        if gate_start == desc.get('min') and gate_end == desc.get('max'):
+            continue  # still at the full default range -- never customized
+        active_gates[channel] = gate_start
+
+    try:
+        result = anndata_gates.save_gates_to_anndata(
+            dataset.table.source, datasource, active_gates,
+            table_name=table_name, imageid_column=imageid_column)
+    except ValueError as exc:
+        return jsonify(success=False, error=str(exc)), 400
+
+    return jsonify(success=True, **result)
+
+
+@gating_bp.route('/get_gates_from_anndata', methods=['GET'])
+def get_gates_from_anndata():
+    """Read-only counterpart of /save_gates_to_anndata -- lets the sidebar
+    pick up gates already present in adata.uns[table_name] (e.g. set up
+    outside Plexora before import) the first time a datasource with no
+    Plexora-side saved gating list is opened."""
+    datasource = request.args.get('datasource')
+    table_name = request.args.get('table_name') or 'gates'
+
+    try:
+        dataset = api.dataset(datasource)
+    except KeyError:
+        return jsonify(success=False, error="Unknown datasource"), 400
+    # A SpatialData datasource's gates go into the uns of the one table it
+    # was imported from, using the same codec -- see anndata_gates._open_group.
+    if dataset.source_kind not in ('anndata', 'spatialdata'):
+        return jsonify(success=False, error="Not an AnnData or SpatialData datasource"), 400
+    imageid_column = dataset.schema.image_id
+    if not imageid_column:
+        # Read-only path: no image id recorded means no gates to find, which
+        # is an ordinary empty result rather than something to demand of the
+        # user. The save path asks; this one does not.
+        return jsonify(success=True, image_id=datasource, gates={})
+
+    try:
+        result = anndata_gates.load_gates_from_anndata(
+            dataset.table.source, datasource,
+            table_name=table_name, imageid_column=imageid_column)
+    except ValueError as exc:
+        return jsonify(success=False, error=str(exc)), 400
+
+    return jsonify(success=True, **result)
+
+
+@gating_bp.route('/get_uploaded_gating_csv_values', methods=['GET'])
+def get_gating_csv_values():
+    datasource = request.args.get('datasource')
+    file_path = _files(datasource) / 'uploaded_gates.csv'
+    if file_path.is_file() == False:
+        abort(422)
+    csv = pl.read_csv(file_path)
+    obj = csv.to_dicts()
+    return api.json_response(obj)