reader-workbench 1.0.0__py3-none-any.whl

reader_workbench/domains/cytometry/analysis/gating.py

@@ -0,0 +1,175 @@
+"""Sequential gating and statistical summaries for cytometry events."""
+
+from __future__ import annotations
+
+import math
+
+import polars as pl
+
+from reader_workbench.domains.cytometry.analysis.events import (
+    _METADATA_COLUMNS,
+    CytometryAnalysis,
+    CytometryAnalysisError,
+    GateSpec,
+    ThresholdSpec,
+    _require_columns,
+)
+
+
+def analyze_events(
+    event_table: pl.DataFrame,
+    *,
+    gate: GateSpec,
+    threshold: ThresholdSpec,
+    group_column: str | None = None,
+) -> CytometryAnalysis:
+    """Apply sequential gates and compute per-sample and group summaries in Polars."""
+
+    required_columns = [threshold.channel, "sample_id"]
+    if gate.cells_enabled:
+        required_columns.extend((gate.cells_x_channel, gate.cells_y_channel))
+    if gate.singlets_enabled:
+        required_columns.extend((gate.singlet_x_channel, gate.singlet_y_channel))
+    _require_columns(event_table, required_columns)
+
+    cells_mask = pl.lit(True)
+    if gate.cells_enabled:
+        _validate_interval("cells X", gate.cells_x_range)
+        _validate_interval("cells Y", gate.cells_y_range)
+        cells_x = pl.col(gate.cells_x_channel).cast(pl.Float64, strict=False)
+        cells_y = pl.col(gate.cells_y_channel).cast(pl.Float64, strict=False)
+        cells_mask = (
+            cells_x.is_finite()
+            & cells_y.is_finite()
+            & cells_x.is_between(*gate.cells_x_range, closed="both")
+            & cells_y.is_between(*gate.cells_y_range, closed="both")
+        )
+
+    singlet_mask = pl.lit(True)
+    if gate.singlets_enabled:
+        _validate_interval("singlet ratio", gate.singlet_ratio_range)
+        singlet_x = pl.col(gate.singlet_x_channel).cast(pl.Float64, strict=False)
+        singlet_y = pl.col(gate.singlet_y_channel).cast(pl.Float64, strict=False)
+        ratio = singlet_y / singlet_x
+        singlet_mask = ratio.is_finite() & ratio.is_between(*gate.singlet_ratio_range, closed="both")
+
+    cells_mask_column = "__reader_cells_mask"
+    gate_mask_column = "__reader_gate_mask"
+    work = event_table.with_columns(
+        cells_mask.alias(cells_mask_column),
+        (cells_mask & singlet_mask).alias(gate_mask_column),
+    )
+    gated_events = work.filter(pl.col(gate_mask_column)).drop(cells_mask_column, gate_mask_column)
+    if gated_events.is_empty():
+        raise CytometryAnalysisError("No events remain after gating. Adjust ranges.")
+
+    if group_column is not None:
+        if not isinstance(group_column, str) or not group_column.strip():
+            raise CytometryAnalysisError("Group column must be a non-empty string or null.")
+        group_column = group_column.strip()
+        _require_columns(event_table, (group_column,))
+    metadata_columns = list(
+        dict.fromkeys(
+            (
+                *[column for column in _METADATA_COLUMNS if column in event_table.columns],
+                *([group_column] if group_column else []),
+            )
+        )
+    )
+    counts = work.group_by("sample_id", maintain_order=True).agg(
+        *[pl.col(column).first().alias(column) for column in metadata_columns],
+        pl.len().alias("n_total_events"),
+        pl.col(cells_mask_column).sum().cast(pl.Int64).alias("n_cells_gate"),
+        pl.col(gate_mask_column).sum().cast(pl.Int64).alias("n_singlets"),
+    )
+    counts = counts.with_columns(
+        pl.when(pl.col("n_total_events") > 0)
+        .then(100.0 * pl.col("n_cells_gate") / pl.col("n_total_events"))
+        .otherwise(float("nan"))
+        .alias("pct_cells"),
+        pl.when(pl.col("n_cells_gate") > 0)
+        .then(100.0 * pl.col("n_singlets") / pl.col("n_cells_gate"))
+        .otherwise(float("nan"))
+        .alias("pct_singlets_of_cells"),
+        pl.when(pl.col("n_total_events") > 0)
+        .then(100.0 * pl.col("n_singlets") / pl.col("n_total_events"))
+        .otherwise(float("nan"))
+        .alias("pct_final"),
+    )
+
+    threshold_value = _resolve_threshold(gated_events, threshold)
+    fluor_column = "__reader_fluor"
+    gated_for_stats = gated_events.with_columns(
+        pl.col(threshold.channel).cast(pl.Float64, strict=False).alias(fluor_column)
+    )
+    finite_fluor = pl.col(fluor_column).filter(pl.col(fluor_column).is_finite())
+    positive_fluor = finite_fluor.filter(finite_fluor > 0)
+    sample_stats = gated_for_stats.group_by("sample_id", maintain_order=True).agg(
+        finite_fluor.median().alias("fluor_median"),
+        finite_fluor.mean().alias("fluor_mean"),
+        positive_fluor.log().mean().exp().alias("fluor_geomean"),
+        finite_fluor.quantile(0.90, interpolation="linear").alias("fluor_p90"),
+        finite_fluor.quantile(0.99, interpolation="linear").alias("fluor_p99"),
+        (100.0 * (finite_fluor > threshold_value).mean()).alias("pct_positive"),
+    )
+    stats_sample = counts.join(sample_stats, on="sample_id", how="left")
+
+    stats_group = None
+    if group_column is not None:
+        stats_group = stats_sample.group_by(group_column, maintain_order=True).agg(
+            pl.col("sample_id").n_unique().alias("n_samples"),
+            pl.col("fluor_median").mean().alias("fluor_median_mean"),
+            pl.col("fluor_median").std().alias("fluor_median_std"),
+            pl.col("fluor_geomean").mean().alias("fluor_geomean_mean"),
+            pl.col("pct_positive").mean().alias("pct_positive_mean"),
+        )
+
+    all_fluor = pl.col(threshold.channel).cast(pl.Float64, strict=False)
+    finite_all_fluor = all_fluor.filter(all_fluor.is_finite())
+    qc_table = event_table.group_by("sample_id", maintain_order=True).agg(
+        pl.when(finite_all_fluor.len() > 0)
+        .then(100.0 * (finite_all_fluor <= 0).mean())
+        .otherwise(float("nan"))
+        .alias("pct_nonpositive")
+    )
+
+    return CytometryAnalysis(
+        gated_events=gated_events,
+        gate_counts_sample=counts,
+        stats_sample=stats_sample,
+        stats_group=stats_group,
+        qc_table=qc_table,
+        threshold_value=threshold_value,
+    )
+
+
+def _validate_interval(label: str, interval: tuple[float, float]) -> None:
+    low, high = interval
+    if not math.isfinite(low) or not math.isfinite(high) or high < low:
+        raise CytometryAnalysisError(f"{label} range must contain two finite values in ascending order.")
+
+
+def _resolve_threshold(gated_events: pl.DataFrame, threshold: ThresholdSpec) -> float:
+    if threshold.mode == "manual":
+        value = float(threshold.value)
+    elif threshold.mode == "from_control_quantile":
+        if threshold.group_column is None or threshold.control_value is None:
+            raise CytometryAnalysisError("Control thresholding requires a group column and control value.")
+        _require_columns(gated_events, (threshold.group_column, threshold.channel))
+        if not 0.0 <= threshold.quantile <= 1.0:
+            raise CytometryAnalysisError("Control quantile must be between 0 and 1.")
+        control_values = (
+            gated_events.filter(pl.col(threshold.group_column).cast(pl.String) == threshold.control_value)
+            .get_column(threshold.channel)
+            .cast(pl.Float64, strict=False)
+            .drop_nulls()
+        )
+        control_values = control_values.filter(control_values.is_finite())
+        if control_values.is_empty():
+            raise CytometryAnalysisError("No control events are available for thresholding.")
+        value = float(control_values.quantile(threshold.quantile, interpolation="linear"))
+    else:
+        raise CytometryAnalysisError(f"Unknown threshold mode `{threshold.mode}`.")
+    if not math.isfinite(value):
+        raise CytometryAnalysisError("Threshold value must be finite.")
+    return value

reader_workbench/domains/cytometry/analysis/workflow.py

@@ -0,0 +1,248 @@
+"""Explicit normal-lifecycle cytometry gating workflow."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Literal
+
+import pandas as pd
+import polars as pl
+
+from .events import CytometryAnalysisError, GateSpec, ThresholdSpec, prepare_event_table
+from .gating import analyze_events
+
+_NONPOSITIVE_EVALUABLE_EVENTS_COLUMN = "__reader_nonpositive_evaluable_events"
+
+
+@dataclass(frozen=True, slots=True)
+class CytometryQCSpec:
+    minimum_final_events: int
+    minimum_final_percent: float
+    maximum_nonpositive_percent: float
+    nonpositive_scope: Literal["all_events", "gated_events"]
+
+
+@dataclass(frozen=True, slots=True)
+class CytometryGatingRequest:
+    gate: GateSpec
+    threshold: ThresholdSpec
+    group_column: str | None
+    qc: CytometryQCSpec
+
+
+@dataclass(frozen=True, slots=True)
+class CytometryGatingResult:
+    gate_definition: pl.DataFrame
+    gated_events: pl.DataFrame
+    sample_stats: pl.DataFrame
+    group_stats: pl.DataFrame
+    qc: pl.DataFrame
+
+
+def run_cytometry_gating(events: pd.DataFrame | pl.DataFrame, request: CytometryGatingRequest) -> CytometryGatingResult:
+    """Resolve an explicit gating request into typed, persistence-ready tables."""
+
+    _validate_request(request)
+    source = pl.from_pandas(events) if isinstance(events, pd.DataFrame) else events
+    if not isinstance(source, pl.DataFrame):
+        raise TypeError(f"Expected pandas or Polars event data, got {type(events).__name__}.")
+
+    selected_channels: list[str] = []
+    if request.gate.cells_enabled:
+        selected_channels.extend((request.gate.cells_x_channel, request.gate.cells_y_channel))
+    if request.gate.singlets_enabled:
+        selected_channels.extend((request.gate.singlet_x_channel, request.gate.singlet_y_channel))
+    selected_channels.append(request.threshold.channel)
+    metadata_columns = tuple(
+        dict.fromkeys(
+            column
+            for column in (request.group_column, request.threshold.group_column)
+            if isinstance(column, str) and column
+        )
+    )
+    wide = prepare_event_table(
+        source,
+        channels=tuple(dict.fromkeys(selected_channels)),
+        metadata_columns=metadata_columns,
+    )
+    analysis = analyze_events(
+        wide,
+        gate=request.gate,
+        threshold=request.threshold,
+        group_column=request.group_column,
+    )
+    sample_stats = _sample_stats(analysis.stats_sample, request=request, threshold_value=analysis.threshold_value)
+    group_stats = _group_stats(analysis.stats_group, request=request)
+    nonpositive_source = wide if request.qc.nonpositive_scope == "all_events" else analysis.gated_events
+    qc = _qc_table(
+        analysis.gate_counts_sample,
+        _nonpositive_table(nonpositive_source, channel=request.threshold.channel),
+        request=request,
+    )
+    return CytometryGatingResult(
+        gate_definition=_gate_definition(request, threshold_value=analysis.threshold_value),
+        gated_events=analysis.gated_events,
+        sample_stats=sample_stats,
+        group_stats=group_stats,
+        qc=qc,
+    )
+
+
+def _validate_request(request: CytometryGatingRequest) -> None:
+    if request.group_column is not None and (
+        not isinstance(request.group_column, str) or not request.group_column.strip()
+    ):
+        raise CytometryAnalysisError("group_column must be a non-empty string or null.")
+    if request.threshold.mode == "manual":
+        if request.threshold.group_column is not None or request.threshold.control_value is not None:
+            raise CytometryAnalysisError("Manual thresholding may not declare control-group fields.")
+    elif request.threshold.mode == "from_control_quantile":
+        if not request.threshold.group_column or not request.threshold.control_value:
+            raise CytometryAnalysisError("Control thresholding requires explicit group_column and control_value.")
+    else:
+        raise CytometryAnalysisError(f"Unknown threshold mode `{request.threshold.mode}`.")
+    if request.qc.minimum_final_events < 0:
+        raise CytometryAnalysisError("minimum_final_events must be nonnegative.")
+    if request.qc.nonpositive_scope not in {"all_events", "gated_events"}:
+        raise CytometryAnalysisError("nonpositive_scope must be 'all_events' or 'gated_events'.")
+    for name, value in (
+        ("minimum_final_percent", request.qc.minimum_final_percent),
+        ("maximum_nonpositive_percent", request.qc.maximum_nonpositive_percent),
+    ):
+        if not 0.0 <= float(value) <= 100.0:
+            raise CytometryAnalysisError(f"{name} must be between 0 and 100.")
+
+
+def _gate_definition(request: CytometryGatingRequest, *, threshold_value: float) -> pl.DataFrame:
+    return pl.DataFrame(
+        {
+            "definition_id": ["resolved"],
+            "cells_enabled": [request.gate.cells_enabled],
+            "cells_x_channel": [request.gate.cells_x_channel],
+            "cells_x_min": [float(request.gate.cells_x_range[0])],
+            "cells_x_max": [float(request.gate.cells_x_range[1])],
+            "cells_y_channel": [request.gate.cells_y_channel],
+            "cells_y_min": [float(request.gate.cells_y_range[0])],
+            "cells_y_max": [float(request.gate.cells_y_range[1])],
+            "singlets_enabled": [request.gate.singlets_enabled],
+            "singlet_x_channel": [request.gate.singlet_x_channel],
+            "singlet_y_channel": [request.gate.singlet_y_channel],
+            "singlet_ratio_min": [float(request.gate.singlet_ratio_range[0])],
+            "singlet_ratio_max": [float(request.gate.singlet_ratio_range[1])],
+            "fluorescence_channel": [request.threshold.channel],
+            "threshold_mode": [request.threshold.mode],
+            "threshold_value": [float(threshold_value)],
+            "threshold_group_column": [request.threshold.group_column],
+            "threshold_control_value": [request.threshold.control_value],
+            "threshold_quantile": [
+                float(request.threshold.quantile) if request.threshold.mode == "from_control_quantile" else None
+            ],
+            "group_column": [request.group_column],
+            "minimum_final_events": [int(request.qc.minimum_final_events)],
+            "minimum_final_percent": [float(request.qc.minimum_final_percent)],
+            "maximum_nonpositive_percent": [float(request.qc.maximum_nonpositive_percent)],
+            "nonpositive_scope": [request.qc.nonpositive_scope],
+        },
+        schema_overrides={
+            "threshold_group_column": pl.String,
+            "threshold_control_value": pl.String,
+            "threshold_quantile": pl.Float64,
+            "group_column": pl.String,
+        },
+    )
+
+
+def _sample_stats(stats: pl.DataFrame, *, request: CytometryGatingRequest, threshold_value: float) -> pl.DataFrame:
+    group_value = (
+        pl.col(request.group_column).cast(pl.String)
+        if request.group_column is not None
+        else pl.lit(None, dtype=pl.String)
+    )
+    return stats.select(
+        "sample_id",
+        pl.lit(request.group_column, dtype=pl.String).alias("group_column"),
+        group_value.alias("group_value"),
+        "n_total_events",
+        "n_cells_gate",
+        "n_singlets",
+        "pct_cells",
+        "pct_singlets_of_cells",
+        "pct_final",
+        "fluor_median",
+        "fluor_mean",
+        "fluor_geomean",
+        "fluor_p90",
+        "fluor_p99",
+        "pct_positive",
+        pl.lit(request.threshold.channel).alias("fluorescence_channel"),
+        pl.lit(float(threshold_value)).alias("threshold_value"),
+    )
+
+
+def _group_stats(stats: pl.DataFrame | None, *, request: CytometryGatingRequest) -> pl.DataFrame:
+    schema = {
+        "group_column": pl.String,
+        "group_value": pl.String,
+        "n_samples": pl.Int64,
+        "fluor_median_mean": pl.Float64,
+        "fluor_median_std": pl.Float64,
+        "fluor_geomean_mean": pl.Float64,
+        "pct_positive_mean": pl.Float64,
+    }
+    if request.group_column is None or stats is None:
+        return pl.DataFrame(schema=schema)
+    return stats.select(
+        pl.lit(request.group_column).alias("group_column"),
+        pl.col(request.group_column).cast(pl.String).alias("group_value"),
+        "n_samples",
+        "fluor_median_mean",
+        "fluor_median_std",
+        "fluor_geomean_mean",
+        "pct_positive_mean",
+    )
+
+
+def _qc_table(counts: pl.DataFrame, nonpositive: pl.DataFrame, *, request: CytometryGatingRequest) -> pl.DataFrame:
+    joined = counts.select(
+        "sample_id",
+        "n_total_events",
+        "n_cells_gate",
+        "n_singlets",
+        "pct_final",
+    ).join(nonpositive, on="sample_id", how="left")
+    # Keep the persisted percentage finite, but require a real denominator so
+    # even a permissive 100% ceiling cannot pass an unevaluable sample.
+    return (
+        joined.with_columns(
+            pl.col("pct_nonpositive").fill_nan(100.0).fill_null(100.0),
+            pl.col(_NONPOSITIVE_EVALUABLE_EVENTS_COLUMN).fill_null(0).cast(pl.Int64),
+            pl.lit(int(request.qc.minimum_final_events)).alias("minimum_final_events"),
+            pl.lit(float(request.qc.minimum_final_percent)).alias("minimum_final_percent"),
+            pl.lit(float(request.qc.maximum_nonpositive_percent)).alias("maximum_nonpositive_percent"),
+            pl.lit(request.qc.nonpositive_scope).alias("nonpositive_scope"),
+        )
+        .with_columns(
+            (pl.col("n_singlets") >= pl.col("minimum_final_events")).alias("passes_final_events"),
+            (pl.col("pct_final") >= pl.col("minimum_final_percent")).alias("passes_final_percent"),
+            (
+                (pl.col(_NONPOSITIVE_EVALUABLE_EVENTS_COLUMN) > 0)
+                & (pl.col("pct_nonpositive") <= pl.col("maximum_nonpositive_percent"))
+            ).alias("passes_nonpositive"),
+        )
+        .with_columns(
+            (pl.col("passes_final_events") & pl.col("passes_final_percent") & pl.col("passes_nonpositive")).alias(
+                "qc_pass"
+            )
+        )
+        .with_columns(pl.when(pl.col("qc_pass")).then(pl.lit("pass")).otherwise(pl.lit("fail")).alias("qc_status"))
+        .drop(_NONPOSITIVE_EVALUABLE_EVENTS_COLUMN)
+    )
+
+
+def _nonpositive_table(events: pl.DataFrame, *, channel: str) -> pl.DataFrame:
+    fluorescence = pl.col(channel).cast(pl.Float64, strict=False)
+    finite = fluorescence.filter(fluorescence.is_finite())
+    return events.group_by("sample_id", maintain_order=True).agg(
+        finite.len().cast(pl.Int64).alias(_NONPOSITIVE_EVALUABLE_EVENTS_COLUMN),
+        pl.when(finite.len() > 0).then(100.0 * (finite <= 0).mean()).otherwise(100.0).alias("pct_nonpositive"),
+    )

reader_workbench/domains/cytometry/io/__init__.py

@@ -0,0 +1,3 @@
+"""Raw cytometry file parsers."""
+
+__all__: list[str] = []

reader_workbench/domains/cytometry/io/fcs.py

@@ -0,0 +1,135 @@
+"""FCS parsing for cytometry experiments."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Literal
+
+import numpy as np
+import pandas as pd
+
+from reader_workbench.errors import ParseError
+
+
+def _to_float(value) -> float:
+    if value is None:
+        return float("nan")
+    try:
+        return float(value)
+    except Exception:
+        return float("nan")
+
+
+def _parse_pne(value) -> tuple[float, float]:
+    if value is None:
+        return (float("nan"), float("nan"))
+    if isinstance(value, (tuple, list)) and len(value) == 2:
+        return (_to_float(value[0]), _to_float(value[1]))
+    text = str(value).strip()
+    if "," in text:
+        left, right = text.split(",", 1)
+        return (_to_float(left.strip()), _to_float(right.strip()))
+    return (float("nan"), float("nan"))
+
+
+def _clean_text(value) -> str | None:
+    if value is None:
+        return None
+    if isinstance(value, bytes):
+        try:
+            return value.decode("utf-8", errors="ignore")
+        except Exception:
+            return value.decode(errors="ignore")
+    return str(value)
+
+
+def _channel_names(channels: dict[int, dict[str, object]], *, field: str) -> list[str]:
+    names: list[str] = []
+    for key in sorted(channels):
+        meta = channels[key]
+        name = meta.get(field)
+        if name is None:
+            raise ParseError(
+                f"Channel metadata missing field '{field}' for channel {key}. Use channel_name_field: pns or pnn."
+            )
+        names.append(str(name))
+    return names
+
+
+def parse_fcs_file(
+    path: Path,
+    *,
+    channel_name_field: str,
+    channel_map: Mapping[str, str] | None = None,
+    drop_channels: set[str] | None = None,
+    sample_id_from: Literal["stem", "name"] = "stem",
+    time_value: float = 0.0,
+) -> tuple[pd.DataFrame, pd.DataFrame]:
+    try:
+        from flowio import FlowData  # noqa: PLC0415
+    except Exception as exc:  # pragma: no cover - environment-specific
+        raise ParseError("flowio is required for ingest/flow_cytometer. Re-sync the core reader environment.") from exc
+
+    field = str(channel_name_field).lower().strip()
+    mapped_names = {str(k): str(v) for k, v in (channel_map or {}).items()}
+    dropped = {str(channel) for channel in (drop_channels or set())}
+
+    flow = FlowData(str(path))
+    event_count = int(flow.event_count)
+    channel_count = int(flow.channel_count)
+    raw_events = np.asarray(flow.events, dtype=float)
+    if raw_events.size != event_count * channel_count:
+        raise ParseError(
+            f"Unexpected event buffer size for {path.name}: "
+            f"{raw_events.size} values for {event_count} events × {channel_count} channels."
+        )
+
+    values = raw_events.reshape(event_count, channel_count)
+    channel_names = _channel_names(flow.channels, field=field)
+    if len(channel_names) != channel_count:
+        raise ParseError(
+            f"Channel count mismatch: metadata has {len(channel_names)} names, events have {channel_count}."
+        )
+    if mapped_names:
+        remapped = [mapped_names.get(name, name) for name in channel_names]
+        if len(set(remapped)) != len(remapped):
+            raise ParseError("channel_map produces duplicate channel names; ensure a 1:1 mapping.")
+        channel_names = remapped
+
+    wide = pd.DataFrame(values, columns=channel_names)
+    wide["event_index"] = range(event_count)
+    long = wide.melt(id_vars=["event_index"], var_name="channel", value_name="value")
+    if dropped:
+        long = long[~long["channel"].isin(dropped)]
+    sample_id = path.stem if sample_id_from == "stem" else path.name
+    long["sample_id"] = sample_id
+    long["position"] = sample_id
+    long["time"] = float(time_value)
+
+    channel_rows = []
+    channel_indices = sorted(flow.channels)
+    for idx, name in zip(channel_indices, channel_names, strict=False):
+        meta = flow.channels.get(idx, {})
+        pne_decades, pne_zero = _parse_pne(meta.get("pne"))
+        channel_rows.append(
+            {
+                "sample_id": sample_id,
+                "channel_index": int(idx),
+                "channel_name": str(name),
+                "pns": _clean_text(meta.get("pns")),
+                "pnn": _clean_text(meta.get("pnn")),
+                "pnt": _clean_text(meta.get("pnt")),
+                "pnf": _clean_text(meta.get("pnf")),
+                "pnl": _clean_text(meta.get("pnl")),
+                "pnr": _to_float(meta.get("pnr")),
+                "pnb": _to_float(meta.get("pnb")),
+                "png": _to_float(meta.get("png")),
+                "pne_decades": pne_decades,
+                "pne_zero": pne_zero,
+            }
+        )
+    channels_meta = pd.DataFrame(channel_rows)
+    if channels_meta.empty:
+        channels_meta = pd.DataFrame(columns=["sample_id", "channel_index", "channel_name"])
+    return long, channels_meta

reader_workbench/domains/cytometry/plots/__init__.py

@@ -0,0 +1,5 @@
+"""Static cytometry figures."""
+
+from .diagnostic import render_cytometry_diagnostic
+
+__all__ = ["render_cytometry_diagnostic"]