reader-workbench 1.0.0__py3-none-any.whl

reader_workbench/contracts/catalog.py

@@ -0,0 +1,257 @@
+"""Explicit dataframe-contract catalog and contract-surface helpers."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Iterator, Mapping
+from dataclasses import dataclass
+
+import pandas as pd
+
+from reader_workbench.errors import ContractError
+
+from .model import ColumnRule, ContractId, ContractToken, DataFrameContract, DType, validate_df
+
+
+def _dtype_satisfies(*, child: DType, parent: DType) -> bool:
+    """Return whether every value accepted by a child dtype is accepted by its parent."""
+
+    return child == parent or (child == "int" and parent == "float")
+
+
+def _require_compatible_rule(
+    *,
+    child_contract: DataFrameContract,
+    parent_contract: DataFrameContract,
+    child_rule: ColumnRule,
+    parent_rule: ColumnRule,
+) -> None:
+    prefix = (
+        f"contract {child_contract.id!r} is not structurally compatible with parent "
+        f"{parent_contract.id!r}: column {parent_rule.name!r}"
+    )
+    if not _dtype_satisfies(child=child_rule.dtype, parent=parent_rule.dtype):
+        raise ContractError(f"{prefix} has dtype {child_rule.dtype!r}, expected a subtype of {parent_rule.dtype!r}")
+    if not parent_rule.allow_nan and child_rule.allow_nan:
+        raise ContractError(f"{prefix} permits null values forbidden by the parent")
+    if parent_rule.monotone_non_decreasing and not child_rule.monotone_non_decreasing:
+        raise ContractError(f"{prefix} does not preserve the parent's monotone constraint")
+    if parent_rule.nonnegative and not child_rule.nonnegative:
+        raise ContractError(f"{prefix} does not preserve the parent's nonnegative constraint")
+    if parent_rule.allowed_values is not None:
+        if child_rule.allowed_values is None:
+            raise ContractError(f"{prefix} does not preserve the parent's allowed-values constraint")
+        parent_values = {str(value) for value in parent_rule.allowed_values}
+        child_values = {str(value) for value in child_rule.allowed_values}
+        if not child_values <= parent_values:
+            raise ContractError(f"{prefix} permits values outside the parent's allowed set")
+
+
+def _require_structural_compatibility(*, child: DataFrameContract, parent: DataFrameContract) -> None:
+    child_columns = {rule.name: rule for rule in child.columns}
+    parent_columns = {rule.name: rule for rule in parent.columns}
+
+    for name, parent_rule in parent_columns.items():
+        child_rule = child_columns.get(name)
+        if child_rule is None:
+            if parent_rule.required:
+                raise ContractError(
+                    f"contract {child.id!r} is not structurally compatible with parent {parent.id!r}: "
+                    f"missing required column {name!r}"
+                )
+            if child.allow_extra_columns:
+                raise ContractError(
+                    f"contract {child.id!r} is not structurally compatible with parent {parent.id!r}: "
+                    f"optional parent column {name!r} is unconstrained by the child"
+                )
+            continue
+        if parent_rule.required and not child_rule.required:
+            raise ContractError(
+                f"contract {child.id!r} is not structurally compatible with parent {parent.id!r}: "
+                f"required column {name!r} is optional in the child"
+            )
+        _require_compatible_rule(
+            child_contract=child,
+            parent_contract=parent,
+            child_rule=child_rule,
+            parent_rule=parent_rule,
+        )
+
+    if not parent.allow_extra_columns:
+        extras = sorted(set(child_columns) - set(parent_columns))
+        if child.allow_extra_columns or extras:
+            detail = "allows undeclared columns" if child.allow_extra_columns else f"declares extra columns {extras}"
+            raise ContractError(
+                f"contract {child.id!r} is not structurally compatible with parent {parent.id!r}: {detail}"
+            )
+
+    child_keys = [set(key) for key in child.unique_keys if key]
+    for parent_key in (set(key) for key in parent.unique_keys if key):
+        if not any(child_key <= parent_key for child_key in child_keys):
+            raise ContractError(
+                f"contract {child.id!r} is not structurally compatible with parent {parent.id!r}: "
+                f"does not preserve unique key {sorted(parent_key)}"
+            )
+
+
+def _validate_contract_graph(contracts: Mapping[ContractId, DataFrameContract]) -> None:
+    for contract_id, contract in contracts.items():
+        if not isinstance(contract, DataFrameContract):
+            raise ContractError(f"contract catalog entry {contract_id!r} must be a DataFrameContract")
+        if not isinstance(contract.id, str) or not contract.id.strip():
+            raise ContractError("contract ids must be non-empty strings")
+        if contract.id != contract_id:
+            raise ContractError(f"contract catalog key {contract_id!r} does not match contract id {contract.id!r}")
+        if any(not isinstance(rule, ColumnRule) for rule in contract.columns):
+            raise ContractError(f"contract {contract.id!r} columns must contain ColumnRule values")
+        column_names = [rule.name for rule in contract.columns]
+        if any(not isinstance(name, str) or not name.strip() for name in column_names):
+            raise ContractError(f"contract {contract.id!r} column names must be non-empty strings")
+        if len(column_names) != len(set(column_names)):
+            raise ContractError(f"contract {contract.id!r} declares duplicate column names")
+        if type(contract.allow_extra_columns) is not bool:
+            raise ContractError(f"contract {contract.id!r} allow_extra_columns must be bool")
+        columns_by_name = {rule.name: rule for rule in contract.columns}
+        for rule in contract.columns:
+            for field in ("required", "allow_nan", "monotone_non_decreasing", "nonnegative"):
+                if type(getattr(rule, field)) is not bool:
+                    raise ContractError(f"contract {contract.id!r} column {rule.name!r} {field} must be bool")
+            if rule.nonnegative and rule.dtype not in {"int", "float"}:
+                raise ContractError(
+                    f"contract {contract.id!r} column {rule.name!r} uses nonnegative with non-numeric dtype"
+                )
+            if rule.monotone_non_decreasing and rule.dtype not in {"int", "float", "datetime"}:
+                raise ContractError(
+                    f"contract {contract.id!r} column {rule.name!r} uses monotonicity with unordered dtype"
+                )
+        for key in contract.unique_keys:
+            if not key:
+                raise ContractError(f"contract {contract.id!r} unique keys must not be empty")
+            if len(key) != len(set(key)):
+                raise ContractError(f"contract {contract.id!r} unique key {list(key)!r} contains duplicate columns")
+            for name in key:
+                rule = columns_by_name.get(name)
+                if rule is None:
+                    raise ContractError(f"contract {contract.id!r} unique key references unknown column {name!r}")
+                if not rule.required:
+                    raise ContractError(f"contract {contract.id!r} unique key column {name!r} must be required")
+        if contract.primary_index is not None:
+            if not contract.primary_index:
+                raise ContractError(f"contract {contract.id!r} primary index must not be empty")
+            if len(contract.primary_index) != len(set(contract.primary_index)):
+                raise ContractError(f"contract {contract.id!r} primary index contains duplicate columns")
+            for name in contract.primary_index:
+                rule = columns_by_name.get(name)
+                if rule is None:
+                    raise ContractError(f"contract {contract.id!r} primary index references unknown column {name!r}")
+                if not rule.required:
+                    raise ContractError(f"contract {contract.id!r} primary index column {name!r} must be required")
+        for parent in contract.parents:
+            if parent == contract.id:
+                raise ContractError(f"contract {contract.id!r} cannot list itself as a parent")
+            if parent not in contracts:
+                raise ContractError(f"contract {contract.id!r} references unknown parent {parent!r}")
+
+    visited: set[ContractId] = set()
+    active: set[ContractId] = set()
+    path: list[ContractId] = []
+
+    def visit(contract_id: ContractId) -> None:
+        if contract_id in active:
+            cycle_start = path.index(contract_id)
+            cycle = [*path[cycle_start:], contract_id]
+            raise ContractError("contract lineage cycle detected: " + " -> ".join(cycle))
+        if contract_id in visited:
+            return
+        active.add(contract_id)
+        path.append(contract_id)
+        for parent in contracts[contract_id].parents:
+            visit(parent)
+        path.pop()
+        active.remove(contract_id)
+        visited.add(contract_id)
+
+    for contract_id in sorted(contracts):
+        visit(contract_id)
+
+    for child in contracts.values():
+        for parent_id in child.parents:
+            _require_structural_compatibility(child=child, parent=contracts[parent_id])
+
+
+@dataclass(frozen=True)
+class OutputContractSurface:
+    minimum: ContractToken
+    runtime_mode: str = "fixed"  # fixed | promoted | passthrough
+    promoted: tuple[ContractId, ...] = ()
+    note: str | None = None
+
+    def render(self) -> str:
+        if self.minimum == "none":
+            return "none"
+        parts = [self.minimum]
+        if self.runtime_mode == "promoted" and self.promoted:
+            parts.append(f"runtime may promote to {', '.join(self.promoted)}")
+        elif self.runtime_mode == "passthrough":
+            hint = f" (e.g. {', '.join(self.promoted)})" if self.promoted else ""
+            parts.append(f"runtime preserves stricter compatible input contracts{hint}")
+        if self.note:
+            parts.append(self.note)
+        return "; ".join(parts)
+
+
+class ContractCatalog:
+    """Immutable contract catalog with explicit lineage and validation helpers."""
+
+    def __init__(self, contracts: Mapping[ContractId, DataFrameContract]) -> None:
+        normalized = dict(contracts)
+        _validate_contract_graph(normalized)
+        self._contracts = normalized
+
+    @classmethod
+    def from_contracts(cls, contracts: Iterable[DataFrameContract]) -> ContractCatalog:
+        normalized: dict[ContractId, DataFrameContract] = {}
+        for contract in contracts:
+            if contract.id in normalized:
+                raise ContractError(f"duplicate contract id {contract.id!r}")
+            normalized[contract.id] = contract
+        return cls(normalized)
+
+    def all(self) -> tuple[DataFrameContract, ...]:
+        return tuple(self._contracts[key] for key in sorted(self._contracts))
+
+    def ids(self) -> tuple[ContractId, ...]:
+        return tuple(sorted(self._contracts))
+
+    def __contains__(self, contract_id: object) -> bool:
+        return isinstance(contract_id, str) and contract_id in self._contracts
+
+    def get(self, contract_id: ContractToken | None) -> DataFrameContract | None:
+        if contract_id in (None, "none"):
+            return None
+        return self._contracts.get(contract_id)
+
+    def require(self, contract_id: ContractId) -> DataFrameContract:
+        try:
+            return self._contracts[contract_id]
+        except KeyError as exc:
+            raise ContractError(f"unknown contract id {contract_id!r}") from exc
+
+    def iter_lineage(self, contract_id: ContractId) -> Iterator[ContractId]:
+        seen: set[ContractId] = set()
+        stack: list[ContractId] = [contract_id]
+        while stack:
+            current = stack.pop()
+            if current in seen:
+                continue
+            contract = self.require(current)
+            seen.add(current)
+            yield current
+            stack.extend(reversed(contract.parents))
+
+    def satisfies(self, *, actual: ContractToken | None, expected: ContractId) -> bool:
+        if actual in (None, "none"):
+            return False
+        return expected in set(self.iter_lineage(actual))
+
+    def validate(self, df: pd.DataFrame, *, contract_id: ContractId, where: str) -> None:
+        validate_df(df, self.require(contract_id), where=where)

reader_workbench/contracts/model.py

@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Literal
+
+import pandas as pd
+
+from reader_workbench.errors import ContractError
+
+type ContractId = str
+DType = Literal["string", "int", "float", "bool", "category", "datetime"]
+type ContractToken = ContractId | Literal["none"]
+
+
+@dataclass(frozen=True)
+class ColumnRule:
+    name: str
+    dtype: DType
+    required: bool = True
+    allow_nan: bool = False
+    monotone_non_decreasing: bool = False
+    nonnegative: bool = False
+    allowed_values: tuple[str, ...] | None = None
+
+    def __post_init__(self) -> None:
+        if self.allowed_values is not None:
+            object.__setattr__(self, "allowed_values", tuple(self.allowed_values))
+
+
+@dataclass(frozen=True)
+class DataFrameContract:
+    id: ContractId
+    description: str
+    columns: tuple[ColumnRule, ...]
+    unique_keys: tuple[tuple[str, ...], ...]
+    parents: tuple[ContractId, ...] = ()
+    domain: str | None = None
+    kind: str | None = None
+    primary_index: tuple[str, ...] | None = None
+    notes: str | None = None
+    allow_extra_columns: bool = True
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "columns", tuple(self.columns))
+        object.__setattr__(self, "unique_keys", tuple(tuple(key) for key in self.unique_keys))
+        object.__setattr__(self, "parents", tuple(self.parents))
+        if self.primary_index is not None:
+            object.__setattr__(self, "primary_index", tuple(self.primary_index))
+
+
+def _is_dtype(series: pd.Series, want: DType) -> bool:
+    if want == "string":
+        return pd.api.types.is_string_dtype(series) or pd.api.types.is_object_dtype(series)
+    if want == "int":
+        return pd.api.types.is_integer_dtype(series)
+    if want == "float":
+        return pd.api.types.is_float_dtype(series) or pd.api.types.is_integer_dtype(series)
+    if want == "bool":
+        return pd.api.types.is_bool_dtype(series)
+    if want == "category":
+        return pd.api.types.is_categorical_dtype(series)
+    if want == "datetime":
+        return pd.api.types.is_datetime64_any_dtype(series)
+    return False
+
+
+def validate_df(df: pd.DataFrame, contract: DataFrameContract, *, where: str) -> None:
+    """Assert df matches the contract exactly; raise ContractError on first failure."""
+    cols = set(df.columns)
+    declared = {rule.name for rule in contract.columns}
+    if not contract.allow_extra_columns:
+        extras = sorted(cols - declared)
+        if extras:
+            raise ContractError(f"[{where}] contract {contract.id}: unexpected columns {extras}")
+
+    for rule in contract.columns:
+        if rule.required and rule.name not in cols:
+            raise ContractError(f"[{where}] contract {contract.id}: missing required column '{rule.name}'")
+
+    for rule in contract.columns:
+        if rule.name not in cols:
+            continue
+        s = df[rule.name]
+        if not _is_dtype(s, rule.dtype):
+            raise ContractError(
+                f"[{where}] contract {contract.id}: column '{rule.name}' has dtype {s.dtype} but expected {rule.dtype}"
+            )
+        if not rule.allow_nan and s.isna().any():
+            raise ContractError(
+                f"[{where}] contract {contract.id}: column '{rule.name}' contains NaN but allow_nan=false"
+            )
+        if rule.nonnegative and (pd.to_numeric(s, errors="coerce") < 0).any():
+            raise ContractError(f"[{where}] contract {contract.id}: column '{rule.name}' must be nonnegative")
+        if rule.monotone_non_decreasing and not s.dropna().is_monotonic_increasing:
+            raise ContractError(
+                f"[{where}] contract {contract.id}: column '{rule.name}' must be monotone non-decreasing"
+            )
+        if rule.allowed_values is not None:
+            bad = sorted(set(map(str, s.dropna().astype(str))) - set(map(str, rule.allowed_values)))
+            if bad:
+                raise ContractError(
+                    f"[{where}] contract {contract.id}: column '{rule.name}' contains values outside allowed set: {bad[:5]}"
+                )
+
+    for key in contract.unique_keys:
+        if not key:
+            continue
+        if df.duplicated(subset=key, keep=False).any():
+            raise ContractError(f"[{where}] contract {contract.id}: uniqueness violated for key {key}")

reader_workbench/domains/__init__.py

@@ -0,0 +1 @@
+"""Domain-owned protocol and data semantics."""

reader_workbench/domains/cytometry/__init__.py

@@ -0,0 +1,3 @@
+"""Cytometry-domain package root."""
+
+__all__: list[str] = []

reader_workbench/domains/cytometry/analysis/__init__.py

@@ -0,0 +1,29 @@
+"""Public event-table preparation and analysis for cytometry workflows."""
+
+from reader_workbench.domains.cytometry.analysis.events import (
+    CytometryAnalysis,
+    CytometryAnalysisError,
+    GateSpec,
+    ThresholdSpec,
+    prepare_event_table,
+)
+from reader_workbench.domains.cytometry.analysis.gating import analyze_events
+from reader_workbench.domains.cytometry.analysis.workflow import (
+    CytometryGatingRequest,
+    CytometryGatingResult,
+    CytometryQCSpec,
+    run_cytometry_gating,
+)
+
+__all__ = [
+    "CytometryAnalysis",
+    "CytometryAnalysisError",
+    "CytometryGatingRequest",
+    "CytometryGatingResult",
+    "CytometryQCSpec",
+    "GateSpec",
+    "ThresholdSpec",
+    "analyze_events",
+    "prepare_event_table",
+    "run_cytometry_gating",
+]

reader_workbench/domains/cytometry/analysis/events.py

@@ -0,0 +1,182 @@
+"""Polars-native preparation for tidy cytometry event tables."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+from typing import Literal
+
+import polars as pl
+
+_REQUIRED_TIDY_COLUMNS = ("channel", "value", "sample_id", "event_index")
+_METADATA_COLUMNS = ("treatment", "design_id", "sample_label")
+
+
+class CytometryAnalysisError(ValueError):
+    """Raised when a cytometry event table cannot support the requested analysis."""
+
+
+@dataclass(frozen=True, slots=True)
+class GateSpec:
+    """Sequential cells and singlets gates for an event-wide table."""
+
+    cells_x_channel: str
+    cells_y_channel: str
+    cells_x_range: tuple[float, float]
+    cells_y_range: tuple[float, float]
+    singlet_x_channel: str
+    singlet_y_channel: str
+    singlet_ratio_range: tuple[float, float]
+    cells_enabled: bool = True
+    singlets_enabled: bool = True
+
+
+@dataclass(frozen=True, slots=True)
+class ThresholdSpec:
+    """Positive-event threshold applied to the selected fluorescence channel."""
+
+    channel: str
+    value: float = 0.0
+    mode: Literal["manual", "from_control_quantile"] = "manual"
+    group_column: str | None = None
+    control_value: str | None = None
+    quantile: float = 0.99
+
+
+@dataclass(frozen=True, slots=True)
+class CytometryAnalysis:
+    """Polars-native event, count, summary, and quality-control tables."""
+
+    gated_events: pl.DataFrame
+    gate_counts_sample: pl.DataFrame
+    stats_sample: pl.DataFrame
+    stats_group: pl.DataFrame | None
+    qc_table: pl.DataFrame
+    threshold_value: float
+
+
+EventFrame = pl.DataFrame | pl.LazyFrame
+
+
+def prepare_event_table(
+    frame: EventFrame,
+    *,
+    channels: Sequence[str],
+    metadata_columns: Sequence[str] = (),
+) -> pl.DataFrame:
+    """Pivot declared channels from a tidy event table without inferring policy."""
+
+    available = _frame_columns(frame)
+    missing_tidy = [column for column in _REQUIRED_TIDY_COLUMNS if column not in available]
+    if missing_tidy:
+        raise CytometryAnalysisError(
+            "Cytometry event data is missing required column(s): " + ", ".join(missing_tidy) + "."
+        )
+
+    selected_channels = tuple(dict.fromkeys(str(channel) for channel in channels if str(channel)))
+    if not selected_channels:
+        raise CytometryAnalysisError("Select at least one cytometry channel.")
+
+    requested_metadata = tuple(dict.fromkeys(str(column).strip() for column in metadata_columns if str(column).strip()))
+    missing_metadata = [column for column in requested_metadata if column not in available]
+    if missing_metadata:
+        raise CytometryAnalysisError(
+            "Missing requested cytometry metadata column(s): " + ", ".join(missing_metadata) + "."
+        )
+    retained_metadata = list(
+        dict.fromkeys((*[column for column in _METADATA_COLUMNS if column in available], *requested_metadata))
+    )
+    index_columns = ["sample_id", "event_index", *retained_metadata]
+    pivot_key_columns = [*index_columns, "channel"]
+    projected_columns = [*index_columns, "channel", "value"]
+    event_identity_columns = ["sample_id", "event_index"]
+    metadata_consistency_columns = {
+        column: f"__reader_event_metadata_n_unique__{column}" for column in retained_metadata
+    }
+    query = _as_lazy(frame).select(projected_columns)
+    if metadata_consistency_columns:
+        query = query.with_columns(
+            *[
+                pl.col(column).n_unique().over(event_identity_columns).alias(consistency_column)
+                for column, consistency_column in metadata_consistency_columns.items()
+            ]
+        )
+    query = query.filter(pl.col("channel").is_in(selected_channels))
+
+    channel_profile = query.select(
+        pl.len().alias("row_count"),
+        pl.struct(pivot_key_columns).n_unique().alias("unique_pivot_key_count"),
+        pl.col("channel").unique().sort().implode().alias("channels"),
+        pl.col("channel").n_unique().over(event_identity_columns).min().alias("minimum_event_selected_channel_count"),
+        *[pl.col(consistency_column).max() for consistency_column in metadata_consistency_columns.values()],
+    ).collect()
+    row_count = int(channel_profile.item(0, "row_count"))
+    unique_pivot_key_count = int(channel_profile.item(0, "unique_pivot_key_count"))
+    if row_count:
+        inconsistent_metadata = [
+            column
+            for column, consistency_column in metadata_consistency_columns.items()
+            if int(channel_profile.item(0, consistency_column)) > 1
+        ]
+        if inconsistent_metadata:
+            identity = ", ".join(event_identity_columns)
+            metadata = ", ".join(inconsistent_metadata)
+            raise CytometryAnalysisError(
+                "Cytometry event data contains inconsistent metadata across channel rows for "
+                f"event identity {identity}: {metadata}."
+            )
+    if row_count > unique_pivot_key_count:
+        duplicate_row_count = row_count - unique_pivot_key_count
+        keys = ", ".join(pivot_key_columns)
+        raise CytometryAnalysisError(
+            f"Cytometry event data contains {duplicate_row_count} duplicate pivot key rows across "
+            f"{keys}; each event/channel key must be unique."
+        )
+    if row_count:
+        present_channels = set(channel_profile.item(0, "channels"))
+        missing_channels = [channel for channel in selected_channels if channel not in present_channels]
+        if missing_channels:
+            raise CytometryAnalysisError("Missing channels after pivot: " + ", ".join(missing_channels) + ".")
+        minimum_channel_count = int(channel_profile.item(0, "minimum_event_selected_channel_count"))
+        if minimum_channel_count < len(selected_channels):
+            identity = ", ".join(event_identity_columns)
+            required_channels = ", ".join(selected_channels)
+            raise CytometryAnalysisError(
+                "Cytometry event data is missing selected channels within at least one event; "
+                f"each {identity} must contain all of: {required_channels}."
+            )
+
+    return (
+        query.select(projected_columns)
+        .with_columns(pl.col("value").cast(pl.Float64))
+        .pivot(
+            on="channel",
+            on_columns=selected_channels,
+            values="value",
+            index=index_columns,
+            maintain_order=True,
+        )
+        .collect()
+    )
+
+
+def _as_lazy(frame: EventFrame) -> pl.LazyFrame:
+    if isinstance(frame, pl.LazyFrame):
+        return frame
+    if isinstance(frame, pl.DataFrame):
+        return frame.lazy()
+    raise TypeError(f"Expected a Polars DataFrame or LazyFrame, got {type(frame).__name__}.")
+
+
+def _frame_columns(frame: EventFrame) -> tuple[str, ...]:
+    if isinstance(frame, pl.LazyFrame):
+        return tuple(frame.collect_schema().names())
+    if isinstance(frame, pl.DataFrame):
+        return tuple(frame.columns)
+    raise TypeError(f"Expected a Polars DataFrame or LazyFrame, got {type(frame).__name__}.")
+
+
+def _require_columns(frame: pl.DataFrame, columns: Sequence[str]) -> None:
+    missing = [column for column in dict.fromkeys(columns) if column not in frame.columns]
+    if missing:
+        raise CytometryAnalysisError("Missing cytometry column(s): " + ", ".join(missing) + ".")