reader-workbench 1.0.0__py3-none-any.whl

reader_workbench/plotting/style.py

@@ -0,0 +1,175 @@
+"""Shared palette + subplot layout helpers for plotting modules."""
+
+from __future__ import annotations
+
+import contextlib
+from collections.abc import Iterable
+from dataclasses import dataclass
+
+import matplotlib as mpl
+import matplotlib.pyplot as plt
+
+_PALETTES: dict[str, list[str]] = {
+    "colorblind": [
+        "#0072B2",
+        "#E69F00",
+        "#009E73",
+        "#CC79A7",
+        "#56B4E9",
+        "#D55E00",
+        "#F0E442",
+        "#000000",
+    ],
+    "muted": [
+        "#4878CF",
+        "#6ACC65",
+        "#D65F5F",
+        "#B47CC7",
+        "#C4AD66",
+        "#77BEDB",
+        "#4D4D4D",
+        "#A0A0A0",
+    ],
+    "tableau": [
+        "#4E79A7",
+        "#F28E2B",
+        "#E15759",
+        "#76B7B2",
+        "#59A14F",
+        "#EDC948",
+        "#B07AA1",
+        "#FF9DA7",
+    ],
+}
+
+
+@dataclass(frozen=True)
+class PaletteBook:
+    """Small helper carried in RunContext (see workbench/context.py)."""
+
+    name: str = "colorblind"
+
+    def colors(self, n: int) -> list[str]:
+        if self.name not in _PALETTES:
+            opts = ", ".join(sorted(_PALETTES))
+            raise ValueError(f"Unknown palette '{self.name}'. Available: {opts}")
+        palette = _PALETTES[self.name]
+        if n <= len(palette):
+            return palette[:n]
+        out = []
+        while len(out) < n:
+            out.extend(palette)
+        return out[:n]
+
+
+def available_palettes() -> list[str]:
+    return sorted(_PALETTES)
+
+
+DEFAULT_RC = {
+    "figure_figsize": (5, 5),
+    "savefig_dpi": 300,
+    "axes_spines_top": False,
+    "axes_spines_right": False,
+    "axes_titleweight": "bold",
+    "axes_labelweight": "regular",
+    "axes_grid": True,
+    "grid_alpha": 0.25,
+    "grid_linestyle": "-",
+    "grid_color": "#B0B0B0",
+    "axes_axisbelow": True,
+    "font_size": 13.0,
+    "axes_labelsize": 13.0,
+    "axes_titlesize": 14.0,
+    "xtick_labelsize": 12.0,
+    "ytick_labelsize": 12.0,
+    "legend_fontsize": 12.0,
+    "legend_title_fontsize": 12.0,
+    "xtick_direction": "out",
+    "ytick_direction": "out",
+    "legend_frameon": False,
+    "pdf_fonttype": 42,
+    "pdf_compression": 9,
+    "path_simplify": True,
+    "path_simplify_threshold": 0.0,
+    "agg_path_chunksize": 20000,
+}
+
+
+@contextlib.contextmanager
+def use_style(rc: dict | None = None, color_cycle: Iterable[str] | None = None):
+    """Context manager to push a small, opinionated Matplotlib style."""
+    rc = {**DEFAULT_RC, **(rc or {})}
+    scale = float(rc.pop("font_scale", 1.0))
+
+    def _scaled(key: str) -> float:
+        value = float(rc.get(key, DEFAULT_RC[key]))
+        return value * scale
+
+    with mpl.rc_context():
+        mpl.rcParams.update(
+            {
+                "figure.figsize": rc["figure_figsize"],
+                "savefig.dpi": rc["savefig_dpi"],
+                "axes.spines.top": rc["axes_spines_top"],
+                "axes.spines.right": rc["axes_spines_right"],
+                "axes.titleweight": rc["axes_titleweight"],
+                "axes.labelweight": rc["axes_labelweight"],
+                "axes.grid": rc["axes_grid"],
+                "grid.alpha": rc["grid_alpha"],
+                "grid.linestyle": rc["grid_linestyle"],
+                "grid.color": rc["grid_color"],
+                "axes.axisbelow": rc["axes_axisbelow"],
+                "font.size": _scaled("font_size"),
+                "axes.labelsize": _scaled("axes_labelsize"),
+                "axes.titlesize": _scaled("axes_titlesize"),
+                "xtick.labelsize": _scaled("xtick_labelsize"),
+                "ytick.labelsize": _scaled("ytick_labelsize"),
+                "legend.fontsize": _scaled("legend_fontsize"),
+                "legend.title_fontsize": _scaled("legend_title_fontsize"),
+                "xtick.direction": rc["xtick_direction"],
+                "ytick.direction": rc["ytick_direction"],
+                "legend.frameon": rc["legend_frameon"],
+                "pdf.fonttype": rc["pdf_fonttype"],
+                "pdf.compression": int(rc["pdf_compression"]),
+                "path.simplify": bool(rc["path_simplify"]),
+                "path.simplify_threshold": float(rc["path_simplify_threshold"]),
+                "agg.path.chunksize": int(rc["agg_path_chunksize"]),
+            }
+        )
+        if color_cycle is not None:
+            mpl.rcParams["axes.prop_cycle"] = mpl.cycler(color=list(color_cycle))
+        yield
+
+
+def new_fig_ax(fig_kwargs: dict | None = None):
+    """
+    Consistent figure construction.
+    Only pass kwargs that Matplotlib's Figure/subplots actually understand.
+    Plot- or style-level options must be consumed by the caller and not forwarded.
+    """
+    fig_kwargs = dict(fig_kwargs or {})
+    allowed = {
+        "num",
+        "figsize",
+        "dpi",
+        "facecolor",
+        "edgecolor",
+        "frameon",
+        "clear",
+        "constrained_layout",
+        "layout",
+        "squeeze",
+        "subplot_kw",
+        "gridspec_kw",
+        "sharex",
+        "sharey",
+    }
+    fig_kwargs = {key: value for key, value in fig_kwargs.items() if key in allowed}
+    if "figsize" not in fig_kwargs:
+        fig_kwargs["figsize"] = DEFAULT_RC["figure_figsize"]
+    fig_kwargs.setdefault("constrained_layout", True)
+    return plt.subplots(**fig_kwargs)
+
+
+__all__ = ["DEFAULT_RC", "PaletteBook", "available_palettes", "new_fig_ax", "use_style"]

reader_workbench/plotting/utils.py

@@ -0,0 +1,27 @@
+"""Shared plotting utilities (filesystem + naming helpers)."""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+
+def ensure_dir(path: Path) -> None:
+    path.mkdir(parents=True, exist_ok=True)
+
+
+def slugify(value: str) -> str:
+    value = str(value).strip()
+    value = re.sub(r"[^\w\-.]+", "_", value)
+    return re.sub(r"_{2,}", "_", value).strip("_")
+
+
+def save_figure(fig, output_dir: Path, filename_stub: str, ext: str = "pdf", dpi: int | None = None) -> Path:
+    """
+    Save figures as PDF by default (print-friendly, vector).
+    Use ext="png" if you explicitly need rasters.
+    """
+    ensure_dir(output_dir)
+    out = output_dir / f"{slugify(filename_stub)}.{ext}"
+    fig.savefig(out, bbox_inches="tight", dpi=dpi)
+    return out

reader_workbench/plugins/__init__.py

@@ -0,0 +1 @@
+"""Built-in workbench plugins shipped with Reader."""

reader_workbench/plugins/catalog.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+import importlib
+from typing import cast
+
+from reader_workbench.workbench.assets import AssetCatalog, AssetDescriptor
+
+_CATEGORY_MODULES = {
+    "ingest": "reader_workbench.plugins.manifests.ingest",
+    "transform": "reader_workbench.plugins.manifests.transform",
+    "plot": "reader_workbench.plugins.manifests.plot",
+    "export": "reader_workbench.plugins.manifests.export",
+    "validator": "reader_workbench.plugins.manifests.validator",
+}
+
+
+def _category_descriptors(category: str) -> tuple[AssetDescriptor, ...]:
+    module = importlib.import_module(_CATEGORY_MODULES[category])
+    return cast(tuple[AssetDescriptor, ...], module.BUILTIN_PLUGIN_DESCRIPTORS)
+
+
+def builtin_plugin_catalog(*, categories: set[str] | None = None) -> AssetCatalog:
+    selected = (
+        tuple(_CATEGORY_MODULES)
+        if categories is None
+        else tuple(category for category in _CATEGORY_MODULES if category in categories)
+    )
+    descriptors = [descriptor for category in selected for descriptor in _category_descriptors(category)]
+    return AssetCatalog(descriptors)
+
+
+def builtin_plugin_descriptors(*, categories: set[str] | None = None) -> tuple[AssetDescriptor, ...]:
+    return builtin_plugin_catalog(categories=categories).all()

reader_workbench/plugins/export/_paths.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from reader_workbench.errors import ExecutionError
+from reader_workbench.workbench.paths import resolve_path_within_root
+
+
+def resolve_export_path(raw: str, *, exports_dir: Path) -> Path:
+    path = Path(raw).expanduser()
+    if path.is_absolute():
+        raise ExecutionError("Export paths must be relative to the experiment exports directory")
+    try:
+        resolved = resolve_path_within_root(path, root=exports_dir)
+    except ValueError as exc:
+        raise ExecutionError(
+            "Export paths must stay under the experiment exports directory after resolving symlinks"
+        ) from exc
+    if resolved == exports_dir.resolve():
+        raise ExecutionError("Export paths must identify a file below the experiment exports directory")
+    return resolved

reader_workbench/plugins/export/csv.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from typing import Any
+
+import pandas as pd
+from pydantic import Field
+
+from reader_workbench.errors import ExecutionError
+from reader_workbench.workbench.ports import dataframe_input, file_path_output
+from reader_workbench.workbench.registry import Plugin, PluginConfig
+
+from ._paths import resolve_export_path
+
+
+class ExportCsvCfg(PluginConfig):
+    path: str = Field(..., description="Output CSV path relative to the experiment exports directory.")
+    index: bool = False
+    sep: str = ","
+    na_rep: str | None = None
+
+
+class ExportCsv(Plugin):
+    ConfigModel = ExportCsvCfg
+
+    @classmethod
+    def input_ports(cls):
+        return {"df": dataframe_input("df")}
+
+    @classmethod
+    def output_ports(cls):
+        return {"artifact": file_path_output("artifact")}
+
+    def run(self, ctx, inputs: dict[str, Any], cfg: ExportCsvCfg) -> dict[str, Any]:
+        df = inputs["df"]
+        if not isinstance(df, pd.DataFrame):
+            raise ExecutionError(f"export/csv expects a DataFrame input, got {type(df).__name__}")
+        out_path = resolve_export_path(cfg.path, exports_dir=ctx.exports_dir)
+        out_path.parent.mkdir(parents=True, exist_ok=True)
+        df.to_csv(out_path, index=cfg.index, sep=cfg.sep, na_rep=cfg.na_rep)
+        ctx.logger.info("export • csv → %s", out_path)
+        return {"artifact": out_path}

reader_workbench/plugins/export/xlsx.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+from typing import Any
+
+import pandas as pd
+from pydantic import Field
+
+from reader_workbench.errors import ExecutionError
+from reader_workbench.workbench.ports import dataframe_input, file_path_output
+from reader_workbench.workbench.registry import Plugin, PluginConfig
+
+from ._paths import resolve_export_path
+
+
+class ExportXlsxCfg(PluginConfig):
+    path: str = Field(..., description="Output XLSX path relative to the experiment exports directory.")
+    sheet_name: str = "Sheet1"
+    index: bool = False
+
+
+class ExportXlsx(Plugin):
+    ConfigModel = ExportXlsxCfg
+
+    @classmethod
+    def input_ports(cls):
+        return {"df": dataframe_input("df")}
+
+    @classmethod
+    def output_ports(cls):
+        return {"artifact": file_path_output("artifact")}
+
+    def run(self, ctx, inputs: dict[str, Any], cfg: ExportXlsxCfg) -> dict[str, Any]:
+        df = inputs["df"]
+        if not isinstance(df, pd.DataFrame):
+            raise ExecutionError(f"export/xlsx expects a DataFrame input, got {type(df).__name__}")
+        out_path = resolve_export_path(cfg.path, exports_dir=ctx.exports_dir)
+        out_path.parent.mkdir(parents=True, exist_ok=True)
+        try:
+            with pd.ExcelWriter(out_path, engine="openpyxl") as writer:
+                df.to_excel(writer, index=cfg.index, sheet_name=cfg.sheet_name)
+        except ImportError as e:
+            raise ExecutionError("export/xlsx requires the 'openpyxl' dependency to write .xlsx files.") from e
+        ctx.logger.info("export • xlsx → %s", out_path)
+        return {"artifact": out_path}

reader_workbench/plugins/ingest/_discovery.py

@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
+from pathlib import Path
+
+from reader_workbench.errors import ParseError
+
+from .discovery_policy import DEFAULT_ROOTS, discover_files
+
+
+def auto_pick_discovered_file(
+    files: Sequence[Path],
+    mode: str,
+    *,
+    singular_label: str,
+) -> Path:
+    if mode == "single":
+        if len(files) != 1:
+            raise ParseError(
+                f"Auto-discovery expected exactly one {singular_label}, found "
+                f"{len(files)}:\n- " + "\n- ".join(str(path) for path in files)
+            )
+        return files[0]
+    if mode == "latest":
+        return max(files, key=lambda path: path.stat().st_mtime)
+    raise ParseError(f"Unknown auto_pick mode {mode!r} (expected: single|latest|merge)")
+
+
+def discover_auto_input_files(
+    *,
+    exp_dir: Path,
+    auto_roots: Sequence[str] | None,
+    auto_include: Sequence[str],
+    auto_exclude: Sequence[str],
+    auto_recursive: bool,
+    auto_pick: str,
+    discovery_label: str,
+    singular_label: str,
+) -> list[Path]:
+    roots = list(auto_roots or DEFAULT_ROOTS)
+    files = discover_files(
+        exp_dir,
+        roots=roots,
+        include=list(auto_include),
+        exclude=list(auto_exclude),
+        recursive=auto_recursive,
+    )
+    if not files:
+        raise ParseError(
+            f"No {discovery_label} discovered under {roots} "
+            f"(include={list(auto_include)}, exclude={list(auto_exclude)}).\n"
+            "Hint: put raw files under ./inputs (default), or set auto_roots / reads.raw explicitly."
+        )
+    if auto_pick in {"single", "latest"}:
+        return [auto_pick_discovered_file(files, auto_pick, singular_label=singular_label)]
+    if auto_pick == "merge":
+        return files
+    raise ParseError(f"Unknown auto_pick mode {auto_pick!r} (expected: single|latest|merge)")

reader_workbench/plugins/ingest/discovery_policy.py

@@ -0,0 +1,66 @@
+"""Raw-file autodiscovery policy for ingest plugins."""
+
+from __future__ import annotations
+
+import fnmatch
+from collections.abc import Iterable, Sequence
+from pathlib import Path
+
+from reader_workbench.errors import ParseError
+from reader_workbench.workbench.input_discovery import (
+    DEFAULT_INPUT_EXCLUDE as DEFAULT_EXCLUDE,
+)
+from reader_workbench.workbench.input_discovery import (
+    DEFAULT_INPUT_ROOTS as DEFAULT_ROOTS,
+)
+from reader_workbench.workbench.input_discovery import (
+    DEFAULT_WORKBOOK_INCLUDE as DEFAULT_INCLUDE,
+)
+from reader_workbench.workbench.paths import resolve_path_within_root
+
+
+def _iter_candidates(root: Path, patterns: Sequence[str], recursive: bool) -> Iterable[Path]:
+    if not root.exists():
+        return []
+    if recursive:
+        for path in root.rglob("*"):
+            if path.is_file() and any(fnmatch.fnmatch(path.name, pat) for pat in patterns):
+                yield path
+        return
+    for path in root.glob("*"):
+        if path.is_file() and any(fnmatch.fnmatch(path.name, pat) for pat in patterns):
+            yield path
+
+
+def discover_files(
+    base: Path,
+    *,
+    roots: Sequence[str] | None = None,
+    include: Sequence[str] = DEFAULT_INCLUDE,
+    exclude: Sequence[str] = DEFAULT_EXCLUDE,
+    recursive: bool = False,
+) -> list[Path]:
+    roots = list(roots or DEFAULT_ROOTS)
+    out: list[Path] = []
+    seen: set[Path] = set()
+    for root in roots:
+        try:
+            root_path = resolve_path_within_root(root, root=base)
+        except ValueError as err:
+            raise ParseError(
+                f"auto_roots entry {root!r} must stay under the experiment root after resolving symlinks."
+            ) from err
+        for path in _iter_candidates(root_path, include, recursive):
+            try:
+                confined_path = resolve_path_within_root(path, root=base)
+            except ValueError as err:
+                raise ParseError(
+                    f"Discovered input {path!s} must stay under the experiment root after resolving symlinks."
+                ) from err
+            if any(fnmatch.fnmatch(path.name, pat) for pat in exclude):
+                continue
+            if confined_path in seen:
+                continue
+            out.append(confined_path)
+            seen.add(confined_path)
+    return sorted(out)

reader_workbench/plugins/ingest/flow_cytometer.py

@@ -0,0 +1,139 @@
+"""Flow cytometer ingest for .fcs files (snapshot data)."""
+
+from __future__ import annotations
+
+import importlib.util
+from collections.abc import Mapping
+from contextlib import suppress
+from typing import Literal
+
+import pandas as pd
+from pydantic import Field
+
+from reader_workbench.domains.cytometry.io.fcs import parse_fcs_file
+from reader_workbench.errors import ParseError
+from reader_workbench.plugins.ingest._discovery import discover_auto_input_files
+from reader_workbench.plugins.ingest.discovery_policy import DEFAULT_EXCLUDE
+from reader_workbench.workbench.ports import dataframe_output, file_set_input
+from reader_workbench.workbench.registry import Plugin, PluginConfig, PreflightIssue
+
+DEFAULT_FCS_INCLUDE = ("*.fcs", "*.FCS")
+
+
+class FlowCytometerCfg(PluginConfig):
+    # auto-discovery knobs
+    auto_roots: list[str] | None = None
+    auto_include: list[str] = Field(default_factory=lambda: list(DEFAULT_FCS_INCLUDE))
+    auto_exclude: list[str] = Field(default_factory=lambda: list(DEFAULT_EXCLUDE))
+    auto_pick: Literal["single", "latest", "merge"] = "merge"
+    auto_recursive: bool = False
+
+    # channel naming / output shaping
+    channel_name_field: str = "pns"  # field in FCS channel metadata (e.g., pns or pnn)
+    channel_map: Mapping[str, str] | None = None
+    drop_channels: list[str] | None = None
+    sample_id_from: Literal["stem", "name"] = "stem"
+    time_value: float = 0.0
+
+    # logging
+    print_summary: bool = True
+
+
+class FlowCytometerIngest(Plugin):
+    """Ingest .fcs files into tidy.v1 (snapshot; time is constant)."""
+
+    ConfigModel = FlowCytometerCfg
+
+    @classmethod
+    def input_ports(cls):
+        return {"raw": file_set_input("raw", optional=True)}
+
+    @classmethod
+    def output_ports(cls):
+        return {
+            "df": dataframe_output("df", "tidy.v1"),
+            "channels": dataframe_output("channels", "cytometer.channels.v1"),
+        }
+
+    @classmethod
+    def preflight_readiness(cls, *, exp_dir, cfg: FlowCytometerCfg, reads):
+        issues: list[PreflightIssue] = []
+        if importlib.util.find_spec("flowio") is None:
+            issues.append(
+                PreflightIssue(
+                    kind="dependency",
+                    message="flowio is required for ingest/flow_cytometer. Re-sync the core reader environment.",
+                )
+            )
+        if "raw" not in reads:
+            try:
+                discover_auto_input_files(
+                    exp_dir=exp_dir,
+                    auto_roots=cfg.auto_roots,
+                    auto_include=cfg.auto_include,
+                    auto_exclude=cfg.auto_exclude,
+                    auto_recursive=cfg.auto_recursive,
+                    auto_pick=cfg.auto_pick,
+                    discovery_label=".fcs files",
+                    singular_label=".fcs file",
+                )
+            except ParseError as err:
+                issues.append(PreflightIssue(kind="file", message=str(err)))
+        return tuple(issues)
+
+    @classmethod
+    def resolve_missing_file_inputs(cls, *, exp_dir, cfg: FlowCytometerCfg, inputs):
+        if "raw" in inputs:
+            return {}
+        files = discover_auto_input_files(
+            exp_dir=exp_dir,
+            auto_roots=cfg.auto_roots,
+            auto_include=cfg.auto_include,
+            auto_exclude=cfg.auto_exclude,
+            auto_recursive=cfg.auto_recursive,
+            auto_pick=cfg.auto_pick,
+            discovery_label=".fcs files",
+            singular_label=".fcs file",
+        )
+        return {"raw": tuple(files)}
+
+    def run(self, ctx, inputs, cfg: FlowCytometerCfg):
+        files = tuple(inputs["raw"])
+        channel_map = {str(k): str(v) for k, v in (cfg.channel_map or {}).items()}
+        drop_channels = {str(c) for c in (cfg.drop_channels or [])}
+
+        frames: list[pd.DataFrame] = []
+        channel_meta_frames: list[pd.DataFrame] = []
+        for f in files:
+            long, channels_meta = parse_fcs_file(
+                f,
+                channel_name_field=cfg.channel_name_field,
+                channel_map=channel_map,
+                drop_channels=drop_channels,
+                sample_id_from=cfg.sample_id_from,
+                time_value=cfg.time_value,
+            )
+            frames.append(long)
+            if not channels_meta.empty:
+                channel_meta_frames.append(channels_meta)
+
+        if not frames:
+            raise ParseError("No cytometer frames parsed from selected files")
+        out = pd.concat(frames, ignore_index=True)
+
+        if cfg.print_summary:
+            with suppress(Exception):
+                ctx.logger.info(
+                    "flow_cytometer ingest • files=%d • rows=%d • channels=%d • samples=%d",
+                    len(files),
+                    len(out),
+                    out["channel"].nunique(),
+                    out["sample_id"].nunique(),
+                )
+
+        if channel_meta_frames:
+            channels_meta = pd.concat(channel_meta_frames, ignore_index=True)
+        else:
+            channels_meta = pd.DataFrame(columns=["sample_id", "channel_index", "channel_name"])
+
+        return {"df": out, "channels": channels_meta}