reader-workbench 1.0.0__py3-none-any.whl

reader_workbench/workbench/registry.py

@@ -0,0 +1,333 @@
+from __future__ import annotations
+
+import importlib.metadata as md
+from abc import ABC, abstractmethod
+from collections.abc import Callable, Iterable, Mapping
+from dataclasses import dataclass, replace
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+from pydantic import BaseModel
+
+from reader_workbench.contracts import ContractCatalog, ContractId, OutputContractSurface
+from reader_workbench.errors import ContractError, RegistryError
+from reader_workbench.workbench.assets import AssetCatalog, AssetDescriptor, plugin_category_from_id
+from reader_workbench.workbench.ports import (
+    InputPortSpec,
+    OutputPortSpec,
+    validate_input_ports,
+    validate_output_ports,
+)
+
+
+class PluginConfig(BaseModel):
+    """Base class for per-plugin configs (pydantic v2)."""
+
+    model_config = {"extra": "forbid"}
+
+
+@dataclass(frozen=True)
+class PreflightIssue:
+    kind: str
+    message: str
+
+
+class Plugin(ABC):
+    """Contract-driven plugin interface."""
+
+    ConfigModel = PluginConfig
+
+    def __init__(self) -> None:
+        self._descriptor: AssetDescriptor | None = None
+        self._contracts: ContractCatalog | None = None
+
+    @classmethod
+    @abstractmethod
+    def input_ports(cls) -> Mapping[str, InputPortSpec]:
+        """Typed input port declarations."""
+
+    @classmethod
+    @abstractmethod
+    def output_ports(cls) -> Mapping[str, OutputPortSpec]:
+        """Typed output port declarations."""
+
+    def resolve_output_ports(
+        self,
+        *,
+        inputs: Mapping[str, Any],
+        outputs: Mapping[str, Any],
+        cfg: PluginConfig,
+        where: str,
+    ) -> Mapping[str, OutputPortSpec]:
+        """Resolve runtime output ports; default is the declared minimum."""
+        del inputs, outputs, cfg, where
+        return dict(type(self).output_ports())
+
+    @classmethod
+    def output_port_surfaces(cls) -> Mapping[str, OutputContractSurface]:
+        surfaces: dict[str, OutputContractSurface] = {}
+        for name, port in cls.output_ports().items():
+            surface = port.contract_surface
+            if surface is not None:
+                surfaces[name] = surface
+        return surfaces
+
+    @classmethod
+    def preflight_readiness(
+        cls,
+        *,
+        exp_dir: Path,
+        cfg: PluginConfig,
+        reads: Mapping[str, Any],
+    ) -> tuple[PreflightIssue, ...]:
+        del exp_dir, cfg, reads
+        return ()
+
+    @classmethod
+    def validate_semantic_references(cls, *, experiment: Any, cfg: PluginConfig) -> None:
+        """Validate experiment annotation references owned by this plugin's config."""
+
+        del experiment, cfg
+
+    @classmethod
+    def resolve_missing_file_inputs(
+        cls,
+        *,
+        exp_dir: Path,
+        cfg: PluginConfig,
+        inputs: Mapping[str, Any],
+    ) -> Mapping[str, Path]:
+        """Resolve optional file inputs that were not bound in the graph."""
+        del exp_dir, cfg, inputs
+        return {}
+
+    @classmethod
+    def passthrough_output_ports(
+        cls,
+        *,
+        outputs: Mapping[str, OutputPortSpec],
+        passthrough: Mapping[str, str],
+        promoted_examples: Mapping[str, tuple[ContractId, ...]] | None = None,
+        note: str | None = None,
+    ) -> dict[str, OutputPortSpec]:
+        ports = dict(outputs)
+        promoted_examples = promoted_examples or {}
+        for out_name in passthrough:
+            port = ports.get(out_name)
+            if port is None or port.kind != "dataframe" or port.contract is None:
+                continue
+            ports[out_name] = replace(
+                port,
+                surface=OutputContractSurface(
+                    minimum=port.contract,
+                    runtime_mode="passthrough",
+                    promoted=tuple(promoted_examples.get(out_name, ())),
+                    note=note,
+                ),
+            )
+        return ports
+
+    @classmethod
+    def promoted_output_ports(
+        cls,
+        *,
+        outputs: Mapping[str, OutputPortSpec],
+        promotions: Mapping[str, tuple[ContractId, ...]],
+        note: str | None = None,
+    ) -> dict[str, OutputPortSpec]:
+        ports = dict(outputs)
+        for out_name, promoted in promotions.items():
+            port = ports.get(out_name)
+            if port is None or port.kind != "dataframe" or port.contract is None:
+                continue
+            ports[out_name] = replace(
+                port,
+                surface=OutputContractSurface(
+                    minimum=port.contract,
+                    runtime_mode="promoted",
+                    promoted=tuple(promoted),
+                    note=note,
+                ),
+            )
+        return ports
+
+    def inherit_dataframe_output_ports(
+        self,
+        *,
+        inputs: Mapping[str, Any],
+        outputs: Mapping[str, Any],
+        passthrough: Mapping[str, str],
+        where: str,
+    ) -> dict[str, OutputPortSpec]:
+        """
+        Preserve stricter dataframe contracts across pass-through transforms when
+        the emitted dataframe still validates against the input contract.
+        """
+        resolved = dict(type(self).output_ports())
+        for out_name, in_name in passthrough.items():
+            if out_name not in resolved or in_name not in inputs or out_name not in outputs:
+                continue
+            port = resolved[out_name]
+            if port.kind != "dataframe" or port.contract is None:
+                continue
+            actual = getattr(inputs[in_name], "contract_id", None)
+            if actual in (None, "none"):
+                continue
+            if not self.contracts.satisfies(actual=actual, expected=port.contract):
+                continue
+            if not isinstance(outputs[out_name], pd.DataFrame):
+                continue
+            try:
+                self.contracts.validate(outputs[out_name], contract_id=actual, where=f"{where}:{out_name}")
+            except ContractError:
+                continue
+            resolved[out_name] = replace(port, contract=actual)
+        return resolved
+
+    @abstractmethod
+    def run(self, ctx, inputs: dict[str, Any], cfg: PluginConfig) -> dict[str, Any]:
+        """Execute and return dict of outputs by label."""
+
+    def bind_runtime(self, *, descriptor: AssetDescriptor, contracts: ContractCatalog) -> None:
+        if descriptor.cls is not type(self):
+            raise RegistryError(
+                f"Descriptor {descriptor.plugin_id!r} points to {descriptor.cls.__module__}.{descriptor.cls.__name__}, "
+                f"not {type(self).__module__}.{type(self).__name__}"
+            )
+        self._descriptor = descriptor
+        self._contracts = contracts
+
+    @property
+    def descriptor(self) -> AssetDescriptor:
+        if self._descriptor is None:
+            raise RegistryError(
+                f"Plugin instance {type(self).__module__}.{type(self).__name__} is missing a bound descriptor"
+            )
+        return self._descriptor
+
+    @property
+    def contracts(self) -> ContractCatalog:
+        if self._contracts is None:
+            raise RegistryError(
+                f"Plugin instance {type(self).__module__}.{type(self).__name__} is missing a bound contract catalog"
+            )
+        return self._contracts
+
+    @property
+    def plugin_id(self) -> str:
+        return self.descriptor.plugin_id
+
+    @property
+    def plugin_key(self) -> str:
+        return self.descriptor.key
+
+    @property
+    def plugin_category(self) -> str:
+        return self.descriptor.category
+
+
+class Registry:
+    """Descriptor-driven plugin registry with explicit built-ins and explicit entry points."""
+
+    def __init__(self, *, contracts: ContractCatalog) -> None:
+        self._descriptors: dict[str, AssetDescriptor] = {}
+        self.contracts = contracts
+
+    def register(self, descriptor: AssetDescriptor) -> None:
+        if not issubclass(descriptor.cls, Plugin):
+            raise RegistryError(
+                f"Plugin descriptor {descriptor.plugin_id!r} must point to a Plugin subclass, "
+                f"got {descriptor.cls.__module__}.{descriptor.cls.__name__}"
+            )
+        if descriptor.plugin in self._descriptors:
+            raise RegistryError(f"Duplicate plugin {descriptor.plugin!r}")
+        validate_input_ports(descriptor.cls.input_ports(), where=descriptor.plugin)
+        outputs = validate_output_ports(descriptor.cls.output_ports(), where=descriptor.plugin)
+        surfaces = descriptor.cls.output_port_surfaces()
+        for name, surface in surfaces.items():
+            port = outputs.get(name)
+            if port is None:
+                raise RegistryError(f"{descriptor.plugin}: output port surface {name!r} does not match a declared port")
+            if port.kind != "dataframe":
+                raise RegistryError(f"{descriptor.plugin}: output port surface {name!r} requires a dataframe port")
+            if surface.minimum != port.contract:
+                raise RegistryError(
+                    f"{descriptor.plugin}: output port surface {name!r} minimum {surface.minimum!r} "
+                    f"must match declared contract {port.contract!r}"
+                )
+        self._descriptors[descriptor.plugin] = descriptor
+
+    def categories(self) -> Mapping[str, Mapping[str, type[Plugin]]]:
+        grouped: dict[str, dict[str, type[Plugin]]] = {
+            "ingest": {},
+            "transform": {},
+            "plot": {},
+            "export": {},
+            "validator": {},
+        }
+        for descriptor in self._descriptors.values():
+            category = descriptor.category
+            grouped[category][descriptor.key] = descriptor.cls
+        return grouped
+
+    def catalog(self) -> AssetCatalog:
+        return AssetCatalog(list(self._descriptors.values()))
+
+    def resolve_descriptor(self, plugin: str) -> AssetDescriptor:
+        try:
+            return self._descriptors[plugin]
+        except KeyError:
+            available = ", ".join(sorted(self._descriptors))
+            raise RegistryError(f"Unknown plugin '{plugin}'. Installed: {available}") from None
+
+    def resolve(self, plugin: str) -> type[Plugin]:
+        return self.resolve_descriptor(plugin).cls
+
+
+def _coerce_external_descriptor(loaded: Any, *, ep_name: str) -> AssetDescriptor:
+    if isinstance(loaded, AssetDescriptor):
+        descriptor = loaded
+    elif isinstance(loaded, type):
+        raise RegistryError(
+            f"Entry point {ep_name!r} must expose a plugin descriptor or descriptor factory, not a plugin class"
+        )
+    elif isinstance(loaded, Callable):
+        descriptor = loaded()
+    else:
+        descriptor = loaded
+    if not isinstance(descriptor, AssetDescriptor):
+        raise RegistryError(
+            f"Entry point {ep_name!r} must load an AssetDescriptor or a zero-arg callable returning one"
+        )
+    if ep_name != descriptor.plugin_id:
+        raise RegistryError(f"Entry point {ep_name!r} must match descriptor plugin id {descriptor.plugin_id!r}")
+    return descriptor
+
+
+def load_plugin_catalog(
+    *,
+    contracts: ContractCatalog,
+    builtin_descriptors: Iterable[AssetDescriptor],
+    categories: set[str] | None = None,
+) -> Registry:
+    """Register supplied built-ins and coordinated external plugin entry points."""
+    reg = Registry(contracts=contracts)
+    wanted = set(categories) if categories else None
+
+    selected_builtin_descriptors = tuple(
+        descriptor for descriptor in builtin_descriptors if wanted is None or descriptor.category in wanted
+    )
+    if not selected_builtin_descriptors:
+        raise RegistryError("No built-in plugin descriptors were supplied for the requested categories.")
+    for descriptor in selected_builtin_descriptors:
+        reg.register(descriptor)
+
+    for ep in md.entry_points(group="reader_workbench.plugins"):
+        entry_point_category = plugin_category_from_id(ep.name)
+        if wanted is not None and entry_point_category not in wanted:
+            continue
+        descriptor = _coerce_external_descriptor(ep.load(), ep_name=ep.name)
+        reg.register(descriptor)
+
+    return reg

reader_workbench/workbench/spec_overrides.py

@@ -0,0 +1,215 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import typer
+import yaml
+
+from reader_workbench.errors import ConfigError
+from reader_workbench.workbench.experiment import ResourceCatalog
+from reader_workbench.workbench.graph import (
+    FileRef,
+    InputRef,
+    OutputRef,
+    RecordRef,
+    ResourceRef,
+    select_workbench_specs,
+)
+from reader_workbench.workbench.paths import resolve_path_within_root
+
+
+def build_surface_command(
+    command: str,
+    job_path: Path,
+    *,
+    only: list[str] | None,
+    exclude: list[str] | None,
+    list_only: bool = False,
+    dry_run: bool = False,
+    log_level: str = "INFO",
+    inputs: list[str] | None = None,
+    sets: list[str] | None = None,
+) -> list[str]:
+    parts = ["uv", "run", *command.split(), str(job_path)]
+    if list_only:
+        parts += ["--list"]
+    if only:
+        for value in only:
+            parts += ["--only", value]
+    if exclude:
+        for value in exclude:
+            parts += ["--exclude", value]
+    if dry_run:
+        parts += ["--dry-run"]
+    if log_level and log_level != "INFO":
+        parts += ["--log-level", log_level]
+    for raw in inputs or []:
+        parts += ["--input", raw]
+    for raw in sets or []:
+        parts += ["--set", raw]
+    return parts
+
+
+def select_surface_specs(steps, *, only: list[str], exclude: list[str], kind: str):
+    try:
+        return select_workbench_specs(steps, only=only, exclude=exclude, kind_label=kind)
+    except ConfigError as err:
+        raise typer.BadParameter(f"{err} Use --list to see valid ids.") from err
+
+
+def parse_input_overrides(
+    raw_inputs: list[str],
+    *,
+    root: Path,
+    resources: ResourceCatalog,
+) -> dict[str, InputRef]:
+    overrides: dict[str, InputRef] = {}
+    for raw in raw_inputs:
+        if "=" not in raw:
+            raise typer.BadParameter("--input expects KEY=VALUE")
+        key, value = raw.split("=", 1)
+        key = key.strip()
+        value = value.strip()
+        if not key:
+            raise typer.BadParameter("--input key cannot be empty")
+        if not value:
+            raise typer.BadParameter("--input value cannot be empty")
+        overrides[key] = _coerce_cli_input_ref(yaml.safe_load(value), root=root, resources=resources)
+    return overrides
+
+
+def parse_set_overrides(raw_sets: list[str]) -> list[tuple[str, object]]:
+    overrides: list[tuple[str, object]] = []
+    for raw in raw_sets:
+        if "=" not in raw:
+            raise typer.BadParameter("--set expects PATH=VALUE")
+        path, value_raw = raw.split("=", 1)
+        path = path.strip()
+        if not path:
+            raise typer.BadParameter("--set path cannot be empty")
+        overrides.append((path, yaml.safe_load(value_raw)))
+    return overrides
+
+
+def apply_step_overrides(
+    steps,
+    *,
+    input_overrides: dict[str, InputRef],
+    set_overrides: list[tuple[str, object]],
+    root: Path,
+    resources: ResourceCatalog,
+):
+    updated = []
+    for step in steps:
+        if hasattr(step, "model_copy"):
+            cloned = step.model_copy(deep=True)
+            reads = dict(cloned.reads or {})
+            with_block = dict(cloned.with_ or {})
+            writes = dict(cloned.writes or {})
+        else:
+            reads = dict(step.reads or {})
+            with_block = dict(step.with_ or {})
+            writes = dict(step.writes or {})
+        if input_overrides:
+            reads.update(input_overrides)
+        for path, value in set_overrides:
+            parts = [item for item in path.split(".") if item]
+            if not parts:
+                raise typer.BadParameter("--set path cannot be empty")
+            section = parts[0]
+            if section not in {"reads", "with", "writes"}:
+                raise typer.BadParameter("--set path must start with reads., with., or writes.")
+            if section in {"reads", "writes"}:
+                if len(parts) != 2:
+                    raise typer.BadParameter(f"--set {section} expects a single key (e.g., {section}.foo=bar)")
+                target = reads if section == "reads" else writes
+                target[parts[1]] = (
+                    _coerce_cli_input_ref(value, root=root, resources=resources)
+                    if section == "reads"
+                    else _coerce_cli_output_ref(value)
+                )
+            else:
+                if len(parts) < 2:
+                    raise typer.BadParameter("--set with.* requires a key (e.g., with.foo=bar)")
+                _set_nested(with_block, parts[1:], value)
+        if hasattr(step, "model_copy"):
+            cloned.reads = reads
+            cloned.with_ = with_block
+            cloned.writes = writes
+            updated.append(cloned)
+            continue
+        payload = {
+            "id": step.id,
+            "plugin": step.plugin,
+            "reads": reads,
+            "with_": with_block,
+            "writes": writes,
+            "source_recipe": getattr(step, "source_recipe", None),
+        }
+        if hasattr(step, "kind"):
+            payload["kind"] = step.kind
+        updated.append(step.__class__(**payload))
+    return updated
+
+
+def _coerce_cli_input_ref(value, *, root: Path, resources: ResourceCatalog) -> InputRef:
+    if isinstance(value, (RecordRef, FileRef, ResourceRef)):
+        return value
+    if isinstance(value, dict):
+        record = value.get("record")
+        file_path = value.get("file")
+        resource_id = value.get("resource")
+        populated = [item for item in (record, file_path, resource_id) if item is not None]
+        if len(populated) != 1:
+            raise typer.BadParameter("reads.* must declare exactly one of record, file, or resource")
+        if isinstance(record, str) and record.strip():
+            return RecordRef(record_id=record.strip())
+        if isinstance(file_path, str) and file_path.strip():
+            path = Path(file_path.strip()).expanduser()
+            try:
+                path = resolve_path_within_root(path, root=root)
+            except ValueError as err:
+                raise typer.BadParameter(
+                    "reads.* file bindings must stay under the experiment root after resolving symlinks"
+                ) from err
+            return FileRef(path=path)
+        if isinstance(resource_id, str) and resource_id.strip():
+            return _resolve_cli_resource_ref(resource_id.strip(), resources=resources)
+        raise typer.BadParameter("reads.* binding values must be non-empty strings")
+    if isinstance(value, str) and value.strip():
+        return RecordRef(record_id=value.strip())
+    raise typer.BadParameter("reads.* expects a YAML/JSON mapping like {record: ...}, {file: ...}, or {resource: ...}")
+
+
+def _resolve_cli_resource_ref(resource_id: str, *, resources: ResourceCatalog) -> ResourceRef:
+    if not resource_id:
+        raise typer.BadParameter("resource bindings require a non-empty resource id")
+    try:
+        resource = resources.require_file(resource_id)
+    except ValueError as err:
+        raise typer.BadParameter(str(err)) from err
+    return ResourceRef(resource_id=resource_id, path=resource.path.resolve())
+
+
+def _coerce_cli_output_ref(value) -> OutputRef:
+    if isinstance(value, OutputRef):
+        return value
+    if isinstance(value, dict):
+        record = value.get("record")
+        if isinstance(record, str) and record.strip():
+            return OutputRef(record_id=record.strip())
+        raise typer.BadParameter("writes.* must declare {record: ...}")
+    if isinstance(value, str) and value.strip():
+        return OutputRef(record_id=value.strip())
+    raise typer.BadParameter("writes.* expects a record id or a {record: ...} mapping")
+
+
+def _set_nested(mapping: dict, keys: list[str], value) -> None:
+    current = mapping
+    for key in keys[:-1]:
+        if key not in current:
+            current[key] = {}
+        if not isinstance(current[key], dict):
+            raise typer.BadParameter(f"--set path invalid (non-mapping at '{key}')")
+        current = current[key]
+    current[keys[-1]] = value

reader_workbench-1.0.0.dist-info/METADATA

@@ -0,0 +1,91 @@
+Metadata-Version: 2.4
+Name: reader-workbench
+Version: 1.0.0
+Summary: Validated, traceable analysis workflows for experimental instrument data
+Author: Eric J. South
+License-Expression: MIT
+Project-URL: Documentation, https://github.com/e-south/reader/tree/main/docs
+Project-URL: Issues, https://github.com/e-south/reader/issues
+Project-URL: Repository, https://github.com/e-south/reader
+Keywords: data-analysis,data-pipelines,scientific-computing,workbench
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering
+Requires-Python: <3.13,>=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.26
+Requires-Dist: matplotlib>=3.8
+Requires-Dist: pandas>=2.1
+Requires-Dist: pydantic>=2.6
+Requires-Dist: pyyaml>=6.0.2
+Requires-Dist: typer>=0.16.0
+Requires-Dist: click>=8.3.3
+Requires-Dist: rich>=13.7
+Requires-Dist: pyarrow>=23.0.1
+Requires-Dist: openpyxl>=3.1
+Requires-Dist: seaborn>=0.13
+Requires-Dist: pillow>=12.3.0
+Requires-Dist: polars>=1.36.1
+Requires-Dist: flowio>=1.4.0
+Requires-Dist: filelock<4,>=3.20.3
+Dynamic: license-file
+
+# ![Reader data workbench](https://raw.githubusercontent.com/e-south/reader/v1.0.0/assets/reader-banner.png)
+
+[![Checks](https://github.com/e-south/reader/actions/workflows/checks.yaml/badge.svg?branch=main)](https://github.com/e-south/reader/actions/workflows/checks.yaml)
+[![Python 3.12](https://img.shields.io/badge/python-3.12-3776AB.svg)](https://www.python.org/downloads/release/python-3120/)
+[![MIT license](https://img.shields.io/badge/license-MIT-3D8068.svg)](https://github.com/e-south/reader/blob/main/LICENSE)
+
+Reader ingests instrument files and experiment metadata, applies declared
+transformations, and writes validated records, plots, exports, and notebooks.
+Every unit of work—including a cross-experiment aggregate—has an owned directory
+beneath `experiments/`, with source material in `inputs/`, a `reader/v8`
+contract in `config.yaml`, and generated artifacts in `outputs/`.
+The distribution and import package are named `reader-workbench` and
+`reader_workbench`; the installed command remains `reader`.
+
+## Install
+
+Install Reader as a command-line tool:
+
+```bash
+uv tool install reader-workbench
+```
+
+For development, install from a checkout:
+
+```bash
+git clone https://github.com/e-south/reader.git
+cd reader
+uv sync --locked --group dev
+uv run reader demo
+```
+
+The installed command is `reader`:
+
+```bash
+reader demo
+reader protocols
+reader init ./experiments/my_experiment --protocol plate_reader/single_reporter_screen
+```
+
+The demo prints a guided command tour. It does not execute a pipeline or write
+files. Every generated starter can be inspected and validated before data is
+added.
+
+## Learn more
+
+- [Getting started](https://github.com/e-south/reader/blob/main/docs/guides/getting_started.md) — install Reader, run the
+  demo, and scaffold a first experiment.
+- [Common tasks](https://github.com/e-south/reader/blob/main/docs/guides/common_routes.md) — shortest commands for
+  discovery, validation, execution, and automation.
+- [Python API](https://github.com/e-south/reader/blob/main/docs/core/python_api.md) — typed, task-oriented experiment and
+  plugin interfaces for integrations.
+- [Documentation index](https://github.com/e-south/reader/blob/main/docs/README.md) — complete user, reference, and
+  maintainer documentation.
+
+Reader is available under the [MIT license](https://github.com/e-south/reader/blob/main/LICENSE).