reader-workbench 1.0.0__py3-none-any.whl

reader_workbench/workbench/dop/builtins.py

@@ -0,0 +1,261 @@
+from __future__ import annotations
+
+from functools import cache
+
+from .model import DataClassSpec, DopRegistry, ReadySpec
+
+DOP_SCHEMA = "reader.dop/v1"
+
+BUILTIN_DATA_CLASSES: tuple[DataClassSpec, ...] = (
+    DataClassSpec(
+        id="plate_reader_screen",
+        label="Plate-reader screen",
+        summary="Well-level plate-reader assay data with explicit channel, treatment, control, and plate/well semantics.",
+        decision_order=10,
+        protocol_candidates=(
+            "plate_reader/dual_reporter_screen",
+            "plate_reader/single_reporter_screen",
+            "plate_reader/growth_screen",
+        ),
+        minimum_capture=(
+            "raw plate-reader workbook or export",
+            "sample map with measured well coverage",
+            "channel labels and any denominator or derived-measurement meaning",
+            "treatment and control semantics",
+            "plate and well identifiers, plus any source-declared replicate and design identifiers",
+        ),
+        stop_conditions=(
+            "well coordinates or sample positions conflict",
+            "treatment or control meaning is incomplete",
+            "channel labels drift from the selected protocol",
+            "nearest protocol would silently change control semantics",
+        ),
+        transfer_rules=(
+            "stage the original workbook or export under inputs/",
+            "bind metadata files through resources",
+            "regenerate plots, exports, and records from source inputs",
+        ),
+        verification=(
+            "config schema and protocol binding validate",
+            "declared raw files and metadata resources exist",
+            "records catalog captures generated dataframe evidence",
+        ),
+    ),
+    DataClassSpec(
+        id="flow_cytometry_panel",
+        label="Flow-cytometry panel",
+        summary="Cytometry panel data with explicit FCS discovery, channel naming, and sample metadata.",
+        decision_order=20,
+        protocol_candidates=("cytometry/flow_panel",),
+        minimum_capture=(
+            "raw FCS files and their discovery policy",
+            "channel naming field",
+            "sample metadata",
+            "required panel metadata columns",
+        ),
+        stop_conditions=(
+            "FCS file discovery or mapping is ambiguous",
+            "channel naming field is unknown",
+            "sample metadata cannot be joined to events",
+        ),
+        transfer_rules=(
+            "stage raw FCS material under inputs/",
+            "declare FCS discovery under protocol inputs and bind metadata files through resources",
+            "keep generated review notebooks under outputs/notebooks/",
+        ),
+        verification=(
+            "declared or discovered FCS files resolve",
+            "configured metadata columns are present",
+            "records catalog captures panel dataframe evidence",
+        ),
+    ),
+    DataClassSpec(
+        id="four_state_logic_analysis",
+        label="Four-state logic analysis",
+        summary="Logic-response assay data with explicit response and intensity channels plus four ordered states.",
+        decision_order=30,
+        protocol_candidates=("logic/four_state_vector_screen",),
+        minimum_capture=(
+            "raw assay files",
+            "metadata map",
+            "response and intensity channel choices",
+            "reference design",
+            "ordered 00/10/01/11 states",
+        ),
+        stop_conditions=(
+            "reference design cannot be reconstructed",
+            "ordered state values are missing or contradictory",
+            "response or intensity channel choices are ambiguous",
+        ),
+        transfer_rules=(
+            "stage raw files under inputs/",
+            "encode ordered state spaces in annotations or metadata resources",
+            "regenerate four-state vector summaries from source inputs",
+        ),
+        verification=(
+            "logic reference config validates",
+            "ordered state-space annotations are present",
+            "records catalog captures vector summary evidence",
+        ),
+    ),
+    DataClassSpec(
+        id="record_collection_analysis",
+        label="Record-collection analysis",
+        summary="A new Reader experiment that analyzes exact dataframe-record revisions from prior experiments.",
+        decision_order=40,
+        protocol_candidates=("logic/four_state_vector_collection", "plate_reader/four_state_event_window"),
+        minimum_capture=(
+            "source experiment ids",
+            "exact dataframe record ids",
+            "analysis semantics",
+            "expected records and review outputs",
+        ),
+        stop_conditions=(
+            "source experiment ids are unknown",
+            "source records are missing, changed, or contract-incompatible",
+            "the selected collection protocol would change analysis meaning",
+        ),
+        transfer_rules=(
+            "declare source records as resources of kind record",
+            "reference source experiments instead of copying generated outputs",
+            "persist derived records and review artifacts through the canonical engine",
+        ),
+        verification=(
+            "source record contracts and exact revisions validate",
+            "derived records are present in the aggregate experiment catalog",
+            "generic Reader verify passes for the aggregate experiment",
+        ),
+    ),
+    DataClassSpec(
+        id="unsupported_long_tail_assay",
+        label="Unsupported long-tail assay",
+        summary="Assay data that does not yet fit an existing executable protocol contract.",
+        decision_order=50,
+        protocol_candidates=("workbench/generic",),
+        minimum_capture=(
+            "raw source path",
+            "intended analysis",
+            "required metadata",
+            "missing protocol decision",
+            "owner for follow-up",
+        ),
+        stop_conditions=(
+            "nearest protocol would change assay meaning",
+            "required metadata is unknown",
+            "execution contract is still being discovered",
+        ),
+        transfer_rules=(
+            "keep the experiment draft or template until semantics are clear",
+            "stage raw files without pretending they are runnable",
+            "add a protocol only after the metadata and execution contract stabilize",
+        ),
+        verification=(
+            "draft/template config shape validates with --no-files",
+            "missing protocol or metadata contract is documented",
+            "no generated outputs are treated as source material",
+        ),
+    ),
+)
+
+BUILTIN_READY_SPECS: tuple[ReadySpec, ...] = (
+    ReadySpec(
+        id="classified",
+        label="Classified",
+        summary="Dataset has a selected DOP data class and protocol candidate set.",
+        required_evidence=(
+            "DOP data class id",
+            "candidate reader protocol ids",
+            "reason the selected class fits the dataset",
+        ),
+        commands=("uv run reader dop classes",),
+    ),
+    ReadySpec(
+        id="metadata_ready",
+        label="Metadata ready",
+        summary="Required semantics for the selected data class have been captured before execution.",
+        required_evidence=(
+            "dataset identity",
+            "raw provenance",
+            "assay semantics",
+            "sample map",
+            "control semantics",
+            "canonical labels",
+            "requested outputs when they differ from protocol defaults",
+        ),
+        commands=("uv run reader protocols <protocol-id> --example-config",),
+    ),
+    ReadySpec(
+        id="staged",
+        label="Staged",
+        summary="Raw files, metadata resources, and hand-authored notes live in the standard experiment layout.",
+        required_evidence=(
+            "raw files under inputs/",
+            "resources entries for consumed files",
+            "hand-authored notes under notebooks/ when present",
+            "no copied generated outputs used as source material",
+        ),
+        commands=("uv run reader validate <config|dir|index> --no-files --format json",),
+    ),
+    ReadySpec(
+        id="preflight_ok",
+        label="Preflight OK",
+        summary="Schema, protocol binding, declared files, and dependencies pass reader validation.",
+        required_evidence=(
+            "config schema is reader/v8",
+            "protocol binding resolves",
+            "declared files and resources exist",
+            "runtime dependencies are available",
+        ),
+        accepted_readiness_states=("runnable", "uncataloged_outputs_present", "catalog_ready", "records_ready"),
+        commands=("uv run reader validate <config|dir|index> --format json",),
+    ),
+    ReadySpec(
+        id="runnable",
+        label="Runnable",
+        summary="The experiment is active and can run from authored source inputs.",
+        required_evidence=(
+            "reader readiness state allows run",
+            "run capability is true",
+            "next command is explicit",
+        ),
+        accepted_readiness_states=("runnable", "uncataloged_outputs_present", "catalog_ready", "records_ready"),
+        required_capabilities=("run",),
+        commands=("uv run reader run <config|dir|index> --dry-run --format json",),
+    ),
+    ReadySpec(
+        id="records_ready",
+        label="Records ready",
+        summary="Generated dataframe and file-bundle evidence is present in the records catalog.",
+        required_evidence=(
+            "records catalog exists",
+            "all current records use the verifiable schema",
+            "source and generated file digests match",
+            "config and exact upstream record revisions match",
+        ),
+        accepted_readiness_states=("records_ready",),
+        required_capabilities=("records", "verify"),
+        commands=("uv run reader verify <config|dir|index>",),
+    ),
+    ReadySpec(
+        id="review_ready",
+        label="Review ready",
+        summary="Records exist and review surfaces such as plots or notebooks can be inspected deliberately.",
+        required_evidence=(
+            "records catalog exists",
+            "selected plots or notebook scaffold are protocol-compatible",
+            "unresolved metadata assumptions remain visible in the handoff",
+        ),
+        accepted_readiness_states=("records_ready",),
+        required_capabilities=("records", "plot", "notebook_scaffold"),
+        commands=(
+            "uv run reader verify <config|dir|index>",
+            "uv run reader plot <config|dir|index> --list",
+            "uv run reader notebook <config|dir|index> --mode none",
+        ),
+    ),
+)
+
+
+@cache
+def builtin_dop_registry() -> DopRegistry:
+    return DopRegistry(data_classes=BUILTIN_DATA_CLASSES, ready_specs=BUILTIN_READY_SPECS)

reader_workbench/workbench/dop/model.py

@@ -0,0 +1,209 @@
+from __future__ import annotations
+
+from collections.abc import Iterable
+from dataclasses import dataclass
+
+
+def _clean_string(value: str, *, field_name: str) -> str:
+    if not isinstance(value, str) or not value.strip():
+        raise ValueError(f"{field_name} must be a non-empty string.")
+    return value.strip()
+
+
+def _clean_tuple(values: Iterable[str], *, field_name: str, allow_empty: bool = False) -> tuple[str, ...]:
+    cleaned = tuple(str(value).strip() for value in values if str(value).strip())
+    if not allow_empty and not cleaned:
+        raise ValueError(f"{field_name} must include at least one value.")
+    if len(set(cleaned)) != len(cleaned):
+        raise ValueError(f"{field_name} must not include duplicate values.")
+    return cleaned
+
+
+@dataclass(frozen=True)
+class DataClassSpec:
+    id: str
+    label: str
+    summary: str
+    decision_order: int
+    protocol_candidates: tuple[str, ...]
+    minimum_capture: tuple[str, ...]
+    stop_conditions: tuple[str, ...]
+    transfer_rules: tuple[str, ...]
+    verification: tuple[str, ...]
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "id", _clean_string(self.id, field_name="DataClassSpec.id"))
+        object.__setattr__(self, "label", _clean_string(self.label, field_name="DataClassSpec.label"))
+        object.__setattr__(self, "summary", _clean_string(self.summary, field_name="DataClassSpec.summary"))
+        if not isinstance(self.decision_order, int) or self.decision_order < 0:
+            raise ValueError("DataClassSpec.decision_order must be a non-negative integer.")
+        object.__setattr__(
+            self,
+            "protocol_candidates",
+            _clean_tuple(self.protocol_candidates, field_name="DataClassSpec.protocol_candidates"),
+        )
+        object.__setattr__(
+            self,
+            "minimum_capture",
+            _clean_tuple(self.minimum_capture, field_name="DataClassSpec.minimum_capture"),
+        )
+        object.__setattr__(
+            self,
+            "stop_conditions",
+            _clean_tuple(self.stop_conditions, field_name="DataClassSpec.stop_conditions"),
+        )
+        object.__setattr__(
+            self,
+            "transfer_rules",
+            _clean_tuple(self.transfer_rules, field_name="DataClassSpec.transfer_rules"),
+        )
+        object.__setattr__(
+            self,
+            "verification",
+            _clean_tuple(self.verification, field_name="DataClassSpec.verification"),
+        )
+
+    def to_payload(self) -> dict[str, object]:
+        return {
+            "id": self.id,
+            "label": self.label,
+            "summary": self.summary,
+            "decision_order": self.decision_order,
+            "protocol_candidates": list(self.protocol_candidates),
+            "minimum_capture": list(self.minimum_capture),
+            "stop_conditions": list(self.stop_conditions),
+            "transfer_rules": list(self.transfer_rules),
+            "verification": list(self.verification),
+        }
+
+
+@dataclass(frozen=True)
+class ReadySpec:
+    id: str
+    label: str
+    summary: str
+    required_evidence: tuple[str, ...]
+    accepted_readiness_states: tuple[str, ...] = ()
+    required_capabilities: tuple[str, ...] = ()
+    commands: tuple[str, ...] = ()
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "id", _clean_string(self.id, field_name="ReadySpec.id"))
+        object.__setattr__(self, "label", _clean_string(self.label, field_name="ReadySpec.label"))
+        object.__setattr__(self, "summary", _clean_string(self.summary, field_name="ReadySpec.summary"))
+        object.__setattr__(
+            self,
+            "required_evidence",
+            _clean_tuple(self.required_evidence, field_name="ReadySpec.required_evidence"),
+        )
+        object.__setattr__(
+            self,
+            "accepted_readiness_states",
+            _clean_tuple(
+                self.accepted_readiness_states,
+                field_name="ReadySpec.accepted_readiness_states",
+                allow_empty=True,
+            ),
+        )
+        object.__setattr__(
+            self,
+            "required_capabilities",
+            _clean_tuple(
+                self.required_capabilities,
+                field_name="ReadySpec.required_capabilities",
+                allow_empty=True,
+            ),
+        )
+        object.__setattr__(
+            self,
+            "commands",
+            _clean_tuple(self.commands, field_name="ReadySpec.commands", allow_empty=True),
+        )
+
+    def to_payload(self) -> dict[str, object]:
+        return {
+            "id": self.id,
+            "label": self.label,
+            "summary": self.summary,
+            "required_evidence": list(self.required_evidence),
+            "accepted_readiness_states": list(self.accepted_readiness_states),
+            "required_capabilities": list(self.required_capabilities),
+            "commands": list(self.commands),
+        }
+
+
+class DopRegistry:
+    def __init__(self, *, data_classes: Iterable[DataClassSpec], ready_specs: Iterable[ReadySpec]):
+        data_class_items = tuple(sorted(data_classes, key=lambda item: item.decision_order))
+        ready_spec_items = tuple(ready_specs)
+        self._data_classes = data_class_items
+        self._ready_specs = ready_spec_items
+        self._data_classes_by_id = _index_by_id(data_class_items, kind="DOP data class")
+        self._ready_specs_by_id = _index_by_id(ready_spec_items, kind="DOP ready spec")
+        orders = [item.decision_order for item in data_class_items]
+        if len(set(orders)) != len(orders):
+            raise ValueError("DOP data class decision_order values must be unique.")
+
+    def data_classes(self) -> tuple[DataClassSpec, ...]:
+        return self._data_classes
+
+    def ready_specs(self) -> tuple[ReadySpec, ...]:
+        return self._ready_specs
+
+    def data_class(self, data_class_id: str) -> DataClassSpec:
+        key = str(data_class_id).strip()
+        try:
+            return self._data_classes_by_id[key]
+        except KeyError:
+            options = ", ".join(sorted(self._data_classes_by_id)) or "—"
+            raise ValueError(f"Unknown DOP data class {data_class_id!r}. Available classes: {options}") from None
+
+    def ready_spec(self, ready_spec_id: str) -> ReadySpec:
+        key = str(ready_spec_id).strip()
+        try:
+            return self._ready_specs_by_id[key]
+        except KeyError:
+            options = ", ".join(sorted(self._ready_specs_by_id)) or "—"
+            raise ValueError(f"Unknown DOP ready spec {ready_spec_id!r}. Available specs: {options}") from None
+
+    def data_classes_for_protocol(self, protocol_id: str) -> tuple[DataClassSpec, ...]:
+        key = str(protocol_id).strip()
+        return tuple(item for item in self._data_classes if key in item.protocol_candidates)
+
+    def validate_protocol_refs(self, protocol_ids: Iterable[str]) -> None:
+        known = {str(protocol_id).strip() for protocol_id in protocol_ids if str(protocol_id).strip()}
+        referenced = {
+            protocol_id for data_class in self._data_classes for protocol_id in data_class.protocol_candidates
+        }
+        missing = sorted(referenced - known)
+        if missing:
+            raise ValueError("DOP registry references unknown protocol ids: " + ", ".join(missing))
+
+    def validate_ready_refs(self, *, readiness_states: Iterable[str], capability_keys: Iterable[str]) -> None:
+        known_states = {str(state).strip() for state in readiness_states if str(state).strip()}
+        known_capabilities = {str(capability).strip() for capability in capability_keys if str(capability).strip()}
+        missing_states = sorted(
+            state for spec in self._ready_specs for state in spec.accepted_readiness_states if state not in known_states
+        )
+        missing_capabilities = sorted(
+            capability
+            for spec in self._ready_specs
+            for capability in spec.required_capabilities
+            if capability not in known_capabilities
+        )
+        errors = []
+        if missing_states:
+            errors.append("unknown readiness states: " + ", ".join(missing_states))
+        if missing_capabilities:
+            errors.append("unknown readiness capabilities: " + ", ".join(missing_capabilities))
+        if errors:
+            raise ValueError("DOP ready specs reference " + "; ".join(errors))
+
+
+def _index_by_id(items: Iterable[DataClassSpec] | Iterable[ReadySpec], *, kind: str):
+    indexed = {}
+    for item in items:
+        if item.id in indexed:
+            raise ValueError(f"Duplicate {kind} id {item.id!r}.")
+        indexed[item.id] = item
+    return indexed

reader_workbench/workbench/engine/__init__.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+from reader_workbench.workbench.registry import load_plugin_catalog
+
+from .contracts import (
+    _assert_input_ports,
+    _assert_output_ports,
+    _resolve_output_labels,
+    _resolve_runtime_output_ports,
+)
+from .execution import execute_step, run_steps
+from .inputs import _resolve_inputs
+from .invocations import ExecutionResult, ProducedRecordRevision, SelectedSteps
+from .planning import build_next_steps, explain
+from .runtime import run_job, run_spec
+from .setup import build_run_context, configure_logger, normalize_log_level, resolve_palette_book, slice_pipeline_steps
+from .validation import validate, validation_summary
+
+__all__ = [
+    "_assert_input_ports",
+    "_assert_output_ports",
+    "_resolve_inputs",
+    "_resolve_output_labels",
+    "_resolve_runtime_output_ports",
+    "build_run_context",
+    "build_next_steps",
+    "configure_logger",
+    "ExecutionResult",
+    "execute_step",
+    "explain",
+    "load_plugin_catalog",
+    "normalize_log_level",
+    "ProducedRecordRevision",
+    "resolve_palette_book",
+    "run_job",
+    "run_steps",
+    "run_spec",
+    "SelectedSteps",
+    "slice_pipeline_steps",
+    "validate",
+    "validation_summary",
+]

reader_workbench/workbench/engine/_shared.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from typing import Any
+
+from reader_workbench.runtime import ReaderRuntime, builtin_runtime
+from reader_workbench.workbench.decl import WorkbenchDecl
+from reader_workbench.workbench.graph import resolve_workbench
+
+
+def digest_cfg(plugin_cfg: Any) -> str:
+    if hasattr(plugin_cfg, "model_dump"):
+        payload = plugin_cfg.model_dump(mode="json")
+    elif isinstance(plugin_cfg, dict):
+        payload = plugin_cfg
+    else:
+        payload = json.loads(json.dumps(plugin_cfg, default=str))
+    raw = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8")
+    return "sha256:" + hashlib.sha256(raw).hexdigest()
+
+
+def needs_plot_palette(steps: list[Any], palette: str | None) -> bool:
+    if palette is None:
+        return False
+    return any(getattr(step, "plugin", "").startswith("plot/") for step in steps)
+
+
+def collect_categories(steps: list[Any]) -> set[str]:
+    categories: set[str] = set()
+    for step in steps:
+        plugin = getattr(step, "plugin", "")
+        if "/" in plugin:
+            categories.add(plugin.split("/", 1)[0])
+    return categories
+
+
+def has_cytometry_step(decl: WorkbenchDecl, *, runtime: ReaderRuntime | None = None) -> bool:
+    return pipeline_has_plugin(decl, runtime=runtime, domain="cytometry")
+
+
+def pipeline_has_plugin(
+    decl: WorkbenchDecl,
+    *,
+    runtime: ReaderRuntime | None = None,
+    plugin: str | None = None,
+    domain: str | None = None,
+    family: str | None = None,
+    tag: str | None = None,
+) -> bool:
+    pipeline = list(resolve_workbench(decl).pipeline)
+    if not pipeline:
+        return False
+
+    registry = None
+    if domain is not None or family is not None or tag is not None:
+        runtime = runtime or builtin_runtime()
+        registry = runtime.plugins
+
+    for step in pipeline:
+        step_plugin = str(getattr(step, "plugin", ""))
+        if plugin is not None and step_plugin != plugin:
+            continue
+        if domain is None and family is None and tag is None:
+            return True
+        if registry is None:
+            continue
+        descriptor = registry.resolve_descriptor(step_plugin)
+        if domain is not None and descriptor.domain != domain:
+            continue
+        if family is not None and descriptor.family != family:
+            continue
+        if tag is not None and tag not in descriptor.tags:
+            continue
+        return True
+    return False