reader-workbench 1.0.0__py3-none-any.whl

reader_workbench/workbench/graph/nodes.py

@@ -0,0 +1,102 @@
+from __future__ import annotations
+
+from collections import Counter
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+from typing import Any
+
+from reader_workbench.errors import ConfigError
+from reader_workbench.workbench.graph.refs import (
+    InputRef,
+    OutputRef,
+    input_ref_to_dict,
+    output_ref_to_dict,
+)
+from reader_workbench.workbench.ontology import (
+    WorkbenchItemKind,
+    WorkbenchPluginStepKind,
+    WorkbenchSurfaceSemantics,
+    get_workbench_surface_semantics,
+)
+
+
+@dataclass(frozen=True)
+class RecipeSource:
+    recipe: str
+    with_: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class PluginStep:
+    kind: WorkbenchPluginStepKind
+    id: str
+    plugin: str
+    reads: dict[str, InputRef] = field(default_factory=dict)
+    with_: dict[str, Any] = field(default_factory=dict)
+    writes: dict[str, OutputRef] = field(default_factory=dict)
+    source_recipe: RecipeSource | None = None
+
+    @property
+    def semantics(self) -> WorkbenchSurfaceSemantics:
+        return get_workbench_surface_semantics(self.kind)
+
+    @property
+    def plugin_category(self) -> str:
+        return self.plugin.split("/", 1)[0] if "/" in self.plugin else self.plugin
+
+    def to_dict(self) -> dict[str, Any]:
+        payload = {
+            "id": self.id,
+            "plugin": self.plugin,
+            "reads": {key: input_ref_to_dict(value) for key, value in (self.reads or {}).items()},
+            "with": dict(self.with_ or {}),
+            "writes": {key: output_ref_to_dict(value) for key, value in (self.writes or {}).items()},
+        }
+        if self.source_recipe is not None:
+            payload["source_recipe"] = {
+                "recipe": self.source_recipe.recipe,
+                "with": dict(self.source_recipe.with_ or {}),
+            }
+        return payload
+
+
+WorkbenchItem = PluginStep
+
+
+@dataclass(frozen=True)
+class Workbench:
+    pipeline: tuple[PluginStep, ...] = ()
+    plots: tuple[PluginStep, ...] = ()
+    exports: tuple[PluginStep, ...] = ()
+
+    def by_kind(self, kind: WorkbenchItemKind) -> tuple[WorkbenchItem, ...]:
+        if kind == "pipeline":
+            return self.pipeline
+        if kind == "plot":
+            return self.plots
+        if kind == "export":
+            return self.exports
+        raise ValueError(f"Unknown workbench item kind {kind!r}")
+
+    def all_specs(self) -> tuple[WorkbenchItem, ...]:
+        return self.pipeline + self.plots + self.exports
+
+    def plugin_steps(self) -> tuple[PluginStep, ...]:
+        return self.pipeline + self.plots + self.exports
+
+    def plugin_categories(self) -> set[str]:
+        return {item.plugin_category for item in self.plugin_steps()}
+
+    def counts(self) -> dict[WorkbenchItemKind, int]:
+        return {
+            "pipeline": len(self.pipeline),
+            "plot": len(self.plots),
+            "export": len(self.exports),
+        }
+
+
+def ensure_unique_workbench_ids(*collections: Sequence[WorkbenchItem]) -> None:
+    ids: list[str] = [item.id for collection in collections for item in collection if item.id]
+    dupes = sorted(item_id for item_id, count in Counter(ids).items() if count > 1)
+    if dupes:
+        raise ConfigError(f"Duplicate step/spec id(s) across pipeline/plots/exports: {dupes}")

reader_workbench/workbench/graph/normalize.py

@@ -0,0 +1,177 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from reader_workbench.errors import ConfigError
+from reader_workbench.workbench.decl import (
+    FileInputDecl,
+    PluginStepDecl,
+    RecordCollectionInputDecl,
+    RecordInputDecl,
+    RecordOutputDecl,
+    ResourceInputDecl,
+    WorkbenchDecl,
+)
+from reader_workbench.workbench.experiment import ResourceCatalog
+from reader_workbench.workbench.graph.nodes import PluginStep, Workbench, ensure_unique_workbench_ids
+from reader_workbench.workbench.graph.refs import (
+    FileRef,
+    InputRef,
+    OutputRef,
+    RecordCollectionRef,
+    RecordRef,
+    ResourceRef,
+    SourceRecordRef,
+)
+from reader_workbench.workbench.ontology import WorkbenchPluginStepKind, get_workbench_surface_semantics
+from reader_workbench.workbench.paths import resolve_path_within_root
+
+
+def resolve_workbench(decl: WorkbenchDecl) -> Workbench:
+    pipeline = tuple(_resolve_plugin_steps(decl, kind="pipeline", specs=decl.pipeline.steps))
+    plots = tuple(_resolve_plugin_steps(decl, kind="plot", specs=decl.plots.specs))
+    exports = tuple(_resolve_plugin_steps(decl, kind="export", specs=decl.exports.specs))
+    ensure_unique_workbench_ids(pipeline, plots, exports)
+    return Workbench(pipeline=pipeline, plots=plots, exports=exports)
+
+
+def materialize_workbench(decl: WorkbenchDecl) -> dict[str, list[dict[str, Any]]]:
+    workbench = resolve_workbench(decl)
+    return {
+        "pipeline": [item.to_dict() for item in workbench.pipeline],
+        "plots": [item.to_dict() for item in workbench.plots],
+        "exports": [item.to_dict() for item in workbench.exports],
+    }
+
+
+def select_workbench_specs(
+    specs: list[PluginStep],
+    *,
+    only: list[str],
+    exclude: list[str],
+    kind_label: str,
+) -> list[PluginStep]:
+    ids = [item.id for item in specs]
+    available = set(ids)
+    if only:
+        only_ids = set(only)
+        missing = sorted(only_ids - available)
+        if missing:
+            raise ConfigError(f"Unknown {kind_label} id(s): {missing}.")
+        selected = [item for item in specs if item.id in only_ids]
+    else:
+        selected = list(specs)
+    if exclude:
+        exclude_ids = set(exclude)
+        missing = sorted(exclude_ids - available)
+        if missing:
+            raise ConfigError(f"Unknown {kind_label} id(s): {missing}.")
+        selected = [item for item in selected if item.id not in exclude_ids]
+    return selected
+
+
+def normalize_input_binding(
+    binding: Any,
+    *,
+    root: Path,
+    resources: ResourceCatalog,
+    section: str,
+    step_id: str,
+    key: str,
+) -> InputRef:
+    if isinstance(binding, RecordInputDecl):
+        return RecordRef(record_id=binding.record_id)
+    if isinstance(binding, FileInputDecl):
+        try:
+            path = resolve_path_within_root(binding.path, root=root)
+        except ValueError as err:
+            raise ConfigError(
+                f"{section} {step_id}: reads '{key}' must stay under the experiment root after resolving symlinks."
+            ) from err
+        return FileRef(path=path)
+    if isinstance(binding, ResourceInputDecl):
+        try:
+            resource = resources.require_file(binding.resource_id)
+        except ValueError as err:
+            raise ConfigError(
+                f"{section} {step_id}: reads '{key}' references unknown resource '{binding.resource_id}'."
+            ) from err
+        return ResourceRef(resource_id=binding.resource_id, path=resource.path)
+    if isinstance(binding, RecordCollectionInputDecl):
+        records: list[SourceRecordRef] = []
+        for resource_id in binding.resource_ids:
+            try:
+                resource = resources.require_record(resource_id)
+            except ValueError as err:
+                raise ConfigError(
+                    f"{section} {step_id}: reads '{key}' references unknown record resource '{resource_id}'."
+                ) from err
+            records.append(
+                SourceRecordRef(
+                    resource_id=resource_id,
+                    experiment_id=resource.experiment_id,
+                    record_id=resource.record_id,
+                    experiment_root=resource.experiment_root,
+                    outputs_dir=resource.outputs_dir,
+                )
+            )
+        return RecordCollectionRef(records=tuple(records))
+    raise ConfigError(f"{section} {step_id}: reads '{key}' uses unsupported binding type {type(binding).__name__}")
+
+
+def normalize_output_binding(binding: RecordOutputDecl) -> OutputRef:
+    return OutputRef(record_id=binding.record_id)
+
+
+def _resolve_plugin_steps(
+    decl: WorkbenchDecl,
+    *,
+    kind: WorkbenchPluginStepKind,
+    specs: tuple[PluginStepDecl, ...],
+) -> list[PluginStep]:
+    semantics = get_workbench_surface_semantics(kind)
+    root = decl.experiment.root
+    resources = decl.experiment_semantics.resources
+    seen: set[str] = set()
+    resolved: list[PluginStep] = []
+    for step in specs or ():
+        step_id = step.id
+        if not step_id or not isinstance(step_id, str):
+            raise ConfigError(f"Every {semantics.section} spec must include an id.")
+        if step_id in seen:
+            raise ConfigError(f"{semantics.section} contains duplicate spec id(s): {step_id}")
+        seen.add(step_id)
+        plugin = step.plugin
+        if not plugin or not isinstance(plugin, str):
+            raise ConfigError(f"{semantics.section} {step_id}: plugin must be a non-empty string")
+        if "/" not in plugin:
+            raise ConfigError(f"{semantics.section} {step_id}: plugin must be 'category/key'")
+        category = plugin.split("/", 1)[0]
+        expected_category = semantics.expected_plugin_category
+        if kind == "pipeline" and category in {"plot", "export"}:
+            raise ConfigError(f"pipeline {step_id}: plot/export plugins are not allowed in pipeline.")
+        if expected_category is not None and category != expected_category:
+            raise ConfigError(f"{semantics.section} {step_id}: plugin must be {expected_category}/*")
+        resolved.append(
+            PluginStep(
+                kind=kind,
+                id=step_id,
+                plugin=plugin,
+                reads={
+                    key: normalize_input_binding(
+                        value,
+                        root=root,
+                        resources=resources,
+                        section=semantics.section,
+                        step_id=step_id,
+                        key=key,
+                    )
+                    for key, value in (step.reads or {}).items()
+                },
+                with_=dict(step.with_ or {}),
+                writes={key: normalize_output_binding(value) for key, value in (step.writes or {}).items()},
+                source_recipe=step.source_recipe,
+            )
+        )
+    return resolved

reader_workbench/workbench/graph/refs.py

@@ -0,0 +1,148 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+
+@dataclass(frozen=True)
+class RecordRef:
+    record_id: str
+
+
+@dataclass(frozen=True)
+class FileRef:
+    path: Path
+
+
+@dataclass(frozen=True)
+class ResourceRef:
+    resource_id: str
+    path: Path
+
+
+@dataclass(frozen=True)
+class SourceRecordRef:
+    resource_id: str
+    experiment_id: str
+    record_id: str
+    experiment_root: Path
+    outputs_dir: Path
+
+
+@dataclass(frozen=True)
+class RecordCollectionRef:
+    records: tuple[SourceRecordRef, ...]
+
+
+InputRef = RecordRef | FileRef | ResourceRef | SourceRecordRef | RecordCollectionRef
+
+
+@dataclass(frozen=True)
+class OutputRef:
+    record_id: str
+
+
+@dataclass(frozen=True)
+class ProvenanceInput:
+    label: str
+    ref: InputRef
+    discovery_policy: str | None = None
+
+
+def input_ref_display(ref: InputRef) -> str:
+    if isinstance(ref, RecordRef):
+        return ref.record_id
+    if isinstance(ref, ResourceRef):
+        return f"resource({ref.resource_id})"
+    if isinstance(ref, SourceRecordRef):
+        return f"record_resource({ref.resource_id}:{ref.experiment_id}:{ref.record_id})"
+    if isinstance(ref, RecordCollectionRef):
+        return "record_collection(" + ", ".join(item.resource_id for item in ref.records) + ")"
+    return f"file({ref.path})"
+
+
+def output_ref_display(ref: OutputRef) -> str:
+    return ref.record_id
+
+
+def input_ref_to_dict(ref: InputRef) -> dict[str, Any]:
+    if isinstance(ref, RecordRef):
+        return {"record": ref.record_id}
+    if isinstance(ref, ResourceRef):
+        return {"resource": ref.resource_id, "path": str(ref.path)}
+    if isinstance(ref, SourceRecordRef):
+        return {
+            "record_resource": ref.resource_id,
+            "experiment": ref.experiment_id,
+            "record": ref.record_id,
+        }
+    if isinstance(ref, RecordCollectionRef):
+        return {
+            "record_collection": [
+                {
+                    "resource": item.resource_id,
+                    "experiment": item.experiment_id,
+                    "record": item.record_id,
+                }
+                for item in ref.records
+            ]
+        }
+    return {"file": str(ref.path)}
+
+
+def output_ref_to_dict(ref: OutputRef) -> dict[str, str]:
+    return {"record": ref.record_id}
+
+
+def provenance_input_to_dict(binding: ProvenanceInput) -> dict[str, Any]:
+    return {"label": binding.label, **input_ref_to_dict(binding.ref)}
+
+
+def input_ref_from_dict(payload: dict[str, Any]) -> InputRef:
+    if not isinstance(payload, dict):
+        raise TypeError("input binding payload must be a mapping")
+    keys = set(payload)
+    if keys == {"record"}:
+        record = payload["record"]
+        if isinstance(record, str) and record:
+            return RecordRef(record_id=record)
+        raise ValueError("record binding must include a non-empty record id")
+    if keys == {"file"}:
+        file_path = payload["file"]
+        if isinstance(file_path, str) and file_path:
+            return FileRef(path=Path(file_path))
+        raise ValueError("file binding must include a non-empty path")
+    if keys == {"resource", "path"}:
+        resource_id = payload["resource"]
+        path = payload["path"]
+        if not isinstance(resource_id, str) or not resource_id:
+            raise ValueError("resource binding must include a non-empty resource id")
+        if not isinstance(path, str) or not path:
+            raise ValueError(f"resource binding {resource_id!r} must include a non-empty path")
+        return ResourceRef(resource_id=resource_id, path=Path(path))
+    raise ValueError(
+        "input binding payload must contain exactly one reference shape: {'record'}, {'file'}, or {'resource', 'path'}"
+    )
+
+
+def output_ref_from_dict(payload: dict[str, Any]) -> OutputRef:
+    if not isinstance(payload, dict):
+        raise TypeError("output binding payload must be a mapping")
+    if set(payload) != {"record"}:
+        raise ValueError("output binding payload must contain only 'record'")
+    record = payload.get("record")
+    if not isinstance(record, str) or not record:
+        raise ValueError("output binding payload must include record")
+    return OutputRef(record_id=record)
+
+
+def provenance_input_from_dict(payload: dict[str, Any]) -> ProvenanceInput:
+    if not isinstance(payload, dict):
+        raise TypeError("provenance input payload must be a mapping")
+    label = payload.get("label")
+    if not isinstance(label, str) or not label:
+        raise ValueError("provenance input payload must include label")
+    ref_payload = dict(payload)
+    ref_payload.pop("label", None)
+    return ProvenanceInput(label=label, ref=input_ref_from_dict(ref_payload))

reader_workbench/workbench/input_discovery.py

@@ -0,0 +1,19 @@
+"""Neutral defaults shared by protocol authoring and ingest discovery mechanics."""
+
+DEFAULT_INPUT_ROOTS = ("./inputs", "./raw", "./raw_data")
+DEFAULT_WORKBOOK_INCLUDE = ("*.xlsx", "*.XLSX")
+DEFAULT_INPUT_EXCLUDE = (
+    "~$*",
+    "._*",
+    "#*#",
+    "*.tmp",
+    "*.temp",
+    "*.bak",
+    "metadata.*",
+    "metadata_filtered.*",
+    "sample_map.*",
+    "sample_metadata.*",
+    "plate_map.*",
+)
+
+__all__ = ["DEFAULT_INPUT_EXCLUDE", "DEFAULT_INPUT_ROOTS", "DEFAULT_WORKBOOK_INCLUDE"]

reader_workbench/workbench/inspection/__init__.py

@@ -0,0 +1,3 @@
+"""Inspection helpers for experiment discovery, reporting, and readiness."""
+
+__all__: list[str] = []

reader_workbench/workbench/inspection/catalogs.py

@@ -0,0 +1,128 @@
+from __future__ import annotations
+
+from collections import Counter
+from copy import deepcopy
+
+from reader_workbench.workbench.graph import resolve_workbench
+
+from .experiments import experiment_identity_payload
+from .runtime import export_output_summaries, plot_output_summaries, record_producer_map, spec_step_payload
+
+
+def _sorted_counter(values: list[str]) -> dict[str, int]:
+    counts = Counter(value for value in values if value)
+    return dict(sorted(counts.items()))
+
+
+def workbench_surface_catalog_payload(
+    *,
+    experiment: dict[str, object],
+    protocol: str,
+    kind: str,
+    only: list[str],
+    exclude: list[str],
+    entries: list[dict[str, object]],
+) -> dict[str, object]:
+    payload_key = f"{kind}s"
+    summary_key = payload_key
+    plugin_ids = [str(entry.get("plugin") or "") for entry in entries]
+    domains = [
+        str(semantics.get("domain") or "")
+        for semantics in (entry.get("semantics") for entry in entries)
+        if isinstance(semantics, dict)
+    ]
+    families = [
+        str(semantics.get("family") or "")
+        for semantics in (entry.get("semantics") for entry in entries)
+        if isinstance(semantics, dict)
+    ]
+    return {
+        "experiment": deepcopy(experiment),
+        "catalog": {
+            "kind": kind,
+            "protocol": protocol,
+        },
+        "selection": {
+            "only": list(only),
+            "exclude": list(exclude),
+        },
+        "summary": {
+            summary_key: len(entries),
+            "by_plugin": _sorted_counter(plugin_ids),
+            "by_domain": _sorted_counter(domains),
+            "by_family": _sorted_counter(families),
+        },
+        payload_key: deepcopy(entries),
+    }
+
+
+def plugin_registry_payload(
+    *,
+    descriptors,
+    category: str | None,
+    domain: str | None,
+    family: str | None,
+    protocol: str | None,
+) -> dict[str, object]:
+    plugins = [
+        {
+            "category": descriptor.category,
+            "domain": descriptor.domain,
+            "family": descriptor.family,
+            "key": descriptor.key,
+            "plugin": descriptor.plugin,
+            "summary": descriptor.summary,
+            "class": f"{descriptor.cls.__module__}.{descriptor.cls.__name__}",
+        }
+        for descriptor in descriptors
+    ]
+    return {
+        "selection": {
+            "category": category,
+            "domain": domain,
+            "family": family,
+            "protocol": protocol,
+        },
+        "summary": {
+            "plugins": len(plugins),
+            "by_category": _sorted_counter([str(item["category"]) for item in plugins]),
+            "by_domain": _sorted_counter([str(item["domain"]) for item in plugins]),
+            "by_family": _sorted_counter([str(item["family"]) for item in plugins]),
+        },
+        "plugins": plugins,
+    }
+
+
+def workbench_surface_specs_payload(
+    *,
+    job_path,
+    decl,
+    runtime,
+    bound_protocol,
+    selected,
+    kind: str,
+    only: list[str],
+    exclude: list[str],
+) -> dict[str, object]:
+    workbench = resolve_workbench(decl)
+    record_producers = record_producer_map(workbench.plugin_steps(), runtime=runtime)
+    summary_lookup = (
+        plot_output_summaries(bound_protocol) if kind == "plot" else export_output_summaries(bound_protocol)
+    )
+    entries = [
+        spec_step_payload(
+            step,
+            summary=summary_lookup.get(step.id, "—"),
+            runtime=runtime,
+            record_producers=record_producers,
+        )
+        for step in selected
+    ]
+    return workbench_surface_catalog_payload(
+        experiment=experiment_identity_payload(job_path=job_path, decl=decl, protocol_id=bound_protocol.id),
+        protocol=bound_protocol.id,
+        kind=kind,
+        only=only,
+        exclude=exclude,
+        entries=entries,
+    )

reader_workbench/workbench/inspection/common.py

@@ -0,0 +1,92 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+
+def count_visible_files(path: Path) -> int:
+    if not path.exists():
+        return 0
+    return sum(1 for item in path.rglob("*") if item.is_file() and not item.name.startswith("."))
+
+
+def count_visible_glob(path: Path, pattern: str) -> int:
+    if not path.exists():
+        return 0
+    return sum(1 for item in path.rglob(pattern) if item.is_file() and not item.name.startswith("."))
+
+
+def visible_relative_files(path: Path, *, base: Path, limit: int = 8) -> list[str]:
+    if not path.exists():
+        return []
+    files = sorted(item for item in path.rglob("*") if item.is_file() and not item.name.startswith("."))
+    return [format_relative_path(item, base=base) for item in files[:limit]]
+
+
+def format_relative_path(path: Path, *, base: Path) -> str:
+    try:
+        return str(path.relative_to(base))
+    except ValueError:
+        return str(path)
+
+
+def resolve_output_subdir(outputs_dir: Path, subdir: str) -> Path:
+    return outputs_dir if subdir in ("", ".", "./") else outputs_dir / subdir
+
+
+def summarize_outputs_dir(
+    outputs_dir: Path,
+    *,
+    plots_subdir: str = "plots",
+    exports_subdir: str = "exports",
+    notebooks_subdir: str = "notebooks",
+) -> dict[str, int]:
+    plots_dir = resolve_output_subdir(outputs_dir, plots_subdir)
+    exports_dir = resolve_output_subdir(outputs_dir, exports_subdir)
+    notebooks_dir = resolve_output_subdir(outputs_dir, notebooks_subdir)
+    artifacts_dir = outputs_dir / "artifacts"
+    return {
+        "records": count_visible_glob(artifacts_dir, "*.parquet"),
+        "plots": count_visible_files(plots_dir),
+        "exports": count_visible_files(exports_dir),
+        "notebooks": count_visible_files(notebooks_dir),
+    }
+
+
+def preview_output_files(path: Path, *, base: Path, limit: int = 4) -> str:
+    files = visible_relative_files(path, base=base, limit=limit)
+    if not files:
+        return "—"
+    remaining = count_visible_files(path) - len(files)
+    preview = ", ".join(files)
+    if remaining > 0:
+        preview += f", … (+{remaining} more)"
+    return preview
+
+
+def render_compact_value(value) -> str:
+    if value is None:
+        return "null"
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, (int, float)):
+        return str(value)
+    if isinstance(value, str):
+        return value
+    if isinstance(value, tuple):
+        value = list(value)
+    try:
+        return json.dumps(value, ensure_ascii=True, sort_keys=True)
+    except TypeError:
+        return str(value)
+
+
+def flatten_binding_rows(value, *, prefix: str = "") -> list[tuple[str, str]]:
+    rows: list[tuple[str, str]] = []
+    if isinstance(value, dict):
+        for key in sorted(value):
+            child_path = f"{prefix}.{key}" if prefix else str(key)
+            rows.extend(flatten_binding_rows(value[key], prefix=child_path))
+        return rows
+    rows.append((prefix or "value", render_compact_value(value)))
+    return rows