reader-workbench 1.0.0__py3-none-any.whl

reader_workbench/workbench/config/model.py

@@ -0,0 +1,274 @@
+from __future__ import annotations
+
+from importlib import import_module
+from pathlib import Path
+from typing import Annotated, Any, Literal
+
+from pydantic import BaseModel, Field, field_validator, model_validator
+
+from reader_workbench.errors import ConfigError
+from reader_workbench.workbench.dop import builtin_dop_registry
+
+
+class ExperimentSpec(BaseModel):
+    id: str
+    title: str | None = None
+    lifecycle: str = "active"
+
+    model_config = {"extra": "forbid"}
+
+    @field_validator("id", mode="after")
+    @classmethod
+    def _validate_id(cls, v: str) -> str:
+        if not isinstance(v, str) or not v.strip():
+            raise ConfigError("experiment.id must be a non-empty string")
+        return v
+
+    @field_validator("lifecycle", mode="after")
+    @classmethod
+    def _validate_lifecycle(cls, v: str) -> str:
+        if not isinstance(v, str) or not v.strip():
+            raise ConfigError("experiment.lifecycle must be a non-empty string")
+        lifecycle = v.strip().lower()
+        allowed = {"active", "draft", "template"}
+        if lifecycle not in allowed:
+            raise ConfigError(f"experiment.lifecycle must be one of: {', '.join(sorted(allowed))}")
+        return lifecycle
+
+
+ReplicateKind = Literal["biological", "technical", "mixed", "unknown", "not_applicable"]
+
+
+class EvidenceSpec(BaseModel):
+    data_class: str
+    data_class_reason: str
+    replicate_kind: ReplicateKind
+    replicate_identity_field: str | None = None
+
+    model_config = {"extra": "forbid"}
+
+    @field_validator("data_class", mode="after")
+    @classmethod
+    def _validate_data_class(cls, value: str) -> str:
+        cleaned = value.strip()
+        if not cleaned:
+            raise ValueError("evidence.data_class must be a non-empty string")
+        builtin_dop_registry().data_class(cleaned)
+        return cleaned
+
+    @field_validator("data_class_reason", mode="after")
+    @classmethod
+    def _validate_data_class_reason(cls, value: str) -> str:
+        cleaned = value.strip()
+        if not cleaned:
+            raise ValueError("evidence.data_class_reason must be a non-empty string")
+        return cleaned
+
+    @field_validator("replicate_identity_field", mode="after")
+    @classmethod
+    def _validate_replicate_identity_field(cls, value: str | None) -> str | None:
+        if value is None:
+            return None
+        cleaned = value.strip()
+        if not cleaned:
+            raise ValueError("evidence.replicate_identity_field must be a non-empty string when provided")
+        return cleaned
+
+    @model_validator(mode="after")
+    def _validate_replicate_identity(self) -> EvidenceSpec:
+        if self.replicate_kind == "not_applicable" and self.replicate_identity_field is not None:
+            raise ValueError("evidence.replicate_identity_field cannot be set when replicate_kind is not_applicable")
+        return self
+
+
+class PathsSpec(BaseModel):
+    outputs: str = "./outputs"
+    plots: str = "plots"
+    exports: str = "exports"
+    notebooks: str = "notebooks"
+
+    model_config = {"extra": "forbid"}
+
+
+class PlottingSpec(BaseModel):
+    palette: str | None = "colorblind"
+
+    model_config = {"extra": "forbid"}
+
+
+class FileResourceSpec(BaseModel):
+    kind: Literal["file"]
+    path: str
+
+    model_config = {"extra": "forbid"}
+
+
+class RecordResourceSpec(BaseModel):
+    kind: Literal["record"]
+    experiment: str
+    record: str
+
+    model_config = {"extra": "forbid"}
+
+    @field_validator("experiment", "record", mode="after")
+    @classmethod
+    def _validate_nonempty(cls, value: str) -> str:
+        if not isinstance(value, str) or not value.strip():
+            raise ValueError("record resource experiment and record values must be non-empty strings")
+        return value.strip()
+
+
+ResourceSpec = Annotated[FileResourceSpec | RecordResourceSpec, Field(discriminator="kind")]
+
+
+class ResourcesSpec(BaseModel):
+    by_id: dict[str, ResourceSpec] = Field(default_factory=dict)
+
+    model_config = {"extra": "forbid"}
+
+
+class AnnotationLabelSpec(BaseModel):
+    source: str
+    values: dict[str, str] = Field(default_factory=dict)
+    output: str | None = None
+
+    model_config = {"extra": "forbid"}
+
+
+class AnnotationOrderSpec(BaseModel):
+    column: str
+    values: list[str] = Field(default_factory=list)
+
+    model_config = {"extra": "forbid"}
+
+
+class AnnotationCollectionSpec(BaseModel):
+    column: str
+    items: dict[str, list[str]] = Field(default_factory=dict)
+
+    model_config = {"extra": "forbid"}
+
+
+class AnnotationOrderedStateSpaceSpec(BaseModel):
+    column: str
+    state_order: list[str]
+    values: dict[str, str]
+    case_sensitive: bool = True
+
+    model_config = {"extra": "forbid"}
+
+    @field_validator("column", mode="after")
+    @classmethod
+    def _validate_column(cls, value: str) -> str:
+        if not isinstance(value, str) or not value.strip():
+            raise ValueError("ordered state-space column must be a non-empty string")
+        return value.strip()
+
+    @model_validator(mode="after")
+    def _validate_states(self) -> AnnotationOrderedStateSpaceSpec:
+        if not self.state_order:
+            raise ValueError("state_order must be a non-empty list")
+        state_ids = self.state_order
+        if any(not state_id.strip() for state_id in state_ids):
+            raise ValueError("state_order must contain non-empty strings")
+        if len(set(state_ids)) != len(state_ids):
+            raise ValueError("state ids must be unique")
+        if set(self.values) != set(state_ids):
+            raise ValueError("values must have exactly the ids declared by state_order")
+        source_values = [self.values[state_id] for state_id in state_ids]
+        if any(not value.strip() for value in source_values):
+            raise ValueError("values must map state ids to non-empty strings")
+        compared = source_values if self.case_sensitive else [value.strip().casefold() for value in source_values]
+        if len(set(compared)) != len(compared):
+            sensitivity = "true" if self.case_sensitive else "false"
+            raise ValueError(f"source values must be unique under case_sensitive={sensitivity}")
+        return self
+
+
+class AnnotationSpec(BaseModel):
+    labels: dict[str, AnnotationLabelSpec] = Field(default_factory=dict)
+    orders: dict[str, AnnotationOrderSpec] = Field(default_factory=dict)
+    collections: dict[str, AnnotationCollectionSpec] = Field(default_factory=dict)
+    ordered_state_spaces: dict[str, AnnotationOrderedStateSpaceSpec] = Field(default_factory=dict)
+
+    model_config = {"extra": "forbid"}
+
+
+class PlotOutputsSpec(BaseModel):
+    profile: str | None = None
+    include: list[str] = Field(default_factory=list)
+    exclude: list[str] = Field(default_factory=list)
+    views: dict[str, dict[str, Any]] = Field(default_factory=dict)
+
+    model_config = {"extra": "forbid"}
+
+
+class ExportOutputsSpec(BaseModel):
+    include: list[str] = Field(default_factory=list)
+    exclude: list[str] = Field(default_factory=list)
+    artifacts: dict[str, dict[str, Any]] = Field(default_factory=dict)
+
+    model_config = {"extra": "forbid"}
+
+
+class OutputsSpec(BaseModel):
+    plots: PlotOutputsSpec = Field(default_factory=PlotOutputsSpec)
+    exports: ExportOutputsSpec = Field(default_factory=ExportOutputsSpec)
+
+    model_config = {"extra": "forbid"}
+
+
+class ProtocolBindingSpec(BaseModel):
+    id: str
+    inputs: dict[str, Any] = Field(default_factory=dict)
+    analysis: dict[str, Any] = Field(default_factory=dict)
+    outputs: OutputsSpec = Field(default_factory=OutputsSpec)
+
+    model_config = {"extra": "forbid"}
+
+    @field_validator("id", mode="after")
+    @classmethod
+    def _validate_id(cls, v: str) -> str:
+        if not isinstance(v, str) or not v.strip():
+            raise ConfigError("protocol.id must be a non-empty string")
+        return v
+
+
+class ReaderSpec(BaseModel):
+    schema_: str = Field(alias="schema")
+    experiment: ExperimentSpec
+    protocol: ProtocolBindingSpec
+    paths: PathsSpec = Field(default_factory=PathsSpec)
+    plotting: PlottingSpec = Field(default_factory=PlottingSpec)
+    resources: ResourcesSpec = Field(default_factory=ResourcesSpec)
+    annotations: AnnotationSpec = Field(default_factory=AnnotationSpec)
+    evidence: EvidenceSpec | None = None
+
+    model_config = {"extra": "forbid"}
+
+    @field_validator("schema_", mode="after")
+    @classmethod
+    def _validate_schema(cls, v: str) -> str:
+        if v != "reader/v8":
+            raise ConfigError("Config schema must be 'reader/v8'. This repo only supports reader/v8.")
+        return v
+
+    @model_validator(mode="after")
+    def _validate_evidence_protocol_candidate(self) -> ReaderSpec:
+        if self.evidence is None:
+            return self
+
+        data_class = builtin_dop_registry().data_class(self.evidence.data_class)
+        protocol_id = self.protocol.id.strip()
+        if protocol_id not in data_class.protocol_candidates:
+            valid_candidates = ", ".join(data_class.protocol_candidates)
+            raise ValueError(
+                f"evidence.data_class {data_class.id!r} does not admit protocol.id {protocol_id!r}. "
+                f"Valid protocol candidates: {valid_candidates}"
+            )
+        return self
+
+    @classmethod
+    def load(cls, path: Path) -> ReaderSpec:
+        load_module = import_module("reader_workbench.workbench.config.load")
+        return load_module.load_reader_spec(path, cls=cls)

reader_workbench/workbench/context.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from reader_workbench.plotting.style import PaletteBook
+    from reader_workbench.protocols import BoundProtocol
+    from reader_workbench.workbench.experiment import ExperimentSemantics
+
+
+@dataclass(frozen=True)
+class RunContext:
+    exp_dir: Path
+    outputs_dir: Path
+    artifacts_dir: Path
+    plots_dir: Path
+    exports_dir: Path
+    records_path: Path
+    logger: logging.Logger
+    palette_book: PaletteBook | None
+    experiment: ExperimentSemantics | None = None
+    protocol: BoundProtocol | None = None
+    config_digest: str = ""

reader_workbench/workbench/decl/__init__.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+import importlib
+
+_EXPORTS = {
+    "build": {"build_workbench_decl", "load_workbench_decl"},
+    "model": {
+        "ExperimentDecl",
+        "FileInputDecl",
+        "InputBindingDecl",
+        "PipelineDecl",
+        "PluginStepDecl",
+        "RecipeSourceDecl",
+        "RecordCollectionInputDecl",
+        "RecordInputDecl",
+        "RecordOutputDecl",
+        "ResourceInputDecl",
+        "SurfaceDecl",
+        "WorkbenchDecl",
+    },
+}
+
+__all__ = tuple(sorted({name for names in _EXPORTS.values() for name in names}))
+
+
+def __getattr__(name: str):
+    for module_name, names in _EXPORTS.items():
+        if name in names:
+            module = importlib.import_module(f"reader_workbench.workbench.decl.{module_name}")
+            return getattr(module, name)
+    raise AttributeError(name)

reader_workbench/workbench/decl/build.py

@@ -0,0 +1,190 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from reader_workbench.errors import ConfigError
+from reader_workbench.protocols.model import ProtocolBinding, ProtocolCatalog
+from reader_workbench.workbench.config import (
+    FileResourceSpec,
+    ProtocolBindingSpec,
+    ReaderSpec,
+    RecordResourceSpec,
+    ResourceSpec,
+    reader_spec_digest,
+)
+from reader_workbench.workbench.experiment import (
+    AnnotationCollections,
+    AnnotationCollectionSpec,
+    AnnotationLabels,
+    AnnotationLabelSpec,
+    AnnotationOrders,
+    AnnotationOrderSpec,
+    AnnotationSemantics,
+    ExperimentEvidence,
+    ExperimentSemantics,
+    FileResourceEntry,
+    OrderedStateSpaces,
+    OrderedStateSpaceSpec,
+    OutputLayout,
+    RecordResourceEntry,
+    ResourceCatalog,
+)
+from reader_workbench.workbench.experiments import ExperimentCatalog
+from reader_workbench.workbench.paths import resolve_path_within_root
+
+from .model import ExperimentDecl, PipelineDecl, SurfaceDecl, WorkbenchDecl
+
+
+def load_workbench_decl(path: Path, *, protocols: ProtocolCatalog) -> WorkbenchDecl:
+    spec = ReaderSpec.load(path)
+    return build_workbench_decl(spec, source_path=path, protocols=protocols)
+
+
+def build_workbench_decl(
+    spec: ReaderSpec,
+    *,
+    source_path: Path,
+    protocols: ProtocolCatalog,
+) -> WorkbenchDecl:
+    root = source_path.parent.resolve()
+    experiment = ExperimentDecl(
+        id=spec.experiment.id,
+        title=spec.experiment.title or spec.experiment.id,
+        lifecycle=spec.experiment.lifecycle,
+        root=root,
+    )
+    layout = OutputLayout(
+        outputs_dir=_resolve_outputs_dir(spec.paths.outputs, root=root),
+        plots_subdir=_validate_output_subdir(spec.paths.plots, key="plots"),
+        exports_subdir=_validate_output_subdir(spec.paths.exports, key="exports"),
+        notebooks_subdir=_validate_output_subdir(spec.paths.notebooks, key="notebooks"),
+    )
+    resources = _bind_resources(spec.resources.by_id or {}, root=root)
+    protocol_binding = _bind_protocol(spec.protocol)
+    bound_protocol = protocols.bind(protocol_binding)
+    compiled = bound_protocol.compile()
+
+    experiment_semantics = ExperimentSemantics(
+        protocol=protocol_binding,
+        annotations=AnnotationSemantics(
+            labels=AnnotationLabels(
+                by_id={
+                    key: AnnotationLabelSpec(
+                        source=value.source,
+                        values=dict(value.values or {}),
+                        output=value.output,
+                    )
+                    for key, value in (spec.annotations.labels or {}).items()
+                }
+            ),
+            orders=AnnotationOrders(
+                by_id={
+                    key: AnnotationOrderSpec(column=value.column, values=list(value.values or []))
+                    for key, value in (spec.annotations.orders or {}).items()
+                }
+            ),
+            collections=AnnotationCollections(
+                by_id={
+                    key: AnnotationCollectionSpec(
+                        column=value.column,
+                        items={item_key: list(item_values) for item_key, item_values in (value.items or {}).items()},
+                    )
+                    for key, value in (spec.annotations.collections or {}).items()
+                }
+            ),
+            ordered_state_spaces=OrderedStateSpaces(
+                by_id={
+                    key: OrderedStateSpaceSpec(
+                        column=value.column,
+                        state_order=tuple(value.state_order),
+                        source_values=dict(value.values),
+                        case_sensitive=bool(value.case_sensitive),
+                    )
+                    for key, value in (spec.annotations.ordered_state_spaces or {}).items()
+                }
+            ),
+        ),
+        resources=resources,
+        layout=layout,
+        protocol_program=compiled.semantic_program,
+        evidence=(
+            ExperimentEvidence(
+                data_class=spec.evidence.data_class,
+                data_class_reason=spec.evidence.data_class_reason,
+                replicate_kind=spec.evidence.replicate_kind,
+                replicate_identity_field=spec.evidence.replicate_identity_field,
+            )
+            if spec.evidence is not None
+            else None
+        ),
+    )
+
+    return WorkbenchDecl(
+        experiment=experiment,
+        experiment_semantics=experiment_semantics,
+        plotting_palette=spec.plotting.palette if spec.plotting else None,
+        pipeline=PipelineDecl(runtime=dict(compiled.runtime or {}), steps=tuple(compiled.pipeline)),
+        plots=SurfaceDecl(specs=tuple(compiled.plots)),
+        exports=SurfaceDecl(specs=tuple(compiled.exports)),
+        config_digest=reader_spec_digest(spec),
+    )
+
+
+def _resolve_outputs_dir(raw: str, *, root: Path) -> Path:
+    if not isinstance(raw, str) or not raw.strip():
+        raise ConfigError("paths.outputs must be a non-empty string path")
+    try:
+        return resolve_path_within_root(raw, root=root)
+    except ValueError as err:
+        raise ConfigError("paths.outputs must stay under the experiment root after resolving symlinks.") from err
+
+
+def _validate_output_subdir(raw: str, *, key: str) -> str:
+    if raw is None:
+        raise ConfigError(f"paths.{key} must be a string subdirectory (use '.' to flatten).")
+    if not isinstance(raw, str):
+        raise ConfigError(f"paths.{key} must be a string subdirectory")
+    subdir = Path(raw)
+    if subdir.is_absolute():
+        raise ConfigError(f"paths.{key} must be relative to paths.outputs, not absolute.")
+    normalized = Path(".") / subdir
+    if any(part == ".." for part in normalized.parts):
+        raise ConfigError(f"paths.{key} must stay under paths.outputs and may not escape via '..'.")
+    return raw
+
+
+def _bind_resources(resources: dict[str, ResourceSpec], *, root: Path) -> ResourceCatalog:
+    bound: dict[str, FileResourceEntry | RecordResourceEntry] = {}
+    experiment_catalog: ExperimentCatalog | None = None
+    for resource_id, resource in resources.items():
+        if isinstance(resource, FileResourceSpec):
+            try:
+                path = resolve_path_within_root(resource.path, root=root)
+            except ValueError as err:
+                raise ConfigError(
+                    f"resources.{resource_id}.path must stay under the experiment root after resolving symlinks."
+                ) from err
+            bound[resource_id] = FileResourceEntry(kind="file", path=path)
+            continue
+        if isinstance(resource, RecordResourceSpec):
+            experiment_catalog = experiment_catalog or ExperimentCatalog.from_experiment_root(root)
+            location = experiment_catalog.resolve(resource.experiment)
+            bound[resource_id] = RecordResourceEntry(
+                kind="record",
+                experiment_id=location.id,
+                record_id=resource.record,
+                experiment_root=location.root,
+                outputs_dir=location.outputs_dir,
+            )
+            continue
+        raise ConfigError(f"resources.{resource_id} has unsupported kind {resource.kind!r}")
+    return ResourceCatalog(by_id=bound)
+
+
+def _bind_protocol(spec: ProtocolBindingSpec) -> ProtocolBinding:
+    return ProtocolBinding(
+        id=spec.id,
+        inputs=dict(spec.inputs or {}),
+        analysis=dict(spec.analysis or {}),
+        outputs=spec.outputs.model_dump(exclude_none=True),
+    )

reader_workbench/workbench/decl/model.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from reader_workbench.workbench.experiment import ExperimentSemantics
+
+
+@dataclass(frozen=True)
+class ExperimentDecl:
+    id: str
+    title: str
+    lifecycle: str
+    root: Path
+
+
+@dataclass(frozen=True)
+class RecordInputDecl:
+    record_id: str
+
+
+@dataclass(frozen=True)
+class FileInputDecl:
+    path: str
+
+
+@dataclass(frozen=True)
+class ResourceInputDecl:
+    resource_id: str
+
+
+@dataclass(frozen=True)
+class RecordCollectionInputDecl:
+    resource_ids: tuple[str, ...]
+
+
+InputBindingDecl = RecordInputDecl | FileInputDecl | ResourceInputDecl | RecordCollectionInputDecl
+
+
+@dataclass(frozen=True)
+class RecordOutputDecl:
+    record_id: str
+
+
+@dataclass(frozen=True)
+class RecipeSourceDecl:
+    recipe: str
+    with_: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class PluginStepDecl:
+    id: str
+    plugin: str
+    reads: dict[str, InputBindingDecl] = field(default_factory=dict)
+    writes: dict[str, RecordOutputDecl] = field(default_factory=dict)
+    with_: dict[str, Any] = field(default_factory=dict)
+    source_recipe: RecipeSourceDecl | None = None
+
+
+@dataclass(frozen=True)
+class PipelineDecl:
+    runtime: dict[str, Any] = field(default_factory=dict)
+    steps: tuple[PluginStepDecl, ...] = ()
+
+
+@dataclass(frozen=True)
+class SurfaceDecl:
+    specs: tuple[PluginStepDecl, ...] = ()
+
+
+@dataclass(frozen=True)
+class WorkbenchDecl:
+    experiment: ExperimentDecl
+    experiment_semantics: ExperimentSemantics
+    plotting_palette: str | None
+    pipeline: PipelineDecl
+    plots: SurfaceDecl
+    exports: SurfaceDecl
+    config_digest: str = ""

reader_workbench/workbench/dop/__init__.py

@@ -0,0 +1,12 @@
+from .builtins import BUILTIN_DATA_CLASSES, BUILTIN_READY_SPECS, DOP_SCHEMA, builtin_dop_registry
+from .model import DataClassSpec, DopRegistry, ReadySpec
+
+__all__ = [
+    "BUILTIN_DATA_CLASSES",
+    "BUILTIN_READY_SPECS",
+    "DOP_SCHEMA",
+    "DataClassSpec",
+    "DopRegistry",
+    "ReadySpec",
+    "builtin_dop_registry",
+]