reader-workbench 1.0.0__py3-none-any.whl

reader_workbench/workbench/audit/experiments.py

@@ -0,0 +1,307 @@
+from __future__ import annotations
+
+import io
+import tempfile
+import time
+from dataclasses import asdict, dataclass
+from pathlib import Path
+
+from rich.console import Console
+
+from reader_workbench.errors import ReaderError
+from reader_workbench.runtime import builtin_runtime
+from reader_workbench.workbench.decl import load_workbench_decl
+from reader_workbench.workbench.engine import run_spec
+from reader_workbench.workbench.engine.validation import validation_summary
+from reader_workbench.workbench.experiments import discover_experiment_configs
+from reader_workbench.workbench.graph import resolve_workbench
+from reader_workbench.workbench.inspection.runtime import record_producer_map, workbench_record_verification_scope
+from reader_workbench.workbench.records import verify_record_store
+
+from .staging import stage_audit_workspace
+
+
+@dataclass(slots=True)
+class AuditResult:
+    config: str
+    name: str
+    lifecycle: str
+    status: str
+    phase: str
+    seconds: float
+    detail: str | None
+    expected_plots: int
+    expected_exports: int
+
+
+def discover_year_dirs(root: Path) -> list[str]:
+    return sorted(path.name for path in root.iterdir() if path.is_dir() and path.name.isdigit() and len(path.name) == 4)
+
+
+def discover_configs(root: Path, years: set[str]) -> list[Path]:
+    configs = discover_experiment_configs(root, include_scaffolds=False)
+    selected: list[Path] = []
+    for config in configs:
+        try:
+            rel = config.relative_to(root)
+        except ValueError:
+            continue
+        if not rel.parts:
+            continue
+        if years and rel.parts[0] not in years:
+            continue
+        selected.append(config)
+    return sorted(selected)
+
+
+def verify_outputs(decl, runtime) -> tuple[int, int, str | None]:
+    workbench = resolve_workbench(decl)
+    verification_scope = workbench_record_verification_scope(workbench, runtime=runtime)
+    layout = decl.experiment_semantics.layout
+    outputs = layout.outputs_dir
+    store = runtime.record_store(
+        outputs,
+        plots_subdir=layout.plots_subdir,
+        exports_subdir=layout.exports_subdir,
+        experiment_root=decl.experiment.root,
+        create=False,
+    )
+    latest_ids = {record.record_id for record in store.iter_latest_records()}
+    expected_plot_ids = {f"plot:{plot.id}" for plot in workbench.plots}
+    expected_export_ids = {f"export:{export.id}" for export in workbench.exports}
+    expected_dataframe_ids = set(record_producer_map(workbench.plugin_steps(), runtime=runtime))
+
+    if (outputs / "manifests" / "records.json").exists() is False:
+        return len(expected_plot_ids), len(expected_export_ids), "records.json missing"
+    missing_dataframe_ids = sorted(expected_dataframe_ids - latest_ids)
+    if missing_dataframe_ids:
+        return (
+            len(expected_plot_ids),
+            len(expected_export_ids),
+            f"missing declared dataframe records: {missing_dataframe_ids}",
+        )
+
+    missing_plot_ids = sorted(expected_plot_ids - latest_ids)
+    if missing_plot_ids:
+        return len(expected_plot_ids), len(expected_export_ids), f"missing plot records: {missing_plot_ids}"
+    missing_export_ids = sorted(expected_export_ids - latest_ids)
+    if missing_export_ids:
+        return len(expected_plot_ids), len(expected_export_ids), f"missing export records: {missing_export_ids}"
+
+    plots_dir = outputs / layout.plots_subdir
+    exports_dir = outputs / layout.exports_subdir
+    if expected_plot_ids and not any(path.is_file() for path in plots_dir.rglob("*")):
+        return len(expected_plot_ids), len(expected_export_ids), "expected plot files were not created"
+    if expected_export_ids and not any(path.is_file() for path in exports_dir.rglob("*")):
+        return len(expected_plot_ids), len(expected_export_ids), "expected export files were not created"
+
+    verification = verify_record_store(
+        store,
+        experiment_root=decl.experiment.root,
+        expected_config_digest=decl.config_digest,
+        scope=verification_scope,
+    )
+    if verification["status"] != "ok":
+        issues = list(verification.get("issues") or [])
+        for record in verification.get("records") or []:
+            issues.extend(record.get("issues") or [])
+        first = issues[0] if issues else {"code": "verification.failed", "reason": "unknown verification failure"}
+        return (
+            len(expected_plot_ids),
+            len(expected_export_ids),
+            f"reader verify failed: {first['code']} • {first['reason']}",
+        )
+
+    return len(expected_plot_ids), len(expected_export_ids), None
+
+
+def audit_config(config_path: Path, *, include_non_active: bool, runtime) -> AuditResult:
+    start = time.perf_counter()
+    try:
+        decl = load_workbench_decl(config_path, protocols=runtime.protocols)
+    except ReaderError as exc:
+        return AuditResult(
+            config=str(config_path),
+            name=config_path.parent.name,
+            lifecycle="unknown",
+            status="failed",
+            phase="config",
+            seconds=time.perf_counter() - start,
+            detail=str(exc),
+            expected_plots=0,
+            expected_exports=0,
+        )
+    lifecycle = decl.experiment.lifecycle
+    rel_config = str(config_path)
+    if lifecycle != "active" and not include_non_active:
+        return AuditResult(
+            config=rel_config,
+            name=config_path.parent.name,
+            lifecycle=lifecycle,
+            status="skipped",
+            phase="lifecycle",
+            seconds=time.perf_counter() - start,
+            detail=f"skipped non-active lifecycle: {lifecycle}",
+            expected_plots=0,
+            expected_exports=0,
+        )
+
+    source_summary = validation_summary(
+        decl,
+        check_files=True,
+        exp_root=decl.experiment.root,
+        runtime=runtime,
+    )
+    if source_summary["status"] != "ok":
+        return AuditResult(
+            config=rel_config,
+            name=config_path.parent.name,
+            lifecycle=lifecycle,
+            status="failed",
+            phase="validate",
+            seconds=time.perf_counter() - start,
+            detail="; ".join(source_summary["errors"]) or "validation failed",
+            expected_plots=0,
+            expected_exports=0,
+        )
+
+    with tempfile.TemporaryDirectory(prefix="reader-local-audit-") as tmpdir:
+        try:
+            staged_config = stage_audit_workspace(
+                config_path=config_path,
+                target_root=Path(tmpdir),
+                resources=decl.experiment_semantics.resources,
+                runtime=runtime,
+            )
+            staged_decl = load_workbench_decl(staged_config, protocols=runtime.protocols)
+        except (ReaderError, OSError) as exc:
+            return AuditResult(
+                config=rel_config,
+                name=config_path.parent.name,
+                lifecycle=lifecycle,
+                status="failed",
+                phase="validate",
+                seconds=time.perf_counter() - start,
+                detail=str(exc),
+                expected_plots=0,
+                expected_exports=0,
+            )
+        summary = validation_summary(
+            staged_decl, check_files=True, exp_root=staged_decl.experiment.root, runtime=runtime
+        )
+        if summary["status"] != "ok":
+            return AuditResult(
+                config=rel_config,
+                name=config_path.parent.name,
+                lifecycle=lifecycle,
+                status="failed",
+                phase="validate",
+                seconds=time.perf_counter() - start,
+                detail="; ".join(summary["errors"]) or "validation failed",
+                expected_plots=0,
+                expected_exports=0,
+            )
+
+        try:
+            run_spec(
+                staged_decl,
+                include_pipeline=True,
+                include_plots=True,
+                include_exports=True,
+                runtime=runtime,
+                console=Console(file=io.StringIO(), force_terminal=False, color_system=None),
+                log_level="ERROR",
+                verbose=False,
+            )
+        except Exception as exc:  # pragma: no cover - exercised by live audit runs
+            return AuditResult(
+                config=rel_config,
+                name=config_path.parent.name,
+                lifecycle=lifecycle,
+                status="failed",
+                phase="run",
+                seconds=time.perf_counter() - start,
+                detail=str(exc),
+                expected_plots=0,
+                expected_exports=0,
+            )
+
+        expected_plots, expected_exports, verify_error = verify_outputs(staged_decl, runtime)
+        if verify_error is not None:
+            return AuditResult(
+                config=rel_config,
+                name=config_path.parent.name,
+                lifecycle=lifecycle,
+                status="failed",
+                phase="verify",
+                seconds=time.perf_counter() - start,
+                detail=verify_error,
+                expected_plots=expected_plots,
+                expected_exports=expected_exports,
+            )
+
+        return AuditResult(
+            config=rel_config,
+            name=config_path.parent.name,
+            lifecycle=lifecycle,
+            status="passed",
+            phase="complete",
+            seconds=time.perf_counter() - start,
+            detail=None,
+            expected_plots=expected_plots,
+            expected_exports=expected_exports,
+        )
+
+
+def render_text(results: list[AuditResult]) -> str:
+    lines = []
+    for result in results:
+        detail = f" :: {result.detail}" if result.detail else ""
+        lines.append(
+            f"[{result.status.upper():7}] {result.name} "
+            f"(lifecycle={result.lifecycle}, phase={result.phase}, "
+            f"plots={result.expected_plots}, exports={result.expected_exports}, "
+            f"seconds={result.seconds:.2f}){detail}"
+        )
+    counts: dict[str, int] = {}
+    for result in results:
+        counts[result.status] = counts.get(result.status, 0) + 1
+    summary = ", ".join(f"{key}={counts[key]}" for key in sorted(counts))
+    lines.append(f"Summary: {summary}")
+    return "\n".join(lines)
+
+
+def audit_experiments(
+    root: Path,
+    *,
+    years: tuple[str, ...] | None = None,
+    include_non_active: bool = False,
+    fail_fast: bool = False,
+) -> dict[str, object]:
+    root = root.resolve()
+    if not root.exists() or not root.is_dir():
+        raise ValueError(f"Experiments root not found: {root}")
+    selected_years = discover_year_dirs(root) if years is None else [str(year) for year in years]
+    if not selected_years:
+        raise ValueError(f"No numeric year directories found under {root}. Use --years to select specific directories.")
+    runtime = builtin_runtime()
+    configs = discover_configs(root, set(selected_years))
+
+    results: list[AuditResult] = []
+    for config_path in configs:
+        result = audit_config(config_path, include_non_active=include_non_active, runtime=runtime)
+        results.append(result)
+        if fail_fast and result.status == "failed":
+            break
+
+    return {
+        "root": str(root),
+        "years": sorted(selected_years),
+        "summary": {
+            "experiments": len(results),
+            "passed": sum(1 for item in results if item.status == "passed"),
+            "failed": sum(1 for item in results if item.status == "failed"),
+            "skipped": sum(1 for item in results if item.status == "skipped"),
+        },
+        "results": [asdict(result) for result in results],
+    }

reader_workbench/workbench/audit/staging.py

@@ -0,0 +1,187 @@
+from __future__ import annotations
+
+import json
+import os
+import shutil
+from pathlib import Path
+
+from reader_workbench.errors import ConfigError, RecordError
+from reader_workbench.runtime import ReaderRuntime
+from reader_workbench.workbench.experiment import ResourceCatalog
+from reader_workbench.workbench.experiments import find_experiments_root
+from reader_workbench.workbench.graph import SourceRecordRef
+from reader_workbench.workbench.records import (
+    record_paths,
+    record_to_dict,
+    verify_record_artifact_integrity,
+)
+from reader_workbench.workbench.records.store import RECORD_CATALOG_SCHEMA_VERSION
+
+
+def stage_experiment(source_dir: Path, target_dir: Path) -> Path:
+    shutil.copytree(
+        source_dir,
+        target_dir,
+        ignore=shutil.ignore_patterns("outputs", "__pycache__", ".DS_Store"),
+        symlinks=True,
+    )
+    _retarget_internal_symlinks(source_dir=source_dir, target_dir=target_dir)
+    return target_dir / "config.yaml"
+
+
+def stage_audit_workspace(
+    *,
+    config_path: Path,
+    target_root: Path,
+    resources: ResourceCatalog,
+    runtime: ReaderRuntime,
+) -> Path:
+    """Stage one writable experiment and exact read-only snapshots of its record resources."""
+
+    record_resources = [resource for resource in resources.by_id.values() if resource.kind == "record"]
+    try:
+        source_experiments_root = find_experiments_root(config_path.parent)
+    except ConfigError:
+        if record_resources:
+            raise
+        source_experiments_root = None
+    staged_experiments_root = target_root / "experiments"
+    relative_experiment_dir = (
+        config_path.parent.resolve().relative_to(source_experiments_root)
+        if source_experiments_root is not None
+        else Path(config_path.parent.name)
+    )
+    staged_config = stage_experiment(
+        config_path.parent,
+        staged_experiments_root / relative_experiment_dir,
+    )
+
+    resources_by_provider: dict[tuple[Path, Path], set[str]] = {}
+    for resource in record_resources:
+        provider_key = (resource.experiment_root.resolve(), resource.outputs_dir.resolve())
+        resources_by_provider.setdefault(provider_key, set()).add(resource.record_id)
+    if resources_by_provider and source_experiments_root is None:
+        raise ConfigError("Cross-experiment record resources require a canonical experiments/ workspace")
+    for (provider_root, provider_outputs), record_ids in sorted(
+        resources_by_provider.items(),
+        key=lambda item: (str(item[0][0]), str(item[0][1])),
+    ):
+        _stage_record_provider(
+            provider_root=provider_root,
+            provider_outputs=provider_outputs,
+            record_ids=record_ids,
+            source_experiments_root=source_experiments_root,
+            staged_experiments_root=staged_experiments_root,
+            staged_target_dir=staged_config.parent,
+            runtime=runtime,
+        )
+    return staged_config
+
+
+def _stage_record_provider(
+    *,
+    provider_root: Path,
+    provider_outputs: Path,
+    record_ids: set[str],
+    source_experiments_root: Path,
+    staged_experiments_root: Path,
+    staged_target_dir: Path,
+    runtime: ReaderRuntime,
+) -> None:
+    staged_provider_dir = _stage_catalog_config(
+        source_root=provider_root,
+        source_experiments_root=source_experiments_root,
+        staged_experiments_root=staged_experiments_root,
+        staged_target_dir=staged_target_dir,
+    )
+    relative_outputs = provider_outputs.relative_to(provider_root)
+    staged_outputs = staged_provider_dir / relative_outputs
+    source_store = runtime.record_store(
+        provider_outputs,
+        experiment_root=provider_root,
+        create=False,
+    )
+    catalog = {
+        "schema_version": RECORD_CATALOG_SCHEMA_VERSION,
+        "provenance_epoch_id": source_store.provenance_epoch_id(),
+        "latest": {},
+        "history": {},
+    }
+    source_records = {record.record_id: record for record in source_store.iter_latest_records()}
+    copied_paths: set[Path] = set()
+    for record_id in sorted(record_ids):
+        record = source_records.get(record_id)
+        if record is None:
+            raise RecordError(f"Source record {record_id!r} is missing from {provider_root.name!r}")
+        verify_record_artifact_integrity(record, outputs_dir=provider_outputs)
+        payload = record_to_dict(record, outputs_dir=provider_outputs)
+        catalog["latest"][record_id] = payload
+        catalog["history"][record_id] = [payload]
+        for source_path in record_paths(record):
+            relative_path = source_path.resolve(strict=True).relative_to(provider_outputs)
+            if relative_path in copied_paths:
+                continue
+            copied_paths.add(relative_path)
+            staged_path = staged_outputs / relative_path
+            staged_path.parent.mkdir(parents=True, exist_ok=True)
+            shutil.copy2(source_path, staged_path)
+        for evidence in record.inputs:
+            if not isinstance(evidence.ref, SourceRecordRef):
+                continue
+            _stage_catalog_config(
+                source_root=evidence.ref.experiment_root,
+                source_experiments_root=source_experiments_root,
+                staged_experiments_root=staged_experiments_root,
+                staged_target_dir=staged_target_dir,
+            )
+
+    staged_manifests = staged_outputs / "manifests"
+    staged_manifests.mkdir(parents=True, exist_ok=True)
+    (staged_manifests / "records.json").write_text(
+        json.dumps(catalog, indent=2, sort_keys=True) + "\n",
+        encoding="utf-8",
+    )
+
+
+def _stage_catalog_config(
+    *,
+    source_root: Path,
+    source_experiments_root: Path,
+    staged_experiments_root: Path,
+    staged_target_dir: Path,
+) -> Path:
+    staged_dir = staged_experiments_root / source_root.resolve().relative_to(source_experiments_root)
+    if staged_dir == staged_target_dir:
+        return staged_dir
+    staged_dir.mkdir(parents=True, exist_ok=True)
+    staged_config = staged_dir / "config.yaml"
+    if not staged_config.exists():
+        shutil.copy2(source_root / "config.yaml", staged_config)
+    return staged_dir
+
+
+def _retarget_internal_symlinks(*, source_dir: Path, target_dir: Path) -> None:
+    """Map confined source symlinks into the staged tree without following links while copying."""
+
+    source_root = source_dir.resolve(strict=True)
+    target_root = target_dir.resolve(strict=True)
+    for current, dirnames, filenames in os.walk(target_root, followlinks=False):
+        current_path = Path(current)
+        for name in [*dirnames, *filenames]:
+            staged_link = current_path / name
+            if not staged_link.is_symlink():
+                continue
+            relative_link = staged_link.relative_to(target_root)
+            source_link = source_root / relative_link
+            try:
+                source_target = source_link.resolve(strict=True)
+                relative_target = source_target.relative_to(source_root)
+            except (OSError, RuntimeError, ValueError):
+                # External, dangling, or cyclic links remain links. Staged
+                # validation can then reject them without copytree traversing them.
+                continue
+            staged_target = target_root / relative_target
+            link_target = os.path.relpath(staged_target, start=staged_link.parent)
+            target_is_directory = source_target.is_dir()
+            staged_link.unlink()
+            staged_link.symlink_to(link_target, target_is_directory=target_is_directory)

reader_workbench/workbench/cli/__init__.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+from . import audit as _audit  # noqa: F401
+from . import demo as _demo  # noqa: F401
+from . import dop as _dop  # noqa: F401
+from . import experiments as _experiments  # noqa: F401
+from . import maintenance as _maintenance  # noqa: F401
+from . import notebooks as notebook_commands
+from . import protocols as _protocols  # noqa: F401
+from . import shared
+from . import surfaces as _surfaces  # noqa: F401
+from . import verification as _verification  # noqa: F401
+from .audit import audit_app
+from .experiments import config, explain, inspect, ls, run, validate
+from .helpers import infer_job_path as _infer_job_path
+from .main import main
+from .maintenance import maintain_app
+from .notebooks import _launch_marimo, notebook
+from .protocols import init, protocols
+from .shared import THEME, app, console, subprocess
+from .surfaces import export, plot, plugins, records, steps
+from .verification import verify
+
+__all__ = [
+    "THEME",
+    "_infer_job_path",
+    "_launch_marimo",
+    "audit_app",
+    "app",
+    "config",
+    "console",
+    "explain",
+    "export",
+    "init",
+    "inspect",
+    "ls",
+    "maintain_app",
+    "main",
+    "notebook",
+    "notebook_commands",
+    "plot",
+    "plugins",
+    "protocols",
+    "records",
+    "run",
+    "shared",
+    "steps",
+    "subprocess",
+    "validate",
+    "verify",
+]

reader_workbench/workbench/cli/_lazy.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+from functools import cache
+from importlib import import_module
+
+
+@cache
+def load(module_name: str):
+    return import_module(module_name)