ml-datarefinery 0.9.4__py3-none-any.whl

datarefinery/__init__.py

@@ -0,0 +1,9 @@
+# Copyright (c) 2026 Pointmatic
+# SPDX-License-Identifier: Apache-2.0
+
+__version__ = "0.9.4"
+
+from datarefinery.core.datarefinery import DataRefinery, materialize
+from datarefinery.core.instance import Instance
+
+__all__ = ["DataRefinery", "Instance", "__version__", "materialize"]

datarefinery/__main__.py

@@ -0,0 +1,8 @@
+# Copyright (c) 2026 Pointmatic
+# SPDX-License-Identifier: Apache-2.0
+"""Entry point for ``python -m datarefinery``."""
+
+from datarefinery.cli.app import main_entry
+
+if __name__ == "__main__":
+    main_entry()

datarefinery/cache/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) 2026 Pointmatic
+# SPDX-License-Identifier: Apache-2.0

datarefinery/cache/atomic.py

@@ -0,0 +1,89 @@
+# Copyright (c) 2026 Pointmatic
+# SPDX-License-Identifier: Apache-2.0
+"""FR-5 atomic temp-then-promote and `FAILED` marker.
+
+`atomic_promote(temp, final)` uses `os.replace` to swap a fully populated
+temp directory into its final location atomically. A cross-device
+mismatch is caught up-front so the EXDEV failure surfaces with a
+"same-filesystem" message rather than deep inside the runner. On
+failure, `mark_failed(temp, exc, stage)` writes a JSON `FAILED` marker
+into the temp dir capturing the stage, exception type, message, and
+traceback for diagnostic recovery.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import traceback
+from datetime import UTC, datetime
+from pathlib import Path
+
+from datarefinery.core.errors import MaterializeError
+
+FAILED_MARKER = "FAILED"
+
+
+def _device_id(path: Path) -> int:
+    """Return `st_dev` for `path`. Wrapped so tests can monkey-patch the cross-device guard."""
+    return os.stat(path).st_dev
+
+
+def atomic_promote(temp_dir: Path, final_dir: Path) -> None:
+    """Atomically promote `temp_dir` to `final_dir` via `os.replace`.
+
+    Raises `MaterializeError` if `temp_dir` does not exist, if temp and
+    final live on different filesystems (`os.replace` would raise
+    `EXDEV`), or if the underlying rename fails. `final_dir.parent` is
+    created if missing; the parent directory chain ends one level above
+    the eventual instance.
+    """
+    if not temp_dir.is_dir():
+        raise MaterializeError(f"temp dir does not exist: {temp_dir}")
+
+    final_parent = final_dir.parent
+    final_parent.mkdir(parents=True, exist_ok=True)
+
+    temp_dev = _device_id(temp_dir.parent)
+    final_dev = _device_id(final_parent)
+    if temp_dev != final_dev:
+        raise MaterializeError(
+            f"cannot atomically promote across filesystems: "
+            f"temp_dir={temp_dir} (st_dev={temp_dev}), "
+            f"final_dir={final_dir} (st_dev={final_dev}). "
+            f"DataRefinery requires the cache root and the temp dir to "
+            f"share a filesystem; configure --cache-root accordingly."
+        )
+
+    try:
+        os.replace(temp_dir, final_dir)
+    except OSError as exc:
+        raise MaterializeError(
+            f"atomic promote failed for {temp_dir} -> {final_dir}: {exc}"
+        ) from exc
+
+
+def mark_failed(temp_dir: Path, exc: BaseException, stage: str) -> None:
+    """Write a `FAILED` JSON marker into `temp_dir` capturing the failure context.
+
+    No-op when `temp_dir` does not exist (e.g., it was already promoted
+    or deleted before the runner caught the failure). The marker is a
+    diagnostic artifact; it never blocks failure propagation in the
+    runner.
+    """
+    if not temp_dir.is_dir():
+        return
+
+    payload = {
+        "stage": stage,
+        "exc_type": type(exc).__name__,
+        "message": str(exc),
+        "traceback": "".join(traceback.format_exception(type(exc), exc, exc.__traceback__)),
+        "marked_at": (datetime.now(UTC).isoformat().replace("+00:00", "Z")),
+    }
+
+    marker = temp_dir / FAILED_MARKER
+    marker.write_text(
+        json.dumps(payload, indent=2, sort_keys=True),
+        encoding="utf-8",
+    )

datarefinery/cache/cleaner.py

@@ -0,0 +1,140 @@
+# Copyright (c) 2026 Pointmatic
+# SPDX-License-Identifier: Apache-2.0
+"""FR-21 cache cleaner: library API only.
+
+The CLI verb wraps this in Phase D. The library here exposes
+`CleanSelector` plus `clean(cache_root, selector, *, force=False)`. The
+selector is intersection-style across the `by_*` filters; `orphans` adds
+old temp dirs to the target set; `all=True` requires `force=True` and
+clears every direct child of `<cache-root>/instances/`.
+"""
+
+from __future__ import annotations
+
+import shutil
+import time
+from collections.abc import Iterable
+from dataclasses import dataclass
+from pathlib import Path
+
+from datarefinery.cache.layout import (
+    TMP_DIR_NAME,
+    instances_root,
+)
+from datarefinery.core.errors import CacheError
+
+
+@dataclass(frozen=True, slots=True)
+class CleanSelector:
+    """Declarative selector for `clean(...)`."""
+
+    by_recipe_hash: str | None = None
+    by_input_hash: str | None = None
+    by_seed: int | None = None
+    by_age_days: float | None = None
+    orphans: bool = False
+    orphan_age_days: float = 1.0
+    all: bool = False
+
+
+@dataclass(frozen=True, slots=True)
+class CleanReport:
+    removed: tuple[Path, ...]
+    skipped: tuple[tuple[Path, str], ...]
+
+
+def clean(
+    cache_root: Path,
+    selector: CleanSelector,
+    *,
+    force: bool = False,
+) -> CleanReport:
+    """Remove cache entries matching `selector`.
+
+    `selector.all=True` requires `force=True` and removes every direct
+    child of `<cache-root>/instances/` (including the `.tmp/` orphans
+    dir). The `by_*` filters compose intersection-style: each one
+    narrows the candidate set further. `orphans=True` independently
+    targets temp dirs older than `orphan_age_days`.
+    """
+    if selector.all:
+        if not force:
+            raise CacheError("clean(all=True) requires force=True")
+        return _clean_everything(cache_root)
+
+    instances = instances_root(cache_root)
+    if not instances.is_dir():
+        return CleanReport(removed=(), skipped=())
+
+    targets: list[Path] = []
+
+    if selector.orphans:
+        targets.extend(_orphan_temp_dirs(cache_root, selector.orphan_age_days))
+
+    instance_filters_active = (
+        selector.by_recipe_hash is not None
+        or selector.by_input_hash is not None
+        or selector.by_seed is not None
+        or selector.by_age_days is not None
+    )
+    if instance_filters_active:
+        candidates = list(_iter_instance_dirs(instances))
+        if selector.by_recipe_hash is not None:
+            prefix = selector.by_recipe_hash[:16]
+            candidates = [p for p in candidates if p.parent.parent.name == prefix]
+        if selector.by_input_hash is not None:
+            prefix = selector.by_input_hash[:16]
+            candidates = [p for p in candidates if p.parent.name == prefix]
+        if selector.by_seed is not None:
+            candidates = [p for p in candidates if p.name == str(selector.by_seed)]
+        if selector.by_age_days is not None:
+            cutoff = time.time() - selector.by_age_days * 86400
+            candidates = [p for p in candidates if p.stat().st_mtime < cutoff]
+        targets.extend(candidates)
+
+    return _remove_paths(targets)
+
+
+def _iter_instance_dirs(instances: Path) -> Iterable[Path]:
+    """Yield every `<recipe>/<input>/<seed>/` directory, skipping `.tmp/`."""
+    for recipe_shard in instances.iterdir():
+        if recipe_shard.name.startswith(".") or not recipe_shard.is_dir():
+            continue
+        for input_shard in recipe_shard.iterdir():
+            if not input_shard.is_dir():
+                continue
+            for seed_dir in input_shard.iterdir():
+                if seed_dir.is_dir():
+                    yield seed_dir
+
+
+def _orphan_temp_dirs(cache_root: Path, age_days: float) -> Iterable[Path]:
+    tmp_root = instances_root(cache_root) / TMP_DIR_NAME
+    if not tmp_root.is_dir():
+        return
+    cutoff = time.time() - age_days * 86400
+    for entry in tmp_root.iterdir():
+        if entry.is_dir() and entry.stat().st_mtime < cutoff:
+            yield entry
+
+
+def _clean_everything(cache_root: Path) -> CleanReport:
+    instances = instances_root(cache_root)
+    if not instances.is_dir():
+        return CleanReport(removed=(), skipped=())
+    return _remove_paths(list(instances.iterdir()))
+
+
+def _remove_paths(paths: list[Path]) -> CleanReport:
+    removed: list[Path] = []
+    skipped: list[tuple[Path, str]] = []
+    for path in paths:
+        try:
+            if path.is_dir():
+                shutil.rmtree(path)
+            else:
+                path.unlink()
+            removed.append(path)
+        except OSError as exc:
+            skipped.append((path, str(exc)))
+    return CleanReport(removed=tuple(removed), skipped=tuple(skipped))

datarefinery/cache/identity.py

@@ -0,0 +1,54 @@
+# Copyright (c) 2026 Pointmatic
+# SPDX-License-Identifier: Apache-2.0
+"""FR-4 cache identity: `CacheKey` + `compute_cache_key`.
+
+The cache key is the triple (recipe_hash, input_hash, seed). Cache
+directory paths use only the first 16 hex characters of `recipe_hash`
+and `input_hash` (`.short`); the full hash is recorded in
+`manifest.json`. See `project-essentials.md` "Cache identity is the
+reproducibility contract - invalidations are ceremonious."
+"""
+
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Mapping
+from dataclasses import dataclass
+
+from datarefinery.recipe.canonical import to_canonical_bytes
+from datarefinery.recipe.models import Recipe
+
+
+@dataclass(frozen=True, slots=True)
+class CacheKey:
+    """Identity tuple for a materialized instance."""
+
+    recipe_hash: str
+    input_hash: str
+    seed: int
+
+    @property
+    def short(self) -> str:
+        """First 16 hex characters of `recipe_hash` (cache directory shard)."""
+        return self.recipe_hash[:16]
+
+
+def compute_cache_key(
+    recipe: Recipe,
+    raw_input_hashes: Mapping[str, str],
+    seed: int,
+) -> CacheKey:
+    """Compute the cache key for a (recipe, inputs, seed) triple.
+
+    `raw_input_hashes` maps each input source name to a SHA-256 hex
+    digest of that source's content. The combined `input_hash` is
+    order-independent: keys are sorted by source name before
+    concatenation.
+    """
+    recipe_hash = hashlib.sha256(to_canonical_bytes(recipe)).hexdigest()
+
+    parts = [f"{name}={raw_input_hashes[name]};" for name in sorted(raw_input_hashes)]
+    payload = "".join(parts).encode("utf-8")
+    input_hash = hashlib.sha256(payload).hexdigest()
+
+    return CacheKey(recipe_hash=recipe_hash, input_hash=input_hash, seed=seed)

datarefinery/cache/layout.py

@@ -0,0 +1,83 @@
+# Copyright (c) 2026 Pointmatic
+# SPDX-License-Identifier: Apache-2.0
+"""Cache directory layout helpers under `<cache-root>/instances/...`.
+
+Layout (per tech-spec):
+
+    <cache-root>/
+    └── instances/
+        ├── .tmp/<run-id>/                  # in-flight runs; promoted via os.replace
+        └── <recipe-hash16>/<input-hash16>/<seed>/
+            ├── manifest.json
+            ├── dataset/
+            ├── fitted_statistics/
+            └── report/
+                ├── report.md
+                ├── drift.json
+                └── visualizations/
+
+The 16-char shards come from `CacheKey.short` (recipe) and the first 16
+chars of `input_hash`; the full hashes are recorded in `manifest.json`.
+"""
+
+from __future__ import annotations
+
+import secrets
+from datetime import UTC, datetime
+from pathlib import Path
+
+from datarefinery.cache.identity import CacheKey
+
+INSTANCES_DIR = "instances"
+TMP_DIR_NAME = ".tmp"
+MANIFEST_FILE = "manifest.json"
+RECIPE_FILE = "recipe.json"
+DATASET_SUBDIR = "dataset"
+FITTED_STATS_SUBDIR = "fitted_statistics"
+REPORT_SUBDIR = "report"
+
+
+def instances_root(cache_root: Path) -> Path:
+    """Root for all materialized instances and temp dirs."""
+    return cache_root / INSTANCES_DIR
+
+
+def instance_dir(cache_root: Path, key: CacheKey) -> Path:
+    """Final path for a materialized instance under the cache root."""
+    return instances_root(cache_root) / key.recipe_hash[:16] / key.input_hash[:16] / str(key.seed)
+
+
+def tmp_dir(cache_root: Path, run_id: str) -> Path:
+    """Temp directory for an in-flight run, atomically promoted via os.replace."""
+    return instances_root(cache_root) / TMP_DIR_NAME / run_id
+
+
+def manifest_path(instance: Path) -> Path:
+    return instance / MANIFEST_FILE
+
+
+def recipe_path(instance: Path) -> Path:
+    return instance / RECIPE_FILE
+
+
+def dataset_dir(instance: Path) -> Path:
+    return instance / DATASET_SUBDIR
+
+
+def fitted_stats_dir(instance: Path) -> Path:
+    return instance / FITTED_STATS_SUBDIR
+
+
+def report_dir(instance: Path) -> Path:
+    return instance / REPORT_SUBDIR
+
+
+def make_run_id() -> str:
+    """Return `<utc_iso_compact>-<8hex>`, e.g. `20260507T143022Z-deadbeef`.
+
+    Lexicographically sortable down to the second; the 8-hex random
+    suffix makes outputs unique under burst/concurrent calls within the
+    same second.
+    """
+    stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
+    return f"{stamp}-{secrets.token_hex(4)}"

datarefinery/cli/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) 2026 Pointmatic
+# SPDX-License-Identifier: Apache-2.0

datarefinery/cli/_exit_codes.py

@@ -0,0 +1,56 @@
+# Copyright (c) 2026 Pointmatic
+# SPDX-License-Identifier: Apache-2.0
+"""Exception-to-exit-code mapping for the DataRefinery CLI.
+
+Per tech-spec:
+
+| code | meaning                                                       |
+|------|---------------------------------------------------------------|
+| 0    | success                                                       |
+| 1    | user/recipe error (Recipe, Validation, Contract, Materialize) |
+| 2    | system error (Plugin, Cache, environment, uncaught)           |
+| 130  | SIGINT / Ctrl-C                                               |
+"""
+
+from __future__ import annotations
+
+from datarefinery.core.errors import (
+    CacheError,
+    ContractError,
+    DataRefineryError,
+    MaterializeError,
+    PluginError,
+    RecipeError,
+    ValidationError,
+)
+
+EXIT_OK = 0
+EXIT_USER = 1
+EXIT_SYSTEM = 2
+EXIT_INTERRUPT = 130
+
+_USER_ERROR_TYPES: tuple[type[DataRefineryError], ...] = (
+    RecipeError,
+    ValidationError,
+    ContractError,
+    MaterializeError,
+)
+_SYSTEM_ERROR_TYPES: tuple[type[DataRefineryError], ...] = (
+    PluginError,
+    CacheError,
+)
+
+
+def exit_code_for(exc: BaseException) -> int:
+    """Return the documented CLI exit code for `exc`."""
+    if isinstance(exc, KeyboardInterrupt):
+        return EXIT_INTERRUPT
+    if isinstance(exc, _USER_ERROR_TYPES):
+        return EXIT_USER
+    if isinstance(exc, _SYSTEM_ERROR_TYPES):
+        return EXIT_SYSTEM
+    if isinstance(exc, DataRefineryError):
+        # Unknown future DataRefineryError subclass — treat as user-facing
+        # by default; widen the explicit tuples above when adding new types.
+        return EXIT_USER
+    return EXIT_SYSTEM

datarefinery/cli/app.py

@@ -0,0 +1,191 @@
+# Copyright (c) 2026 Pointmatic
+# SPDX-License-Identifier: Apache-2.0
+"""Typer CLI entry point for DataRefinery."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+from typing import Annotated
+
+import click
+import typer
+from rich.console import Console
+from rich.panel import Panel
+
+from datarefinery import __version__
+from datarefinery.cli._exit_codes import (
+    EXIT_INTERRUPT,
+    EXIT_OK,
+    EXIT_SYSTEM,
+    exit_code_for,
+)
+from datarefinery.cli.commands.check_cmd import check as check_cmd
+from datarefinery.cli.commands.clean_cmd import clean_command as clean_cmd
+from datarefinery.cli.commands.init_cmd import init as init_cmd
+from datarefinery.cli.commands.inspect_cmd import inspect as inspect_cmd
+from datarefinery.cli.commands.materialize_cmd import materialize as materialize_cmd
+from datarefinery.cli.commands.report_cmd import report as report_cmd
+from datarefinery.cli.commands.status_cmd import status as status_cmd
+from datarefinery.cli.commands.validate_cmd import validate as validate_cmd
+from datarefinery.core.config import RuntimeConfig
+from datarefinery.core.errors import DataRefineryError
+from datarefinery.logging import get_logger
+
+app = typer.Typer(
+    name="datarefinery",
+    help="DataRefinery — recipe-driven data preparation and caching for ML.",
+    no_args_is_help=False,
+    add_completion=False,
+)
+
+
+def _version_callback(value: bool) -> None:
+    if value:
+        typer.echo(__version__)
+        raise typer.Exit()
+
+
+@app.callback(invoke_without_command=True)
+def main(
+    ctx: typer.Context,
+    version: Annotated[
+        bool,
+        typer.Option(
+            "--version",
+            callback=_version_callback,
+            is_eager=True,
+            help="Show the package version and exit.",
+        ),
+    ] = False,
+    cache_root: Annotated[
+        Path | None,
+        typer.Option(
+            "--cache-root",
+            help="Root directory for the cache (env: DATAREFINERY_CACHE_ROOT).",
+        ),
+    ] = None,
+    log_level: Annotated[
+        str | None,
+        typer.Option(
+            "--log-level",
+            help="Log level (env: DATAREFINERY_LOG_LEVEL).",
+        ),
+    ] = None,
+    log_target: Annotated[
+        str | None,
+        typer.Option(
+            "--log-target",
+            help="Log routing target; reserved no-op stub (env: DATAREFINERY_LOG_TARGET).",
+        ),
+    ] = None,
+    plugin_path: Annotated[
+        list[Path] | None,
+        typer.Option(
+            "--plugin-path",
+            help="Extra plugin discovery path; repeatable "
+            "(env: DATAREFINERY_PLUGIN_PATH, PATH-style).",
+        ),
+    ] = None,
+    workers: Annotated[
+        int | None,
+        typer.Option(
+            "--workers",
+            help="Process pool worker count (env: DATAREFINERY_WORKERS).",
+        ),
+    ] = None,
+    seed: Annotated[
+        int | None,
+        typer.Option(
+            "--seed",
+            help="Override the recipe-declared seed (changes cache identity).",
+        ),
+    ] = None,
+    variant: Annotated[
+        str | None,
+        typer.Option("--variant", help="Recipe variant to apply before canonicalization."),
+    ] = None,
+    no_color: Annotated[
+        bool,
+        typer.Option("--no-color", help="Disable colored output."),
+    ] = False,
+    quiet: Annotated[
+        bool,
+        typer.Option("--quiet", "-q", help="Suppress non-essential output."),
+    ] = False,
+    verbose: Annotated[
+        bool,
+        typer.Option("--verbose", "-v", help="Verbose output."),
+    ] = False,
+) -> None:
+    """DataRefinery — recipe-driven data preparation and caching for ML."""
+    config = RuntimeConfig.resolve(
+        cache_root=cache_root,
+        log_level=log_level,
+        log_target=log_target,
+        plugin_path=plugin_path,
+        workers=workers,
+    )
+    state = ctx.ensure_object(dict)
+    state["config"] = config
+    state["seed"] = seed
+    state["variant"] = variant
+    state["no_color"] = no_color
+    state["quiet"] = quiet
+    state["verbose"] = verbose
+
+    get_logger("cli")
+    return None
+
+
+app.command("check", help="Report environment soundness (FR-18).")(check_cmd)
+app.command("validate", help="Validate a recipe (FR-2).")(validate_cmd)
+app.command("init", help="Scaffold a starter recipe from raw inputs (FR-17).")(init_cmd)
+app.command(
+    "materialize",
+    help="Run the pipeline end-to-end against the recipe's inputs (FR-3).",
+)(materialize_cmd)
+app.command(
+    "status",
+    help="Summarize a materialized instance or resolve a recipe to one (FR-19).",
+)(status_cmd)
+app.command(
+    "report",
+    help="Re-render report.md, drift.json, and reporting visualizations (FR-15).",
+)(report_cmd)
+app.command(
+    "inspect",
+    help="Read-only views of a materialized instance (FR-20).",
+)(inspect_cmd)
+app.command(
+    "clean",
+    help="Remove cached instances and orphan temp directories (FR-21).",
+)(clean_cmd)
+
+
+def _render_error(message: str, *, title: str) -> None:
+    Console(stderr=True).print(Panel(message, title=title, border_style="red", expand=False))
+
+
+def main_entry() -> None:
+    """Console-script entry point with DataRefinery's exit-code mapping."""
+    try:
+        app(standalone_mode=False)
+    except click.exceptions.Exit as exc:
+        sys.exit(exc.exit_code)
+    except click.exceptions.UsageError as exc:
+        exc.show()
+        sys.exit(EXIT_SYSTEM)
+    except click.exceptions.ClickException as exc:
+        exc.show()
+        sys.exit(exc.exit_code)
+    except (click.exceptions.Abort, KeyboardInterrupt):
+        _render_error("Interrupted.", title="Aborted")
+        sys.exit(EXIT_INTERRUPT)
+    except DataRefineryError as exc:
+        _render_error(str(exc) or type(exc).__name__, title=type(exc).__name__)
+        sys.exit(exit_code_for(exc))
+    except Exception as exc:
+        _render_error(f"{type(exc).__name__}: {exc}", title="Internal Error")
+        sys.exit(EXIT_SYSTEM)
+    sys.exit(EXIT_OK)

datarefinery/cli/commands/__init__.py

@@ -0,0 +1,8 @@
+# Copyright (c) 2026 Pointmatic
+# SPDX-License-Identifier: Apache-2.0
+"""Subcommand modules for the ``datarefinery`` CLI.
+
+Each verb lives in its own ``<verb>_cmd.py`` so the verb name does not
+collide with Python keyword-adjacent identifiers and stays readable in
+import paths (per tech-spec).
+"""