duckpipe 0.1.0__py3-none-any.whl

duckpipe/__init__.py

@@ -0,0 +1,41 @@
+"""DuckPipe: a serverless-first, DuckDB-native pipeline orchestrator.
+
+No scheduler daemon, no central metadata database required, no broker
+-- a run is a process that starts, does work, records what it did to a
+``.duckdb`` file, and exits::
+
+    from duckpipe import task, run
+
+    @task
+    def extract():
+        return duckdb.sql("select * from read_parquet('data.parquet')")
+
+    @task(cache=True)
+    def transform(rel=extract):  # `rel=extract` infers the dependency
+        return rel.filter("amount > 0")
+
+    if __name__ == "__main__":
+        run(__file__)
+
+Then from a shell: ``duckpipe run pipeline.py``.
+"""
+
+from __future__ import annotations
+
+from duckpipe.dag import DAG, CycleError, build_dag
+from duckpipe.remote import StateLockedError
+from duckpipe.scheduler import RunSummary, run
+from duckpipe.task import Task, task
+
+__all__ = [
+    "task",
+    "run",
+    "build_dag",
+    "Task",
+    "DAG",
+    "CycleError",
+    "RunSummary",
+    "StateLockedError",
+]
+
+__version__ = "0.1.0"

duckpipe/cli.py

@@ -0,0 +1,393 @@
+"""``duckpipe`` command-line interface.
+
+Every subcommand is a thin wrapper over the same public functions
+(``duckpipe.run``, ``duckpipe.build_dag``) any other trigger -- cron, CI,
+a Lambda handler -- would call directly (DESIGN.md sec 5, sec 9).
+"""
+
+from __future__ import annotations
+
+import functools
+import re
+from collections.abc import Callable
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from duckpipe.dag import CycleError, DuplicateTaskNameError, build_dag
+from duckpipe.fingerprint import resolve_fingerprints
+from duckpipe.remote import StateLockedError
+from duckpipe.scheduler import (
+    UpstreamNotCachedError,
+    UpstreamNotReadyError,
+    _default_db_path,
+    would_skip,
+)
+from duckpipe.scheduler import run as run_pipeline
+from duckpipe.state import is_ducklake
+
+app = typer.Typer(
+    name="duckpipe",
+    help="A serverless-first, DuckDB-native pipeline orchestrator.",
+    no_args_is_help=True,
+    pretty_exceptions_enable=False,
+)
+console = Console()
+
+_STATUS_STYLE = {"success": "green", "skipped": "yellow", "failed": "red"}
+
+# A malformed pipeline (a cycle, a duplicate task name, a plain typo that
+# blows up at import time), a locked state_uri, or a scoped (--only) run
+# dispatched out of order is a user-actionable situation, not a DuckPipe
+# bug -- it should read as one short line, not a wall of framework frames.
+_USER_FACING_ERRORS = (
+    CycleError,
+    DuplicateTaskNameError,
+    ImportError,
+    SyntaxError,
+    StateLockedError,
+    UpstreamNotReadyError,
+    UpstreamNotCachedError,
+    ValueError,
+)
+
+_DUCKLAKE_FILE_PREFIXES = ("ducklake:sqlite:", "ducklake:duckdb:")
+
+
+def _state_probably_exists(db_path: str | Path) -> bool:
+    """A cheap existence check that never opens (and so never creates) the
+    store, for commands like `show`/`stats` that shouldn't conjure a
+    fresh, empty state file just by asking about one that isn't there yet.
+    """
+    if is_ducklake(db_path):
+        for prefix in _DUCKLAKE_FILE_PREFIXES:
+            if db_path.startswith(prefix):
+                return Path(db_path[len(prefix) :]).exists()
+        return True  # a live catalog (e.g. Postgres) -- can't check cheaply
+    return Path(db_path).exists()
+
+
+_MERMAID_STATUS_CLASS = {
+    "success": "success",
+    "skipped": "skipped",
+    "failed": "failed",
+    "upstream_failed": "failed",
+}
+_MERMAID_CLASS_DEFS = {
+    "success": "classDef success fill:#d4f7dc,stroke:#2f9e44,color:#1a1a1a",
+    "skipped": "classDef skipped fill:#fff3cd,stroke:#d9a400,color:#1a1a1a",
+    "failed": "classDef failed fill:#f8d7da,stroke:#d64545,color:#1a1a1a",
+}
+
+
+def _mermaid_node_id(name: str) -> str:
+    # Task names are usually valid identifiers already (a function's own
+    # __name__, or an explicit name=...), but a Mermaid node id has its
+    # own, stricter rules -- sanitize rather than assume, so an unusual
+    # name= never produces a broken diagram.
+    return "t_" + re.sub(r"[^A-Za-z0-9_]", "_", name)
+
+
+def _mermaid_diagram(order: list, last_status: dict[str, tuple[str, str]]) -> str:
+    """A `flowchart TD` a task's real dependency edges, one node per task
+    (labeled with its real name, not the sanitized id), colored by last
+    recorded status when state exists -- pastes straight into any
+    Markdown that renders Mermaid (GitHub, GitLab, many others)."""
+    lines = ["flowchart TD"]
+    for t in order:
+        label = t.name.replace('"', """)
+        lines.append(f'    {_mermaid_node_id(t.name)}["{label}"]')
+    for t in order:
+        for up in t.upstream_tasks():
+            lines.append(f"    {_mermaid_node_id(up.name)} --> {_mermaid_node_id(t.name)}")
+
+    used_classes: list[str] = []
+    for t in order:
+        status = last_status.get(t.name, (None, None))[0]
+        cls = _MERMAID_STATUS_CLASS.get(status)
+        if cls:
+            lines.append(f"    class {_mermaid_node_id(t.name)} {cls}")
+            if cls not in used_classes:
+                used_classes.append(cls)
+    for cls in used_classes:
+        lines.append(f"    {_MERMAID_CLASS_DEFS[cls]}")
+    return "\n".join(lines)
+
+
+def _friendly_errors[F: Callable[..., object]](command: F) -> F:
+    @functools.wraps(command)
+    def wrapper(*args: object, **kwargs: object) -> object:
+        try:
+            return command(*args, **kwargs)
+        except typer.Exit:
+            raise
+        except _USER_FACING_ERRORS as exc:
+            console.print(f"[red]error:[/red] {exc}")
+            raise typer.Exit(code=1) from None
+        except Exception as exc:  # noqa: BLE001 - last-resort CLI-friendly fallback
+            console.print(f"[red]error:[/red] {type(exc).__name__}: {exc}")
+            raise typer.Exit(code=1) from None
+
+    return wrapper
+
+
+@app.command()
+@_friendly_errors
+def run(
+    pipeline: Annotated[Path, typer.Argument(exists=True, help="Path to a pipeline .py module")],
+    db: Annotated[
+        str | None,
+        typer.Option(
+            "--db",
+            help="State file path, or a ducklake:... catalog "
+            "(default: duckpipe.db next to the pipeline)",
+        ),
+    ] = None,
+    state_uri: Annotated[
+        str | None,
+        typer.Option("--state-uri", help="fsspec URI to sync state to/from (S3/GCS/Azure/local)"),
+    ] = None,
+    force: Annotated[
+        bool, typer.Option("--force", help="Ignore fingerprints; re-run every task")
+    ] = False,
+    max_workers: Annotated[
+        int | None, typer.Option("--max-workers", help="Cap concurrent tasks")
+    ] = None,
+    no_lock: Annotated[
+        bool,
+        typer.Option(
+            "--no-lock",
+            help="With --state-uri, skip the advisory lock and allow racing (not recommended)",
+        ),
+    ] = False,
+    only: Annotated[
+        str | None,
+        typer.Option("--only", help="Run just this one task -- for distributed dispatch"),
+    ] = None,
+    run_id: Annotated[
+        str | None,
+        typer.Option("--run-id", help="Shared run id so --only calls for one run group together"),
+    ] = None,
+    data_path: Annotated[
+        str | None,
+        typer.Option(
+            "--data-path", help="DuckLake DATA_PATH (only with --db ducklake:...; usually auto)"
+        ),
+    ] = None,
+) -> None:
+    """Run a pipeline module end-to-end, or (with --only) exactly one task."""
+    summary = run_pipeline(
+        pipeline,
+        db_path=db,
+        state_uri=state_uri,
+        force=force,
+        max_workers=max_workers,
+        lock=not no_lock,
+        only=only,
+        run_id=run_id,
+        data_path=data_path,
+    )
+    table = Table(title=f"run {summary.run_id}  ({summary.db_path})")
+    table.add_column("task")
+    table.add_column("status")
+    table.add_column("detail")
+    for name, status in summary.statuses.items():
+        style = _STATUS_STYLE.get(status, "")
+        rendered = f"[{style}]{status}[/{style}]" if style else status
+        table.add_row(name, rendered, summary.errors.get(name, ""))
+    console.print(table)
+    if not summary.success:
+        raise typer.Exit(code=1)
+
+
+@app.command()
+@_friendly_errors
+def show(
+    pipeline: Annotated[Path, typer.Argument(exists=True, help="Path to a pipeline .py module")],
+    db: Annotated[
+        str | None, typer.Option("--db", help="State file to read last-run status from")
+    ] = None,
+    as_json: Annotated[
+        bool,
+        typer.Option(
+            "--json", help="Machine-readable output -- topological order + next-run preview"
+        ),
+    ] = False,
+    mermaid: Annotated[
+        bool,
+        typer.Option(
+            "--mermaid", help="Print a Mermaid flowchart of the DAG, colored by last status"
+        ),
+    ] = False,
+) -> None:
+    """Print the resolved DAG, each task's last-run status, and what would
+    happen if you ran it again right now -- a dry-run preview of the same
+    fingerprint check `run` itself uses, so you can see what's stale
+    before spending the time to re-run it. `--json` is the discovery
+    primitive a coordinator dispatching `--only <task>` calls needs
+    (DESIGN.md sec 8): topological order plus which tasks would skip.
+    `--mermaid` prints a flowchart instead -- paste it into any Markdown
+    that renders Mermaid."""
+    if as_json and mermaid:
+        raise ValueError("--json and --mermaid are two different output modes -- pick one")
+    dag = build_dag(pipeline)
+    order = dag.topological_order()
+    fingerprints = resolve_fingerprints(order)
+    db_path: str | Path = db or _default_db_path(pipeline)
+
+    last_status: dict[str, tuple[str, str]] = {}
+    next_run: dict[str, str] = {}
+    if _state_probably_exists(db_path):
+        from duckpipe.state import StateStore
+
+        # read_only: `show` never needs to write, and must never block on
+        # (or be blocked by) a pipeline that's still running.
+        with StateStore(db_path, read_only=True) as store:
+            for t in order:
+                row = store.last_status(t.name)
+                if row:
+                    last_status[t.name] = (row[0], str(row[1]))
+                if would_skip(store, t, fingerprints[t.name]):
+                    next_run[t.name] = "skip (unchanged)"
+                elif not t.cache:
+                    next_run[t.name] = "run (not cached)"
+                else:
+                    next_run[t.name] = "run (changed)"
+    else:
+        next_run = {t.name: "run (no prior state)" for t in order}
+
+    if mermaid:
+        print(_mermaid_diagram(order, last_status))
+        return
+
+    if as_json:
+        import json
+
+        print(
+            json.dumps(
+                [
+                    {
+                        "task": t.name,
+                        "depends_on": sorted(u.name for u in t.upstream_tasks()),
+                        "next_run": next_run[t.name],
+                    }
+                    for t in order
+                ]
+            )
+        )
+        return
+
+    table = Table(title=f"DAG: {pipeline}")
+    table.add_column("task")
+    table.add_column("depends on")
+    table.add_column("last status")
+    table.add_column("last run")
+    table.add_column("next run")
+    for t in order:
+        deps = ", ".join(sorted(u.name for u in t.upstream_tasks())) or "-"
+        status, ts = last_status.get(t.name, ("-", "-"))
+        style = _STATUS_STYLE.get(status, "")
+        rendered = f"[{style}]{status}[/{style}]" if style else status
+        next_style = "yellow" if next_run[t.name].startswith("skip") else ""
+        next_rendered = (
+            f"[{next_style}]{next_run[t.name]}[/{next_style}]" if next_style else next_run[t.name]
+        )
+        table.add_row(t.name, deps, rendered, ts, next_rendered)
+    console.print(table)
+
+
+@app.command()
+@_friendly_errors
+def compact(
+    state_uri: Annotated[str, typer.Argument(help="fsspec URI of the state file to compact")],
+    no_lock: Annotated[
+        bool, typer.Option("--no-lock", help="Skip the advisory lock (not recommended)")
+    ] = False,
+) -> None:
+    """Fold pending per-task deltas from `--only` runs into the canonical
+    state file, and clean them up. Nothing needs this for correctness --
+    every invocation already absorbs pending deltas itself -- but a purely
+    distributed workflow (many `--only` workers, no whole-run ever) never
+    otherwise re-uploads the canonical file, so its `.pending/` directory
+    only grows. Run this periodically (e.g. from cron) if that's your
+    workflow."""
+    from duckpipe.remote import compact as compact_state
+
+    compact_state(state_uri, lock=not no_lock)
+    console.print(f"[green]compacted[/green] {state_uri}")
+
+
+@app.command()
+@_friendly_errors
+def stats(
+    db: Annotated[str, typer.Argument(help="Path to a state file, or a ducklake:... catalog")],
+    limit: Annotated[int, typer.Option(help="Number of recent runs to show")] = 20,
+    snapshots: Annotated[
+        bool,
+        typer.Option(
+            "--snapshots", help="Show DuckLake snapshot history (time travel) instead of stats"
+        ),
+    ] = False,
+) -> None:
+    """Show recent pipeline runs and per-task duration stats from the state
+    file. Against a DuckLake-backed store (``--db ducklake:...``), pass
+    ``--snapshots`` to see every commit instead -- the time-travel-over-
+    run-history payoff of that backend (DESIGN.md sec 8, Phase 3b)."""
+    from duckpipe.state import StateStore
+
+    # read_only: never blocks on, or is blocked by, a pipeline still
+    # running against the same state file.
+    with StateStore(db, read_only=True) as store:
+        if snapshots:
+            _print_snapshots(store)
+            return
+
+        runs = store.con.execute(
+            "SELECT run_id, module_path, started_at, ended_at, status "
+            "FROM pipeline_runs ORDER BY started_at DESC LIMIT ?",
+            [limit],
+        ).fetchall()
+        runs_table = Table(title="recent pipeline runs")
+        for col in ("run_id", "module", "started_at", "ended_at", "status"):
+            runs_table.add_column(col)
+        for row in runs:
+            runs_table.add_row(*(str(v) for v in row))
+        console.print(runs_table)
+
+        task_stats = store.con.execute(
+            "SELECT task_name, runs, avg_duration_ms, max_duration_ms, skipped_count, failed_count "
+            "FROM v_task_stats ORDER BY avg_duration_ms DESC"
+        ).fetchall()
+        stats_table = Table(title="per-task stats (from v_task_stats)")
+        for col in ("task", "runs", "avg_ms", "max_ms", "skipped", "failed"):
+            stats_table.add_column(col)
+        for row in task_stats:
+            stats_table.add_row(*(f"{v:.1f}" if isinstance(v, float) else str(v) for v in row))
+        console.print(stats_table)
+
+        if store.is_ducklake:
+            console.print("[dim]DuckLake-backed -- see full history with --snapshots[/dim]")
+
+
+def _print_snapshots(store) -> None:
+    columns = ("snapshot_id", "snapshot_time", "author", "commit_message")
+    table = Table(title="snapshot history (time travel)")
+    for col in columns:
+        table.add_column(col)
+    for snap in store.snapshots():
+        table.add_row(*(str(snap[col]) if snap[col] is not None else "-" for col in columns))
+    console.print(table)
+    console.print(
+        "[dim]query any table AT (VERSION => snapshot_id) to see state as of that point[/dim]"
+    )
+
+
+def main() -> None:
+    app()
+
+
+if __name__ == "__main__":
+    main()

duckpipe/dag.py

@@ -0,0 +1,164 @@
+"""Build a DAG from a pipeline module.
+
+A pipeline is just a Python module (DESIGN.md sec 5); DuckPipe discovers
+its tasks by importing it once and scanning the module namespace for
+``Task`` instances -- module-level attributes directly, and one level into
+list/tuple/set/dict values, which is where the "plain Python loop
+generating uniquely identified task instances" fan-out pattern (sec 4)
+naturally puts its tasks.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+from types import ModuleType
+
+from duckpipe.task import Task
+
+
+class CycleError(RuntimeError):
+    pass
+
+
+class DuplicateTaskNameError(RuntimeError):
+    pass
+
+
+@dataclass
+class DAG:
+    module: ModuleType
+    tasks: dict[str, Task] = field(default_factory=dict)
+
+    def topological_order(self) -> list[Task]:
+        indegree = {name: 0 for name in self.tasks}
+        children: dict[str, list[str]] = {name: [] for name in self.tasks}
+        for t in self.tasks.values():
+            for up in t.upstream_tasks():
+                indegree[t.name] += 1
+                children[up.name].append(t.name)
+
+        ready = sorted(name for name, deg in indegree.items() if deg == 0)
+        order: list[Task] = []
+        while ready:
+            name = ready.pop(0)
+            order.append(self.tasks[name])
+            newly_ready = []
+            for child in children[name]:
+                indegree[child] -= 1
+                if indegree[child] == 0:
+                    newly_ready.append(child)
+            ready = sorted(ready + newly_ready)
+
+        if len(order) != len(self.tasks):
+            remaining = sorted(set(self.tasks) - {t.name for t in order})
+            raise CycleError(f"cycle detected among tasks: {remaining}")
+        return order
+
+    def roots(self) -> list[Task]:
+        return [t for t in self.tasks.values() if not t.upstream_tasks()]
+
+
+def _discover_tasks(module: ModuleType) -> dict[str, Task]:
+    found: dict[str, Task] = {}
+
+    def add(t: Task) -> None:
+        if t.name in found and found[t.name] is not t:
+            raise DuplicateTaskNameError(
+                f"two distinct tasks are both named {t.name!r}; "
+                "give one an explicit name=... to disambiguate"
+            )
+        found[t.name] = t
+
+    for value in vars(module).values():
+        if isinstance(value, Task):
+            add(value)
+        elif isinstance(value, list | tuple | set):
+            for item in value:
+                if isinstance(item, Task):
+                    add(item)
+        elif isinstance(value, dict):
+            for item in value.values():
+                if isinstance(item, Task):
+                    add(item)
+
+    return found
+
+
+def _package_context(path: Path) -> tuple[Path, str] | None:
+    """If ``path`` sits inside a real Python package (an ``__init__.py``
+    beside it, and so on up the tree), return ``(root_parent, dotted_name)``
+    so it can be imported through the normal import system -- the only way
+    a relative import between sibling task files (``from .extract import
+    extract``) resolves correctly. Returns ``None`` for a standalone
+    script (no ``__init__.py`` beside it) -- the common case, and the one
+    ``load_module`` leaves entirely unchanged.
+    """
+    if not (path.parent / "__init__.py").exists():
+        return None
+    parts = [] if path.name == "__init__.py" else [path.stem]
+    current = path.parent
+    while (current / "__init__.py").exists():
+        parts.append(current.name)
+        current = current.parent
+    parts.reverse()
+    return current, ".".join(parts)
+
+
+def load_module(path: str | Path) -> ModuleType:
+    """Import a pipeline file. A plain standalone script (no adjacent
+    ``__init__.py``) always loads fresh, under a unique synthetic name --
+    calling this twice on the same edited-in-place file never serves a
+    stale cached version. A file that's a real package member instead
+    goes through ``importlib.import_module`` so relative imports between
+    sibling task-definition files resolve exactly like any other Python
+    package -- ordinary ``sys.modules`` caching then applies too, the
+    same as importing that package any other way. Splitting a pipeline's
+    tasks across multiple files needs no DuckPipe-specific mechanism
+    either way (DESIGN.md tenet #3): it's just Python composition, with
+    one entrypoint module DuckPipe is pointed at.
+    """
+    path = Path(path).resolve()
+    package = _package_context(path)
+    if package is not None:
+        root_parent, dotted_name = package
+        root_parent_str = str(root_parent)
+        if root_parent_str not in sys.path:
+            sys.path.insert(0, root_parent_str)
+        return importlib.import_module(dotted_name)
+
+    module_name = f"duckpipe_pipeline_{path.stem}_{abs(hash(str(path)))}"
+    spec = importlib.util.spec_from_file_location(module_name, path)
+    if spec is None or spec.loader is None:
+        raise ImportError(f"could not load pipeline module from {path}")
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+def build_dag(source: str | Path | ModuleType) -> DAG:
+    """Import (if needed) a pipeline and resolve its task graph.
+
+    Raises ``CycleError`` immediately if the graph is malformed, so callers
+    never have to discover that mid-run.
+    """
+    module = source if isinstance(source, ModuleType) else load_module(source)
+    tasks = _discover_tasks(module)
+
+    # A task may reference an upstream Task that isn't itself bound to a
+    # module-level name (e.g. constructed inline) -- pull those in too so
+    # topological_order() sees the full graph.
+    frontier = list(tasks.values())
+    while frontier:
+        t = frontier.pop()
+        for up in t.upstream_tasks():
+            if up.name not in tasks:
+                tasks[up.name] = up
+                frontier.append(up)
+
+    dag = DAG(module=module, tasks=tasks)
+    dag.topological_order()  # raises CycleError early if malformed
+    return dag

duckpipe/fingerprint.py

@@ -0,0 +1,68 @@
+"""Content fingerprinting for tasks (DESIGN.md tenet #6, open question #2).
+
+A task's fingerprint hashes together:
+
+- its own function source code
+- its declarative config (retries, cache, cache_backend, depends_on names)
+- the resolved fingerprints of its upstream tasks
+- any values in ``extra_fingerprint`` the user opted to include
+
+It deliberately never hashes a task's *return value* -- fingerprinting is
+data-blind, per tenet #2. One accepted consequence (open question #2,
+resolved for v1): an upstream *external* state change -- a source file
+that changed on disk without any task code changing -- is invisible to
+it. ``extra_fingerprint`` is the documented, opt-in escape hatch: pass
+anything hashable (a file's mtime, a config dict, an API version string)
+that should also invalidate the cache when it changes.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import inspect
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from duckpipe.task import Task
+
+
+def fingerprint_source(func: Callable[..., Any]) -> str:
+    try:
+        source = inspect.getsource(func)
+    except (OSError, TypeError):
+        # Dynamically generated function with no retrievable source (e.g.
+        # built via exec/eval) -- fall back to bytecode identity.
+        source = repr(func.__code__.co_code)
+    return hashlib.sha256(source.encode("utf-8")).hexdigest()
+
+
+def _combine(*parts: str) -> str:
+    h = hashlib.sha256()
+    for p in parts:
+        h.update(p.encode("utf-8"))
+        h.update(b"\0")
+    return h.hexdigest()
+
+
+def fingerprint_task(task: Task, upstream_fingerprints: dict[str, str]) -> str:
+    config_repr = repr(
+        (
+            task.retries,
+            task.cache,
+            task.cache_backend,
+            sorted(t.name for t in task.depends_on),
+            [repr(v) for v in task.extra_fingerprint],
+        )
+    )
+    upstream_repr = [f"{name}={fp}" for name, fp in sorted(upstream_fingerprints.items())]
+    return _combine(task.source_fingerprint, config_repr, *upstream_repr)
+
+
+def resolve_fingerprints(order: list[Task]) -> dict[str, str]:
+    """Compute every task's fingerprint in topological order."""
+    resolved: dict[str, str] = {}
+    for t in order:
+        upstream_fps = {up.name: resolved[up.name] for up in t.upstream_tasks()}
+        resolved[t.name] = fingerprint_task(t, upstream_fps)
+    return resolved