rubedo 0.1.0__py3-none-any.whl

rubedo/__init__.py

@@ -0,0 +1,44 @@
+"""
+Rubedo: A local-first batch processing engine.
+
+This package provides a framework for defining DAG pipelines over collections of 
+coordinates with content-addressed caching, durable run history, and surgical invalidation.
+"""
+from .spec import (
+    step,
+    source,
+    pipeline,
+    describe,
+    PipelineSpec,
+    StepSpec,
+    PipelineBuilder,
+)
+from .models import Filtered, ProcessResult, RunSummary
+from .selection import Selection
+from .sources import Source, SourceItem, FolderSource, CsvSource
+from .invalidation import invalidate
+from .runner import plan, run, RunPlan
+from .progress import TerminalProgress
+
+__all__ = [
+    "invalidate",
+    "Filtered",
+    "ProcessResult",
+    "RunSummary",
+    "Selection",
+    "Source",
+    "SourceItem",
+    "FolderSource",
+    "CsvSource",
+    "step",
+    "source",
+    "pipeline",
+    "describe",
+    "PipelineSpec",
+    "StepSpec",
+    "PipelineBuilder",
+    "run",
+    "plan",
+    "RunPlan",
+    "TerminalProgress",
+]

rubedo/cli.py

@@ -0,0 +1,143 @@
+import argparse
+import json
+import sys
+from rich.console import Console
+from rich.table import Table
+
+from .db import get_session
+from .queries import get_recent_runs, get_run_summary, get_run_failures
+from .invalidation import invalidate
+from .selection import Selection
+
+console = Console()
+
+def cmd_ls(args):
+    with get_session() as session:
+        runs = get_recent_runs(session, limit=args.limit)
+    
+    if not runs:
+        console.print("No runs found.")
+        return
+        
+    table = Table(show_header=True, header_style="bold magenta")
+    table.add_column("ID", style="dim")
+    table.add_column("Pipeline")
+    table.add_column("Status")
+    table.add_column("Created / Reused")
+    table.add_column("Failed")
+    table.add_column("Started At")
+    
+    for r in runs:
+        pipeline = r.pipeline_id or "(none)"
+        stats = f"{r.created_count} / {r.reused_count}"
+        table.add_row(
+            r.id, 
+            pipeline, 
+            r.status, 
+            stats, 
+            str(r.failed_count), 
+            r.started_at
+        )
+    console.print(table)
+
+
+def cmd_show(args):
+    with get_session() as session:
+        run = get_run_summary(session, args.run_id)
+        if not run:
+            console.print(f"[red]Run {args.run_id} not found.[/red]")
+            sys.exit(1)
+            
+        if args.failed:
+            failures = get_run_failures(session, args.run_id)
+            if args.json:
+                print(json.dumps(failures, indent=2))
+                return
+            
+            if not failures:
+                console.print("No failures recorded for this run.")
+                return
+                
+            table = Table(title=f"Failures for Run {args.run_id}", show_header=True)
+            table.add_column("Step")
+            table.add_column("Coordinate")
+            table.add_column("Error Type")
+            table.add_column("Message")
+            
+            for f in failures:
+                table.add_row(
+                    str(f["step_name"]), 
+                    str(f["coordinate"]), 
+                    str(f["error_type"]), 
+                    str(f["error_message"])
+                )
+            console.print(table)
+            return
+
+    if args.json:
+        print(run.model_dump_json(indent=2))
+        return
+        
+    console.print(f"[bold]Run ID:[/bold] {run.id}")
+    console.print(f"[bold]Pipeline:[/bold] {run.pipeline_id or '(none)'}")
+    console.print(f"[bold]Status:[/bold] {run.status}")
+    console.print(f"[bold]Started At:[/bold] {run.started_at}")
+    console.print(f"[bold]Finished At:[/bold] {run.finished_at or 'N/A'}")
+    console.print(f"[bold]Summary:[/bold] Created: {run.created_count}, Reused: {run.reused_count}, Failed: {run.failed_count}, Blocked: {run.blocked_count}, Filtered: {run.filtered_count}")
+    
+    if run.by_step:
+        table = Table(title="Step Outcomes", show_header=True)
+        table.add_column("Step")
+        table.add_column("Created")
+        table.add_column("Reused")
+        table.add_column("Failed")
+        table.add_column("Blocked")
+        table.add_column("Filtered")
+        for step_name, counts in run.by_step.items():
+            table.add_row(
+                step_name,
+                str(counts.get("created", 0)),
+                str(counts.get("reused", 0)),
+                str(counts.get("failed", 0)),
+                str(counts.get("blocked", 0)),
+                str(counts.get("filtered", 0))
+            )
+        console.print(table)
+
+
+def cmd_invalidate(args):
+    try:
+        selection = Selection.parse(args.selection)
+    except Exception as e:
+        console.print(f"[red]Error parsing selection:[/red] {e}")
+        sys.exit(1)
+        
+    result = invalidate(selection, args.reason)
+    console.print(f"Invalidated [bold green]{result['invalidated_count']}[/bold green] materializations.")
+    console.print(f"New Run ID recorded for invalidation: [cyan]{result['run_id']}[/cyan]")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Rubedo Read-Only Ops CLI")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+    
+    parser_ls = subparsers.add_parser("ls", help="List recent runs")
+    parser_ls.add_argument("--limit", type=int, default=50, help="Number of runs to show")
+    parser_ls.set_defaults(func=cmd_ls)
+    
+    parser_show = subparsers.add_parser("show", help="Show details for a specific run")
+    parser_show.add_argument("run_id", help="The Run ID to show")
+    parser_show.add_argument("--json", action="store_true", help="Output as JSON")
+    parser_show.add_argument("--failed", action="store_true", help="Show failure details")
+    parser_show.set_defaults(func=cmd_show)
+    
+    parser_inv = subparsers.add_parser("invalidate", help="Invalidate materializations by selection query")
+    parser_inv.add_argument("selection", help="Selection query (e.g., 'pipeline:my-pipe step:extract')")
+    parser_inv.add_argument("--reason", required=True, help="Reason for invalidation")
+    parser_inv.set_defaults(func=cmd_invalidate)
+    
+    args = parser.parse_args()
+    args.func(args)
+
+if __name__ == "__main__":
+    main()

rubedo/db.py

@@ -0,0 +1,76 @@
+"""
+Database initialization and session management.
+
+Provides utilities for setting up the SQLite database, ensuring the directory
+is gitignored, configuring WAL mode for concurrency, and providing sessions.
+"""
+import os
+from .util import _ensure_gitignore
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker, Session
+from typing import Optional, Any
+from .models import Base
+
+engine: Any = None
+SessionLocal: Any = None
+
+
+def init_db(db_path: Optional[str] = None):
+    """
+    Initialize the database engine and create tables.
+
+    Args:
+        db_path (str, optional): The database URL or file path. If None, uses
+            RUBEDO_DB_PATH, or RUBEDO_HOME/rubedo.sqlite, or the default
+            '.rubedo/rubedo.sqlite' — in that precedence order.
+    """
+    global engine, SessionLocal
+    if engine is not None:
+        try:
+            engine.dispose()
+        except Exception:
+            pass
+
+    if db_path is None:
+        db_path = os.environ.get("RUBEDO_DB_PATH") or os.path.join(
+            os.environ.get("RUBEDO_HOME", ".rubedo"), "rubedo.sqlite"
+        )
+    dir_path = (
+        db_path.replace("sqlite:///", "")
+        if db_path.startswith("sqlite:///")
+        else db_path
+    )
+
+    db_dir = os.path.dirname(dir_path)
+    if db_dir:
+        os.makedirs(db_dir, exist_ok=True)
+        _ensure_gitignore(db_dir)
+    if db_path.startswith("sqlite:///"):
+        engine_url = db_path
+    else:
+        engine_url = f"sqlite:///{db_path}"
+    engine = create_engine(engine_url)
+    
+    if engine_url.startswith("sqlite"):
+        from sqlalchemy import event
+        @event.listens_for(engine, "connect")
+        def set_sqlite_pragma(dbapi_connection, connection_record):
+            cursor = dbapi_connection.cursor()
+            cursor.execute("PRAGMA journal_mode=WAL")
+            cursor.execute("PRAGMA busy_timeout=5000")
+            cursor.close()
+            
+    Base.metadata.create_all(bind=engine)
+    SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+
+
+def get_session() -> Session:
+    """
+    Get a new SQLAlchemy session, initializing the DB if necessary.
+
+    Returns:
+        Session: A new database session.
+    """
+    if SessionLocal is None:
+        init_db()
+    return SessionLocal()

rubedo/execution.py

@@ -0,0 +1,397 @@
+"""Execute phase: running step functions.
+
+No database access — inputs come in as refs, results go out as
+ExecutionOutcome values for the ledger to persist.
+"""
+
+import concurrent.futures
+import threading
+import time
+import traceback
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple
+
+import loky
+
+
+from .hashing import hash_json, hash_bytes
+from .models import Filtered, ProcessResult
+from .planning import (
+    EphemeralRef,
+    MatRef,
+    StepDecision,
+    _build_step_params,
+    _step_accepts_params,
+    expand_anchor_address,
+    expand_child_coord,
+    expand_child_identity,
+)
+from .spec import StepSpec
+from .sources import Source
+from .store import read_materialization_output
+
+
+class _RateLimiter:
+    """Paces calls evenly across a step's worker threads."""
+
+    def __init__(self, count: int, period_seconds: float):
+        """
+        Initialize the rate limiter.
+
+        Args:
+            count (int): Number of allowed calls per period.
+            period_seconds (float): Time period in seconds.
+        """
+        self.min_interval = period_seconds / count
+        self._lock = threading.Lock()
+        self._next_free = 0.0
+
+    def acquire(self):
+        """Wait until it is safe to proceed according to the rate limit."""
+        with self._lock:
+            now = time.monotonic()
+            wait = self._next_free - now
+            self._next_free = max(now, self._next_free) + self.min_interval
+        if wait > 0:
+            time.sleep(wait)
+
+
+class _RunMemo:
+    """Per-run memo so an ephemeral step runs at most once per coordinate.
+
+    Reentrant lock: chained skip_cache steps resolve recursively on the
+    same worker thread. Exceptions are memoized too, so every consumer of
+    a failed util sees the same failure.
+    """
+
+    def __init__(self):
+        """Initialize the run memoizer with a per-key locking scheme."""
+        self._lock = threading.Lock()
+        self._values: Dict[Tuple[str, str], Tuple[str, Any]] = {}
+
+    def compute(self, key: Tuple[str, str], producer: Callable[[], Any]) -> Any:
+        """
+        Compute or retrieve a memoized value for the given key.
+
+        Args:
+            key: (step_name, coordinate) — see _compute_ephemeral's call site.
+            producer: A zero-argument function that produces the value.
+
+        Returns:
+            Any: The produced or cached value.
+        """
+        with self._lock:
+            state = self._values.get(key)
+            if state is None:
+                event = threading.Event()
+                self._values[key] = ("computing", event)
+            elif state[0] == "done":
+                kind, value = state[1]
+                if kind == "err":
+                    raise value
+                return value
+            else:
+                event = state[1]
+
+        if state is not None:
+            event.wait()
+            kind, value = self._values[key][1]
+            if kind == "err":
+                raise value
+            return value
+
+        try:
+            val = producer()
+            res = ("ok", val)
+        except Exception as e:
+            res = ("err", e)
+
+        with self._lock:
+            self._values[key] = ("done", res)
+            event.set()
+
+        kind, value = res
+        if kind == "err":
+            raise value
+        return value
+
+
+@dataclass
+class ExecutionOutcome:
+    """Represents the final result of attempting to execute a step for a coordinate."""
+    decision: StepDecision
+    success: bool
+    result: Any = None
+    error_trace: Optional[str] = None
+    attempts: int = 1
+    attempt_errors: List[str] = field(default_factory=list)
+    # An expand step's cache anchor (the full yielded list, addressed by the
+    # parent): stored so a re-run can skip the fn, but it is not a lane — no
+    # status, count, edge, or coord_step_mats entry.
+    is_anchor: bool = False
+
+
+def _resolve_parent_value(
+    ref, step_sources: Dict[str, Source], params: Optional[dict], memo: _RunMemo
+):
+    """
+    Resolve the output value of a parent step, computing it lazily if ephemeral.
+
+    Args:
+        ref: The reference to the parent output (MatRef or EphemeralRef).
+        step_sources: step name -> the Source it reads (root/skip_cache roots).
+        params (Optional[dict]): Run parameters.
+        memo (_RunMemo): The run memoizer for ephemeral steps.
+
+    Returns:
+        Any: The resolved output value.
+    """
+    if isinstance(ref, EphemeralRef):
+        return _compute_ephemeral(ref, step_sources, params, memo)
+    return read_materialization_output(ref)
+
+
+def _compute_ephemeral(
+    ref: EphemeralRef, step_sources: Dict[str, Source], params: Optional[dict],
+    memo: _RunMemo,
+):
+    """Lazily compute a skip_cache step's value, at most once per run."""
+
+    def produce():
+        step = ref.step
+        if not step.depends_on:
+            args = [step_sources[step.name].load(ref.item)]
+            kwargs = {}
+        else:
+            args = []
+            kwargs = {
+                dep: _resolve_parent_value(
+                    ref.parent_refs[dep], step_sources, params, memo
+                )
+                for dep in step.depends_on
+            }
+        if _step_accepts_params(step):
+            kwargs["params"] = _build_step_params(step, params)
+        result = step.fn(*args, **kwargs)
+        if isinstance(result, Filtered):
+            raise RuntimeError(
+                f"skip_cache step '{step.name}' returned Filtered: filtering "
+                "is a cacheable decision, so filter steps must be materialized"
+            )
+        # Consumers of materialized steps receive the unwrapped value;
+        # keep the contract identical (minus the serialization round-trip)
+        if isinstance(result, ProcessResult):
+            result = result.value
+        return result
+
+    return memo.compute((ref.step.name, ref.item.coordinate), produce)
+
+
+def _materialized_ancestors(parent_refs: Dict[str, Any]) -> Dict[int, MatRef]:
+    """Nearest materialized ancestors, skipping through ephemeral hops."""
+    out: Dict[int, MatRef] = {}
+    for ref in parent_refs.values():
+        if isinstance(ref, EphemeralRef):
+            out.update(_materialized_ancestors(ref.parent_refs))
+        else:
+            out[ref.id] = ref
+    return out
+
+
+def _validate_output(step: StepSpec, value: Any) -> None:
+    """Run data quality assertions on a step's output."""
+    if step.output_model is not None:
+        step.output_model.model_validate(value)
+    if step.assertions:
+        for assertion in step.assertions:
+            assertion(value)
+
+
+def _execute_step(
+    step: StepSpec,
+    decisions: List[StepDecision],
+    step_sources: Dict[str, Source],
+    params: Optional[dict],
+    accepts_params: bool,
+    workers: Optional[int],
+    memo: _RunMemo,
+) -> Iterator[ExecutionOutcome]:
+    """Run the step function for each execute decision; yield as completed.
+
+    Honors the step's rate limit (shared across workers, retries included)
+    and retry policy (only exceptions matching retry_on are retried).
+    `step_sources` maps a step name to the Source it reads (root/skip_cache
+    roots), so each root loads from its own source in a multi-source pipeline.
+    """
+    limiter = _RateLimiter(*step.rate_limit) if step.rate_limit else None
+    params_hash = hash_json(params or {})
+
+    def call(decision: StepDecision, pool: Optional[Any] = None):
+        # Root steps get the source payload positionally; dependent steps
+        # get parent outputs by parameter name. Either kind may declare
+        # `params`. A *root expand* is itself a source — it reads no payload.
+        if not step.depends_on:
+            if step.shape == "expand":
+                args = []
+            else:
+                assert decision.item is not None
+                args = [step_sources[step.name].load(decision.item)]
+            kwargs = {}
+        elif step.shape == "reduce":
+            args = []
+            kwargs = {
+                dep: {
+                    lane: _resolve_parent_value(
+                        ref, step_sources, params, memo
+                    )
+                    for lane, ref in decision.parent_mats[dep].items()
+                }
+                for dep in step.depends_on
+            }
+        else:
+            args = []
+            kwargs = {
+                dep: _resolve_parent_value(
+                    decision.parent_mats[dep], step_sources, params, memo
+                )
+                for dep in step.depends_on
+            }
+        if accepts_params:
+            kwargs["params"] = _build_step_params(step, params)
+            
+        if pool is not None:
+            return pool.submit(step.fn, *args, **kwargs).result()
+        return step.fn(*args, **kwargs)
+
+    def _expand_outcomes(
+        decision: StepDecision, values: List[Any], attempt: int, attempt_errors: List[str]
+    ) -> List[ExecutionOutcome]:
+        """Fan one parent lane's yielded payloads into content-addressed lanes.
+
+        A dependent expand emits the cache anchor first (the child content
+        hashes, addressed by the parent so a re-run can skip the fn). A *root*
+        expand (a source) has no parent to cache against, so it writes no
+        anchor and always re-runs. Then one child per distinct payload — each a
+        content-addressed lane `row-<hash>`; identical payloads collapse.
+        """
+        seen: set = set()
+        children: List[tuple] = []  # (child_hash, value)
+        for value in values:
+            _validate_output(step, value)
+            if isinstance(value, bytes):
+                child_hash = "b:" + hash_bytes(value)
+            else:
+                child_hash = hash_json(value)
+            if child_hash in seen:
+                continue  # identical payload — one lane
+            seen.add(child_hash)
+            children.append((child_hash, value))
+
+        outcomes: List[ExecutionOutcome] = []
+        if step.depends_on:
+            # Anchor: the child hashes, addressed by the parent. Not a lane.
+            parent_hash = decision.parent_mats[step.depends_on[0]].output_content_hash
+            anchor = StepDecision(
+                coordinate=decision.coordinate,
+                action="execute",
+                input_hash=parent_hash,
+                output_address=expand_anchor_address(
+                    step, parent_hash, params_hash, accepts_params
+                ),
+                parent_mats=decision.parent_mats,
+            )
+            outcomes.append(
+                ExecutionOutcome(
+                    anchor, True, result=[h for h, _ in children],
+                    attempts=attempt, attempt_errors=attempt_errors, is_anchor=True,
+                )
+            )
+
+        for child_hash, value in children:
+            input_hash, child_address = expand_child_identity(
+                step, child_hash, params_hash, accepts_params
+            )
+            child = StepDecision(
+                coordinate=expand_child_coord(child_hash),
+                action="execute",
+                input_hash=input_hash,
+                output_address=child_address,
+                parent_mats=decision.parent_mats,
+            )
+            outcomes.append(
+                ExecutionOutcome(
+                    child, True, result=value, attempts=attempt,
+                    attempt_errors=attempt_errors,
+                )
+            )
+        return outcomes
+
+    def process(  # type: ignore
+        decision: StepDecision, pool: Optional[Any] = None
+    ) -> List[ExecutionOutcome]:
+        attempt_errors: List[str] = []
+        delay = step.retry_delay
+        for attempt in range(1, step.retries + 2):
+            if limiter:
+                limiter.acquire()
+            try:
+                result = call(decision, pool)
+                if step.shape == "expand":
+                    # Consume the iterable inside the try so a failing
+                    # generator body is caught and retried like any other.
+                    return _expand_outcomes(
+                        decision, list(result), attempt, attempt_errors
+                    )
+                _validate_output(step, result)
+                return [
+                    ExecutionOutcome(
+                        decision,
+                        True,
+                        result=result,
+                        attempts=attempt,
+                        attempt_errors=attempt_errors,
+                    )
+                ]
+            except Exception as e:
+                trace = traceback.format_exc()
+                retryable = attempt <= step.retries and isinstance(e, step.retry_on)
+                if not retryable:
+                    return [
+                        ExecutionOutcome(
+                            decision,
+                            False,
+                            error_trace=trace,
+                            attempts=attempt,
+                            attempt_errors=attempt_errors,
+                        )
+                    ]
+                attempt_errors.append(trace)
+                if delay > 0:
+                    time.sleep(delay)
+                delay *= step.retry_backoff
+
+
+
+    # Never spin up more workers (or subprocesses) than there is work.
+    pool_size = min(workers or step.workers, len(decisions))
+    if pool_size < 1:
+        return
+
+    # Two layers, on purpose. The thread pool is the orchestrator: it drives
+    # the retry loop and the parent-shared rate limiter for every lane. A
+    # process pool, when requested, is only where the CPU-bound step body
+    # runs — retries and rate limiting must stay in the parent, so process
+    # steps still need the thread layer to feed them.
+    process_pool = (
+        loky.ProcessPoolExecutor(max_workers=pool_size)
+        if step.executor == "process"
+        else None
+    )
+    try:
+        with concurrent.futures.ThreadPoolExecutor(max_workers=pool_size) as executor:
+            futures = [executor.submit(process, d, process_pool) for d in decisions]
+            for future in concurrent.futures.as_completed(futures):
+                yield from future.result()
+    finally:
+        if process_pool is not None:
+            process_pool.shutdown()