pipeloom 0.0.1__py3-none-any.whl

pipeloom/__init__.py

@@ -0,0 +1,75 @@
+"""
+pipeloom
+=======
+
+Reusable scaffolding for ETL-style, multi-threaded pipelines with:
+
+- One SQLite writer thread in WAL mode
+- Workers publishing progress/results over a Queue
+- Rich progress (sticky overall + transient per-task)
+- Typer CLI
+
+Top-level exports are provided for convenience so you can do:
+
+    from pipeloom import (
+        run_pipeline,
+        SQLiteWriter,
+        TaskDef,
+        MsgTaskStarted,  # or alias: MsgTaskStart
+        MsgTaskProgress,
+        MsgTaskFinished,
+        SENTINEL,
+        make_overall_progress,
+        make_task_progress,
+        preregister_task_bars,
+        console,
+        logger
+    )
+"""
+
+from .engine import run_pipeline
+from .messages import (
+    SENTINEL,
+    MsgTaskFinished,
+    MsgTaskProgress,
+    MsgTaskStarted,
+    TaskDef,
+)
+from .progress import (
+    make_overall_progress,
+    make_task_progress,
+    preregister_task_bars,
+)
+from .rlog import (
+    console,
+    logger,
+    setup_logging,
+)  # shared Console and logger, useful for advanced usage
+from .writer import SQLiteWriter
+
+# Ergonomic alias to match common naming used by callers
+MsgTaskStart = MsgTaskStarted
+
+__all__ = [
+    # Core orchestration
+    "run_pipeline",
+    # Writer
+    "SQLiteWriter",
+    # Messages
+    "TaskDef",
+    "MsgTaskStarted",
+    "MsgTaskStart",  # alias
+    "MsgTaskProgress",
+    "MsgTaskFinished",
+    "SENTINEL",
+    # Progress helpers
+    "make_overall_progress",
+    "make_task_progress",
+    "preregister_task_bars",
+    # Advanced: shared Console
+    "console",
+    "logger",
+    "setup_logging",
+]
+
+__version__ = "0.0.1"

pipeloom/cli.py

@@ -0,0 +1,73 @@
+"""
+cli.py
+======
+
+Typer-powered CLI that exposes a simple `pipeloom demo` command and serves as
+a blueprint for wiring your own task definitions and worker functions.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from pathlib import Path
+
+import typer
+
+from .engine import run_pipeline
+from .messages import TaskDef
+from .rlog import setup_logging
+
+LOG = logging.getLogger("pipeloom")
+
+app = typer.Typer(
+    help="Threaded ETL scaffolding with a single-writer SQLite (WAL).",
+    add_completion=False,
+    rich_markup_mode="rich",
+)
+
+
+@app.command(name="demo")
+def demo(
+    db: Path = typer.Option(Path("./wal_demo.db"), help="SQLite DB path."),
+    wal: bool = typer.Option(True, help="Use SQLite WAL mode."),
+    num_tasks: int = typer.Option(20, min=1, help="How many demo tasks to run."),
+    workers: int = typer.Option(
+        os.cpu_count() or 4, min=1, help="Max concurrent worker threads."
+    ),
+    verbose: int = typer.Option(
+        1, "--verbose", "-v", count=True, help="Increase log verbosity (-v, -vv)."
+    ),
+    log_file: Path | None = typer.Option(None, help="Optional log file path."),
+    store_task_status: bool = typer.Option(
+        True,
+        help="Persist task status to SQLite (disable to only use DB for domain data).",
+    ),
+) -> None:
+    """
+    Run a demonstration pipeline that exercises the entire stack:
+
+    - Worker threads emit progress and results to a Queue
+    - A single writer thread updates SQLite in WAL mode
+    - Rich renders per-task and overall progress cleanly
+    """
+    setup_logging(verbose, log_file)
+    LOG.info(
+        "Starting demo (tasks=%s, workers=%s, WAL=%s) → DB: %s",
+        num_tasks,
+        workers,
+        wal,
+        db,
+    )
+
+    tasks = [
+        TaskDef(task_id=i, name=f"task-{i}", steps=20) for i in range(1, num_tasks + 1)
+    ]
+
+    run_pipeline(
+        db_path=db,
+        tasks=tasks,
+        workers=workers,
+        wal=wal,
+        store_task_status=store_task_status,
+    )

pipeloom/db.py

@@ -0,0 +1,94 @@
+"""
+db.py
+=====
+
+Low-level, thread-bound SQLite helpers (connection + schema + checkpoint).
+
+Design rules:
+-------------
+- Create **one** connection in the writer thread; never share it across threads.
+- Prefer WAL for on-disk DBs to allow concurrent readers during writes.
+- Keep pragmas pragmatic: good performance without wrecking durability.
+"""
+
+from __future__ import annotations
+
+import sqlite3
+from pathlib import Path
+from typing import Literal
+
+
+def connect(db_path: Path, *, wal: bool = True) -> sqlite3.Connection:
+    """
+    Create and configure a **thread-bound** SQLite connection.
+
+    Must be called from the writer thread. The connection is not safe to pass
+    to worker threads.
+
+    Parameters
+    ----------
+    db_path
+        Path to SQLite database (may be :memory:).
+    wal
+        Enable WAL mode for file-backed databases.
+
+    Returns
+    -------
+    sqlite3.Connection
+    """
+    conn = sqlite3.connect(db_path, timeout=60, check_same_thread=True)
+
+    # Journal mode:
+    if db_path == Path(":memory:"):
+        conn.execute("PRAGMA journal_mode=MEMORY")
+    else:
+        conn.execute(f"PRAGMA journal_mode={'WAL' if wal else 'DELETE'}")
+
+    # Pragmas: balanced defaults suitable for ETL-like workloads.
+    conn.execute("PRAGMA synchronous=NORMAL")
+    conn.execute("PRAGMA foreign_keys=ON")
+    conn.execute("PRAGMA temp_store=MEMORY")
+    conn.execute("PRAGMA cache_size=-100000")  # ~100MB page cache (negative means KB)
+    conn.execute("PRAGMA wal_autocheckpoint=1000")  # periodic auto-checkpoints
+
+    return conn
+
+
+def init_schema(conn: sqlite3.Connection, *, store_task_status: bool) -> None:
+    """
+    Create optional tables used for observability (task_runs).
+
+    If you set `store_task_status=False`, this becomes a no-op and the DB is
+    entirely yours for domain tables (e.g., ETL outputs).
+    """
+    if not store_task_status:
+        return
+    conn.executescript(
+        """
+        CREATE TABLE IF NOT EXISTS task_runs (
+          id INTEGER PRIMARY KEY,
+          name TEXT NOT NULL,
+          status TEXT NOT NULL,          -- queued|running|done|error|cancelled
+          progress REAL NOT NULL DEFAULT 0.0,  -- fraction 0..1
+          message TEXT,
+          started_at TEXT,
+          finished_at TEXT,
+          result TEXT
+        );
+        CREATE INDEX IF NOT EXISTS idx_task_runs_status ON task_runs(status);
+        """
+    )
+    conn.commit()
+
+
+def wal_checkpoint(
+    conn: sqlite3.Connection,
+    mode: Literal["PASSIVE", "FULL", "RESTART", "TRUNCATE"] = "TRUNCATE",
+) -> None:
+    """
+    Manually checkpoint the WAL into the main DB and (optionally) truncate -wal.
+
+    Useful at shutdown or between phases to keep the main file tidy.
+    """
+    conn.commit()
+    conn.execute(f"PRAGMA wal_checkpoint({mode});").fetchone()

pipeloom/engine.py

@@ -0,0 +1,175 @@
+"""
+engine.py
+=========
+
+Pipeline orchestration:
+
+- Builds two Rich progress managers (overall sticky / per-task transient).
+- Pre-registers per-task bars to avoid UI races.
+- Starts the SQLite writer thread.
+- Submits work to a ThreadPoolExecutor.
+- Signals the writer with SENTINEL and joins while the transient progress
+  is still live, so bars vanish before the final frame is rendered.
+"""
+
+from __future__ import annotations
+
+import logging
+import queue
+import random
+import signal
+import threading
+import time
+from collections.abc import Callable, Iterable
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import UTC, datetime
+from pathlib import Path
+
+from .messages import (
+    SENTINEL,
+    MsgTaskFinished,
+    MsgTaskProgress,
+    MsgTaskStarted,
+    TaskDef,
+)
+from .progress import make_overall_progress, make_task_progress, preregister_task_bars
+from .writer import SQLiteWriter
+
+LOG = logging.getLogger("pipeloom.engine")
+
+
+def default_worker(task: TaskDef, msg_q: queue.Queue[object]) -> None:
+    """
+    Demo worker function.
+    Replace this in your projects with real work (ETL steps, API calls, etc.).
+    """
+    started = datetime.now(UTC).isoformat()
+    msg_q.put(MsgTaskStarted(task_id=task.task_id, name=task.name, started_at=started))
+    try:
+        for step in range(1, task.steps + 1):
+            time.sleep(0.05 + random.random() * 0.05)  # simulate variable work
+            msg_q.put(
+                MsgTaskProgress(
+                    task_id=task.task_id,
+                    step=step,
+                    total=task.steps,
+                    message=f"step {step}/{task.steps}",
+                )
+            )
+        finished = datetime.now(UTC).isoformat()
+        msg_q.put(
+            MsgTaskFinished(
+                task_id=task.task_id,
+                status="done",
+                finished_at=finished,
+                result=f"ok:{task.name}",
+                message="completed",
+            )
+        )
+    except Exception as e:
+        finished = datetime.now(UTC).isoformat()
+        msg_q.put(
+            MsgTaskFinished(
+                task_id=task.task_id,
+                status="error",
+                finished_at=finished,
+                result=None,
+                message=str(e),
+            )
+        )
+
+
+def run_pipeline(
+    db_path: Path,
+    tasks: Iterable[TaskDef],
+    *,
+    workers: int,
+    wal: bool = True,
+    store_task_status: bool = True,
+    worker_fn: Callable[[TaskDef, queue.Queue[object]], None] = default_worker,
+) -> None:
+    """
+    Execute an ETL-style workload using a single-writer SQLite backend.
+
+    Parameters
+    ----------
+    db_path
+        Target SQLite database path.
+    tasks
+        Iterable of TaskDef instances (one per unit of work).
+    workers
+        Maximum number of concurrent worker threads.
+    wal
+        Enable WAL mode for the writer (recommended for file-backed DBs).
+    store_task_status
+        If True, create and maintain the `task_runs` table for observability.
+    worker_fn
+        Callable invoked for each TaskDef (signature: (task, queue) -> None).
+
+    Notes
+    -----
+    The writer thread is shut down *before* the per-task progress context exits,
+    guaranteeing that all bars are removed and the final frame shows only the
+    “All tasks 100%” line.
+    """
+    all_tasks = list(tasks)
+    n = len(all_tasks)
+
+    # Workers → Queue → Writer
+    msg_q: queue.Queue[object] = queue.Queue(maxsize=max(64, workers * 8))
+
+    # Gentle shutdown: we don't hard-kill threads; we let in-flight tasks finish.
+    stop_event = threading.Event()
+
+    def handle_sigint(signum, frame):  # type: ignore[override]
+        LOG.warning("SIGINT received; finishing in-flight tasks, then exiting…")
+        stop_event.set()
+
+    signal.signal(signal.SIGINT, handle_sigint)
+
+    # Build progress managers that share the same Console (see rlog.py).
+    overall_p = make_overall_progress()
+    task_p = make_task_progress()
+
+    with overall_p:
+        overall = overall_p.add_task("[cyan]All tasks", total=n)
+        with task_p:
+            # Pre-register bars to avoid creation races.
+            bar_map = preregister_task_bars(task_p, n)
+
+            # Start the writer *after* bars are created, passing the mapping.
+            writer = SQLiteWriter(
+                db_path=db_path,
+                msg_q=msg_q,
+                wal=wal,
+                store_task_status=store_task_status,
+                task_progress=task_p,
+                task_bar_map=bar_map,
+            )
+            writer.start()
+
+            start = time.time()
+
+            # Submit work to a thread pool.
+            with ThreadPoolExecutor(
+                max_workers=workers, thread_name_prefix="worker"
+            ) as ex:
+                futures = [ex.submit(worker_fn, t, msg_q) for t in all_tasks]
+                try:
+                    for fut in as_completed(futures):
+                        _ = fut.result()  # surface exceptions
+                        overall_p.advance(
+                            overall, 1
+                        )  # advance only when a task finishes
+                        if stop_event.is_set():
+                            break
+                except KeyboardInterrupt:
+                    LOG.warning("KeyboardInterrupt; letting in-flight tasks finish…")
+
+            # Shut down writer while the transient progress is still alive
+            # so the last bar removal appears before the final frame.
+            msg_q.put(SENTINEL)
+            writer.join(timeout=30)
+            task_p.refresh()
+
+    LOG.info("Elapsed: %.2fs", time.time() - start)

pipeloom/messages.py

@@ -0,0 +1,82 @@
+"""
+messages.py
+===========
+
+Typed message contracts used on the in-memory **Queue** between worker threads
+and the single SQLite writer thread.
+
+Why messages?
+-------------
+Workers should **never** touch SQLite directly (SQLite connections are not
+thread-safe across threads). Instead, workers publish *intent* via small,
+immutable dataclasses. The writer is the only component that owns a SQLite
+connection and it executes the requested changes in a single, serialized place.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class TaskDef:
+    """
+    Static definition of a unit of work.
+
+    Attributes
+    ----------
+    task_id:
+        Stable numeric identifier for the task. Used as the primary key in
+        SQLite and as the key for progress bar lookups.
+    name:
+        Display-friendly name shown in logs and progress UI.
+    steps:
+        Number of progress steps the *demo* worker will simulate.
+        In real code, you can ignore this and emit MsgTaskProgress at your own cadence.
+    """
+
+    task_id: int
+    name: str
+    steps: int = 20
+
+
+@dataclass(frozen=True)
+class MsgTaskStarted:
+    """
+    Signal that a task has begun.
+    Posted by a worker thread as soon as the task is admitted for work.
+    """
+
+    task_id: int
+    name: str
+    started_at: str  # ISO 8601 UTC string (avoid tz-naive datetimes over queues)
+
+
+@dataclass(frozen=True)
+class MsgTaskProgress:
+    """
+    Incremental progress signal.
+    The writer translates this into a fractional progress column and updates the Rich bar.
+    """
+
+    task_id: int
+    step: int
+    total: int
+    message: str = ""
+
+
+@dataclass(frozen=True)
+class MsgTaskFinished:
+    """
+    Final status + optional result payload for a completed or failed task.
+    """
+
+    task_id: int
+    status: str  # "done" | "error" | "cancelled"
+    finished_at: str  # ISO 8601 UTC string
+    result: str | None = None
+    message: str = ""
+
+
+# Special object placed on the queue to request the writer to shut down cleanly.
+SENTINEL: object = object()

pipeloom/progress.py

@@ -0,0 +1,83 @@
+"""
+progress.py
+===========
+
+Factory helpers for Rich progress managers.
+
+We intentionally separate **overall** progress (sticky, remains visible at 100%)
+from **per-task** progress (transient, disappears once all tasks complete).
+Both share the same Console (see rlog.console).
+"""
+
+from __future__ import annotations
+
+from rich.progress import (
+    BarColumn,
+    MofNCompleteColumn,
+    Progress,
+    SpinnerColumn,
+    TaskID,
+    TaskProgressColumn,
+    TimeElapsedColumn,
+    TimeRemainingColumn,
+)
+
+from .rlog import console
+
+
+def make_overall_progress() -> Progress:
+    """
+    Create the overall persistent progress manager.
+
+    This remains visible after completion and is ideal for a single “All tasks: 100%”
+    summary line once per-task bars disappear.
+    """
+    return Progress(
+        SpinnerColumn(),
+        "[progress.description]{task.description}",
+        BarColumn(),
+        TaskProgressColumn(),
+        MofNCompleteColumn(),
+        "•",
+        TimeElapsedColumn(),
+        "•",
+        TimeRemainingColumn(),
+        transient=False,  # keep final frame visible
+        console=console,
+        refresh_per_second=8,  # throttle repaints to avoid log spam jitter
+        disable=not console.is_terminal,
+    )
+
+
+def make_task_progress() -> Progress:
+    """
+    Create the transient per-task progress manager.
+
+    This hides when the context exits so your terminal remains tidy.
+    """
+    return Progress(
+        SpinnerColumn(),
+        "[progress.description]{task.description}",
+        BarColumn(),
+        TaskProgressColumn(),
+        TimeElapsedColumn(),
+        transient=True,  # auto-hide on exit
+        console=console,
+        refresh_per_second=8,
+        disable=not console.is_terminal,
+    )
+
+
+def preregister_task_bars(task_progress: Progress, num_tasks: int) -> dict[int, TaskID]:
+    """
+    Pre-create one Rich bar per task and return a mapping {task_id -> TaskID}.
+
+    Why up-front?
+    -------------
+    If a worker publishes a MsgTaskProgress before the writer has created the bar,
+    updates can be lost. Pre-registering eliminates this race completely.
+    """
+    mapping: dict[int, TaskID] = {}
+    for i in range(1, num_tasks + 1):
+        mapping[i] = task_progress.add_task(f"[bold]task-{i}", total=100)
+    return mapping

pipeloom/rlog.py

@@ -0,0 +1,64 @@
+"""
+rlog.py
+=======
+
+Rich-aware logging. We create **one Console** and share it between RichHandler
+(logging) and Rich Progress (live bars). Without a shared console, Rich would
+render duplicate live regions and your terminal would flicker or show
+repeated “All tasks” bars.
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+from rich.console import Console
+from rich.logging import RichHandler
+
+# Single shared console for logs and progress.
+console = Console(log_path=False)
+
+logger = logging.getLogger("pipeloom")
+
+
+def setup_logging(verbose: int, log_file: Path | None = None) -> None:
+    """
+    Configure logging.
+
+    Parameters
+    ----------
+    verbose
+        0 = WARNING, 1 = INFO, 2+ = DEBUG
+    log_file
+        If provided, a plain (non-Rich) file handler is added for CI/grep.
+    """
+    level = (
+        logging.WARNING
+        if verbose <= 0
+        else logging.INFO if verbose == 1 else logging.DEBUG
+    )
+    handlers: list[logging.Handler] = [
+        RichHandler(
+            rich_tracebacks=True,
+            show_time=False,
+            show_level=True,
+            show_path=False,
+            console=console,  # IMPORTANT: share with Progress
+            markup=True,
+        )
+    ]
+
+    if log_file:
+        fh = logging.FileHandler(log_file, encoding="utf-8")
+        fh.setLevel(level)
+        fh.setFormatter(
+            logging.Formatter(
+                fmt="%(asctime)s %(levelname)s [%(name)s] %(message)s",
+                datefmt="%Y-%m-%d %H:%M:%S",
+            )
+        )
+        handlers.append(fh)
+
+    logging.basicConfig(level=level, format="%(message)s", handlers=handlers)
+    logger.setLevel(level)

pipeloom/writer.py

@@ -0,0 +1,194 @@
+"""
+writer.py
+=========
+
+The **single** SQLite writer thread. It:
+
+- Opens and owns the only SQLite connection (configured for WAL).
+- Consumes message objects from a thread-safe Queue.
+- Writes task status/progress to SQLite (optional).
+- Updates/removes Rich per-task progress bars safely.
+
+Why a single writer?
+--------------------
+SQLite allows multiple writers *serially*, but a single connection and a single
+owning thread is the least surprising, highest-reliability approach. It also
+avoids cross-thread connection misuse, which is a common source of bugs.
+"""
+
+from __future__ import annotations
+
+import logging
+import queue
+import sqlite3
+import threading
+
+from rich.progress import Progress, TaskID
+
+from . import db as dbmod
+from .messages import SENTINEL, MsgTaskFinished, MsgTaskProgress, MsgTaskStarted
+
+LOG = logging.getLogger("pipeloom.writer")
+
+
+class SQLiteWriter(threading.Thread):
+    """
+    Dedicated thread that exclusively writes to SQLite and manages per-task bars.
+
+    Parameters
+    ----------
+    db_path
+        Path to the SQLite database file (or ':memory:').
+    msg_q
+        Thread-safe queue from which this writer consumes message objects.
+    wal
+        Whether to enable WAL on file-backed databases.
+    store_task_status
+        Toggle persistence of task status into the `task_runs` table.
+    task_progress
+        Rich Progress manager used for per-task bars (transient).
+    task_bar_map
+        Pre-registered mapping: task_id -> Rich TaskID (prevents render races).
+    """
+
+    def __init__(
+        self,
+        db_path,
+        msg_q: queue.Queue[object],
+        *,
+        wal: bool = True,
+        store_task_status: bool = True,
+        task_progress: Progress | None = None,
+        task_bar_map: dict[int, TaskID] | None = None,
+    ) -> None:
+        super().__init__(daemon=True)
+        self.db_path = db_path
+        self.msg_q = msg_q
+        self._use_wal = wal
+        self._store = store_task_status
+        self._progress = task_progress
+        self._progress_tasks: dict[int, TaskID] = dict(task_bar_map or {})
+        self._conn: sqlite3.Connection | None = None
+        self._stop = threading.Event()
+
+    # --- message handlers --------------------------------------------------------------
+    def _on_started(self, m: MsgTaskStarted) -> None:
+        """Persist that a task has started (if enabled)."""
+        assert self._conn is not None
+        if self._store:
+            self._conn.execute(
+                """
+                INSERT INTO task_runs (id,name,status,progress,started_at,message)
+                VALUES (?,?, 'running', 0.0, ?, '')
+                ON CONFLICT(id) DO UPDATE SET
+                  name=excluded.name, status='running', progress=0.0, started_at=excluded.started_at, message=''
+                """,
+                (m.task_id, m.name, m.started_at),
+            )
+            self._conn.commit()
+
+    def _on_progress(self, m: MsgTaskProgress) -> None:
+        """Update DB progress as fraction and drive the Rich bar."""
+        assert self._conn is not None
+        pct = round(100.0 * (m.step / max(1, m.total)), 2)
+
+        if self._store:
+            self._conn.execute(
+                "UPDATE task_runs SET progress=?, message=? WHERE id=?",
+                (pct / 100.0, m.message, m.task_id),
+            )
+            self._conn.commit()
+
+        # IMPORTANT: Rich TaskID can be 0; do not use `if tid:` or walrus in condition.
+        if self._progress:
+            tid = self._progress_tasks.get(m.task_id)
+            if tid is not None:
+                self._progress.update(tid, completed=int(pct))
+
+    def _on_finished(self, m: MsgTaskFinished) -> None:
+        """Write final status and remove the per-task bar cleanly."""
+        assert self._conn is not None
+        if self._store:
+            final = 1.0 if m.status == "done" else 0.0
+            self._conn.execute(
+                """
+                UPDATE task_runs SET status=?, finished_at=?, progress=?, result=?, message=? WHERE id=?
+                """,
+                (m.status, m.finished_at, final, m.result, m.message, m.task_id),
+            )
+            self._conn.commit()
+
+        if self._progress:
+            tid = self._progress_tasks.pop(m.task_id, None)
+            if tid is not None:
+                self._progress.update(tid, completed=100)
+                self._progress.remove_task(tid)
+                self._progress.refresh()
+
+    # --- thread run loop ---------------------------------------------------------------
+    def run(self) -> None:
+        """
+        Lifecycle:
+        - open connection
+        - initialize schema (optional)
+        - consume messages until SENTINEL arrives
+        - checkpoint and close
+        """
+        try:
+            self._conn = dbmod.connect(self.db_path, wal=self._use_wal)
+            dbmod.init_schema(self._conn, store_task_status=self._store)
+            LOG.info("DB writer started → %s (WAL=%s)", self.db_path, self._use_wal)
+
+            while not self._stop.is_set():
+                try:
+                    item = self.msg_q.get(
+                        timeout=0.5
+                    )  # small timeout to enable graceful exit
+                except queue.Empty:
+                    continue
+
+                if item is SENTINEL:
+                    self.msg_q.task_done()
+                    break
+
+                if isinstance(item, MsgTaskStarted):
+                    self._on_started(item)
+                elif isinstance(item, MsgTaskProgress):
+                    self._on_progress(item)
+                elif isinstance(item, MsgTaskFinished):
+                    self._on_finished(item)
+                else:
+                    LOG.warning("Unknown message: %r", type(item))
+
+                self.msg_q.task_done()
+
+        finally:
+            # Best-effort cleanup & checkpoint
+            try:
+                if self._conn:
+                    self._conn.execute("ANALYZE;")
+            except Exception:
+                LOG.debug("ANALYZE failed", exc_info=True)
+
+            if self._conn:
+                with _Suppress(sqlite3.OperationalError):
+                    dbmod.wal_checkpoint(self._conn, "TRUNCATE")
+                    self._conn.close()
+                self._conn = None
+
+            LOG.info("DB writer stopped")
+
+
+class _Suppress:
+    """Minimal suppress helper to avoid importing contextlib."""
+
+    def __init__(self, *exc_types):
+        self.exc_types = exc_types
+
+    def __enter__(self):
+        return None
+
+    def __exit__(self, exc_type, exc, tb):
+        return exc_type is not None and issubclass(
+            exc_type, self.exc_types or (Exception,)
+        )

pipeloom-0.0.1.dist-info/METADATA

@@ -0,0 +1,379 @@
+Metadata-Version: 2.4
+Name: pipeloom
+Version: 0.0.1
+Summary: Threaded ETL scaffolding with a single-writer SQLite (WAL) backend, Rich progress, and a Typer CLI.
+Author-email: Caleb Grant <grantcaleb22@gmail.com>
+License-Expression: GPL-3.0-or-later
+Project-URL: Homepage, https://github.com/geocoug/pipeloom
+Project-URL: Repository, https://github.com/geocoug/pipeloom
+Project-URL: Issues, https://github.com/geocoug/pipeloom/issues
+Requires-Python: ==3.13.*
+Description-Content-Type: text/markdown
+Requires-Dist: rich>=14.1.0
+Requires-Dist: typer>=0.16.0
+
+# pipeloom
+
+Threaded ETL scaffolding with a **single-writer SQLite (WAL)** backend, **Rich** progress, and a **Typer** CLI.
+
+## Why
+
+- **SQLite WAL**: readers don’t block while the writer appends; perfect for local ETL and small services.
+- **Single writer**: one connection, one thread → no cross-thread SQLite bugs.
+- **Structured messaging**: workers never touch the DB; they put typed messages on a queue.
+- **Clean UX**: sticky overall bar, transient per-task bars, shared Console to prevent flicker.
+
+## Quickstart
+
+```bash
+uv run pipeloom --db ./wal_demo.db --num-tasks 10 -vv
+```
+
+## Use in your ETL
+
+1. Define tasks (`TaskDef(...)`).
+2. Provide a `worker_fn(task, msg_q)` that emits:
+   - `MsgTaskStarted`
+   - `MsgTaskProgress` (at your own cadence)
+   - `MsgTaskFinished`
+3. Call `run_pipeline(db_path, tasks, workers=..., wal=True, store_task_status=True)`
+
+> If you don’t want the task_runs table, pass `store_task_status=False`.
+
+## Design Notes
+
+- All SQLite access happens in SQLiteWriter (one thread, one connection).
+- Two progress managers:
+  - overall: remains after completion
+  - per-task: disappears when done
+- We pre-register per-task bars to avoid races.
+- Rich TaskID can be 0 (falsy!) — always check tid is not None.
+
+## Components
+
+```mermaid
+flowchart LR
+  subgraph App["Your App / CLI (Typer)"]
+    CLI["CLI (Typer)"]
+    Engine["Engine (orchestrator)"]
+  end
+
+  subgraph Work["Worker Threads (N)"]
+    W1["Worker #1"]
+    Wn["Worker #N"]
+  end
+
+  subgraph Writer["SQLite Writer Thread (single owner)"]
+    Q["Queue[object]"]
+    SW["SQLiteWriter (owns one sqlite3.Connection)"]
+  end
+
+  subgraph DB["SQLite Database"]
+    WAL["db-wal / db-shm"]
+    Main["main.db file"]
+  end
+
+  CLI --> Engine
+  Engine -->|ThreadPoolExecutor| Work
+  Work -->|MsgTaskStarted/Progress/Finished| Q
+  Engine -->|start| SW
+  SW -->|consume| Q
+  SW -->|INSERT/UPDATE| DB
+  DB <--> WAL
+```
+
+### Why this shape works
+
+- Only the writer thread touches SQLite — no cross-thread connections.
+- WAL makes reads cheap during writes (readers don’t block while the writer appends).
+- Workers talk to the writer via a simple Queue using typed messages.
+
+## Lifecycle
+
+```mermaid
+sequenceDiagram
+  participant CLI as CLI (Typer)
+  participant ENG as Engine
+  participant TPool as ThreadPoolExecutor
+  participant W as Workers (N)
+  participant Q as Queue
+  participant WR as SQLiteWriter
+  participant DB as SQLite (WAL)
+
+  CLI->>ENG: parse args, build tasks
+  ENG->>WR: start (creates connection + schema)
+  ENG->>TPool: submit N workers
+  loop each task
+    W->>Q: MsgTaskStarted
+    W->>Q: MsgTaskProgress (many)
+    W->>Q: MsgTaskFinished
+  end
+  WR->>DB: write status/progress (optional)
+  ENG->>WR: put SENTINEL (shutdown request)
+  WR->>DB: WAL checkpoint(TRUNCATE), close
+  WR-->>ENG: join()
+  ENG-->>CLI: exit (overall bar stays at 100%)
+```
+
+## Writer state machine
+
+```mermaid
+stateDiagram-v2
+  [*] --> Boot
+  Boot --> Running: open conn + init schema
+  Running --> Running: handle message
+  Running --> ShutdownRequested: receive SENTINEL
+  ShutdownRequested --> Teardown: ANALYZE + checkpoint + close
+  Teardown --> [*]
+```
+
+## How to extend
+
+You’ve got two knobs:
+
+1. What work gets done (your worker function).
+2. What gets stored (domain tables and, optionally, task_runs).
+
+Below are pragmatic patterns.
+
+1. Use your own worker function
+
+    Workers never touch SQLite. They publish messages to the queue. Replace the default with your own.
+
+    ```py
+    # my_pipeline.py
+    from pipeloom.messages import TaskDef, MsgTaskStarted, MsgTaskProgress, MsgTaskFinished
+    from datetime import UTC, datetime
+    import time
+
+    def my_worker(task: TaskDef, q) -> None:
+        started = datetime.now(UTC).isoformat()
+        q.put(MsgTaskStarted(task_id=task.task_id, name=task.name, started_at=started))
+
+        try:
+            # Do real work here: extract → transform → load
+            total = 3
+            # Extract
+            data = extract(task)            # your code
+            q.put(MsgTaskProgress(task.task_id, 1, total, "extracted"))
+
+            # Transform
+            records = transform(data)       # your code
+            q.put(MsgTaskProgress(task.task_id, 2, total, "transformed"))
+
+            # Load (NOT via SQLite here — emit a domain message instead; see below)
+            q.put(MsgTaskProgress(task.task_id, 3, total, "loaded"))
+
+            finished = datetime.now(UTC).isoformat()
+            q.put(MsgTaskFinished(task.task_id, "done", finished, result=f"ok:{task.name}"))
+
+        except Exception as e:
+            finished = datetime.now(UTC).isoformat()
+            q.put(MsgTaskFinished(task.task_id, "error", finished, message=str(e)))
+    ```
+
+    Then run:
+
+    ```py
+    from pathlib import Path
+    from pipeloom.engine import run_pipeline
+    from pipeloom.messages import TaskDef
+
+    tasks = [TaskDef(i, f"etl-{i}") for i in range(1, 11)]
+    run_pipeline(
+        db_path=Path("etl.db"),
+        tasks=tasks,
+        workers=4,
+        wal=True,
+        store_task_status=True,  # keep it on for observability
+        worker_fn=my_worker,     # <- your worker
+    )
+    ```
+
+2. Store your own data (domain tables)
+
+    Right now, the writer only persists task_runs (toggle with store_task_status). To persist domain data, add new message types and handle them in the writer. Keep DB writes centralized.
+
+    Define a domain message:
+
+    ```py
+    # my_messages.py
+    from dataclasses import dataclass
+
+    @dataclass(frozen=True)
+    class MsgUpsertRecord:
+        table: str
+        key: str
+        payload: dict  # or a typed dataclass you marshal to a tuple
+    ```
+
+    Emit from your worker:
+
+    ```py
+    q.put(MsgUpsertRecord(table="users", key="abc123", payload={"name": "Caleb", "active": 1}))
+    ```
+
+    Handle in the writer:
+
+    ```py
+    # in your SQLiteWriter.run loop
+    from my_messages import MsgUpsertRecord
+
+    elif isinstance(item, MsgUpsertRecord):
+        self._on_upsert(item)
+
+    # and implement:
+    def _on_upsert(self, m: MsgUpsertRecord) -> None:
+        assert self._conn is not None
+        # Example UPSERT using a prepared SQL per table; avoid dynamic SQL when possible.
+        self._conn.execute(
+            "INSERT INTO users(key, name, active) VALUES (?, ?, ?) "
+            "ON CONFLICT(key) DO UPDATE SET name=excluded.name, active=excluded.active",
+            (m.key, m.payload["name"], m.payload["active"]),
+        )
+        self._conn.commit()
+    ```
+
+    Create your domain schema in a small helper or migration step:
+
+    ```sql
+    CREATE TABLE IF NOT EXISTS users (
+       key TEXT PRIMARY KEY,
+       name TEXT NOT NULL,
+       active INTEGER NOT NULL
+    );
+    ```
+
+    > [!NOTE]
+    > Keep schema creation in a dedicated module (like db_schema.py) and call it from the writer just after init_schema(...).
+
+3. Idempotency & retries (don’t overthink it)
+
+   - Use natural keys + ON CONFLICT DO UPDATE to make “retries” cheap.
+   - For large loads, prefer batch messages (e.g., MsgUpsertBatch) to reduce commit overhead.
+   - If a worker fails, you’ll still get a MsgTaskFinished(..., status="error"). You can query task_runs later and re-enqueue specific tasks.
+
+4. Backpressure & throughput
+
+   - The queue is bounded: maxsize=max(64, workers * 8). If you push heavy domain data, consider:
+   - Using batching (MsgUpsertBatch), say 1,000–10,000 rows per message.
+   - Increasing maxsize if writer keeps up, or reducing if you want memory pressure to be explicit.
+   - Keep wal_autocheckpoint=1000. For huge ingest, raise it; for tiny writes, it’s fine.
+
+5. Tuning SQLite (safe defaults vs. speed)
+
+   Current pragmas are conservative for ETL:
+
+   - journal_mode=WAL
+   - synchronous=NORMAL (fewer fsyncs; still safe enough)
+   - temp_store=MEMORY
+   - cache_size=-100000 (~100 MB)
+   - wal_autocheckpoint=1000
+
+   Tradeoffs:
+
+   - For maximum durability, bump to synchronous=FULL.
+   - For speed on scratch DBs, you can use PRAGMA synchronous=OFF (⚠️ crash can corrupt last transaction).
+
+6. Customize the progress UI
+
+   - Per-task bars are pre-registered to avoid races.
+   - You can change columns or add text with task_progress.update(tid, description="…").
+   - To group tasks, give names like "extract:users", "transform:users", etc.
+
+7. Multiprocessing or asyncio (when to switch)
+
+   - Multiprocessing: if your work is CPU-bound and the GIL bites. You’ll need:
+   - multiprocessing.Queue for messages.
+   - A small shim to update Rich progress from the main process only.
+   - Asyncio: if your work is network-bound and you want extreme concurrency.
+   - Keep the same “single-writer” idea — but the writer becomes an asyncio.Task with aiosqlite.
+
+   For most local ETL, threaded workers + single writer are enough.
+
+8. Observability (recommended: keep task_runs)
+
+    You’ll thank yourself later. Handy queries:
+
+    ```sql
+    -- Stuck or failed tasks in the last 24h
+    SELECT id, name, status, progress, message, started_at, finished_at
+    FROM task_runs
+    WHERE (status <> 'done' OR progress < 1.0)
+    AND (datetime(started_at) > datetime('now','-1 day'));
+
+    -- Histogram of statuses
+    SELECT status, COUNT(*) FROM task_runs GROUP BY status;
+    ```
+
+    If a project really doesn’t need it, set --store-task-status false (or store_task_status=False in code).
+
+9. Example: replace the demo with a real ETL
+
+    Worker: extract from API → transform rows → send a batch to writer.
+
+    ```py
+    from dataclasses import dataclass
+    from pipeloom.messages import MsgTaskStarted, MsgTaskProgress, MsgTaskFinished
+    from datetime import UTC, datetime
+    import time
+
+    @dataclass(frozen=True)
+    class MsgUpsertUsersBatch:
+        rows: list[tuple[str, str, int]]  # key, name, active
+
+    def user_worker(task, q):
+        q.put(MsgTaskStarted(task.task_id, task.name, datetime.now(UTC).isoformat()))
+        try:
+            users = fetch_users_page(task.task_id)   # your I/O
+            rows = [(u["id"], u["name"], int(u["active"])) for u in users]
+            q.put(MsgTaskProgress(task.task_id, 1, 2, "downloaded"))
+            q.put(MsgUpsertUsersBatch(rows))
+            q.put(MsgTaskProgress(task.task_id, 2, 2, "stored"))
+            q.put(MsgTaskFinished(task.task_id, "done", datetime.now(UTC).isoformat()))
+        except Exception as e:
+            q.put(MsgTaskFinished(task.task_id, "error", datetime.now(UTC).isoformat(), message=str(e)))
+    ```
+
+    Writer handler:
+
+    ```py
+    def _on_upsert_users_batch(self, m: MsgUpsertUsersBatch) -> None:
+        assert self._conn is not None
+        self._conn.executemany(
+            "INSERT INTO users(key, name, active) VALUES (?, ?, ?) "
+            "ON CONFLICT(key) DO UPDATE SET name=excluded.name, active=excluded.active",
+            m.rows,
+        )
+        self._conn.commit()
+
+    Wire it in the writer’s dispatch:
+
+    elif isinstance(item, MsgUpsertUsersBatch):
+        self._on_upsert_users_batch(item)
+    ```
+
+10. Safety checklist
+
+    - ✅ Never share a SQLite connection across threads.
+    - ✅ Always pre-register per-task bars; check tid is not None (TaskID 0 is valid).
+    - ✅ Keep one Console for Logging + Progress.
+    - ✅ Use UPSERT for idempotency; batch inserts when possible.
+    - ✅ Shut down writer before transient progress exits (to remove bars).
+    - ✅ Keep task_runs on unless you have a strong reason not to.
+
+## How to Extend
+
+The core design is message-driven. To add new features:
+
+1. Add a new message type
+    - Create a @dataclass in messages.py representing the event.
+    - Example: MsgTaskError to log errors per task.
+2. Handle it in SQLiteWriter.run()
+    - Add a new elif isinstance(m, MsgTaskError): self._on_error(m).
+3. Update the database schema
+    - Modify the SQLite initialization in SQLiteWriter._init_db() to add new columns/tables.
+    - Consider migrations if you want backward compatibility.
+4. Update your workers to emit the new message type when appropriate.
+
+Because the SQLiteWriter owns all database writes, you keep DB operations safe and consistent no matter how many worker threads/processes you spawn.

pipeloom-0.0.1.dist-info/RECORD

@@ -0,0 +1,13 @@
+pipeloom/__init__.py,sha256=QWvMcKQvllkUkHYpwfEwldWvs4DGSm4ib26udHjqCE0,1591
+pipeloom/cli.py,sha256=ab4EAHymYQd7sowkb58iKT8SeqPkelOMFokbvQZNLFA,1998
+pipeloom/db.py,sha256=7J2TaR4JTfqLtlHeEJSMJG7bmRMkUtjbShEccPgSi4w,2801
+pipeloom/engine.py,sha256=kQKSWl7lkhpz876twLeycep905sijaZmZtgnuj23Z28,5652
+pipeloom/messages.py,sha256=TYDhOYBs6TMjStvIYYaNsitwXhPatCVR1LHKQJ2MjAA,2071
+pipeloom/progress.py,sha256=4HzPLMnvLm2WDUK3oBV9F1rPi4FgwsVqPc-btYRGCsU,2302
+pipeloom/rlog.py,sha256=bQymOyvChTA_XQX80K6fE0vEtsQw5y5zDczlR1C1qBg,1690
+pipeloom/writer.py,sha256=toDXDsInvNho0CU2ct8joUGQCpTFGCH4uAPi-RBKmHk,6676
+pipeloom-0.0.1.dist-info/METADATA,sha256=C7e-X_qLVwUDvHlOZKu8JzutLA33h7VKGJqY_5pW3mU,12712
+pipeloom-0.0.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+pipeloom-0.0.1.dist-info/entry_points.txt,sha256=KswZgdPH8r_naw3yTUhzyL9CLY8wow4LQ6mB_R_N_0U,46
+pipeloom-0.0.1.dist-info/top_level.txt,sha256=FpG3qrEqzP846AnHvqAhvdeLosylZrsUJsi7KjtKmaM,9
+pipeloom-0.0.1.dist-info/RECORD,,

pipeloom-0.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

pipeloom-0.0.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+pipeloom = pipeloom.cli:app

pipeloom-0.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+pipeloom