footprinter-cli 1.0.0__py3-none-any.whl

footprinter/ingest/pipe_runner.py

@@ -0,0 +1,217 @@
+"""Pipe runner — pipe dispatch, iteration, timing, error aggregation.
+
+Runs pipes in order, delegates data-source pipes to adapters and
+processing pipes to ProcessingPipeline.  Handles timing, error
+classification, and fatal-error halting.
+"""
+
+from __future__ import annotations
+
+import logging
+import sqlite3
+from datetime import datetime
+from typing import TYPE_CHECKING, Callable, Dict, List, Optional
+
+from footprinter.ingest.adapters.protocol import PipeContext
+
+if TYPE_CHECKING:
+    from footprinter.ingest.processing import ProcessingPipeline
+
+logger = logging.getLogger(__name__)
+
+
+class PipeRunner:
+    """Runs pipes in order, handles timing, manages error aggregation.
+
+    Receives its adapter registry, pipeline definitions, and valid pipe
+    list from the orchestrator (composition root). Does not import pipe
+    definitions directly.
+    """
+
+    def __init__(
+        self,
+        processing: ProcessingPipeline,
+        get_db: Callable,
+        config: Dict,
+        config_path: str,
+        adapter_registry: Dict[str, type],
+        pipelines: Dict[str, List[str]],
+        all_pipes: List[str],
+        user_pipes: Optional[List[str]] = None,
+        connector_pipe_map: Optional[Dict[str, str]] = None,
+    ):
+        self.processing = processing
+        self._get_db = get_db
+        self.config = config
+        self.config_path = config_path
+        self.adapter_registry = adapter_registry
+        self.pipelines = pipelines
+        self.all_pipes = all_pipes
+        # User-selectable subset for error messages. Falls back to all_pipes
+        # when omitted (legacy call sites) — error messages then show every
+        # pipe including post-processing, as before.
+        self.user_pipes = user_pipes if user_pipes is not None else all_pipes
+        self._connector_pipe_map = connector_pipe_map or {}
+        self.full_mode = False
+
+    def run_pipe(
+        self,
+        pipe: str,
+        on_progress: Optional[Callable] = None,
+        last_run: Optional[datetime] = None,
+    ) -> Dict:
+        """Run a single pipe.
+
+        Dispatches to the adapter registry for data-source pipes,
+        or to ProcessingPipeline for processing pipes.  If the pipe
+        belongs to an uninstalled connector, returns a skip result with
+        install instructions.
+
+        Returns:
+            Dict with pipe results including elapsed_seconds.
+        """
+        logger.info(f"Running pipe: {pipe}")
+        start_time = datetime.now()
+
+        result = {"stage": pipe, "status": "unknown"}
+
+        try:
+            adapter_cls = self.adapter_registry.get(pipe)
+            if adapter_cls is not None:
+                adapter = adapter_cls()
+                db = self._get_db()
+                ctx = PipeContext(
+                    source_config=self.config,
+                    config_path=self.config_path,
+                    full_mode=self.full_mode,
+                    last_run=last_run,
+                    on_progress=on_progress,
+                )
+                pipe_result = adapter.run(db, ctx)
+                elapsed = (datetime.now() - start_time).total_seconds()
+                pipe_result.elapsed_seconds = round(elapsed, 1)
+                result = pipe_result.to_dict()
+            elif self.processing.is_processing_pipe(pipe):
+                db = self._get_db()
+                pipe_result = self.processing.run_phase(pipe, db)
+                elapsed = (datetime.now() - start_time).total_seconds()
+                pipe_result.elapsed_seconds = round(elapsed, 1)
+                result = pipe_result.to_dict()
+            else:
+                # Check if this pipe belongs to an uninstalled connector
+                connector_name = self._find_connector_for_pipe(pipe)
+                if connector_name:
+                    result = {
+                        "stage": pipe,
+                        "status": "skipped",
+                        "reason": "not installed",
+                        "hint": f"run: fp connect install {connector_name}",
+                    }
+                else:
+                    logger.error(f"Unknown pipe: {pipe}")
+                    result = {
+                        "stage": pipe,
+                        "status": "error",
+                        "error": f"Unknown pipe: {pipe}",
+                    }
+
+            result["stage"] = pipe
+            result["status"] = result.get("status", "completed")
+
+        except ImportError as e:
+            logger.warning(f"Pipe {pipe} skipped — missing dependency: {e}")
+            result = {
+                "stage": pipe,
+                "status": "skipped",
+                "reason": f"Not installed: {e}",
+                "error_type": "missing_dependency",
+            }
+        except sqlite3.OperationalError as e:
+            logger.error(f"Database error in pipe {pipe}: {e}")
+            result = {
+                "stage": pipe,
+                "status": "error",
+                "error": str(e),
+                "error_type": "database",
+            }
+        except FileNotFoundError as e:
+            logger.error(f"Config/file error in pipe {pipe}: {e}")
+            result = {
+                "stage": pipe,
+                "status": "error",
+                "error": str(e),
+                "error_type": "config",
+            }
+        # Intentional broad catch: last-resort after specific
+        # ImportError, OperationalError, FileNotFoundError handlers
+        except Exception as e:
+            logger.error(f"Error in pipe {pipe}: {e}")
+            result = {
+                "stage": pipe,
+                "status": "error",
+                "error": str(e),
+                "error_type": "runtime",
+            }
+
+        elapsed = (datetime.now() - start_time).total_seconds()
+        result["elapsed_seconds"] = round(elapsed, 1)
+        logger.info(f"Pipe {pipe} completed in {elapsed:.1f}s")
+
+        return result
+
+    def validate_pipes(self, pipes: List[str]) -> None:
+        """Raise ValueError for unknown pipe names. Pure check, no side effects.
+
+        Exposed so callers that need to fail before starting UI output
+        (progress bars, headers) can pre-flight without duplicating the
+        unknown-pipe rule.
+        """
+        unknown = [s for s in pipes if s not in self.all_pipes]
+        if unknown:
+            raise ValueError(
+                f"Unknown pipe(s): {', '.join(unknown)}. "
+                f"Valid pipes: {', '.join(self.user_pipes)}"
+            )
+
+    def run_pipes(
+        self,
+        pipes: List[str],
+        on_pipe_start: Optional[Callable] = None,
+        on_pipe_end: Optional[Callable] = None,
+        on_progress: Optional[Callable] = None,
+        pipe_hook: Optional[Callable] = None,
+        last_run: Optional[datetime] = None,
+    ) -> List[Dict]:
+        """Run multiple pipes in order.
+
+        Raises ValueError for unknown pipe names. Stops on fatal errors
+        (database/config error_type), continues on runtime errors.
+        """
+        self.validate_pipes(pipes)
+
+        results = []
+
+        for pipe in pipes:
+            if on_pipe_start:
+                on_pipe_start(pipe)
+
+            if pipe_hook:
+                result = pipe_hook(pipe, on_progress=on_progress)
+            else:
+                result = self.run_pipe(pipe, on_progress=on_progress, last_run=last_run)
+            results.append(result)
+
+            if on_pipe_end:
+                on_pipe_end(pipe, result)
+
+            # Stop pipeline on fatal errors (database/config); runtime errors continue
+            if result.get("status") == "error":
+                if result.get("error_type") in ("database", "config"):
+                    logger.error(f"Fatal error in {pipe}: {result.get('error', 'unknown')}")
+                    break
+
+        return results
+
+    def _find_connector_for_pipe(self, pipe: str) -> str | None:
+        """Find the connector name that owns a given pipe, if any."""
+        return self._connector_pipe_map.get(pipe)

footprinter/ingest/processing.py

@@ -0,0 +1,165 @@
+"""
+Processing module — access resolution and pipeline framework.
+
+Primary role: ``run_access_resolution`` stamps visibility and permissions
+on ingested entities, with last-run-based incremental processing.
+Also provides the ``ProcessingPipeline`` framework for phase registration
+and dispatch.
+"""
+
+from __future__ import annotations
+
+import logging
+import sqlite3
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Callable, Dict, List, Optional
+
+from footprinter.ingest.adapters.protocol import ErrorType, PipeResult
+
+if TYPE_CHECKING:
+    from footprinter.ingest.database import Database
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Last-run helpers (backed by ingests table)
+# ---------------------------------------------------------------------------
+
+
+def _read_last_run(conn: sqlite3.Connection, pipe: str) -> Optional[str]:
+    """Read the last-completed timestamp for a pipe."""
+    row = conn.execute(
+        "SELECT completed_at FROM ingests WHERE pipe = ? AND status = 'completed' ORDER BY completed_at DESC LIMIT 1",
+        (pipe,),
+    ).fetchone()
+    if row is None:
+        return None
+    return row[0] if isinstance(row, tuple) else row["completed_at"]
+
+
+# ---------------------------------------------------------------------------
+# Access resolution runner
+# ---------------------------------------------------------------------------
+
+
+def run_access_resolution(db: "Database", full_mode: bool = False) -> PipeResult:
+    """Stamp visibility and permissions on entities.
+
+    Args:
+        db: Database instance (needs db.conn).
+        full_mode: If True, recalculate everything. If False, only
+            entities added/modified since the last run.
+
+    Returns:
+        PipeResult with per-entity-type counts in data.
+    """
+    from footprinter.access import ENTITY_META, recalculate_access, stamp_entities
+
+    conn = db.conn
+    last_run = _read_last_run(conn, "access_resolution")
+
+    try:
+        if full_mode or last_run is None:
+            # Full recalculation
+            stats = recalculate_access(conn, "global")
+        else:
+            # Incremental — only entities with indexed_at > last run
+            ids_by_type: Dict[str, list] = {}
+            for entity_type, meta in ENTITY_META.items():
+                table = meta["table"]
+
+                # Not all tables have indexed_at (folders, projects, clients don't)
+                try:
+                    conn.execute(f"SELECT indexed_at FROM {table} LIMIT 0")
+                except sqlite3.OperationalError:
+                    continue
+
+                where = "indexed_at > ?"
+                if meta["has_status"]:
+                    where += " AND status != 'removed'"
+
+                rows = conn.execute(f"SELECT id FROM {table} WHERE {where}", (last_run,)).fetchall()
+                ids = [r["id"] if isinstance(r, sqlite3.Row) else r[0] for r in rows]
+
+                if ids:
+                    ids_by_type[entity_type] = ids
+
+            stats = stamp_entities(conn, ids_by_type)
+    except Exception as e:  # Intentional broad catch: last-resort for access resolution; pipeline must continue
+        logger.error("Access resolution error: %s", e, exc_info=True)
+        return PipeResult.make_error("access_resolution", str(e), ErrorType.RUNTIME)
+    else:
+        return PipeResult.completed("access_resolution", **stats)
+
+
+# Identity mapping — pipe names map directly to phase names.
+PIPE_TO_PHASE: Dict[str, str] = {}
+
+
+@dataclass
+class PhaseSpec:
+    """Specification for a single processing phase."""
+
+    name: str
+    skip_guard: Optional[Callable[["Database"], bool]] = None
+    runner: Optional[Callable[["Database"], PipeResult]] = None
+
+
+class ProcessingPipeline:
+    """Pipeline for processing stages.
+
+    Phases are registered with a runner callable and optional skip guard.
+    Execution order follows registration order.
+    """
+
+    def __init__(self) -> None:
+        self._phases: Dict[str, PhaseSpec] = {}
+
+    def register(
+        self,
+        name: str,
+        runner: Optional[Callable[["Database"], PipeResult]] = None,
+        skip_guard: Optional[Callable[["Database"], bool]] = None,
+    ) -> None:
+        """Register a processing phase."""
+        self._phases[name] = PhaseSpec(
+            name=name,
+            skip_guard=skip_guard,
+            runner=runner,
+        )
+
+    def is_processing_pipe(self, pipe_name: str) -> bool:
+        """Check if a pipe name maps to a registered processing phase."""
+        phase_name = PIPE_TO_PHASE.get(pipe_name, pipe_name)
+        return phase_name in self._phases
+
+    @property
+    def phase_names(self) -> List[str]:
+        """Return phase names in registration order."""
+        return list(self._phases.keys())
+
+    def run_phase(self, pipe_name: str, db: "Database") -> PipeResult:
+        """Execute a processing phase by pipe name.
+
+        Applies skip guard, then calls runner directly.
+        """
+        phase_name = PIPE_TO_PHASE.get(pipe_name, pipe_name)
+        spec = self._phases.get(phase_name)
+
+        if spec is None:
+            return PipeResult.make_error(pipe_name, f"Unknown processing phase: {phase_name}")
+
+        if spec.runner is None:
+            return PipeResult.make_error(pipe_name, f"No runner registered for phase: {phase_name}")
+
+        # Check skip guard
+        if spec.skip_guard is not None:
+            try:
+                should_skip = spec.skip_guard(db)
+                if should_skip:
+                    return PipeResult.skipped(pipe_name, f"Skip guard triggered for {phase_name}")
+            except Exception as e:
+                logger.warning(f"Skip guard for {phase_name} raised {type(e).__name__}: {e}; proceeding")
+
+        return spec.runner(db)

footprinter/ingest/registry.py

@@ -0,0 +1,201 @@
+"""Pipe registry — the "phone book" for pipes.
+
+Knows what pipes exist, which adapter classes implement the core
+data-source pipes, and provides functions to compute pipeline and
+refresh pipe definitions dynamically.  Does NOT run anything — that's
+the orchestrator's job.
+
+v1.0 pipe set
+--------------
+Core (always available): local_folders, local_files, browser, chat.
+Connector pipes are resolved dynamically from installed ConnectorSpecs.
+"""
+
+from footprinter.ingest.adapters import (
+    BrowserAdapter,
+    ChatAdapter,
+    LocalFilesAdapter,
+    LocalFoldersAdapter,
+)
+from footprinter.ingest.adapters.protocol import ErrorType, PipeResult, PipeStatus
+
+# ── Source catalogue ────────────────────────────────────────────────
+
+# Core v1.0 sources (work out of the box)
+CORE_PIPES = [
+    "local_folders",  # Scan ~/Work, ~/Personal folder structure
+    "local_files",  # Index local files
+    "browser",  # Browser history
+    "chat",  # Claude/ChatGPT exports (status only - manual import)
+]
+
+# Not valid CLI targets; excluded from get_all_pipes()
+FUTURE_PIPES = [
+    "project_links",
+    "summaries",
+    "drive_links",
+]
+
+# Post-processing pipes — appended to every pipeline, run after all data-source pipes
+POST_PIPES = [
+    "access_resolution",  # Stamp visibility + permissions on ingested entities
+]
+
+# ── Core source registry (data-source adapters only) ─────────────────
+
+CORE_PIPE_REGISTRY = {
+    "local_folders": LocalFoldersAdapter,
+    "local_files": LocalFilesAdapter,
+    "browser": BrowserAdapter,
+    "chat": ChatAdapter,
+}
+
+# ── Dynamic resolution functions ─────────────────────────────────────
+#
+# These replace the former static PIPELINES, REFRESH_PIPES, and ALL_PIPES
+# dicts. They accept connector_pipelines — a dict mapping connector names
+# to their pipe lists (e.g., {"google": ["drive_folders", "drive_files", "gmail"]}).
+# The orchestrator builds this from ConnectorSpec metadata and passes it in,
+# so this module never imports from connectors/.
+
+
+def get_pipelines(
+    connector_pipes: dict[str, type],
+    connector_pipelines: dict[str, list[str]] | None = None,
+) -> dict[str, list[str]]:
+    """Compute pipeline definitions from core + installed connector pipes.
+
+    Args:
+        connector_pipes: Merged adapter registry from get_connector_pipes().
+        connector_pipelines: Connector name → adapter pipe names. Built by
+            the orchestrator from ConnectorSpec.adapter_entries.
+
+    Returns pipeline name → ordered pipe list.
+    """
+    pipelines: dict[str, list[str]] = {
+        "local": list(CORE_PIPES),
+    }
+
+    # Add a pipeline per connector whose pipes are in connector_pipes
+    for name, pipes in (connector_pipelines or {}).items():
+        installed = [s for s in pipes if s in connector_pipes]
+        if installed:
+            pipelines[name] = installed
+
+    # "all" = core + all installed connector data-source pipes
+    all_pipe_names = list(CORE_PIPES)
+    for name, pipes in pipelines.items():
+        if name == "local":
+            continue
+        for s in pipes:
+            if s not in all_pipe_names:
+                all_pipe_names.append(s)
+    pipelines["all"] = all_pipe_names
+
+    # Append post-processing pipes to every pipeline
+    for name in pipelines:
+        pipelines[name] = pipelines[name] + POST_PIPES
+
+    return pipelines
+
+
+def get_refresh_pipes(
+    connector_pipes: dict[str, type],
+    connector_pipelines: dict[str, list[str]] | None = None,
+) -> dict[str, list[str]]:
+    """Compute refresh pipe mappings from core + installed connector pipes.
+
+    Args:
+        connector_pipes: Merged adapter registry from get_connector_pipes().
+        connector_pipelines: Connector name → adapter pipe names.
+
+    Returns source name → pipe list. Each core source group gets a key,
+    each connector gets a key, and individual connector pipes get keys.
+    """
+    refresh: dict[str, list[str]] = {
+        "local": ["local_folders", "local_files"],
+        "browser": ["browser"],
+        "chat": ["chat"],
+    }
+
+    # Per-connector and per-pipe entries
+    for name, pipes in (connector_pipelines or {}).items():
+        installed = [s for s in pipes if s in connector_pipes]
+        if not installed:
+            continue
+
+        # Connector-level key (e.g., "google")
+        refresh[name] = installed
+
+        # Per-pipe keys and grouped keys (e.g., "gmail", "drive")
+        drive_pipes = []
+        for pipe in installed:
+            if pipe.startswith("drive_"):
+                drive_pipes.append(pipe)
+            else:
+                # Individual pipe key (e.g., "gmail")
+                refresh[pipe] = [pipe]
+
+        if drive_pipes:
+            refresh["drive"] = drive_pipes
+
+    # "all" = everything
+    all_pipe_names = list(CORE_PIPES)
+    for name, pipes in (connector_pipelines or {}).items():
+        for s in pipes:
+            if s in connector_pipes and s not in all_pipe_names:
+                all_pipe_names.append(s)
+    refresh["all"] = all_pipe_names
+
+    # Append post-processing pipes to every refresh group
+    for name in refresh:
+        refresh[name] = refresh[name] + POST_PIPES
+
+    return refresh
+
+
+def get_all_pipes(connector_pipes: dict[str, type]) -> list[str]:
+    """Compute complete list of valid pipe names.
+
+    Includes core pipes and installed connector pipes.
+    FUTURE_PIPES entries are excluded — they are not registered pipes.
+    """
+    result = list(CORE_PIPES)
+    for s in connector_pipes:
+        if s not in result:
+            result.append(s)
+    for s in POST_PIPES:
+        if s not in result:
+            result.append(s)
+    return result
+
+
+def get_user_pipes(connector_pipes: dict[str, type]) -> list[str]:
+    """Compute the user-selectable subset of pipes for CLI error messages.
+
+    Includes core + installed connector data-source pipes. POST_PIPES are
+    excluded — they run implicitly after every pipeline and aren't meant
+    to be invoked directly via ``fp ingest --pipe``.
+    """
+    result = list(CORE_PIPES)
+    for s in connector_pipes:
+        if s not in result:
+            result.append(s)
+    return result
+
+
+# ── Convenience re-exports ───────────────────────────────────────────
+
+__all__ = [
+    "CORE_PIPES",
+    "FUTURE_PIPES",
+    "POST_PIPES",
+    "CORE_PIPE_REGISTRY",
+    "get_pipelines",
+    "get_refresh_pipes",
+    "get_all_pipes",
+    "get_user_pipes",
+    "PipeResult",
+    "PipeStatus",
+    "ErrorType",
+]

footprinter/ingest/run_record.py

@@ -0,0 +1,91 @@
+"""Pure persistence for pipeline run records.
+
+Saves and loads a JSON record of each pipeline run. No heuristics or
+config awareness — warning logic lives in the display layer.
+"""
+
+import json
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Dict, List, Optional
+
+from footprinter.paths import get_last_run_path
+
+SESSION_WINDOW_MINUTES = 10
+
+
+def save_run_record(
+    results: List[Dict],
+    mode: str,
+    started_at: datetime,
+    *,
+    interrupted: bool = False,
+    path: Optional[Path] = None,
+) -> Path:
+    """Write a run record to JSON, merging with recent records.
+
+    If an existing record started within SESSION_WINDOW_MINUTES of
+    ``started_at``, new stages are appended to it (preserving the
+    original ``started_at``).  Otherwise the record is replaced.
+
+    Args:
+        results: List of per-stage result dicts from PipeRunner.
+        mode: Run mode string (e.g. "incremental", "full").
+        started_at: When the pipeline started.
+        interrupted: Whether the run was interrupted (e.g. KeyboardInterrupt).
+        path: Override output path (default: get_last_run_path()).
+
+    Returns:
+        The path the record was written to.
+    """
+    if path is None:
+        path = get_last_run_path()
+
+    completed_at = datetime.now(timezone.utc)
+    total_elapsed = sum(r.get("elapsed_seconds", 0) for r in results)
+
+    # Merge with existing record if within session window
+    existing = load_run_record(path=path)
+    if existing and _within_session_window(existing, started_at):
+        existing["stages"].extend(results)
+        existing["completed_at"] = completed_at.isoformat()
+        existing["total_elapsed_seconds"] = sum(r.get("elapsed_seconds", 0) for r in existing["stages"])
+        existing["interrupted"] = interrupted
+        record = existing
+    else:
+        record = {
+            "started_at": started_at.isoformat(),
+            "completed_at": completed_at.isoformat(),
+            "mode": mode,
+            "interrupted": interrupted,
+            "total_elapsed_seconds": total_elapsed,
+            "stages": results,
+        }
+
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(record, indent=2, default=str))
+    return path
+
+
+def _within_session_window(existing: Dict, new_started_at: datetime) -> bool:
+    """Check if an existing record is within the merge window."""
+    try:
+        existing_start = datetime.fromisoformat(existing["started_at"])
+        return abs(new_started_at - existing_start) <= timedelta(minutes=SESSION_WINDOW_MINUTES)
+    except (KeyError, ValueError):
+        return False
+
+
+def load_run_record(path: Optional[Path] = None) -> Optional[Dict]:
+    """Read a run record from JSON.
+
+    Returns:
+        The parsed record dict, or None if the file doesn't exist.
+    """
+    if path is None:
+        path = get_last_run_path()
+
+    if not path.exists():
+        return None
+
+    return json.loads(path.read_text())