footprinter-cli 1.0.0__py3-none-any.whl

footprinter/ingest/adapters/chat.py

@@ -0,0 +1,57 @@
+"""Chat history adapter.
+
+Wraps ChatIndexer to conform to PipeAdapter protocol.
+Chat imports are manual — this adapter provides read-only status.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, List
+
+from footprinter.ingest.adapters.protocol import ErrorType, PipeContext, PipeResult
+from footprinter.ingest.chat_indexer import ChatIndexer
+
+logger = logging.getLogger(__name__)
+
+
+class ChatAdapter:
+    """Adapter wrapping ChatIndexer for the chat stage."""
+
+    name = "chat"
+    pipe_name = "chat"
+    required_extras: List[str] = []
+
+    def run(self, db: Any, ctx: PipeContext) -> PipeResult:
+        """Report chat history stats (read-only).
+
+        Chat imports are manual via the chat_indexer CLI, so this
+        just reports current counts.
+        """
+        try:
+            manager = ChatIndexer(db)
+            stats = manager.get_stats()
+
+            return PipeResult.info(
+                "chat",
+                note="Chat imports are manual - run chat_indexer import-claude or import-chatgpt",
+                current_chats=stats.get("total_chats", 0),
+                current_messages=stats.get("total_messages", 0),
+                by_account=stats.get("by_account", {}),
+            )
+        except Exception as e:
+            logger.error(f"chat stage failed: {e}")
+            return PipeResult.make_error(
+                "chat",
+                error=str(e),
+                error_type=ErrorType.RUNTIME,
+            )
+
+    def status(self, db: Any) -> Dict[str, Any]:
+        """Return chat and message counts."""
+        cursor = db.conn.cursor()
+        cursor.execute("SELECT COUNT(*) FROM chats")
+        chats = cursor.fetchone()[0]
+        cursor.execute("SELECT COUNT(*) FROM messages")
+        messages = cursor.fetchone()[0]
+        return {"chats": chats, "messages": messages}

footprinter/ingest/adapters/ingest.py

@@ -0,0 +1,146 @@
+"""Shared ingest loop helper for pipeline adapters.
+
+Extracts the common iterate-try-count-log pattern used by Browser, Email,
+DriveFiles, and DriveFolders adapters into a single function.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Callable, Iterable
+
+from footprinter.ingest.adapters.protocol import PipeResult
+
+logger = logging.getLogger(__name__)
+
+
+def ingest_entries(
+    stage: str,
+    entries: Iterable,
+    insert_fn: Callable[[Any], Any],
+    *,
+    count_label: str = "items_indexed",
+    max_logged_errors: int = 5,
+    progress_interval: int | None = None,
+    conn: Any | None = None,
+    batch_size: int = 1000,
+    on_progress: Callable[[int], None] | None = None,
+) -> PipeResult:
+    """Iterate *entries*, calling *insert_fn* per entry with error resilience.
+
+    Returns a PipeResult with:
+    - ``count_label``: number of successful inserts
+    - ``skipped``: number of entries the insert_fn chose not to process
+    - ``errors``: number of failed inserts
+    - Status ``completed`` or ``completed_with_errors``
+
+    **Skip contract:** if *insert_fn* returns ``False`` (identity check, not
+    truthiness), the entry is counted as *skipped* rather than a success.
+    Any other return value (``None``, ``True``, etc.) counts as a success.
+    This lets adapters signal "I intentionally didn't process this" without
+    post-correcting counts.
+
+    **Batch commits:** when *conn* is provided, ``conn.commit()`` is called
+    every *batch_size* successful inserts and once after the loop for any
+    remainder.  On insert error, pending successes are committed before
+    continuing.  When *conn* is ``None``, no commits are issued.
+
+    **Commit failures:** if ``conn.commit()`` itself raises, the error is
+    caught and logged (warning for mid-loop commits, error for the final
+    commit).  Processing continues — uncommitted rows stay in the open
+    transaction and are flushed by the next successful commit or by a
+    retry commit after the loop.  The ``count_label`` value counts entries
+    where *insert_fn* succeeded, not entries durably committed; when
+    ``commit_errors`` is present in the result data, some inserts may not
+    have been persisted.
+
+    Note: 100% failure still returns ``completed_with_errors`` (not ``error``).
+    ``error`` is reserved for stage-level failures (database, config, etc.).
+    ``completed_with_errors`` means the loop completed — individual entries failed.
+
+    Errors are logged up to *max_logged_errors* to avoid flooding.
+    If *progress_interval* is set, logs a progress message every N successes.
+    """
+    success_count = 0
+    skip_count = 0
+    error_count = 0
+    commit_error_count = 0
+    batch_count = 0
+    processed_count = 0
+
+    for entry in entries:
+        try:
+            result = insert_fn(entry)
+            if result is False:
+                skip_count += 1
+            else:
+                success_count += 1
+                batch_count += 1
+                if conn is not None and batch_count >= batch_size:
+                    try:
+                        conn.commit()
+                    except Exception as exc:
+                        commit_error_count += 1
+                        logger.warning(
+                            "%s: batch commit failed (%d pending): %s",
+                            stage,
+                            batch_count,
+                            exc,
+                        )
+                    batch_count = 0
+                if progress_interval and success_count % progress_interval == 0:
+                    logger.info(f"Indexed {success_count} {count_label}...")
+        except Exception as e:
+            error_count += 1
+            if conn is not None and batch_count > 0:
+                try:
+                    conn.commit()
+                except Exception as exc:
+                    commit_error_count += 1
+                    logger.warning(
+                        "%s: error-recovery commit failed (%d pending): %s",
+                        stage,
+                        batch_count,
+                        exc,
+                    )
+                batch_count = 0
+            if error_count <= max_logged_errors:
+                logger.error(f"Error in {stage} ingest: {e}")
+        finally:
+            processed_count += 1
+            if on_progress is not None:
+                on_progress(processed_count)
+
+    if conn is not None and (batch_count > 0 or commit_error_count > 0):
+        try:
+            conn.commit()
+        except Exception as exc:
+            commit_error_count += 1
+            logger.error(
+                "%s: final commit failed (%d pending): %s",
+                stage,
+                batch_count,
+                exc,
+            )
+
+    suppressed = error_count - max_logged_errors
+    if suppressed > 0:
+        logger.warning(f"{stage}: {suppressed} more errors not shown")
+
+    data = {count_label: success_count, "skipped": skip_count, "errors": error_count}
+    if commit_error_count > 0:
+        data["commit_errors"] = commit_error_count
+
+    if error_count > 0 or commit_error_count > 0:
+        error_parts = []
+        if error_count > 0:
+            error_parts.append(f"{error_count} entries failed")
+        if commit_error_count > 0:
+            error_parts.append(f"{commit_error_count} commit errors")
+        return PipeResult.completed_with_errors(
+            stage,
+            error=", ".join(error_parts),
+            **data,
+        )
+
+    return PipeResult.completed(stage, **data)

footprinter/ingest/adapters/local_files.py

@@ -0,0 +1,68 @@
+"""Local files adapter.
+
+Wraps FileIndexer to conform to PipeAdapter protocol.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, List
+
+from footprinter.db import files as files_db
+from footprinter.ingest.adapters.protocol import ErrorType, PipeContext, PipeResult
+from footprinter.ingest.file_indexer import FileIndexer
+from footprinter.source_registry import SourceRegistry
+
+logger = logging.getLogger(__name__)
+
+
+class LocalFilesAdapter:
+    """Adapter wrapping FileIndexer for the local_files stage."""
+
+    name = "local_files"
+    pipe_name = "local_files"
+    required_extras: List[str] = []
+
+    def run(self, db: Any, ctx: PipeContext) -> PipeResult:
+        """Index local files into files table."""
+        try:
+            last_run = None if ctx.full_mode else ctx.last_run
+            indexer = FileIndexer(config_path=ctx.config_path, last_run=last_run, db=db)
+
+            # Build in-memory maps once before ingest
+            registry = SourceRegistry(db.conn)
+            folder_path_map, folder_project_map = files_db.build_folder_maps(db.conn)
+            relationship_maps = {
+                "project_prefix_map": files_db.build_project_prefix_map(db.conn),
+                "folder_path_map": folder_path_map,
+                "folder_project_map": folder_project_map,
+                "remote_source_names": frozenset(registry.remote_source_names()),
+            }
+
+            counts = indexer.index_files(
+                relationship_maps=relationship_maps,
+                on_progress=ctx.on_progress,
+            )
+
+            return PipeResult.completed(
+                "local_files",
+                inserted=counts["inserted"],
+                updated=counts["updated"],
+                skipped=counts["skipped"],
+                errors=counts["errors"],
+                mode="full" if ctx.full_mode else "incremental",
+            )
+        except Exception as e:
+            logger.error(f"local_files stage failed: {e}")
+            return PipeResult.make_error(
+                "local_files",
+                error=str(e),
+                error_type=ErrorType.RUNTIME,
+            )
+
+    def status(self, db: Any) -> Dict[str, Any]:
+        """Return local file count."""
+        cursor = db.conn.cursor()
+        cursor.execute("SELECT COUNT(*) FROM files WHERE source = 'local' AND status != 'removed'")
+        count = cursor.fetchone()[0]
+        return {"local_files": count}

footprinter/ingest/adapters/local_folders.py

@@ -0,0 +1,52 @@
+"""Local folders adapter.
+
+Wraps FolderIndexer to conform to PipeAdapter protocol.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, List
+
+from footprinter.ingest.adapters.protocol import ErrorType, PipeContext, PipeResult
+from footprinter.ingest.folder_indexer import FolderIndexer
+
+logger = logging.getLogger(__name__)
+
+
+class LocalFoldersAdapter:
+    """Adapter wrapping FolderIndexer for the local_folders stage."""
+
+    name = "local_folders"
+    pipe_name = "local_folders"
+    required_extras: List[str] = []
+
+    def run(self, db: Any, ctx: PipeContext) -> PipeResult:
+        """Scan local folder structure into folders."""
+        try:
+            indexer = FolderIndexer(ctx.source_config, db)
+            root_paths = ctx.source_config.get("directories", ["~/Work", "~/Personal"])
+
+            folders = indexer.scan_folders(root_paths)
+            inserted, updated = indexer.save_folders(folders)
+
+            return PipeResult.completed(
+                "local_folders",
+                folders_found=len(folders),
+                inserted=inserted,
+                updated=updated,
+            )
+        except Exception as e:
+            logger.error(f"local_folders stage failed: {e}")
+            return PipeResult.make_error(
+                "local_folders",
+                error=str(e),
+                error_type=ErrorType.RUNTIME,
+            )
+
+    def status(self, db: Any) -> Dict[str, Any]:
+        """Return folders count."""
+        cursor = db.conn.cursor()
+        cursor.execute("SELECT COUNT(*) FROM folders")
+        count = cursor.fetchone()[0]
+        return {"folders": count}

footprinter/ingest/adapters/protocol.py

@@ -0,0 +1,174 @@
+"""Adapter protocol types for the pipeline refactor.
+
+Defines the formal types that all pipe adapters implement:
+- PipeStatus: enum of result statuses matching current orchestrator strings
+- ErrorType: enum of error categories used for halt decisions
+- PipeResult: typed replacement for ad-hoc result dicts
+- PipeContext: typed runtime context replacing the convention-based config dict
+- PipeAdapter: Protocol that all adapters implement
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Protocol, runtime_checkable
+
+if TYPE_CHECKING:
+    from footprinter.ingest.database import Database
+
+
+class PipeStatus(Enum):
+    """Pipe result status.
+
+    Values match the status strings in the current orchestrator result dicts.
+    """
+
+    COMPLETED = "completed"
+    COMPLETED_WITH_ERRORS = "completed_with_errors"
+    SKIPPED = "skipped"
+    ERROR = "error"
+    INFO = "info"
+
+
+class ErrorType(Enum):
+    """Error categories for pipeline halt decisions.
+
+    The orchestrator uses error_type to decide whether to halt the pipeline:
+    database and config errors are fatal; missing_dependency and runtime are not.
+    """
+
+    MISSING_DEPENDENCY = "missing_dependency"
+    DATABASE = "database"
+    CONFIG = "config"
+    RUNTIME = "runtime"
+
+
+@dataclass
+class PipeResult:
+    """Typed result from a pipeline pipe.
+
+    Replaces the ad-hoc Dict[str, Any] returned by orchestrator pipe methods.
+    Factory classmethods reduce boilerplate in adapter implementations.
+    """
+
+    stage: str
+    status: PipeStatus
+    elapsed_seconds: float = 0.0
+    data: Dict[str, Any] = field(default_factory=dict)
+    error: Optional[str] = None
+    error_type: Optional[ErrorType] = None
+
+    # -- Factory classmethods --------------------------------------------------
+
+    @classmethod
+    def completed(cls, stage: str, **data: Any) -> PipeResult:
+        """Create a result indicating the stage completed successfully."""
+        return cls(stage=stage, status=PipeStatus.COMPLETED, data=data)
+
+    @classmethod
+    def completed_with_errors(cls, stage: str, error: str, **data: Any) -> PipeResult:
+        """Create a result indicating the stage completed with non-fatal errors."""
+        return cls(
+            stage=stage,
+            status=PipeStatus.COMPLETED_WITH_ERRORS,
+            data=data,
+            error=error,
+        )
+
+    @classmethod
+    def skipped(cls, stage: str, reason: str, **data: Any) -> PipeResult:
+        """Create a result indicating the stage was skipped."""
+        return cls(
+            stage=stage,
+            status=PipeStatus.SKIPPED,
+            data={"reason": reason, **data},
+        )
+
+    @classmethod
+    def make_error(
+        cls,
+        stage: str,
+        error: str,
+        error_type: Optional[ErrorType] = None,
+        **data: Any,
+    ) -> PipeResult:
+        """Create a result indicating the stage failed with an error."""
+        return cls(
+            stage=stage,
+            status=PipeStatus.ERROR,
+            data=data,
+            error=error,
+            error_type=error_type,
+        )
+
+    @classmethod
+    def info(cls, stage: str, **data: Any) -> PipeResult:
+        """Create an informational result (no processing occurred)."""
+        return cls(stage=stage, status=PipeStatus.INFO, data=data)
+
+    # -- Serialization ---------------------------------------------------------
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Flatten to the dict shape expected by the orchestrator.
+
+        Data keys are spread to the top level first, then reserved fields
+        overlay them so an adapter can't accidentally clobber stage/status.
+        """
+        result = {**self.data}
+        result["stage"] = self.stage
+        result["status"] = self.status.value
+        result["elapsed_seconds"] = self.elapsed_seconds
+        if self.error is not None:
+            result["error"] = self.error
+        if self.error_type is not None:
+            result["error_type"] = self.error_type.value
+        return result
+
+
+@dataclass
+class PipeContext:
+    """Typed runtime context passed to adapter.run().
+
+    Replaces the convention-based Dict[str, Any] config parameter.
+    """
+
+    source_config: Dict[str, Any]
+    config_path: str = ""
+    full_mode: bool = False
+    last_run: Optional[datetime] = None
+    on_progress: Optional[Callable[[int], None]] = None
+
+
+@runtime_checkable
+class PipeAdapter(Protocol):
+    """Protocol that all pipe adapters implement.
+
+    Enables isinstance() validation in the adapter registry.
+    Implementors can use either @property decorators or class attributes
+    for the metadata fields.
+    """
+
+    @property
+    def name(self) -> str:
+        """Human-readable adapter name."""
+        ...
+
+    @property
+    def pipe_name(self) -> str:
+        """Pipe identifier used by the orchestrator."""
+        ...
+
+    @property
+    def required_extras(self) -> List[str]:
+        """Pip extras that must be installed for this adapter to run."""
+        ...
+
+    def run(self, db: Database, ctx: PipeContext) -> PipeResult:
+        """Execute the adapter's pipe."""
+        ...
+
+    def status(self, db: Database) -> Dict[str, Any]:
+        """Return current data counts and health for this pipe."""
+        ...