footprinter-cli 1.0.0rc1__py3-none-any.whl

footprinter/ingest/content_extractors.py

@@ -0,0 +1,117 @@
+"""
+Content extraction from various file types.
+"""
+
+import logging
+from pathlib import Path
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+logging.getLogger("pypdf").setLevel(logging.ERROR)
+
+_pypdf_warned = False
+_docx_warned = False
+
+
+class ContentExtractor:
+    """Extract text content from various file types."""
+
+    def __init__(self, max_preview_length: int = 1000):
+        self.max_preview_length = max_preview_length
+
+    def extract(self, file_path: Path) -> Optional[str]:
+        """
+        Extract content from file based on type.
+
+        Returns preview of file content (first N characters).
+        """
+        try:
+            file_type = file_path.suffix.lower()
+
+            if file_type in [".txt", ".md", ".py", ".js", ".json", ".yaml", ".yml"]:
+                return self._extract_text(file_path)
+            elif file_type == ".pdf":
+                return self._extract_pdf(file_path)
+            elif file_type == ".docx":
+                return self._extract_docx(file_path)
+            else:
+                logger.debug(f"No extractor for {file_type}")
+                return None
+
+        except (
+            Exception
+        ) as e:  # Intentional broad catch: extraction is inherently brittle (encoding, corrupt files, library bugs)
+            logger.error(f"Error extracting content from {file_path}: {e}")
+            return None
+
+    def _extract_text(self, file_path: Path) -> str:
+        """Extract from plain text files."""
+        try:
+            with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
+                content = f.read(self.max_preview_length * 2)  # Read a bit more
+                return content[: self.max_preview_length]
+        except (
+            Exception
+        ) as e:  # Intentional broad catch: extraction is inherently brittle (encoding, corrupt files, library bugs)
+            logger.error(f"Error reading text file {file_path}: {e}")
+            return ""
+
+    def _extract_pdf(self, file_path: Path) -> str:
+        """Extract from PDF files."""
+        try:
+            import pypdf
+
+            with open(file_path, "rb") as f:
+                reader = pypdf.PdfReader(f)
+
+                # Extract first few pages
+                text = ""
+                for page_num in range(min(3, len(reader.pages))):
+                    page = reader.pages[page_num]
+                    text += page.extract_text() + "\n"
+
+                    if len(text) >= self.max_preview_length:
+                        break
+
+                return text[: self.max_preview_length]
+
+        except ImportError:
+            global _pypdf_warned
+            if not _pypdf_warned:
+                logger.warning("pypdf not installed, skipping PDF extraction")
+                _pypdf_warned = True
+            return None
+        except (
+            Exception
+        ) as e:  # Intentional broad catch: extraction is inherently brittle (encoding, corrupt files, library bugs)
+            logger.error(f"Error reading PDF {file_path}: {e}")
+            return ""
+
+    def _extract_docx(self, file_path: Path) -> str:
+        """Extract from DOCX files."""
+        try:
+            import docx
+
+            doc = docx.Document(file_path)
+
+            # Extract paragraphs
+            text = ""
+            for para in doc.paragraphs:
+                text += para.text + "\n"
+
+                if len(text) >= self.max_preview_length:
+                    break
+
+            return text[: self.max_preview_length]
+
+        except ImportError:
+            global _docx_warned
+            if not _docx_warned:
+                logger.warning("python-docx not installed, skipping DOCX extraction")
+                _docx_warned = True
+            return None
+        except (
+            Exception
+        ) as e:  # Intentional broad catch: extraction is inherently brittle (encoding, corrupt files, library bugs)
+            logger.error(f"Error reading DOCX {file_path}: {e}")
+            return ""

footprinter/ingest/database.py

@@ -0,0 +1,36 @@
+"""SQLite database schema and operations for Footprinter."""
+
+import logging
+from pathlib import Path
+
+from footprinter.ingest.db.connector_schema import init_connector_schemas
+from footprinter.ingest.db.schema import SchemaMixin
+from footprinter.paths import get_db_path
+
+logger = logging.getLogger(__name__)
+
+
+class Database(SchemaMixin):
+    """SQLite database connection and schema manager for Footprinter."""
+
+    def __init__(self, db_path: str = None, connector_specs: list = None):
+        if db_path is None:
+            db_path = str(get_db_path())
+        self.db_path = db_path
+        Path(db_path).parent.mkdir(parents=True, exist_ok=True)
+        self.conn = None
+        self.init_db()
+        init_connector_schemas(self.conn, connector_specs or [])
+
+    def close(self):
+        """Close database connection."""
+        if self.conn:
+            self.conn.close()
+
+    def __enter__(self):
+        """Enter context manager, returning self."""
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Exit context manager, closing the database connection."""
+        self.close()

footprinter/ingest/db/__init__.py

@@ -0,0 +1 @@
+"""Database sub-package."""

footprinter/ingest/db/connector_schema.py

@@ -0,0 +1,47 @@
+"""Connector-scope schema extensions.
+
+Connectors declare extra columns via ConnectorSpec.schema_extensions.
+This module applies those declarations using idempotent ALTER TABLE,
+mirroring the pattern in app_schema.py.
+"""
+
+import logging
+import sqlite3
+
+logger = logging.getLogger(__name__)
+
+
+def init_connector_schemas(conn: sqlite3.Connection, connector_specs: list) -> None:
+    """Apply schema extensions for the given connector specs.
+
+    The caller is responsible for filtering to installed connectors.
+    For each spec with schema_extensions, calls register_connector_schema()
+    to add columns via ALTER TABLE.
+    """
+    for spec in connector_specs:
+        if spec.schema_extensions:
+            register_connector_schema(conn, spec.schema_extensions)
+
+
+def register_connector_schema(
+    conn: sqlite3.Connection,
+    extensions: dict[str, list[tuple[str, str]]],
+) -> None:
+    """Add connector-declared columns to existing tables.
+
+    Args:
+        conn: An open sqlite3 connection with base schema already initialized.
+        extensions: Mapping of table_name → [(col_name, col_definition), ...].
+            Example: {"folders": [("web_link", "TEXT")]}
+    """
+    cursor = conn.cursor()
+    for table, columns in extensions.items():
+        for col_name, col_def in columns:
+            try:
+                cursor.execute(f"ALTER TABLE {table} ADD COLUMN {col_name} {col_def}")
+            except sqlite3.OperationalError as e:
+                if "duplicate column" in str(e).lower():
+                    pass  # column already exists
+                else:
+                    raise
+    conn.commit()

footprinter/ingest/db/migration.py

@@ -0,0 +1,315 @@
+"""Database schema migration for pre-existing Footprinter databases.
+
+Extracted for separation of concerns. Contains all
+ALTER TABLE, RENAME, DROP, and data-migration logic needed to upgrade
+databases created before the current DDL.
+
+Only runs on databases that already have tables — fresh installs skip
+this entirely (init_db handles everything via CREATE TABLE IF NOT EXISTS).
+"""
+
+import logging
+import sqlite3
+
+from footprinter.ingest.db.schema import _INGESTS_DDL, ACCESS_CONTROL_TABLES
+
+logger = logging.getLogger(__name__)
+
+_DISPLAY_NAME_BACKFILL = {
+    "files": "name",
+    "folders": "name",
+    "visits": "title",
+    "projects": "project_name",
+    "chats": "title",
+    "messages": "SUBSTR(content, 1, 100)",
+    "emails": "subject",
+    "clients": "name",
+}
+
+
+def migrate_schema(cursor: sqlite3.Cursor) -> None:
+    """Upgrade a pre-existing database to the current schema.
+
+    Adds missing columns, renames legacy columns, drops stale artefacts,
+    and migrates data where needed.  Silently skips tables that don't
+    exist yet and columns that already exist.
+
+    Must run BEFORE ``PRAGMA foreign_keys=ON`` — the browser_visits →
+    visits rename triggers SQLite's schema rewriter which recompiles FK
+    references and fails on stale compiled references with FK enforcement.
+    """
+
+    # ── mcp_read / mcp_view on all entity tables ──
+    for table in ACCESS_CONTROL_TABLES:
+        for col, col_def in [
+            ("mcp_read", "TEXT DEFAULT 'inherit'"),
+            ("mcp_view", "TEXT DEFAULT 'inherit'"),
+        ]:
+            try:
+                cursor.execute(f"ALTER TABLE {table} ADD COLUMN {col} {col_def}")
+            except sqlite3.OperationalError:
+                pass  # table doesn't exist yet or column already exists
+
+    # Drop stale artefacts from chat_conversations → chats rename.
+    for name in (
+        "chat_conversations_ai",
+        "chat_conversations_ad",
+        "chat_conversations_au",
+        "chats_ai",
+        "chats_ad",
+        "chats_au",
+    ):
+        cursor.execute(f"DROP TRIGGER IF EXISTS {name}")
+    cursor.execute("DROP TABLE IF EXISTS chat_conversations_fts")
+    # If browser_visits still exists, the RENAME TO visits below will
+    # trigger SQLite's schema rewriter.  With foreign_keys ON, the
+    # rewriter recompiles FK references and hits the stale compiled
+    # chat_conversations FK.  Dropping chats_fts BEFORE the rename
+    # prevents corruption.  Skip on fresh/already-migrated DBs.
+    cursor.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='browser_visits'")
+    if cursor.fetchone() is not None:
+        for name in ("chats_fts_ai", "chats_fts_ad", "chats_fts_au"):
+            cursor.execute(f"DROP TRIGGER IF EXISTS {name}")
+        cursor.execute("DROP TABLE IF EXISTS chats_fts")
+
+    # Rename indexed_drive_id → remote_file_id, indexed_drive_folder_id → remote_folder_id
+    for old, new, table in [
+        ("indexed_drive_id", "remote_file_id", "files"),
+        ("indexed_drive_folder_id", "remote_folder_id", "folders"),
+    ]:
+        try:
+            cursor.execute(f"ALTER TABLE {table} RENAME COLUMN {old} TO {new}")
+        except sqlite3.OperationalError:
+            pass  # table doesn't exist yet or column already renamed
+
+    # Standardize column naming conventions
+    for old, new, table in [
+        ("last_scanned_at", "scanned_at", "folders"),
+        ("info_vectorized_at", "metadata_vectorized_at", "chats"),
+        ("direct_in_drive", "remote_file_count", "folders"),
+        ("total_in_drive", "remote_file_count_recursive", "folders"),
+        ("last_drive_check", "remote_checked_at", "folders"),
+    ]:
+        try:
+            cursor.execute(f"ALTER TABLE {table} RENAME COLUMN {old} TO {new}")
+        except sqlite3.OperationalError:
+            pass  # table doesn't exist yet or column already renamed
+
+    # Rename artifact_count → file_count (missed in artifacts → files rename)
+    for old, new, table in [
+        ("direct_artifact_count", "direct_file_count", "folders"),
+        ("total_artifact_count", "total_file_count", "folders"),
+    ]:
+        try:
+            cursor.execute(f"ALTER TABLE {table} RENAME COLUMN {old} TO {new}")
+        except sqlite3.OperationalError:
+            pass  # table doesn't exist yet or column already renamed
+
+    # Rename files.content_hash → sha256_hash
+    try:
+        cursor.execute("ALTER TABLE files RENAME COLUMN content_hash TO sha256_hash")
+    except sqlite3.OperationalError:
+        pass  # table doesn't exist yet or column already renamed
+
+    # Data migration: Drive files stored MD5 in content_hash — move to md5_hash
+    try:
+        cursor.execute("""
+            UPDATE files SET md5_hash = sha256_hash, sha256_hash = NULL
+            WHERE source != 'local' AND sha256_hash IS NOT NULL AND md5_hash IS NULL
+        """)
+    except sqlite3.OperationalError:
+        pass  # table doesn't exist yet on fresh install
+
+    # Drop duplicate total_size column (total_size_bytes is canonical)
+    try:
+        cursor.execute("ALTER TABLE folders DROP COLUMN total_size")
+    except sqlite3.OperationalError:
+        pass  # column doesn't exist or already dropped
+
+    # Drop dead columns: written but never read
+    # Include old name (counts_updated_at) for DBs that were never
+    # migrated through the rename step.
+    for col, table in [
+        ("stats_updated_at", "folders"),
+        ("counts_updated_at", "folders"),
+        ("summarized_at", "emails"),
+    ]:
+        try:
+            cursor.execute(f"ALTER TABLE {table} DROP COLUMN {col}")
+        except sqlite3.OperationalError:
+            pass  # column doesn't exist or table missing
+
+    # Drop orphan tables from old schema.
+    for table in ("artifact_sync_state", "file_ai_analysis", "permission_defaults", "visibility_defaults"):
+        cursor.execute(f"DROP TABLE IF EXISTS {table}")
+
+    # Retire dead tracking tables.
+    # Migrate the live watermark row before dropping the table.
+    # Create ingests early if needed — the main DDL is idempotent.
+    cursor.execute(_INGESTS_DDL)
+    try:
+        cursor.execute("SELECT stage, last_completed_at FROM pipeline_watermarks")
+        for row in cursor.fetchall():
+            stage = row[0] if isinstance(row, tuple) else row["stage"]
+            ts = row[1] if isinstance(row, tuple) else row["last_completed_at"]
+            if ts:
+                cursor.execute(
+                    "INSERT INTO ingests (pipe, started_at, completed_at, status) "
+                    "SELECT ?, ?, ?, 'completed' "
+                    "WHERE NOT EXISTS ("
+                    "  SELECT 1 FROM ingests WHERE pipe = ? AND completed_at = ?"
+                    ")",
+                    (stage, ts, ts, stage, ts),
+                )
+    except sqlite3.OperationalError:
+        pass  # table doesn't exist on fresh install
+    cursor.execute("DROP TABLE IF EXISTS pipeline_watermarks")
+    cursor.execute("DROP TABLE IF EXISTS runs")
+
+    # browser_visits columns added with status/client/project support
+    for col, col_def in [
+        ("status", "TEXT DEFAULT 'active'"),
+        ("client_id", "INTEGER"),
+        ("project_id", "INTEGER"),
+    ]:
+        try:
+            cursor.execute(f"ALTER TABLE browser_visits ADD COLUMN {col} {col_def}")
+        except sqlite3.OperationalError:
+            pass  # table doesn't exist yet or column already exists
+
+    # emails: add status column
+    try:
+        cursor.execute("ALTER TABLE emails ADD COLUMN status TEXT DEFAULT 'active'")
+    except sqlite3.OperationalError:
+        pass  # table doesn't exist yet or column already exists
+
+    # files: add client_id
+    try:
+        cursor.execute("ALTER TABLE files ADD COLUMN client_id INTEGER")
+    except sqlite3.OperationalError:
+        pass  # table doesn't exist yet or column already exists
+
+    # Rename browser_visits → visits
+    # Add mcp columns to old table first — the ACCESS_CONTROL_TABLES loop
+    # above targets "visits" which doesn't exist yet on legacy DBs.
+    for col, col_def in [
+        ("mcp_read", "TEXT DEFAULT 'inherit'"),
+        ("mcp_view", "TEXT DEFAULT 'inherit'"),
+    ]:
+        try:
+            cursor.execute(f"ALTER TABLE browser_visits ADD COLUMN {col} {col_def}")
+        except sqlite3.OperationalError:
+            pass  # table gone, or column exists
+    try:
+        for idx in [
+            "idx_browser_time",
+            "idx_browser_browser",
+            "idx_browser_visits_project",
+            "idx_browser_unique",
+            "idx_browser_visits_client",
+            "idx_browser_visits_status",
+            "idx_browser_visits_visibility",
+        ]:
+            cursor.execute(f"DROP INDEX IF EXISTS {idx}")
+        cursor.execute("ALTER TABLE browser_visits RENAME TO visits")
+    except sqlite3.OperationalError:
+        pass  # already renamed or fresh install
+
+    # clients/projects: add status_reason column
+    for table in ("clients", "projects"):
+        try:
+            cursor.execute(f"ALTER TABLE {table} ADD COLUMN status_reason TEXT")
+        except sqlite3.OperationalError:
+            pass  # table doesn't exist yet or column already exists
+
+    # ── standard entity column set ──
+
+    # folders: add status, client_id, indexed_at
+    for col, col_def in [
+        ("status", "TEXT DEFAULT 'active'"),
+        ("client_id", "INTEGER"),
+        ("indexed_at", "DATETIME"),
+    ]:
+        try:
+            cursor.execute(f"ALTER TABLE folders ADD COLUMN {col} {col_def}")
+        except sqlite3.OperationalError:
+            pass  # table doesn't exist yet or column already exists
+
+    # messages: add status
+    try:
+        cursor.execute("ALTER TABLE messages ADD COLUMN status TEXT DEFAULT 'active'")
+    except sqlite3.OperationalError:
+        pass
+
+    # emails: add created_at (no DEFAULT — see visits comment above)
+    try:
+        cursor.execute("ALTER TABLE emails ADD COLUMN created_at DATETIME")
+    except sqlite3.OperationalError:
+        pass
+
+    # visits / browser_visits: add created_at
+    # Note: ALTER TABLE cannot use CURRENT_TIMESTAMP as default
+    # (non-constant), so we add without default. The CREATE TABLE
+    # DDL has the default for fresh DBs.
+    for table in ("visits", "browser_visits"):
+        try:
+            cursor.execute(f"ALTER TABLE {table} ADD COLUMN created_at DATETIME")
+        except sqlite3.OperationalError:
+            pass
+
+    # display_name on all 8 entity tables
+    for table in ACCESS_CONTROL_TABLES:
+        try:
+            cursor.execute(f"ALTER TABLE {table} ADD COLUMN display_name TEXT")
+        except sqlite3.OperationalError:
+            pass
+
+    # Backfill display_name from source columns for existing rows
+    for table, source_col in _DISPLAY_NAME_BACKFILL.items():
+        try:
+            cursor.execute(f"UPDATE {table} SET display_name = {source_col} WHERE display_name IS NULL")
+        except sqlite3.OperationalError:
+            pass  # table doesn't exist yet
+
+    # ── Timestamp column standardization ──
+
+    # Rename chats.updated_at → modified_at (origin timestamp)
+    try:
+        cursor.execute("ALTER TABLE chats RENAME COLUMN updated_at TO modified_at")
+    except sqlite3.OperationalError:
+        pass  # already renamed or table doesn't exist
+
+    # Add updated_at audit column to all 6 entity tables
+    for table in ("files", "folders", "visits", "chats", "messages", "emails"):
+        try:
+            cursor.execute(f"ALTER TABLE {table} ADD COLUMN updated_at DATETIME")
+        except sqlite3.OperationalError:
+            pass  # column already exists or table doesn't exist
+
+    # Add indexed_at to messages (was missing)
+    try:
+        cursor.execute("ALTER TABLE messages ADD COLUMN indexed_at DATETIME")
+    except sqlite3.OperationalError:
+        pass  # column already exists
+
+    # Add vectorized_chunks to messages (matches files pattern)
+    try:
+        cursor.execute("ALTER TABLE messages ADD COLUMN vectorized_chunks INTEGER DEFAULT 0")
+    except sqlite3.OperationalError:
+        pass  # column already exists
+
+    # Backfill new audit columns from existing data
+    _timestamp_backfill = {
+        "files": "UPDATE files SET updated_at = indexed_at WHERE updated_at IS NULL",
+        "folders": "UPDATE folders SET updated_at = indexed_at WHERE updated_at IS NULL",
+        "visits": "UPDATE visits SET updated_at = indexed_at WHERE updated_at IS NULL",
+        "chats": "UPDATE chats SET updated_at = indexed_at WHERE updated_at IS NULL",
+        "emails": "UPDATE emails SET updated_at = indexed_at WHERE updated_at IS NULL",
+        "messages_indexed": "UPDATE messages SET indexed_at = created_at WHERE indexed_at IS NULL",
+        "messages_updated": "UPDATE messages SET updated_at = created_at WHERE updated_at IS NULL",
+    }
+    for label, sql in _timestamp_backfill.items():
+        try:
+            cursor.execute(sql)
+        except sqlite3.OperationalError:
+            pass  # table doesn't exist yet