footprinter-cli 1.0.0__py3-none-any.whl

footprinter/db/sql_utils.py

@@ -0,0 +1,168 @@
+"""Shared SQL helper functions for building dynamic CASE/WHEN clauses,
+pagination utilities, and chunked query execution."""
+
+import sqlite3
+
+# Stay well under SQLite's variable limit (999 on older builds, 32766 on newer).
+_SQLITE_VAR_LIMIT = 500
+
+
+def chunked_query(cursor, sql_template: str, item_ids: list[int]) -> list:
+    """Execute a query in chunks to stay under SQLite's variable limit.
+
+    The *sql_template* must contain a ``{placeholders}`` marker where the
+    ``IN (?, ?, ...)`` list will be inserted.
+    """
+    results = []
+    for i in range(0, len(item_ids), _SQLITE_VAR_LIMIT):
+        chunk = item_ids[i : i + _SQLITE_VAR_LIMIT]
+        placeholders = ",".join("?" * len(chunk))
+        sql = sql_template.format(placeholders=placeholders)
+        cursor.execute(sql, chunk)
+        results.extend(cursor.fetchall())
+    return results
+
+
+def paginate(
+    conn: sqlite3.Connection,
+    count_sql: str,
+    fetch_sql: str,
+    params,
+    *,
+    page: int = 1,
+    limit: int = 50,
+) -> tuple[list, dict]:
+    """Execute a count + paginated fetch and return (rows, pagination_dict)."""
+    total = conn.execute(count_sql, list(params)).fetchone()[0]
+    total_pages = max(1, (total + limit - 1) // limit)
+    offset = (page - 1) * limit
+    rows = conn.execute(fetch_sql, list(params) + [limit, offset]).fetchall()
+    return rows, {"page": page, "limit": limit, "total": total, "total_pages": total_pages}
+
+
+def paginated_response(entity_key: str, items, pagination: dict, **extras) -> dict:
+    """Build a standard paginated response envelope."""
+    return {entity_key: items, "pagination": pagination, **extras}
+
+
+def build_location_case_clauses(
+    config: dict, home: str, path_col: str = "path", prefix: str = ""
+) -> tuple[list[str], list]:
+    """Build CASE/WHEN SQL clauses from config directories.
+    Returns (case_lines: list[str], params: list)."""
+    dirs = config.get("directories", [])
+    case_lines = []
+    params = []
+    for d in dirs:
+        expanded = d.replace("~", home)
+        label = d.rstrip("/").split("/")[-1]
+        if not label.startswith("."):
+            label = label.title()
+        case_lines.append(f"WHEN {path_col} LIKE ? THEN ?")
+        params.extend([expanded + "/%", prefix + label])
+    return case_lines, params
+
+
+def build_remote_source_label_clauses(
+    sources_data: list[dict],
+) -> tuple[list[str], list]:
+    """Build CASE/WHEN clauses for remote source labels.
+    Returns (case_lines: list[str], params: list)."""
+    case_lines = []
+    params = []
+    for s in sources_data:
+        if s["source_type"] == "remote":
+            case_lines.append("WHEN source = ? THEN ?")
+            params.extend([s["name"], f"Drive (indexed): {s['account']}"])
+    return case_lines, params
+
+
+def build_status_filter(
+    status: "str | list[str] | None",
+    *,
+    column: str,
+    default_exclude: "list[str] | None" = None,
+    default_include: "list[str] | None" = None,
+) -> tuple[list[str], list]:
+    """Build a status filter clause for dynamic WHERE construction.
+
+    Returns (conditions, params) where conditions is a list of 0 or 1
+    SQL fragments suitable for extending a WHERE clause.
+
+    Parameters
+    ----------
+    status : str, list[str], or None
+        ``None`` → apply default filter.
+        ``"all"`` → no filter (bypass defaults).
+        Single string → exact match.
+        List of strings → IN clause.  Empty list → no filter.
+    column : str
+        Fully qualified column reference (e.g. ``"file.status"``).
+    default_exclude : list[str], optional
+        Statuses to exclude when ``status is None``.
+    default_include : list[str], optional
+        Statuses to include when ``status is None``.
+
+    ``default_exclude`` and ``default_include`` are mutually exclusive.
+    If both are provided, ``default_exclude`` takes precedence.
+    """
+    if status == "all":
+        return [], []
+
+    if status is None:
+        if default_exclude:
+            placeholders = ",".join("?" for _ in default_exclude)
+            return [f"{column} NOT IN ({placeholders})"], list(default_exclude)
+        if default_include:
+            placeholders = ",".join("?" for _ in default_include)
+            return [f"{column} IN ({placeholders})"], list(default_include)
+        return [], []
+
+    if isinstance(status, list):
+        if not status:
+            return [], []
+        placeholders = ",".join("?" for _ in status)
+        return [f"{column} IN ({placeholders})"], list(status)
+
+    # Single string — exact match
+    return [f"{column} = ?"], [status]
+
+
+def split_query_terms(query: str) -> list[str]:
+    """Split query on whitespace, dropping terms shorter than 2 chars."""
+    return [t for t in query.split() if len(t) >= 2]
+
+
+def build_fts5_query(terms: list[str]) -> str:
+    """Build an FTS5 query with AND semantics and prefix matching."""
+    sanitized = [term.replace('"', "") for term in terms]
+    return " ".join(f'"{term}"*' for term in sanitized if len(term) >= 2)
+
+
+def build_term_conditions(
+    columns: list[str],
+    terms: list[str],
+) -> tuple[str, list[str]]:
+    """Build AND-ed LIKE conditions: every term must appear in at least one column."""
+    groups = []
+    params: list[str] = []
+    for term in terms:
+        like = f"%{term}%"
+        col_parts = [f"{col} LIKE ?" for col in columns]
+        groups.append(f"({' OR '.join(col_parts)})")
+        params.extend([like] * len(columns))
+    return " AND ".join(groups), params
+
+
+def build_remote_account_case_clauses(
+    sources_data: list[dict],
+) -> tuple[list[str], list]:
+    """Build CASE/WHEN clauses mapping remote source names to accounts.
+    Returns (case_lines: list[str], params: list)."""
+    case_lines = []
+    params = []
+    for s in sources_data:
+        if s["source_type"] == "remote":
+            case_lines.append("WHEN ? THEN ?")
+            params.extend([s["name"], s["account"]])
+    return case_lines, params

footprinter/db/status.py

@@ -0,0 +1,320 @@
+"""System status queries — table counts, config presence, last-indexed timestamp.
+
+Includes both MCP-oriented status and the legacy get_stats.
+"""
+
+import sqlite3
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+
+def _safe_count(cursor: sqlite3.Cursor, query: str) -> int:
+    """Execute a COUNT query, returning 0 if the table doesn't exist."""
+    try:
+        cursor.execute(query)
+        return cursor.fetchone()[0]
+    except sqlite3.OperationalError:
+        return 0
+
+
+def _safe_query(conn: sqlite3.Connection, query: str, *, default: Any = None) -> Any:
+    """Execute a query and return the first column of the first row.
+
+    Returns ``default`` if the table doesn't exist or the query returns no rows.
+    """
+    try:
+        row = conn.execute(query).fetchone()
+        return row[0] if row else default
+    except sqlite3.OperationalError:
+        return default
+
+
+def _safe_fetchall(conn: sqlite3.Connection, query: str) -> list[sqlite3.Row]:
+    """Execute a query and return all rows, or [] on missing table."""
+    try:
+        return conn.execute(query).fetchall()
+    except sqlite3.OperationalError:
+        return []
+
+
+# -- Hidden-client NOT EXISTS clause (reused across source queries) -----------
+_NOT_HIDDEN_CLIENT = (
+    "NOT EXISTS (  SELECT 1 FROM clients client  WHERE client.id = {alias}.client_id AND client.mcp_view = 'hidden')"
+)
+
+
+def get_mcp_status(conn: sqlite3.Connection) -> dict:
+    """Return MCP-oriented status: per-source counts excluding hidden clients.
+
+    Designed for ``role=Role.VIEWER`` callers — hidden-client rows are
+    excluded from emails, chats, messages, and browser counts.
+    """
+    # -- Per-table counts and last-sync timestamps ----------------------------
+    tables: Dict[str, Dict[str, Optional[str]]] = {
+        "files": {
+            "count": (
+                "SELECT COUNT(*) FROM files WHERE status != 'removed' AND COALESCE(mcp_view, 'inherit') != 'hidden'"
+            ),
+            "latest": (
+                "SELECT MAX(indexed_at) FROM files "
+                "WHERE status != 'removed' "
+                "AND COALESCE(mcp_view, 'inherit') != 'hidden'"
+            ),
+        },
+        "emails": {
+            "count": (
+                "SELECT COUNT(*) FROM emails email "
+                "WHERE email.status != 'removed' "
+                f"AND {_NOT_HIDDEN_CLIENT.format(alias='email')}"
+            ),
+            "latest": (
+                "SELECT MAX(indexed_at) FROM emails email "
+                "WHERE email.status != 'removed' "
+                f"AND {_NOT_HIDDEN_CLIENT.format(alias='email')}"
+            ),
+        },
+        "chats": {
+            "count": (
+                "SELECT COUNT(*) FROM chats chat "
+                "WHERE COALESCE(chat.status, 'active') != 'removed' "
+                f"AND {_NOT_HIDDEN_CLIENT.format(alias='chat')}"
+            ),
+            "latest": (
+                "SELECT MAX(modified_at) FROM chats chat "
+                "WHERE COALESCE(chat.status, 'active') != 'removed' "
+                f"AND {_NOT_HIDDEN_CLIENT.format(alias='chat')}"
+            ),
+        },
+        "messages": {
+            "count": (
+                "SELECT COUNT(*) FROM messages message "
+                "WHERE NOT EXISTS ("
+                "  SELECT 1 FROM chats chat"
+                "  JOIN clients client ON client.id = chat.client_id"
+                "  WHERE chat.id = message.chat_id AND client.mcp_view = 'hidden'"
+                ")"
+            ),
+            "latest": (
+                "SELECT MAX(created_at) FROM messages message "
+                "WHERE NOT EXISTS ("
+                "  SELECT 1 FROM chats chat"
+                "  JOIN clients client ON client.id = chat.client_id"
+                "  WHERE chat.id = message.chat_id AND client.mcp_view = 'hidden'"
+                ")"
+            ),
+        },
+        "browser": {
+            "count": (
+                "SELECT COUNT(*) FROM visits bv "
+                "WHERE bv.status != 'removed' "
+                f"AND {_NOT_HIDDEN_CLIENT.format(alias='bv')}"
+            ),
+            "latest": (
+                "SELECT MAX(visit_time) FROM visits bv "
+                "WHERE bv.status != 'removed' "
+                f"AND {_NOT_HIDDEN_CLIENT.format(alias='bv')}"
+            ),
+        },
+        "projects": {
+            "count": "SELECT COUNT(*) FROM projects",
+            "latest": "SELECT MAX(created_at) FROM projects",
+        },
+        "clients": {
+            "count": ("SELECT COUNT(*) FROM clients WHERE COALESCE(mcp_view, 'inherit') != 'hidden'"),
+            "latest": None,
+        },
+    }
+
+    sources: Dict[str, dict] = {}
+    for table, queries in tables.items():
+        count = _safe_query(conn, queries["count"], default=0)
+        latest = _safe_query(conn, queries["latest"]) if queries["latest"] else None
+        if count == 0 and latest is None:
+            # Confirm count isn't masking a missing-table error
+            try:
+                conn.execute(queries["count"])
+            except sqlite3.OperationalError:
+                sources[table] = {"count": 0, "last_sync": None, "error": "table not found"}
+                continue
+        sources[table] = {"count": count, "last_sync": latest}
+
+    # -- File breakdown by source ---------------------------------------------
+    rows = _safe_fetchall(
+        conn,
+        """
+        SELECT source, COUNT(*) as count, COALESCE(SUM(size_bytes), 0) as size
+        FROM files WHERE status != 'removed'
+        AND COALESCE(mcp_view, 'inherit') != 'hidden'
+        GROUP BY source
+    """,
+    )
+    files_by_source = {r["source"]: {"count": r["count"], "size_bytes": r["size"]} for r in rows}
+
+    # -- File breakdown by status ---------------------------------------------
+    rows = _safe_fetchall(
+        conn,
+        """
+        SELECT status, COUNT(*) as count FROM files GROUP BY status
+    """,
+    )
+    files_by_status = {r["status"]: r["count"] for r in rows}
+
+    # -- Project count by status ----------------------------------------------
+    rows = _safe_fetchall(
+        conn,
+        """
+        SELECT status, COUNT(*) as count FROM projects GROUP BY status
+    """,
+    )
+    projects_by_status = {r["status"]: r["count"] for r in rows}
+
+    # -- Emails by client (excludes hidden clients and removed emails) --------
+    rows = _safe_fetchall(
+        conn,
+        """
+        SELECT COALESCE(client.name, '(unassigned)') AS client_name, COUNT(*) as count
+        FROM emails email LEFT JOIN clients client ON email.client_id = client.id
+        WHERE email.status != 'removed'
+          AND (client.mcp_view IS NULL OR client.mcp_view != 'hidden')
+        GROUP BY client_name
+    """,
+    )
+    emails_by_client = {r["client_name"]: r["count"] for r in rows}
+
+    # -- Chats by client (excludes hidden clients and removed chats) ----------
+    rows = _safe_fetchall(
+        conn,
+        """
+        SELECT COALESCE(client.name, '(unassigned)') AS client_name, COUNT(*) as count
+        FROM chats chat LEFT JOIN clients client ON chat.client_id = client.id
+        WHERE chat.status != 'removed'
+          AND (client.mcp_view IS NULL OR client.mcp_view != 'hidden')
+        GROUP BY client_name
+    """,
+    )
+    chats_by_client = {r["client_name"]: r["count"] for r in rows}
+
+    return {
+        "sources": sources,
+        "files_by_source": files_by_source,
+        "files_by_status": files_by_status,
+        "projects_by_status": projects_by_status,
+        "emails_by_client": emails_by_client,
+        "chats_by_client": chats_by_client,
+    }
+
+
+def get_system_status(conn: sqlite3.Connection, config_path: Path) -> dict:
+    """Return system status dict with table counts, data presence, and config check.
+
+    Parameters
+    ----------
+    conn : sqlite3.Connection
+        Database connection with row_factory = sqlite3.Row.
+    config_path : Path
+        Absolute path to config.yaml (caller decides where it lives).
+
+    Returns
+    -------
+    dict with keys: has_data, config_exists, counts, total, last_indexed
+    """
+    cur = conn.cursor()
+
+    counts = {
+        "files": _safe_count(cur, "SELECT COUNT(*) FROM files WHERE status != 'removed'"),
+        "folders": _safe_count(cur, "SELECT COUNT(*) FROM folders WHERE status != 'removed'"),
+        "visits": _safe_count(cur, "SELECT COUNT(*) FROM visits"),
+        "emails": _safe_count(cur, "SELECT COUNT(*) FROM emails"),
+        "messages": _safe_count(cur, "SELECT COUNT(*) FROM messages"),
+        "projects": _safe_count(cur, "SELECT COUNT(*) FROM projects"),
+    }
+
+    try:
+        cur.execute("SELECT MAX(indexed_at) FROM files")
+        last_indexed = cur.fetchone()[0]
+    except sqlite3.OperationalError:
+        last_indexed = None
+
+    total = sum(counts.values())
+
+    return {
+        "has_data": total > 0,
+        "config_exists": config_path.exists(),
+        "counts": counts,
+        "total": total,
+        "last_indexed": last_indexed,
+    }
+
+
+# ---------------------------------------------------------------------------
+# Write/aggregate operations
+# ---------------------------------------------------------------------------
+
+
+def get_stats(conn: sqlite3.Connection) -> Dict[str, Any]:
+    """Get database statistics."""
+    cursor = conn.cursor()
+
+    cursor.execute("SELECT COUNT(*) as count FROM files WHERE status != 'removed'")
+    files_count = cursor.fetchone()["count"]
+
+    cursor.execute("SELECT COUNT(DISTINCT content_type) as count FROM files WHERE status != 'removed'")
+    content_types_count = cursor.fetchone()["count"]
+
+    cursor.execute("SELECT COUNT(*) as count FROM visits")
+    urls_count = cursor.fetchone()["count"]
+
+    cursor.execute("SELECT COUNT(DISTINCT browser) as count FROM visits")
+    browsers_count = cursor.fetchone()["count"]
+
+    cursor.execute(
+        """
+        SELECT MAX(modified_at) as latest_file
+        FROM files
+        WHERE modified_at IS NOT NULL AND status != 'removed'
+    """
+    )
+    latest_file = cursor.fetchone()["latest_file"]
+
+    cursor.execute("SELECT MAX(visit_time) as latest_visit FROM visits")
+    latest_visit = cursor.fetchone()["latest_visit"]
+
+    # Query remote source names directly from the sources table
+    rows = cursor.execute("SELECT name FROM sources WHERE source_type = 'remote'").fetchall()
+    remote_sources = [r["name"] for r in rows]
+
+    if remote_sources:
+        remote_ph = ",".join("?" * len(remote_sources))
+        cursor.execute(
+            f"SELECT COUNT(*) as count FROM files WHERE source IN ({remote_ph}) AND status != 'removed'",
+            remote_sources,
+        )
+        remote_files_count = cursor.fetchone()["count"]
+
+        cursor.execute(
+            f"SELECT COUNT(DISTINCT source) as count FROM files WHERE source IN ({remote_ph}) AND status != 'removed'",
+            remote_sources,
+        )
+        remote_sources_count = cursor.fetchone()["count"]
+    else:
+        remote_files_count = 0
+        remote_sources_count = 0
+
+    cursor.execute("SELECT COUNT(*) as count FROM emails")
+    emails_count = cursor.fetchone()["count"]
+
+    cursor.execute("SELECT COUNT(DISTINCT account) as count FROM emails")
+    email_accounts_count = cursor.fetchone()["count"]
+
+    return {
+        "total_files": files_count,
+        "content_types": content_types_count,
+        "total_urls": urls_count,
+        "browsers": browsers_count,
+        "latest_file_modified": latest_file,
+        "latest_visit": latest_visit,
+        "remote_files": remote_files_count,
+        "remote_sources": remote_sources_count,
+        "total_emails": emails_count,
+        "email_accounts": email_accounts_count,
+    }

footprinter/db/uploads.py

@@ -0,0 +1,70 @@
+"""Upload tracking — CRUD for the uploads table."""
+
+import json
+import sqlite3
+from typing import Any, Dict, List, Optional
+
+
+def create_upload(conn: sqlite3.Connection, data: Dict[str, Any]) -> int:
+    """Create a new upload record."""
+    cursor = conn.cursor()
+    cursor.execute(
+        """
+        INSERT INTO uploads
+        (filename, file_hash, file_size, type, source, status, metadata)
+        VALUES (?, ?, ?, ?, ?, ?, ?)
+    """,
+        (
+            data["filename"],
+            data["file_hash"],
+            data.get("file_size"),
+            data["type"],
+            data.get("source"),
+            data.get("status", "pending"),
+            json.dumps(data.get("metadata", {})),
+        ),
+    )
+    return cursor.lastrowid
+
+
+def update_upload(conn: sqlite3.Connection, upload_id: int, **kwargs) -> None:
+    """Update an upload record with results."""
+    allowed = {
+        "status",
+        "items_added",
+        "items_updated",
+        "items_total",
+        "completed_at",
+        "error_message",
+    }
+    updates, values = [], []
+    for field, value in kwargs.items():
+        if field in allowed:
+            updates.append(f"{field} = ?")
+            values.append(value)
+    if updates:
+        values.append(upload_id)
+        cursor = conn.cursor()
+        cursor.execute(f"UPDATE uploads SET {', '.join(updates)} WHERE id = ?", values)
+        conn.commit()
+
+
+def get_upload_by_hash(conn: sqlite3.Connection, file_hash: str) -> Optional[Dict]:
+    """Check if a file was already uploaded."""
+    cursor = conn.cursor()
+    cursor.execute("SELECT * FROM uploads WHERE file_hash = ?", (file_hash,))
+    row = cursor.fetchone()
+    return dict(row) if row else None
+
+
+def get_recent_uploads(conn: sqlite3.Connection, upload_type: Optional[str] = None, limit: int = 10) -> List[Dict]:
+    """Get recent uploads, optionally filtered by type."""
+    cursor = conn.cursor()
+    if upload_type:
+        cursor.execute(
+            "SELECT * FROM uploads WHERE type = ? ORDER BY uploaded_at DESC LIMIT ?",
+            (upload_type, limit),
+        )
+    else:
+        cursor.execute("SELECT * FROM uploads ORDER BY uploaded_at DESC LIMIT ?", (limit,))
+    return [dict(row) for row in cursor.fetchall()]

footprinter/ingest/adapters/__init__.py

@@ -0,0 +1,33 @@
+"""Pipeline adapter types and concrete adapters.
+
+Re-exports the core types and built-in source adapters:
+
+    from footprinter.ingest.adapters import PipeAdapter, PipeResult
+    from footprinter.ingest.adapters import LocalFoldersAdapter, BrowserAdapter
+
+Connector adapters (Drive, Gmail) live in connectors/google/adapters/.
+"""
+
+from footprinter.ingest.adapters.browser import BrowserAdapter
+from footprinter.ingest.adapters.chat import ChatAdapter
+from footprinter.ingest.adapters.local_files import LocalFilesAdapter
+from footprinter.ingest.adapters.local_folders import LocalFoldersAdapter
+from footprinter.ingest.adapters.protocol import (
+    ErrorType,
+    PipeAdapter,
+    PipeContext,
+    PipeResult,
+    PipeStatus,
+)
+
+__all__ = [
+    "BrowserAdapter",
+    "ChatAdapter",
+    "PipeAdapter",
+    "PipeContext",
+    "ErrorType",
+    "LocalFilesAdapter",
+    "LocalFoldersAdapter",
+    "PipeResult",
+    "PipeStatus",
+]

footprinter/ingest/adapters/browser.py

@@ -0,0 +1,54 @@
+"""Browser adapter.
+
+Wraps BrowserManager to conform to PipeAdapter protocol.
+"""
+
+from __future__ import annotations
+
+import logging
+from functools import partial
+from typing import Any, Dict, List
+
+from footprinter.db import browser as browser_db
+from footprinter.ingest.adapters.ingest import ingest_entries
+from footprinter.ingest.adapters.protocol import ErrorType, PipeContext, PipeResult
+from footprinter.ingest.browser_indexer import BrowserManager
+
+logger = logging.getLogger(__name__)
+
+
+class BrowserAdapter:
+    """Adapter wrapping BrowserManager for the browser stage."""
+
+    name = "browser"
+    pipe_name = "browser"
+    required_extras: List[str] = []
+
+    def run(self, db: Any, ctx: PipeContext) -> PipeResult:
+        """Index browser history into visits table."""
+        try:
+            last_run = None if ctx.full_mode else ctx.last_run
+            manager = BrowserManager(ctx.source_config, since=last_run)
+            result = ingest_entries(
+                "browser",
+                manager.parse_all(),
+                partial(browser_db.insert_visit, db.conn),
+                count_label="urls_indexed",
+                conn=db.conn,
+                on_progress=ctx.on_progress,
+            )
+            return result
+        except Exception as e:
+            logger.error(f"browser stage failed: {e}")
+            return PipeResult.make_error(
+                "browser",
+                error=str(e),
+                error_type=ErrorType.RUNTIME,
+            )
+
+    def status(self, db: Any) -> Dict[str, Any]:
+        """Return browser visit entry count."""
+        cursor = db.conn.cursor()
+        cursor.execute("SELECT COUNT(*) FROM visits")
+        count = cursor.fetchone()[0]
+        return {"visits": count}