footprinter-cli 1.0.0__py3-none-any.whl

footprinter/services/ingest_service.py

@@ -0,0 +1,231 @@
+"""IngestService — single authority on ingest tracking and FTS optimization.
+
+All ingest operations (pipeline and non-pipeline) flow through this service.
+Wraps PipeRunner for pipeline ingests; called directly for non-pipeline ingests.
+Manages FTS trigger lifecycle around batch runs.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import sqlite3
+from datetime import datetime
+from typing import TYPE_CHECKING, Callable, List, Optional
+
+from footprinter.utils.time import utc_now_iso
+
+if TYPE_CHECKING:
+    from footprinter.ingest.database import Database
+
+log = logging.getLogger(__name__)
+
+
+class IngestService:
+    """Track ingest lifecycle: begin, complete, fail, query history.
+
+    Optionally manages FTS trigger optimization around batch runs
+    when constructed with a ``get_db`` callable.
+    """
+
+    def __init__(
+        self,
+        conn: sqlite3.Connection,
+        get_db: Optional[Callable[[], Database]] = None,
+    ) -> None:
+        self.conn = conn
+        self._get_db = get_db
+
+    def begin(
+        self,
+        pipe: str,
+        mode: str | None = None,
+        trigger: str | None = None,
+    ) -> int:
+        """Insert a running ingest record and return its id."""
+        cursor = self.conn.execute(
+            "INSERT INTO ingests (pipe, started_at, status, mode, trigger) VALUES (?, ?, 'running', ?, ?)",
+            (pipe, utc_now_iso(), mode, trigger),
+        )
+        self.conn.commit()
+        return cursor.lastrowid  # type: ignore[return-value]
+
+    def complete(
+        self,
+        ingest_id: int,
+        result: dict | None = None,
+        metadata: dict | None = None,
+    ) -> None:
+        """Mark an ingest as completed with optional result counts and metadata."""
+        result = result or {}
+        meta_json = json.dumps(metadata) if metadata is not None else None
+        self.conn.execute(
+            "UPDATE ingests SET"
+            " status = 'completed',"
+            " completed_at = ?,"
+            " items_processed = ?,"
+            " items_new = ?,"
+            " items_updated = ?,"
+            " items_skipped = ?,"
+            " errors = ?,"
+            " elapsed_seconds = ?,"
+            " metadata = ?"
+            " WHERE id = ?",
+            (
+                utc_now_iso(),
+                result.get("items_processed", 0),
+                result.get("items_new", 0),
+                result.get("items_updated", 0),
+                result.get("items_skipped", 0),
+                result.get("errors", 0),
+                result.get("elapsed_seconds"),
+                meta_json,
+                ingest_id,
+            ),
+        )
+        self.conn.commit()
+
+    def fail(self, ingest_id: int, error: str) -> None:
+        """Mark an ingest as failed with an error message."""
+        self.conn.execute(
+            "UPDATE ingests SET status = 'failed', completed_at = ?, metadata = ? WHERE id = ?",
+            (
+                utc_now_iso(),
+                json.dumps({"error": error}),
+                ingest_id,
+            ),
+        )
+        self.conn.commit()
+
+    def last_run(self, pipe: str) -> datetime | None:
+        """Return the completed_at timestamp of the most recent successful ingest."""
+        row = self.conn.execute(
+            "SELECT completed_at FROM ingests"
+            " WHERE pipe = ? AND status = 'completed'"
+            " ORDER BY completed_at DESC LIMIT 1",
+            (pipe,),
+        ).fetchone()
+        if row is None:
+            return None
+        return datetime.fromisoformat(row["completed_at"])
+
+    def run_pipe(
+        self,
+        pipe: str,
+        *,
+        mode: str | None = None,
+        trigger: str | None = None,
+        runner,
+        on_progress=None,
+    ) -> dict:
+        """Wrap a PipeRunner.run_pipe call with ingest tracking.
+
+        Creates an ingests record, passes last_run to runner.run_pipe(),
+        then records completion or failure.
+        """
+        ingest_id = self.begin(pipe, mode=mode, trigger=trigger)
+        try:
+            result = runner.run_pipe(pipe, on_progress=on_progress, last_run=self.last_run(pipe))
+            if result.get("status") == "error":
+                self.fail(ingest_id, error=result.get("error", "unknown"))
+            else:
+                self.complete(ingest_id, result=result)
+            return result
+        except Exception as e:
+            self.fail(ingest_id, error=str(e))
+            raise
+
+    def history(self, pipe: str, limit: int = 20) -> list[dict]:
+        """Return recent ingest records for a pipe, most recent first."""
+        rows = self.conn.execute(
+            "SELECT * FROM ingests WHERE pipe = ? ORDER BY started_at DESC LIMIT ?",
+            (pipe, limit),
+        ).fetchall()
+        records = []
+        for row in rows:
+            record = dict(row)
+            if record.get("metadata") is not None:
+                record["metadata"] = json.loads(record["metadata"])
+            records.append(record)
+        return records
+
+    # ── FTS optimization ────────────────────────────────────────────
+
+    def ensure_fts_health(self, full_mode: bool) -> None:
+        """Check FTS health and restore missing triggers.
+
+        Always probes FTS health (both modes). In incremental mode, also
+        restores missing triggers from a prior crash (SIGKILL/OOM during
+        a full-mode run) before the health probe. Trigger restore is
+        skipped in full mode because triggers are dropped anyway.
+
+        No-op if constructed without ``get_db``.
+        """
+        if self._get_db is None:
+            return
+        try:
+            db = self._get_db()
+            if db is None:
+                return
+            if not full_mode:
+                missing_triggers = db.check_fts_triggers()
+                if missing_triggers:
+                    log.info(
+                        "Restoring %d missing FTS triggers from prior crash",
+                        len(missing_triggers),
+                    )
+                    db.create_fts_triggers()
+            fts_health = db.check_fts_health()
+            for table, info in fts_health.items():
+                if info["status"] == "error":
+                    log.warning(
+                        "FTS index corrupted (%s) — run 'fp ingest --repair-fts' to fix",
+                        table,
+                    )
+        except sqlite3.OperationalError as e:
+            log.debug("FTS health probe skipped: %s", e)
+
+    def run_pipes(
+        self,
+        pipes: List[str],
+        *,
+        runner,
+        full_mode: bool = False,
+        on_pipe_start: Optional[Callable] = None,
+        on_pipe_end: Optional[Callable] = None,
+        on_progress: Optional[Callable] = None,
+        pipe_hook: Optional[Callable] = None,
+    ) -> List[dict]:
+        """Run multiple pipes with FTS optimization around the batch.
+
+        In full mode, drops FTS triggers before the first pipe and rebuilds
+        FTS indexes after the last pipe (or on error) to avoid per-row
+        trigger overhead during bulk ingest. FTS optimization requires
+        ``get_db`` — silently skipped if constructed without it.
+        """
+        self.ensure_fts_health(full_mode)
+
+        fts_dropped = False
+        if full_mode and self._get_db is not None:
+            try:
+                db = self._get_db()
+                db.drop_fts_triggers()
+                fts_dropped = True
+            except sqlite3.OperationalError as e:
+                log.warning("Failed to drop FTS triggers: %s", e)
+
+        try:
+            return runner.run_pipes(
+                pipes,
+                on_pipe_start=on_pipe_start,
+                on_pipe_end=on_pipe_end,
+                on_progress=on_progress,
+                pipe_hook=pipe_hook,
+            )
+        finally:
+            if fts_dropped:
+                try:
+                    db = self._get_db()
+                    db.rebuild_fts_indexes()
+                except sqlite3.OperationalError as e:
+                    log.error("Failed to rebuild FTS indexes: %s", e)

footprinter/services/project_service.py

@@ -0,0 +1,262 @@
+"""Project service — get/list with role-based visibility, upsert and soft delete."""
+
+import sqlite3
+from typing import Optional
+
+from footprinter.db import projects as db
+from footprinter.services.access_service import (
+    _read_visibility,
+    filter_result,
+    filter_results_list,
+)
+from footprinter.services.includes import validate_include
+from footprinter.services.roles import Role
+
+VALID_INCLUDES = frozenset({"files", "folders"})
+
+
+def get(
+    conn: sqlite3.Connection,
+    project_id: int,
+    *,
+    role: Role = Role.ADMIN,
+    include: list[str] | None = None,
+) -> dict | None:
+    """Fetch a single project by ID, filtered by role.
+
+    Pass ``include`` to attach nested data:
+    - ``"files"`` — paginated list of files in this project
+    - ``"folders"`` — list of folders in this project
+    """
+    includes = validate_include(include, VALID_INCLUDES)
+    result = db.get_project_detail(conn, project_id)
+    if result is None:
+        return None
+
+    # Attach includes only when caller has full access to this entity
+    is_full = role.sees_all or _read_visibility(result) == "visible"
+    if is_full and includes:
+        if "files" in includes:
+            from footprinter.services import file_service
+
+            resp = file_service.list_(conn, role=role, project_id=project_id)
+            result["files"] = resp["files"]
+        if "folders" in includes:
+            from footprinter.services import folder_service
+
+            resp = folder_service.list_(conn, role=role, project_id=project_id, depth=None)
+            result["folders"] = resp["folders"]
+
+    if role.sees_all:
+        return result
+    return filter_result("project", result)
+
+
+def list_(
+    conn: sqlite3.Connection,
+    *,
+    role: Role = Role.ADMIN,
+    include: list[str] | None = None,
+    status: Optional[str | list[str]] = None,
+    client: Optional[str] = None,
+    project_type: Optional[str] = None,
+    limit: int = 50,
+    page: int = 1,
+) -> dict:
+    """List projects with pagination, filtered by role."""
+    includes = validate_include(include, VALID_INCLUDES)
+    response = db.list_projects(
+        conn,
+        status=status,
+        client=client,
+        project_type=project_type,
+        limit=limit,
+        page=page,
+    )
+
+    # Track which items are fully visible before filtering strips fields
+    visible_ids: set[int] = set()
+    if includes and not role.sees_all:
+        visible_ids = {p["id"] for p in response["projects"] if _read_visibility(p) == "visible"}
+
+    if not role.sees_all:
+        filtered, suppressed = filter_results_list("project", response["projects"])
+        response["projects"] = filtered
+        response["suppressed"] = suppressed
+
+    if includes:
+        for project in response["projects"]:
+            if not role.sees_all and project["id"] not in visible_ids:
+                continue
+            if "files" in includes:
+                from footprinter.services import file_service
+
+                resp = file_service.list_(conn, role=role, project_id=project["id"])
+                project["files"] = resp["files"]
+            if "folders" in includes:
+                from footprinter.services import folder_service
+
+                resp = folder_service.list_(
+                    conn,
+                    role=role,
+                    project_id=project["id"],
+                    depth=None,
+                )
+                project["folders"] = resp["folders"]
+
+    return response
+
+
+def resolve_by_name(
+    conn: sqlite3.Connection,
+    name: str,
+    *,
+    role: Role = Role.ADMIN,
+) -> dict | None:
+    """Resolve a project by fuzzy name match, with navigation data.
+
+    Returns:
+        Full navigation dict for single match.
+        Disambiguation dict for multiple ambiguous matches.
+        None for no match (or hidden-only matches for VIEWER).
+    """
+    rows = db.find_by_name_fuzzy(conn, name)
+    if not rows:
+        return None
+
+    # Filter hidden for VIEWER
+    if not role.sees_all:
+        rows = [r for r in rows if _read_visibility(r) != "hidden"]
+    if not rows:
+        return None
+
+    if len(rows) == 1:
+        return _build_project_navigation(conn, rows[0], role=role)
+
+    # Check exact match (case-insensitive)
+    exact = [r for r in rows if r["project_name"].lower() == name.lower()]
+    if len(exact) == 1:
+        return _build_project_navigation(conn, exact[0], role=role)
+
+    # Disambiguation
+    return _build_disambiguation(rows, "project_name", name, role)
+
+
+def _build_project_navigation(conn: sqlite3.Connection, row: dict, *, role: Role) -> dict:
+    """Build full project navigation dict from a project row."""
+    visibility = _read_visibility(row)
+    if not role.sees_all and visibility == "opaque":
+        return filter_result("project", row)
+
+    nav = db.get_project_navigation(conn, row["id"])
+    result = {**row, **nav}
+
+    if role.sees_all:
+        return result
+
+    # Filter child folders by visibility
+    result["folders"], _ = filter_results_list("folder", result["folders"])
+    return result
+
+
+def _build_disambiguation(rows: list[dict], name_col: str, query: str, role: Role) -> dict:
+    """Build a disambiguation dict from multiple matches."""
+    from footprinter.services.access_service import resolve_inherit_visibility
+
+    matches = []
+    for r in rows:
+        vis = resolve_inherit_visibility(r.get("mcp_view"))
+        if vis == "opaque":
+            matches.append({"id": r["id"], "visibility": "restricted"})
+        else:
+            matches.append({"id": r["id"], "name": r[name_col]})
+    return {
+        "disambiguation": True,
+        "message": f"Multiple matches for '{query}'. Please be more specific.",
+        "matches": matches,
+    }
+
+
+def upsert(
+    conn: sqlite3.Connection,
+    *,
+    project_name: str,
+    role: Role = Role.ADMIN,
+    root_path: Optional[str] = None,
+    client_id: Optional[int] = None,
+    project_type: Optional[str] = None,
+    description: Optional[str] = None,
+    github_url: Optional[str] = None,
+    status: Optional[str] = None,
+    status_reason: Optional[str] = None,
+) -> dict:
+    """Insert or update a project. Matches on root_path first, then project_name.
+
+    Returns dict with ``id`` and ``action`` ("created"|"updated").
+    Raises PermissionError if role cannot write, ValueError on bad input.
+    """
+    if not role.can_write:
+        raise PermissionError("Role does not permit write operations")
+
+    project_name = (project_name or "").strip()
+    if not project_name:
+        raise ValueError("project_name is required")
+
+    existing_id = db.find_project_id_by_key(
+        conn,
+        root_path=root_path,
+        project_name=project_name,
+    )
+
+    if existing_id is None:
+        result = db.create_project(
+            conn,
+            project_name=project_name,
+            root_path=root_path,
+            client_id=client_id,
+            project_type=project_type,
+            description=description,
+            github_url=github_url,
+            status=status or "active",
+        )
+        return {"id": result["id"], "action": "created"}
+
+    update_fields: dict = {}
+    if project_type is not None:
+        update_fields["project_type"] = project_type
+    if root_path is not None:
+        update_fields["root_path"] = root_path
+    if description is not None:
+        update_fields["description"] = description
+    if github_url is not None:
+        update_fields["github_url"] = github_url
+    if client_id is not None:
+        update_fields["client_id"] = client_id
+    if status is not None:
+        update_fields["status"] = status
+        if status_reason is not None:
+            update_fields["status_reason"] = status_reason
+    # Always update name — desired-state semantics
+    update_fields["project_name"] = project_name
+    db.update_project(conn, existing_id, **update_fields)
+    return {"id": existing_id, "action": "updated"}
+
+
+def delete(
+    conn: sqlite3.Connection,
+    project_id: int,
+    *,
+    role: Role = Role.ADMIN,
+) -> dict | None:
+    """Soft-delete a project by setting status to 'removed'.
+
+    Returns ``{"id", "status"}`` on success, ``None`` if not found.
+    Raises PermissionError if role cannot write.
+    """
+    if not role.can_write:
+        raise PermissionError("Role does not permit write operations")
+
+    result = db.update_project(conn, project_id, status="removed", status_reason="cli:delete")
+    if result is None:
+        return None
+    return {"id": project_id, "status": "removed"}

footprinter/services/roles.py

@@ -0,0 +1,25 @@
+"""Role enum for the service layer."""
+
+from enum import Enum
+
+
+class Role(Enum):
+    """Caller role — determines write access and metadata visibility.
+
+    Interface layers assign the role:
+    - CLI passes Role.ADMIN (full access, local user)
+    - MCP passes Role.VIEWER (read-only, filtered metadata)
+    """
+
+    ADMIN = "admin"
+    VIEWER = "viewer"
+
+    @property
+    def can_write(self) -> bool:
+        """Whether this role permits write operations."""
+        return self in (Role.ADMIN,)
+
+    @property
+    def sees_all(self) -> bool:
+        """Whether this role can see all metadata (including sensitive paths)."""
+        return self == Role.ADMIN

footprinter/services/search_service.py

@@ -0,0 +1,177 @@
+"""search_service — multi-source keyword search with visibility filtering.
+
+Orchestrates per-source searches (files, emails, chats, browser) and applies
+role-based visibility filtering + content stripping.
+"""
+
+import sqlite3
+from typing import Optional
+
+from footprinter.db.search import (
+    search_browser_keyword,
+    search_chats_keyword,
+    search_emails_keyword,
+    search_files_keyword,
+)
+from footprinter.db.sql_utils import split_query_terms
+from footprinter.services.access_service import (
+    filter_results_list,
+    strip_content_for_denied,
+)
+from footprinter.services.roles import Role
+from footprinter.visibility import get_source_visibility
+
+DEFAULT_SOURCES = ["files", "emails", "chats", "browser"]
+
+
+def search(
+    conn: sqlite3.Connection,
+    *,
+    role: Role = Role.ADMIN,
+    query: str = "",
+    sources: Optional[list[str]] = None,
+    project: Optional[str] = None,
+    client: Optional[str] = None,
+    date_from: Optional[str] = None,
+    date_to: Optional[str] = None,
+    limit: int = 50,
+    account: Optional[str] = None,
+    sender: Optional[str] = None,
+    days_back: Optional[int] = None,
+    folder: Optional[str] = None,
+    mime_type: Optional[str] = None,
+) -> dict:
+    """Search across indexed sources by keyword.
+
+    Returns dict with per-source result lists and a ``suppressed`` count.
+    VIEWER role: hidden items excluded, opaque items minimized, content
+    stripped for permission-denied items.
+    """
+    if not sources:
+        sources = list(DEFAULT_SOURCES)
+
+    results: dict = {}
+    total_suppressed = 0
+    has_query = bool(query and query.strip())
+    terms = split_query_terms(query) if has_query else []
+    has_query = has_query and bool(terms)
+
+    if "files" in sources:
+        file_results = search_files_keyword(
+            conn,
+            terms=terms,
+            has_query=has_query,
+            project=project,
+            client=client,
+            date_from=date_from,
+            date_to=date_to,
+            account=account,
+            folder=folder,
+            mime_type=mime_type,
+            limit=limit,
+            exclude_hidden=not role.sees_all,
+        )
+        if role.sees_all:
+            results["files"] = file_results
+        else:
+            filtered, suppressed = filter_results_list("file", file_results)
+            results["files"] = filtered
+            total_suppressed += suppressed
+
+    if "emails" in sources:
+        email_results = search_emails_keyword(
+            conn,
+            terms=terms,
+            has_query=has_query,
+            project=project,
+            client=client,
+            date_from=date_from,
+            date_to=date_to,
+            account=account,
+            sender=sender,
+            days_back=days_back,
+            limit=limit,
+            exclude_hidden=not role.sees_all,
+        )
+        if role.sees_all:
+            results["emails"] = email_results
+        else:
+            filtered, suppressed = filter_results_list("email", email_results)
+            strip_content_for_denied("email", filtered)
+            results["emails"] = filtered
+            total_suppressed += suppressed
+
+    if "chats" in sources:
+        chat_results = search_chats_keyword(
+            conn,
+            terms=terms,
+            has_query=has_query,
+            project=project,
+            client=client,
+            date_from=date_from,
+            date_to=date_to,
+            limit=limit,
+            exclude_hidden=not role.sees_all,
+        )
+        if role.sees_all:
+            results["chats"] = chat_results
+        else:
+            filtered, suppressed = filter_results_list("chat", chat_results)
+            strip_content_for_denied("chat", filtered)
+            results["chats"] = filtered
+            total_suppressed += suppressed
+
+    if "browser" in sources:
+        browser_results = _search_browser_with_visibility(
+            conn,
+            terms=terms,
+            has_query=has_query,
+            date_from=date_from,
+            date_to=date_to,
+            limit=limit,
+            role=role,
+        )
+        if browser_results is not None:
+            results["browser"] = browser_results
+
+    if total_suppressed > 0:
+        results["suppressed"] = total_suppressed
+
+    return results
+
+
+def _search_browser_with_visibility(
+    conn: sqlite3.Connection,
+    *,
+    terms: list[str],
+    has_query: bool,
+    date_from: Optional[str] = None,
+    date_to: Optional[str] = None,
+    limit: int = 50,
+    role: Role = Role.ADMIN,
+) -> Optional[list[dict]]:
+    """Search browser visits with source-level visibility gating.
+
+    Returns None if source is hidden. The visibility check is business logic
+    that stays in the service; the SQL query is delegated to db.search.
+    """
+    browser_visibility = None
+    if not role.sees_all:
+        browser_visibility = get_source_visibility(conn, "source:browser")
+        if browser_visibility == "hidden":
+            return None
+
+    raw_results = search_browser_keyword(
+        conn,
+        terms=terms,
+        has_query=has_query,
+        date_from=date_from,
+        date_to=date_to,
+        limit=limit,
+    )
+
+    # Source-level opaque gating
+    if browser_visibility == "opaque":
+        return [{"id": r["id"], "browser": r["browser"]} for r in raw_results]
+
+    return raw_results