footprinter-cli 1.0.0rc1__py3-none-any.whl

footprinter/services/semantic_service.py

@@ -0,0 +1,360 @@
+"""semantic_service — embedding search with FTS5 fallback and access control.
+
+D2 access rule: semantic matches are content-derived, so visible items also
+require mcp_read='allow' (presence in results reveals content).
+"""
+
+import logging
+import sqlite3
+from typing import Dict, List
+
+from footprinter.db.search import (
+    chat_fts5_fallback,
+    enrich_chat_visibility,
+    enrich_file_metadata,
+    file_fts5_fallback,
+)
+from footprinter.services.access_service import (
+    resolve_inherit_permission,
+    resolve_inherit_visibility,
+)
+from footprinter.services.roles import Role
+
+logger = logging.getLogger(__name__)
+
+_VALID_SOURCES = frozenset({"chats", "files", "all"})
+
+_CHAT_FIELDS = {
+    "chat_id",
+    "chat_title",
+    "snippet",
+    "relevance_score",
+    "source",
+    "created_at",
+    "message_id",
+}
+
+_FILE_FIELDS = {
+    "id",
+    "name",
+    "path",
+    "content_type",
+    "size_bytes",
+    "modified_at",
+    "relevance_score",
+    "snippet",
+}
+
+# Search outcome: ok (vector worked), degraded (FTS5 fallback), failed (both crashed)
+_OK = "ok"
+_DEGRADED = "degraded"
+_FAILED = "failed"
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+def semantic_search(
+    conn: sqlite3.Connection,
+    query: str,
+    *,
+    role: Role = Role.ADMIN,
+    source: str = "all",
+    limit: int = 10,
+) -> dict:
+    """Search chats and/or files by semantic similarity.
+
+    Returns dict with source-specific keys (``chats``, ``files``), ``summary``,
+    and optionally ``note`` and ``suppressed``.  Returns ``{"status": ...}``
+    for validation errors.
+    """
+    if not query or len(query) < 3:
+        return {"status": "invalid_query"}
+
+    if source not in _VALID_SOURCES:
+        return {"status": "invalid_source"}
+
+    result: dict = {"query": query}
+    all_notes: list[str] = []
+    total_suppressed = 0
+    chat_status = _OK
+    file_status = _OK
+
+    if source in ("chats", "all"):
+        chats, notes, suppressed, chat_status = _search_chats(
+            conn,
+            query,
+            limit,
+            role,
+        )
+        result["chats"] = chats
+        all_notes.extend(notes)
+        total_suppressed += suppressed
+
+    if source in ("files", "all"):
+        files, notes, suppressed, file_status = _search_files(
+            conn,
+            query,
+            limit,
+            role,
+        )
+        result["files"] = files
+        all_notes.extend(notes)
+        total_suppressed += suppressed
+
+    if total_suppressed > 0:
+        result["suppressed"] = total_suppressed
+
+    summary_parts = []
+    if "chats" in result:
+        summary_parts.append(
+            _build_chat_summary(result["chats"], query, status=chat_status),
+        )
+    if "files" in result:
+        summary_parts.append(
+            _build_file_summary(result["files"], query, status=file_status),
+        )
+    result["summary"] = " ".join(summary_parts)
+
+    if all_notes:
+        result["note"] = " ".join(dict.fromkeys(all_notes))
+
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Chat search
+# ---------------------------------------------------------------------------
+
+
+def _search_chats(
+    conn: sqlite3.Connection,
+    query: str,
+    limit: int,
+    role: Role,
+) -> tuple[list[dict], list[str], int, str]:
+    """Search chats via VectorStore → FTS5 fallback → enrich → filter."""
+    notes: list[str] = []
+    status = _OK
+    results: list[dict] = []
+
+    # Try vector search first
+    try:
+        from footprinter.semantic.vector_store import VectorStore
+
+        store = VectorStore.get_instance()
+        results = store.search_chats(query=query, n_results=limit)
+    except Exception as e:
+        logger.warning("Vector search unavailable (%s), falling back to FTS5", e)
+        status = _DEGRADED
+        try:
+            results = chat_fts5_fallback(conn, query, limit)
+        except Exception as fallback_err:
+            logger.warning("Chat FTS5 fallback failed: %s", fallback_err)
+            return [], ["Chat search failed — try footprinter_search"], 0, _FAILED
+
+    # Enrich with visibility from DB
+    chat_ids = [r.get("chat_id") for r in results if r.get("chat_id")]
+    vis_lookup = enrich_chat_visibility(conn, chat_ids) if chat_ids else {}
+
+    for r in results:
+        db_row = vis_lookup.get(r.get("chat_id"))
+        r["id"] = r.get("chat_id")
+        r["account"] = db_row["account"] if db_row else ""
+        r["mcp_view"] = db_row["mcp_view"] if db_row else "hidden"
+        r["mcp_read"] = db_row["mcp_read"] if db_row else None
+
+    # Access control filtering
+    if role.sees_all:
+        filtered = results
+        suppressed = 0
+    else:
+        # D2: presence in semantic results reveals content — exclude anything
+        # not both visible AND allowed. Fail-closed on null/missing values.
+        filtered = [
+            r for r in results
+            if resolve_inherit_visibility(r.get("mcp_view")) == "visible"
+            and resolve_inherit_permission(r.get("mcp_read")) == "allow"
+        ]
+        suppressed = len(results) - len(filtered)
+
+    # Trim visible results to presentation fields
+    trimmed = [_trim_chat_result(r) if r.get("mcp_view") == "visible" else r for r in filtered]
+
+    if status == _DEGRADED:
+        notes.append("Results are keyword-based (semantic search unavailable)")
+
+    return trimmed, notes, suppressed, status
+
+
+# ---------------------------------------------------------------------------
+# File search
+# ---------------------------------------------------------------------------
+
+
+def _search_files(
+    conn: sqlite3.Connection,
+    query: str,
+    limit: int,
+    role: Role,
+) -> tuple[list[dict], list[str], int, str]:
+    """Search files via VectorStore (+ enrich) or FTS5 fallback → filter."""
+    notes: list[str] = []
+    status = _OK
+    enriched: List[Dict] = []
+    dropped = 0
+
+    try:
+        from footprinter.semantic.vector_store import VectorStore
+
+        store = VectorStore.get_instance()
+        raw_results = store.search_files(query=query, n_results=limit * 3)
+    except Exception as e:
+        logger.warning("Vector search unavailable (%s), falling back to FTS5", e)
+        status = _DEGRADED
+        try:
+            enriched = file_fts5_fallback(conn, query, limit)
+        except Exception as fallback_err:
+            logger.warning("File FTS5 fallback failed: %s", fallback_err)
+            return [], ["File search failed — try footprinter_search"], 0, _FAILED
+    else:
+        for r in raw_results:
+            distance = r.get("distance") or 0
+            r["relevance_score"] = round(max(0, 1 - (distance / 2)), 3)
+            r["snippet"] = r.get("content_snippet", "")
+
+        deduped, dropped = _deduplicate_by_file(raw_results)
+        if dropped > 0:
+            logger.warning(
+                "Dropped %d vector results with missing file_id",
+                dropped,
+            )
+
+        file_ids = [r["file_id"] for r in deduped if r.get("file_id")]
+        if file_ids:
+            db_lookup = enrich_file_metadata(conn, file_ids)
+            for r in deduped:
+                db_row = db_lookup.get(r["file_id"])
+                if db_row:
+                    r.update(db_row)
+            enriched = [r for r in deduped if r.get("id")]
+
+    # Access control filtering
+    if role.sees_all:
+        filtered = enriched
+        suppressed = 0
+    else:
+        # D2: presence in semantic results reveals content — exclude anything
+        # not both visible AND allowed. Fail-closed on null/missing values.
+        filtered = [
+            r for r in enriched
+            if resolve_inherit_visibility(r.get("mcp_view")) == "visible"
+            and resolve_inherit_permission(r.get("mcp_read")) == "allow"
+        ]
+        suppressed = len(enriched) - len(filtered)
+
+    trimmed = [_trim_file_result(r) if r.get("mcp_view") == "visible" else r for r in filtered]
+    trimmed = trimmed[:limit]
+
+    if status == _DEGRADED:
+        notes.append("Results are keyword-based (semantic search unavailable)")
+    if dropped > 0:
+        notes.append(f"Dropped {dropped} results with missing file_id. Run --rebuild-vectors to fix.")
+
+    return trimmed, notes, suppressed, status
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _deduplicate_by_file(results: List[Dict]) -> tuple[List[Dict], int]:
+    """Group by file_id, keep highest-relevance chunk per file."""
+    best: Dict[int, Dict] = {}
+    dropped = 0
+    for r in results:
+        fid = r.get("file_id")
+        if fid is None:
+            dropped += 1
+            continue
+        existing = best.get(fid)
+        if existing is None or r.get("relevance_score", 0) > existing.get("relevance_score", 0):
+            best[fid] = r
+    return list(best.values()), dropped
+
+
+def _trim_chat_result(result: dict) -> dict:
+    return {k: v for k, v in result.items() if k in _CHAT_FIELDS}
+
+
+def _trim_file_result(result: dict) -> dict:
+    return {k: v for k, v in result.items() if k in _FILE_FIELDS}
+
+
+def _build_chat_summary(
+    chats: list[dict],
+    query: str,
+    *,
+    status: str = _OK,
+) -> str:
+    visible = [c for c in chats if c.get("chat_title")]
+    opaque_count = len(chats) - len(visible)
+    count = len(chats)
+    if count > 0:
+        label = "chat" if count == 1 else "chats"
+        top_titles = [c["chat_title"] for c in visible[:3]]
+        summary = f"Found {count} {label} matching '{query}'."
+        if top_titles:
+            summary += f" Top: {', '.join(repr(t) for t in top_titles)}."
+        if opaque_count > 0:
+            summary += f" ({opaque_count} with restricted visibility.)"
+        if status == _DEGRADED:
+            summary += " (keyword match — semantic search was unavailable)"
+    else:
+        if status == _FAILED:
+            summary = f"Chat search failed for '{query}' — try footprinter_search for keyword matching."
+        elif status == _DEGRADED:
+            summary = f"Semantic search unavailable — keyword search returned no chats for '{query}'."
+        else:
+            summary = (
+                f"No chats found for '{query}'. "
+                f"Tips: try different keywords, use footprinter_search "
+                f"for broader keyword matching across files/emails/browser."
+            )
+    return summary
+
+
+def _build_file_summary(
+    files: list[dict],
+    query: str,
+    *,
+    status: str = _OK,
+) -> str:
+    visible = [f for f in files if f.get("name")]
+    opaque_count = len(files) - len(visible)
+    count = len(files)
+    if count > 0:
+        label = "file" if count == 1 else "files"
+        top_names = [f["name"] for f in visible[:3]]
+        summary = f"Found {count} {label} matching '{query}'."
+        if top_names:
+            summary += f" Top: {', '.join(repr(n) for n in top_names)}."
+        if opaque_count > 0:
+            summary += f" ({opaque_count} with restricted visibility.)"
+        if status == _DEGRADED:
+            summary += " (keyword match — semantic search was unavailable)"
+    else:
+        if status == _FAILED:
+            summary = f"File search failed for '{query}' — try footprinter_search for keyword matching."
+        elif status == _DEGRADED:
+            summary = f"Semantic search unavailable — keyword search returned no files for '{query}'."
+        else:
+            summary = (
+                f"No files found for '{query}'. "
+                f"Tips: try different keywords, use footprinter_search "
+                f"for exact keyword matching across file names/paths."
+            )
+    return summary

footprinter/services/status_service.py

@@ -0,0 +1,18 @@
+"""Status service — visibility-aware system status aggregates."""
+
+import sqlite3
+
+from footprinter.db import status as db_status
+from footprinter.paths import get_config_path
+from footprinter.services.roles import Role
+
+
+def get_status(conn: sqlite3.Connection, *, role: Role = Role.ADMIN) -> dict:
+    """Return system status, filtered by role.
+
+    VIEWER gets MCP-oriented counts with hidden-client data excluded.
+    ADMIN gets the full system status including config presence checks.
+    """
+    if role == Role.VIEWER:
+        return db_status.get_mcp_status(conn)
+    return db_status.get_system_status(conn, get_config_path())

footprinter/services/visit_service.py

@@ -0,0 +1,65 @@
+"""Visit (browser history) read service — get/list with role-based visibility filtering."""
+
+import sqlite3
+
+from footprinter.db import browser as db
+from footprinter.services.access_service import filter_result, filter_results_list
+from footprinter.services.roles import Role
+
+
+def get(conn: sqlite3.Connection, entry_id: int, *, role: Role = Role.ADMIN) -> dict | None:
+    """Fetch a single browser visit by ID, filtered by role."""
+    result = db.get_visit(conn, entry_id)
+    if result is None:
+        return None
+    if role.sees_all:
+        return result
+    return filter_result("visit", result)
+
+
+def assign(
+    conn: sqlite3.Connection,
+    entry_id: int,
+    *,
+    role: Role = Role.ADMIN,
+    project_id: int | None = None,
+    client_id: int | None = None,
+) -> dict | None:
+    """Assign a visit to a project and/or client.
+
+    Returns result dict on success, None if not found.
+    Raises PermissionError if role cannot write.
+    """
+    if not role.can_write:
+        raise PermissionError("Role does not permit write operations")
+    result = db.update_visit_relationships(
+        conn,
+        entry_id,
+        project_id=project_id,
+        client_id=client_id,
+    )
+    if result is None:
+        return None
+    resp: dict = {"id": entry_id}
+    if project_id is not None:
+        resp["project_id"] = project_id
+    if client_id is not None:
+        resp["client_id"] = client_id
+    return resp
+
+
+def list_(
+    conn: sqlite3.Connection,
+    *,
+    role: Role = Role.ADMIN,
+    limit: int = 50,
+    page: int = 1,
+) -> dict:
+    """List browser visits with pagination, filtered by role."""
+    response = db.list_visits(conn, limit=limit, page=page)
+    if role.sees_all:
+        return response
+    filtered, suppressed = filter_results_list("visit", response["visits"])
+    response["visits"] = filtered
+    response["suppressed"] = suppressed
+    return response

footprinter/source_registry.py

@@ -0,0 +1,194 @@
+"""
+Source registry — runtime registry for all Footprinter data sources.
+
+Seeded from config.yaml on database init. Provides a query/mutation API
+that all other code can use to discover available sources.
+"""
+
+import json
+import os
+import sqlite3
+from typing import Any, Dict, List, Optional
+
+import yaml
+
+from footprinter.paths import get_config_path
+
+
+class ConfigError(Exception):
+    """Raised when the config file is missing or invalid."""
+
+
+def get_config(config_path: Optional[str] = None) -> dict:
+    """Load configuration from YAML file.
+
+    Checks FOOTPRINTER_CONFIG env var first, then falls back to default path.
+    Raises ConfigError with a friendly message on missing/corrupt files.
+    """
+    path = config_path or str(get_config_path())
+    try:
+        with open(path) as f:
+            return yaml.safe_load(f)
+    except FileNotFoundError:
+        raise ConfigError(f"Config not found: {path}\nRun 'fp setup' to get started.") from None
+    except yaml.YAMLError as e:
+        raise ConfigError(f"Invalid config file: {path}\n{e}") from None
+
+
+def home_path() -> str:
+    """Return the user's home directory path."""
+    return os.path.expanduser("~")
+
+
+def remote_accounts() -> List[str]:
+    """Return list of remote (Drive) account names from config."""
+    config = get_config()
+    seeds = config.get("source_seeds", [])
+    return [s["account"] for s in seeds if s.get("source_type") == "remote" and s.get("account")]
+
+
+class SourceRegistry:
+    """Registry for data sources, backed by the sources table."""
+
+    def __init__(self, conn: sqlite3.Connection):
+        self.conn = conn
+
+    # ------------------------------------------------------------------
+    # Seeding
+    # ------------------------------------------------------------------
+
+    def seed_from_config(self, config_path: Optional[str] = None) -> int:
+        """Seed the sources table from config.yaml source_seeds.
+
+        Uses INSERT OR IGNORE so user edits to existing rows are preserved.
+        Returns the number of rows inserted.
+        """
+        config = get_config(config_path)
+        seeds = config.get("source_seeds", [])
+        inserted = 0
+        cursor = self.conn.cursor()
+        for seed in seeds:
+            config_json = json.dumps(seed.get("config")) if seed.get("config") else None
+            cursor.execute(
+                """
+                INSERT OR IGNORE
+                    INTO sources (name, source_type, adapter, account, label, icon, enabled, config)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+            """,
+                (
+                    seed["name"],
+                    seed["source_type"],
+                    None,
+                    seed.get("account"),
+                    seed.get("label"),
+                    seed.get("icon"),
+                    1 if seed.get("enabled", True) else 0,
+                    config_json,
+                ),
+            )
+            if cursor.rowcount > 0:
+                inserted += 1
+        self.conn.commit()
+        return inserted
+
+    # ------------------------------------------------------------------
+    # Query API
+    # ------------------------------------------------------------------
+
+    def all_source_names(self) -> List[str]:
+        """Return all source names."""
+        cursor = self.conn.cursor()
+        cursor.execute("SELECT name FROM sources ORDER BY name")
+        return [row[0] for row in cursor.fetchall()]
+
+    def all_sources(self) -> List[Dict[str, Any]]:
+        """Return all sources as dicts."""
+        cursor = self.conn.cursor()
+        cursor.execute("SELECT * FROM sources ORDER BY name")
+        columns = [desc[0] for desc in cursor.description]
+        return [dict(zip(columns, row)) for row in cursor.fetchall()]
+
+    def get_source(self, name: str) -> Optional[Dict[str, Any]]:
+        """Return a single source by name, or None."""
+        cursor = self.conn.cursor()
+        cursor.execute("SELECT * FROM sources WHERE name = ?", (name,))
+        row = cursor.fetchone()
+        if row is None:
+            return None
+        columns = [desc[0] for desc in cursor.description]
+        return dict(zip(columns, row))
+
+    def remote_source_names(self) -> List[str]:
+        """Return names of sources with source_type='remote'."""
+        cursor = self.conn.cursor()
+        cursor.execute("SELECT name FROM sources WHERE source_type = 'remote' ORDER BY name")
+        return [row[0] for row in cursor.fetchall()]
+
+    def file_source_names(self) -> List[str]:
+        """Return names of sources with source_type='file'."""
+        cursor = self.conn.cursor()
+        cursor.execute("SELECT name FROM sources WHERE source_type = 'file' ORDER BY name")
+        return [row[0] for row in cursor.fetchall()]
+
+    def source_label(self, name: str) -> Optional[str]:
+        """Return the label for a source, or None if not found."""
+        source = self.get_source(name)
+        return source["label"] if source else None
+
+    def source_account(self, name: str) -> Optional[str]:
+        """Return the account for a source, or None if not found."""
+        source = self.get_source(name)
+        return source["account"] if source else None
+
+    def is_remote_source(self, name: str) -> bool:
+        """Return True if the named source is a remote source."""
+        source = self.get_source(name)
+        return source is not None and source["source_type"] == "remote"
+
+    # ------------------------------------------------------------------
+    # Mutation API
+    # ------------------------------------------------------------------
+
+    def update_label(self, name: str, label: str) -> bool:
+        """Update a source's label. Returns True if a row was updated."""
+        cursor = self.conn.cursor()
+        cursor.execute(
+            "UPDATE sources SET label = ?, updated_at = CURRENT_TIMESTAMP WHERE name = ?",
+            (label, name),
+        )
+        self.conn.commit()
+        return cursor.rowcount > 0
+
+    def set_enabled(self, name: str, enabled: bool) -> bool:
+        """Enable or disable a source. Returns True if a row was updated."""
+        cursor = self.conn.cursor()
+        cursor.execute(
+            "UPDATE sources SET enabled = ?, updated_at = CURRENT_TIMESTAMP WHERE name = ?",
+            (1 if enabled else 0, name),
+        )
+        self.conn.commit()
+        return cursor.rowcount > 0
+
+    def register_source(self, name: str, source_type: str, **kwargs) -> bool:
+        """Register a new source. Returns True if inserted, False if already exists."""
+        cursor = self.conn.cursor()
+        config_json = json.dumps(kwargs.get("config")) if kwargs.get("config") else None
+        cursor.execute(
+            """
+            INSERT OR IGNORE
+                INTO sources (name, source_type, adapter, account, label, icon, enabled, config)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+        """,
+            (
+                name,
+                source_type,
+                kwargs.get("adapter"),
+                kwargs.get("account"),
+                kwargs.get("label"),
+                kwargs.get("icon"),
+                1 if kwargs.get("enabled", True) else 0,
+                config_json,
+            ),
+        )
+        self.conn.commit()
+        return cursor.rowcount > 0

footprinter/utils/__init__.py

@@ -0,0 +1,7 @@
+"""Utility modules for Footprinter."""
+
+from .hash_utils import compute_md5, compute_sha256
+from .mime import mime_to_content_type
+from .time import UTC_FMT, utc_now_iso
+
+__all__ = ["compute_md5", "compute_sha256", "mime_to_content_type", "UTC_FMT", "utc_now_iso"]