footprinter-cli 1.0.0rc1__py3-none-any.whl

footprinter/mcp/extraction.py

@@ -0,0 +1,226 @@
+"""
+Text extraction from document formats for MCP read tool.
+
+Extracts plaintext from binary document data (.docx, .pdf, .xlsx, .pptx, .csv, .tsv).
+Sanitizes output to only include visible text (no comments, tracked changes, formulas).
+"""
+
+import csv
+import io
+import logging
+from typing import Optional, Tuple
+
+logger = logging.getLogger(__name__)
+
+# Map file extensions to extractor types
+EXTENSION_MAP = {
+    ".pdf": "pdf",
+    ".docx": "docx",
+    ".xlsx": "xlsx",
+    ".pptx": "pptx",
+    ".csv": "csv",
+    ".tsv": "tsv",
+}
+
+# Map MIME types to extractor types
+MIME_MAP = {
+    "application/pdf": "pdf",
+    "application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx",
+    "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "xlsx",
+    "application/vnd.openxmlformats-officedocument.presentationml.presentation": "pptx",
+    "text/csv": "csv",
+    "text/tab-separated-values": "tsv",
+    # Google Workspace types (handled as text after Drive export)
+    "application/vnd.google-apps.document": "text",
+    "application/vnd.google-apps.spreadsheet": "text",
+    "application/vnd.google-apps.presentation": "text",
+}
+
+
+def get_extractor_for_file(name: str, mime_type: str = "") -> Optional[str]:
+    """
+    Determine the extractor type for a file.
+
+    Args:
+        name: Filename with extension
+        mime_type: MIME type hint (optional)
+
+    Returns:
+        Extractor type string ('pdf', 'docx', etc.) or None if no extraction needed
+    """
+    # Check MIME type first (more reliable)
+    if mime_type and mime_type in MIME_MAP:
+        return MIME_MAP[mime_type]
+
+    # Fall back to extension
+    ext = "." + name.rsplit(".", 1)[-1].lower() if "." in name else ""
+    return EXTENSION_MAP.get(ext)
+
+
+def extract_text(data: bytes, extractor_type: str) -> Tuple[Optional[str], Optional[str]]:
+    """
+    Extract text from binary document data.
+
+    Args:
+        data: Raw file bytes
+        extractor_type: Type of extractor to use ('pdf', 'docx', etc.)
+
+    Returns:
+        Tuple of (extracted_text, error_message)
+        - On success: (text, None)
+        - On failure: (None, error_description)
+    """
+    extractors = {
+        "pdf": _extract_pdf,
+        "docx": _extract_docx,
+        "xlsx": _extract_xlsx,
+        "pptx": _extract_pptx,
+        "csv": _extract_csv,
+        "tsv": _extract_tsv,
+        "text": _extract_text,
+    }
+
+    extractor = extractors.get(extractor_type)
+    if not extractor:
+        return None, f"Unknown extractor type: {extractor_type}"
+
+    try:
+        text = extractor(data)
+        return text, None
+    except ImportError as e:
+        return None, f"Missing dependency: {e}"
+    except Exception as e:
+        logger.error(f"Extraction error ({extractor_type}): {e}")
+        return None, str(e)
+
+
+def _extract_pdf(data: bytes) -> str:
+    """Extract text from PDF bytes."""
+    import pypdf
+
+    reader = pypdf.PdfReader(io.BytesIO(data))
+    text_parts = []
+
+    for page in reader.pages:
+        page_text = page.extract_text()
+        if page_text:
+            text_parts.append(page_text)
+
+    return "\n\n".join(text_parts)
+
+
+def _extract_docx(data: bytes) -> str:
+    """
+    Extract text from DOCX bytes.
+
+    Only extracts visible paragraph text - no comments, tracked changes, or headers/footers.
+    """
+    import docx
+
+    doc = docx.Document(io.BytesIO(data))
+    text_parts = []
+
+    for para in doc.paragraphs:
+        if para.text.strip():
+            text_parts.append(para.text)
+
+    # Also extract text from tables
+    for table in doc.tables:
+        for row in table.rows:
+            row_text = [cell.text.strip() for cell in row.cells if cell.text.strip()]
+            if row_text:
+                text_parts.append(" | ".join(row_text))
+
+    return "\n\n".join(text_parts)
+
+
+def _extract_xlsx(data: bytes) -> str:
+    """
+    Extract text from XLSX bytes.
+
+    Extracts cell values only - no formulas, comments, or hidden data.
+    """
+    import openpyxl
+
+    wb = openpyxl.load_workbook(io.BytesIO(data), read_only=True, data_only=True)
+    text_parts = []
+
+    for sheet_name in wb.sheetnames:
+        sheet = wb[sheet_name]
+        text_parts.append(f"=== {sheet_name} ===")
+
+        for row in sheet.iter_rows(values_only=True):
+            # Filter out None values and convert to strings
+            row_values = [str(cell) for cell in row if cell is not None]
+            if row_values:
+                text_parts.append(" | ".join(row_values))
+
+    wb.close()
+    return "\n".join(text_parts)
+
+
+def _extract_pptx(data: bytes) -> str:
+    """
+    Extract text from PPTX bytes.
+
+    Extracts text from shapes and text frames - no speaker notes or comments.
+    """
+    from pptx import Presentation
+
+    prs = Presentation(io.BytesIO(data))
+    text_parts = []
+
+    for slide_num, slide in enumerate(prs.slides, 1):
+        slide_text = []
+        text_parts.append(f"--- Slide {slide_num} ---")
+
+        for shape in slide.shapes:
+            if hasattr(shape, "text") and shape.text.strip():
+                slide_text.append(shape.text)
+
+        if slide_text:
+            text_parts.append("\n".join(slide_text))
+
+    return "\n\n".join(text_parts)
+
+
+def _extract_csv(data: bytes) -> str:
+    """Extract text from CSV bytes."""
+    # Try to decode as UTF-8 first, fall back to latin-1
+    try:
+        text = data.decode("utf-8")
+    except UnicodeDecodeError:
+        text = data.decode("latin-1")
+
+    # Parse and format as readable text
+    reader = csv.reader(io.StringIO(text))
+    rows = []
+    for row in reader:
+        if any(cell.strip() for cell in row):
+            rows.append(" | ".join(row))
+
+    return "\n".join(rows)
+
+
+def _extract_tsv(data: bytes) -> str:
+    """Extract text from TSV bytes."""
+    try:
+        text = data.decode("utf-8")
+    except UnicodeDecodeError:
+        text = data.decode("latin-1")
+
+    reader = csv.reader(io.StringIO(text), delimiter="\t")
+    rows = []
+    for row in reader:
+        if any(cell.strip() for cell in row):
+            rows.append(" | ".join(row))
+
+    return "\n".join(rows)
+
+
+def _extract_text(data: bytes) -> str:
+    """Pass-through for already-text content (e.g., Google Workspace exports)."""
+    try:
+        return data.decode("utf-8")
+    except UnicodeDecodeError:
+        return data.decode("latin-1")

footprinter/mcp/server.py

@@ -0,0 +1,39 @@
+"""Footprinter MCP server — permission-gated access to indexed data and content."""
+
+from mcp.server.fastmcp import FastMCP
+
+from footprinter.mcp.tools.navigation import footprinter_client, footprinter_folder, footprinter_project
+from footprinter.mcp.tools.read import footprinter_read
+from footprinter.mcp.tools.search import footprinter_search
+from footprinter.mcp.tools.status import footprinter_status
+
+try:
+    from footprinter.mcp.tools.semantic import _SEMANTIC_AVAILABLE, footprinter_semantic
+except ImportError:
+    _SEMANTIC_AVAILABLE = False
+
+_server = None
+
+
+def _build_server():
+    global _server
+    _server = FastMCP("footprinter")
+    _server.tool()(footprinter_status)
+    _server.tool()(footprinter_search)
+    _server.tool()(footprinter_project)
+    _server.tool()(footprinter_client)
+    _server.tool()(footprinter_folder)
+    if _SEMANTIC_AVAILABLE:
+        _server.tool()(footprinter_semantic)
+    _server.tool()(footprinter_read)
+    return _server
+
+
+def main():
+    """Launch the Footprinter MCP server."""
+    _build_server()
+    _server.run()
+
+
+if __name__ == "__main__":
+    main()

footprinter/mcp/tools/navigation.py

@@ -0,0 +1,70 @@
+"""Navigation tools: projects, clients, folders.
+
+Thin MCP adapters — all query/filtering logic lives in the service layer.
+"""
+
+from pathlib import Path
+
+from footprinter.mcp.db import get_db, handle_db_errors
+from footprinter.mcp.errors import mcp_error
+from footprinter.services import client_service, folder_service, project_service
+from footprinter.services.roles import Role
+
+HOME = str(Path.home())
+
+
+def _shorten(path: str) -> str:
+    if path and path.startswith(HOME):
+        return "~" + path[len(HOME) :]
+    return path or ""
+
+
+@handle_db_errors
+def footprinter_project(project_name: str) -> dict:
+    """Get project metadata, file counts, and basic stats."""
+    with get_db() as conn:
+        result = project_service.resolve_by_name(conn, project_name, role=Role.VIEWER)
+        if result is None:
+            return mcp_error("NOT_FOUND", internal_message=f"project search: {project_name}")
+        if result.get("disambiguation"):
+            return result
+        # Shorten paths for MCP display
+        if "root_path" in result:
+            result["root_path"] = _shorten(result["root_path"])
+        for f in result.get("folders", []):
+            if "path" in f:
+                f["path"] = _shorten(f["path"])
+        return result
+
+
+@handle_db_errors
+def footprinter_client(client_name: str) -> dict:
+    """Get group info with all projects and aggregate stats."""
+    with get_db() as conn:
+        result = client_service.resolve_by_name(conn, client_name, role=Role.VIEWER)
+        if result is None:
+            return mcp_error("NOT_FOUND", internal_message=f"client search: {client_name}")
+        if result.get("disambiguation"):
+            return result
+        # Shorten paths for MCP display
+        for p in result.get("projects", []):
+            if "root_path" in p:
+                p["root_path"] = _shorten(p["root_path"])
+        return result
+
+
+@handle_db_errors
+def footprinter_folder(path: str) -> dict:
+    """Get folder contents and metadata."""
+    if path.startswith("~"):
+        path = HOME + path[1:]
+
+    with get_db() as conn:
+        result = folder_service.get_by_path(conn, path, role=Role.VIEWER)
+        if result is None:
+            return mcp_error("NOT_FOUND", internal_message=f"folder: {path}")
+        if "path" in result:
+            result["path"] = _shorten(result["path"])
+        for sf in result.get("subfolders", []):
+            sf["path"] = _shorten(sf.get("path", ""))
+        return result

footprinter/mcp/tools/read.py

@@ -0,0 +1,75 @@
+"""Read tool: fetch content for permitted items."""
+
+from typing import Literal
+
+from footprinter.mcp.db import get_db, handle_db_errors
+from footprinter.mcp.errors import mcp_error
+from footprinter.services import access_service, content_service
+from footprinter.services.roles import Role
+
+# Map service status codes to MCP error codes
+_STATUS_TO_MCP = {
+    "hidden": "NOT_FOUND",
+    "not_found": "NOT_FOUND",
+    "opaque": "VISIBILITY_RESTRICTED",
+    "denied": "PERMISSION_DENIED",
+    "invalid_type": "INVALID_TYPE",
+    "read_failed": "READ_FAILED",
+    "decode_failed": "DECODE_FAILED",
+    "extraction_failed": "EXTRACTION_FAILED",
+}
+
+
+@handle_db_errors
+def footprinter_read(
+    item_type: str,
+    item_id: int,
+    format: Literal["text", "raw"] = "text",
+) -> dict:
+    """Read content for a file, email, or chat if Claude has permission.
+
+    Args:
+        item_type: 'file', 'email', or 'chat'
+        item_id: Row ID of the item.
+        format: 'text' (default) extracts plaintext from documents,
+                'raw' returns raw decoded content without extraction.
+
+    Returns:
+        Dict with 'content' and 'metadata' on success,
+        or 'error', 'error_code', and 'metadata' on denial/failure.
+    """
+    with get_db() as conn:
+        result = access_service.gate_access(
+            conn,
+            item_type,
+            item_id,
+            role=Role.VIEWER,
+        )
+        status = result.get("status")
+
+        if status != "ok":
+            return mcp_error(
+                _STATUS_TO_MCP.get(status, "READ_FAILED"),
+                metadata=result.get("metadata"),
+                internal_message=result.get("message"),
+            )
+
+        # Email/chat: content already in result
+        if item_type != "file":
+            return {
+                "content": result.get("content", ""),
+                "metadata": result["metadata"],
+            }
+
+        # File: read content via service I/O
+        file_result = content_service.read_file(conn, result["metadata"], format=format)
+        if file_result.get("status") != "ok":
+            return mcp_error(
+                _STATUS_TO_MCP.get(file_result["status"], "READ_FAILED"),
+                metadata=file_result.get("metadata", result["metadata"]),
+                internal_message=file_result.get("message"),
+            )
+        return {
+            "content": file_result["content"],
+            "metadata": file_result["metadata"],
+        }

footprinter/mcp/tools/search.py

@@ -0,0 +1,158 @@
+"""Search tool: query across data sources."""
+
+from pathlib import Path
+from typing import Optional
+
+from footprinter.mcp.db import get_db, handle_db_errors
+from footprinter.services import search_service
+from footprinter.services.roles import Role
+
+HOME = str(Path.home())
+
+# Display names for source keys in summary text
+_SOURCE_LABELS = {
+    "files": ("file", "files"),
+    "emails": ("email", "emails"),
+    "chats": ("chat", "chats"),
+    "browser": ("browser result", "browser results"),
+}
+
+
+def _build_search_summary(results: dict, query: str, sources: list[str]) -> str:
+    """Build a human-readable summary of search results."""
+    found_parts = []
+    empty_parts = []
+
+    for source in sources:
+        items = results.get(source, [])
+        singular, plural = _SOURCE_LABELS.get(source, (source, source))
+        count = len(items)
+        if count > 0:
+            label = singular if count == 1 else plural
+            found_parts.append(f"{count} {label}")
+        else:
+            empty_parts.append(plural)
+
+    total_suppressed = results.get("suppressed", 0)
+
+    if found_parts:
+        query_part = f" matching '{query}'" if query and query.strip() else ""
+        summary = f"Found {', '.join(found_parts)}{query_part}."
+        if empty_parts:
+            summary += f" No {' or '.join(empty_parts)} matched."
+    else:
+        query_part = f" for '{query}'" if query and query.strip() else ""
+        summary = (
+            f"No results{query_part}. "
+            f"Tips: try single keywords, use footprinter_semantic "
+            f"for semantic matching, or browse recent items with date_from/date_to "
+            f"and no query."
+        )
+
+    if total_suppressed > 0:
+        item_word = "item" if total_suppressed == 1 else "items"
+        summary += f" ({total_suppressed} {item_word} hidden by visibility policy)"
+
+    return summary
+
+
+def _shorten_path(path: str) -> str:
+    if path and path.startswith(HOME):
+        return "~" + path[len(HOME) :]
+    return path or ""
+
+
+@handle_db_errors
+def footprinter_search(
+    query: str = "",
+    sources: Optional[list[str]] = None,
+    project: Optional[str] = None,
+    client: Optional[str] = None,
+    date_from: Optional[str] = None,
+    date_to: Optional[str] = None,
+    limit: int = 50,
+    account: Optional[str] = None,
+    sender: Optional[str] = None,
+    days_back: Optional[int] = None,
+    folder: Optional[str] = None,
+    mime_type: Optional[str] = None,
+) -> dict:
+    """Search across indexed sources by keyword. Returns metadata only, no file content.
+
+    SEARCH BEHAVIOR:
+    - Matches against file names, email subjects/senders, chat titles, and
+      browser page titles/URLs depending on which sources are included.
+    - Multi-word queries use AND logic: every term must appear. "project report" only returns
+      items containing both "project" AND "report".
+    - Terms shorter than 2 characters are ignored.
+    - When query is empty, returns the most recent items (sorted by date descending).
+      Combine with date_from/date_to to browse a specific time range.
+
+    QUERY TIPS:
+    - Use 1-3 short, specific keywords. Each additional word narrows results further.
+    - Avoid long natural-language phrases — they produce too many AND terms and return nothing.
+    - Good: "salesforce proposal" — Bad: "nonprofit technology consulting Salesforce partner"
+    - To search by time period, leave query empty and use date_from/date_to.
+
+    WHEN TO USE THIS vs footprinter_semantic:
+    - Use THIS tool for keyword/metadata lookups: finding files by name, emails by subject,
+      chats by title, or browsing recent activity across all sources.
+    - Use footprinter_semantic when you want meaning-based search across
+      chat or file content (e.g., "discussions about authentication architecture").
+
+    SOURCE-SPECIFIC FILTERS:
+    - account: Filter by account name. Applies to emails and files.
+    - sender: Partial match on email sender name or address. Applies to emails only.
+    - days_back: Only include emails from the last N days. Applies to emails only.
+    - folder: Filter files by path prefix (e.g. "~/Work/projects"). Applies to files only.
+    - mime_type: Exact MIME type match (e.g. "application/pdf"). Applies to files only.
+    Source-specific filters are silently ignored when the relevant source is not being searched.
+
+    Args:
+        query: Keyword(s) matched against names, titles, subjects. Empty = list recent.
+        sources: Which sources to search. Default: all.
+            Options: "files", "emails", "chats", "browser".
+        project: Filter to a project name (exact match, applies to files, emails, chats).
+        client: Filter to a client name (exact match, applies to files, emails, chats).
+        date_from: ISO date string lower bound (e.g. "2026-02-01").
+        date_to: ISO date string upper bound (e.g. "2026-02-14").
+        limit: Max results per source (default 50).
+        account: Filter by account (e.g. "personal", "work"). Applies to emails and files.
+        sender: Partial match on sender name or address (e.g. "alice"). Emails only.
+        days_back: Only include emails from the last N days. Emails only.
+        folder: Filter files under this path prefix (e.g. "~/Work/projects"). Files only.
+        mime_type: Exact MIME type filter (e.g. "application/pdf"). Files only.
+
+    Returns:
+        Dict with keys per source (e.g. "files", "emails", "chats", "browser"),
+        each containing a list of result dicts. Includes a "summary" key with
+        a human-readable overview of what was found.
+    """
+    if not sources:
+        sources = ["files", "emails", "chats", "browser"]
+
+    with get_db() as conn:
+        results = search_service.search(
+            conn,
+            role=Role.VIEWER,
+            query=query,
+            sources=sources,
+            project=project,
+            client=client,
+            date_from=date_from,
+            date_to=date_to,
+            limit=limit,
+            account=account,
+            sender=sender,
+            days_back=days_back,
+            folder=folder,
+            mime_type=mime_type,
+        )
+
+    # Shorten file paths for MCP display
+    for f in results.get("files", []):
+        if "path" in f:
+            f["path"] = _shorten_path(f["path"])
+
+    results["summary"] = _build_search_summary(results, query, sources)
+    return results

footprinter/mcp/tools/semantic.py

@@ -0,0 +1,79 @@
+"""Semantic search tool — thin adapter over semantic_service."""
+
+from footprinter.mcp.db import get_db, handle_db_errors
+from footprinter.mcp.errors import mcp_error
+from footprinter.services import semantic_service
+from footprinter.services.roles import Role
+
+_SEMANTIC_AVAILABLE = True
+
+
+@handle_db_errors
+def footprinter_semantic(query: str, source: str = "all", limit: int = 10) -> dict:
+    """Search chats and/or files by meaning using semantic (embedding-based) search.
+
+    SEARCH BEHAVIOR:
+    - Uses vector embeddings to find items with similar meaning, not exact keyword
+      matches. "auth problems" can find chats about "login failures"; "revenue
+      forecast" can find files about "financial projections".
+    - Falls back to FTS5 keyword search per-collection when ML dependencies
+      (ChromaDB, sentence-transformers) are unavailable. A note is included when
+      fallback is active.
+    - The ``source`` parameter controls which collections to search:
+      "chats" (conversations only), "files" (file content only), or "all" (both).
+    - ``limit`` applies per-collection: source="all" with limit=10 returns up to
+      10 chats + 10 files.
+    - Files are chunked during indexing; results are deduplicated so each file
+      appears at most once (best-matching chunk).
+
+    QUERY TIPS:
+    - Natural language works best: describe what you're looking for conceptually.
+    - Ideal query length is 3-10 words. Very short (1 word) or very long (10+ words)
+      queries reduce relevance.
+    - Good: "database migration strategies" — Bad: "database"
+    - Good: "client onboarding process" — Bad: "how did we onboard that new client
+      who came in last month through the partner referral program"
+
+    WHEN TO USE THIS vs footprinter_search:
+    - Use THIS tool for meaning-based search — finding items by topic even when you
+      don't know the exact words used.
+    - Use footprinter_search for exact keyword matches in names/subjects/titles, or
+      to search non-semantic sources (emails, browser history).
+
+    Args:
+        query: Natural language search query (minimum 3 characters).
+        source: Which collection(s) to search: "chats", "files", or "all" (default).
+        limit: Max results per collection (default 10).
+
+    Returns:
+        Dict with source-specific keys. "chats" and/or "files" lists are present
+        based on the source parameter. Only items with both visible access AND
+        read permission appear (semantic matches are content-derived, so
+        presence in results reveals content — per decision D2). Visible+allowed
+        chats have: chat_id, chat_title, snippet (text excerpt showing why it
+        matched), relevance_score, source, created_at, message_id.
+        Visible+allowed files have: id, name, path, content_type, size_bytes,
+        modified_at, relevance_score, snippet (from best-matching chunk).
+        Opaque items have minimal fields only. Hidden and permission-denied
+        items are excluded. Includes "summary" with a human-readable overview.
+    """
+    # Validate before opening DB connection
+    if not query or len(query) < 3:
+        return mcp_error(
+            "QUERY_INVALID",
+            internal_message=f"query too short: {len(query) if query else 0}",
+        )
+    if source not in ("chats", "files", "all"):
+        return mcp_error(
+            "INVALID_INPUT",
+            hint="source must be 'chats', 'files', or 'all'",
+        )
+
+    with get_db() as conn:
+        return semantic_service.semantic_search(
+            conn,
+            query,
+            role=Role.VIEWER,
+            source=source,
+            limit=limit,
+        )

footprinter/mcp/tools/status.py

@@ -0,0 +1,19 @@
+"""Status tool: system overview via status_service.
+
+Thin MCP adapter — all query logic lives in the service layer.
+"""
+
+from footprinter.mcp.db import get_db, handle_db_errors
+from footprinter.paths import is_test_mode
+from footprinter.services import status_service
+from footprinter.services.roles import Role
+
+
+@handle_db_errors
+def footprinter_status() -> dict:
+    """System status: record counts, sync times, and breakdowns for all data sources."""
+    with get_db() as conn:
+        result = status_service.get_status(conn, role=Role.VIEWER)
+        if is_test_mode():
+            result["_sandbox"] = True
+        return result