footprinter-cli 1.0.0rc1__py3-none-any.whl

footprinter/api/entities.py

@@ -0,0 +1,250 @@
+"""Entity read endpoints for Footprinter HTTP API."""
+
+from typing import Optional
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+
+from footprinter.api.db import get_conn
+from footprinter.services import (
+    chat_service,
+    client_service,
+    email_service,
+    file_service,
+    folder_service,
+    project_service,
+    visit_service,
+)
+from footprinter.services.roles import Role
+
+router = APIRouter(tags=["entities"])
+
+
+def _or_404(result, entity_type: str, entity_id: int):
+    """Return result or raise 404."""
+    if result is None:
+        raise HTTPException(status_code=404, detail=f"{entity_type} {entity_id} not found")
+    return result
+
+
+# --- Files ---
+
+
+@router.get("/files")
+def list_files(
+    conn=Depends(get_conn),
+    project_id: Optional[int] = None,
+    source: Optional[str] = Query(None, description="Comma-separated source filter"),
+    status: Optional[str] = Query(None, description="Comma-separated status filter"),
+    content_type: Optional[str] = None,
+    limit: int = 50,
+    page: int = 1,
+):
+    source_list = [s.strip() for s in source.split(",")] if source else None
+    status_list = [s.strip() for s in status.split(",")] if status else None
+    return file_service.list_(
+        conn,
+        role=Role.ADMIN,
+        project_id=project_id,
+        source=source_list,
+        status=status_list,
+        content_type=content_type,
+        limit=limit,
+        page=page,
+    )
+
+
+@router.get("/files/{file_id}")
+def get_file(file_id: int, conn=Depends(get_conn)):
+    return _or_404(file_service.get(conn, file_id, role=Role.ADMIN), "file", file_id)
+
+
+# --- Emails ---
+
+
+@router.get("/emails")
+def list_emails(
+    conn=Depends(get_conn),
+    account: Optional[str] = None,
+    client_id: Optional[int] = None,
+    project_id: Optional[int] = None,
+    query: Optional[str] = None,
+    has_attachments: Optional[bool] = None,
+    sort_by: str = "received_at",
+    order: str = "desc",
+    limit: int = 50,
+    page: int = 1,
+):
+    return email_service.list_(
+        conn,
+        role=Role.ADMIN,
+        account=account,
+        client_id=client_id,
+        project_id=project_id,
+        query=query,
+        has_attachments=has_attachments,
+        sort_by=sort_by,
+        order=order,
+        limit=limit,
+        page=page,
+    )
+
+
+@router.get("/emails/{email_id}")
+def get_email(email_id: int, conn=Depends(get_conn)):
+    return _or_404(email_service.get(conn, email_id, role=Role.ADMIN), "email", email_id)
+
+
+# --- Chats ---
+
+
+@router.get("/chats")
+def list_chats(
+    conn=Depends(get_conn),
+    account: Optional[str] = None,
+    query: Optional[str] = None,
+    sort_by: str = "modified_at",
+    order: str = "desc",
+    status: Optional[str] = Query(None, description="Comma-separated status filter"),
+    limit: int = 50,
+    page: int = 1,
+):
+    status_list = [s.strip() for s in status.split(",")] if status else None
+    return chat_service.list_(
+        conn,
+        role=Role.ADMIN,
+        account=account,
+        query=query,
+        sort_by=sort_by,
+        order=order,
+        status=status_list,
+        limit=limit,
+        page=page,
+    )
+
+
+@router.get("/chats/{chat_id}")
+def get_chat(chat_id: int, conn=Depends(get_conn)):
+    return _or_404(chat_service.get(conn, chat_id, role=Role.ADMIN), "chat", chat_id)
+
+
+# --- Projects ---
+
+
+@router.get("/projects")
+def list_projects(
+    conn=Depends(get_conn),
+    include: Optional[str] = Query(None, description="Comma-separated includes"),
+    status: Optional[str] = Query(None, description="Comma-separated status filter"),
+    client: Optional[str] = None,
+    project_type: Optional[str] = None,
+    limit: int = 50,
+    page: int = 1,
+):
+    include_list = [s.strip() for s in include.split(",")] if include else None
+    status_list = [s.strip() for s in status.split(",")] if status else None
+    return project_service.list_(
+        conn,
+        role=Role.ADMIN,
+        include=include_list,
+        status=status_list,
+        client=client,
+        project_type=project_type,
+        limit=limit,
+        page=page,
+    )
+
+
+@router.get("/projects/{project_id}")
+def get_project(project_id: int, conn=Depends(get_conn), include: Optional[str] = None):
+    include_list = [s.strip() for s in include.split(",")] if include else None
+    return _or_404(
+        project_service.get(conn, project_id, role=Role.ADMIN, include=include_list),
+        "project",
+        project_id,
+    )
+
+
+# --- Clients ---
+
+
+@router.get("/clients")
+def list_clients(
+    conn=Depends(get_conn),
+    include: Optional[str] = Query(None, description="Comma-separated includes"),
+    status: Optional[str] = Query(None, description="Comma-separated status filter"),
+    limit: int = 50,
+    page: int = 1,
+):
+    include_list = [s.strip() for s in include.split(",")] if include else None
+    status_list = [s.strip() for s in status.split(",")] if status else None
+    return client_service.list_(
+        conn,
+        role=Role.ADMIN,
+        include=include_list,
+        status=status_list,
+        limit=limit,
+        page=page,
+    )
+
+
+@router.get("/clients/{client_id}")
+def get_client(client_id: int, conn=Depends(get_conn), include: Optional[str] = None):
+    include_list = [s.strip() for s in include.split(",")] if include else None
+    return _or_404(
+        client_service.get(conn, client_id, role=Role.ADMIN, include=include_list),
+        "client",
+        client_id,
+    )
+
+
+# --- Folders ---
+# NOTE: /folders/by-path MUST be defined before /folders/{folder_id}
+# to avoid FastAPI treating "by-path" as an int parameter.
+
+
+@router.get("/folders/by-path")
+def get_folder_by_path(path: str, conn=Depends(get_conn)):
+    result = folder_service.get_by_path(conn, path, role=Role.ADMIN)
+    if result is None:
+        raise HTTPException(status_code=404, detail=f"folder at path '{path}' not found")
+    return result
+
+
+@router.get("/folders")
+def list_folders(
+    conn=Depends(get_conn),
+    project_id: Optional[int] = None,
+    depth: Optional[int] = 1,
+    include_hidden: bool = False,
+    sort_by: str = "size",
+    limit: int = 50,
+    page: int = 1,
+):
+    return folder_service.list_(
+        conn,
+        role=Role.ADMIN,
+        project_id=project_id,
+        depth=depth,
+        include_hidden=include_hidden,
+        sort_by=sort_by,
+        limit=limit,
+        page=page,
+    )
+
+
+@router.get("/folders/{folder_id}")
+def get_folder(folder_id: int, conn=Depends(get_conn)):
+    return _or_404(folder_service.get(conn, folder_id, role=Role.ADMIN), "folder", folder_id)
+
+
+# --- Visits ---
+
+
+@router.get("/visits")
+def list_visits(conn=Depends(get_conn), limit: int = 50, page: int = 1):
+    return visit_service.list_(conn, role=Role.ADMIN, limit=limit, page=page)
+
+
+@router.get("/visits/{entry_id}")
+def get_visit(entry_id: int, conn=Depends(get_conn)):
+    return _or_404(visit_service.get(conn, entry_id, role=Role.ADMIN), "visit", entry_id)

footprinter/api/search.py

@@ -0,0 +1,47 @@
+"""Search endpoint for Footprinter HTTP API."""
+
+from typing import Optional
+
+from fastapi import APIRouter, Depends, Query
+
+from footprinter.api.db import get_conn
+from footprinter.services import search_service
+from footprinter.services.roles import Role
+
+router = APIRouter(tags=["search"])
+
+
+@router.get("/search")
+def search(
+    conn=Depends(get_conn),
+    query: str = "",
+    sources: Optional[str] = Query(None, description="Comma-separated source filter"),
+    project: Optional[str] = None,
+    client: Optional[str] = None,
+    date_from: Optional[str] = None,
+    date_to: Optional[str] = None,
+    limit: int = 50,
+    account: Optional[str] = None,
+    sender: Optional[str] = None,
+    days_back: Optional[int] = None,
+    folder: Optional[str] = None,
+    mime_type: Optional[str] = None,
+):
+    """Keyword search across indexed content."""
+    source_list = [s.strip() for s in sources.split(",")] if sources else None
+    return search_service.search(
+        conn,
+        role=Role.ADMIN,
+        query=query,
+        sources=source_list,
+        project=project,
+        client=client,
+        date_from=date_from,
+        date_to=date_to,
+        limit=limit,
+        account=account,
+        sender=sender,
+        days_back=days_back,
+        folder=folder,
+        mime_type=mime_type,
+    )

footprinter/api/semantic.py

@@ -0,0 +1,33 @@
+"""Semantic search endpoint for Footprinter HTTP API."""
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+
+from footprinter.api.db import get_conn
+from footprinter.services import semantic_service
+from footprinter.services.roles import Role
+
+router = APIRouter(tags=["semantic"])
+
+_VALID_SOURCES = {"chats", "files", "all"}
+
+
+@router.get("/semantic")
+def semantic_search(
+    conn=Depends(get_conn),
+    query: str = Query(..., min_length=3, description="Search query (minimum 3 characters)"),
+    source: str = Query("all", description="Source to search: chats, files, or all"),
+    limit: int = 10,
+):
+    """Semantic (vector) search across indexed content."""
+    if source not in _VALID_SOURCES:
+        raise HTTPException(
+            status_code=422,
+            detail=f"Invalid source '{source}'. Must be one of: {', '.join(sorted(_VALID_SOURCES))}",
+        )
+    return semantic_service.semantic_search(
+        conn,
+        query,
+        role=Role.ADMIN,
+        source=source,
+        limit=limit,
+    )

footprinter/api/server.py

@@ -0,0 +1,66 @@
+"""Footprinter HTTP API — FastAPI app factory and server entry point."""
+
+from fastapi import FastAPI, Request
+from fastapi.responses import JSONResponse
+
+from footprinter.api.db import DatabaseNotInitializedError
+
+
+def create_app() -> "FastAPI":
+    """Create and configure the FastAPI application.
+
+    Mounts all API routers under ``/api`` and registers error handlers.
+    Semantic router is conditionally mounted if chromadb is available.
+    """
+    from footprinter import __version__
+    from footprinter.api.entities import router as entities_router
+    from footprinter.api.search import router as search_router
+    from footprinter.api.status import router as status_router
+
+    app = FastAPI(
+        title="Footprinter API",
+        version=__version__,
+        description="HTTP API for Footprinter — file archival and AI context system.",
+    )
+
+    # Health check (outside /api prefix)
+    @app.get("/health")
+    def health():
+        return {"status": "ok"}
+
+    # Exception handler for uninitialized DB
+    @app.exception_handler(DatabaseNotInitializedError)
+    async def db_not_initialized_handler(request: Request, exc: DatabaseNotInitializedError):
+        return JSONResponse(
+            status_code=503,
+            content={
+                "detail": "Database not initialized. Run 'fp ingest' to populate.",
+            },
+        )
+
+    # Mount routers
+    app.include_router(status_router, prefix="/api")
+    app.include_router(search_router, prefix="/api")
+    app.include_router(entities_router, prefix="/api")
+
+    # Conditional semantic router
+    try:
+        from footprinter.api.semantic import router as semantic_router
+
+        app.include_router(semantic_router, prefix="/api")
+    except ImportError:
+        pass
+
+    return app
+
+
+def main(host: str = "127.0.0.1", port: int = 8000) -> None:
+    """Start the Footprinter HTTP API server."""
+    import uvicorn
+
+    app = create_app()
+    uvicorn.run(app, host=host, port=port)
+
+
+if __name__ == "__main__":
+    main()

footprinter/api/status.py

@@ -0,0 +1,15 @@
+"""Status endpoint for Footprinter HTTP API."""
+
+from fastapi import APIRouter, Depends
+
+from footprinter.api.db import get_conn
+from footprinter.services import status_service
+from footprinter.services.roles import Role
+
+router = APIRouter(tags=["status"])
+
+
+@router.get("/status")
+def get_status(conn=Depends(get_conn)):
+    """Return system status and data counts."""
+    return status_service.get_status(conn, role=Role.ADMIN)

footprinter/bundled/config.example.yaml

@@ -0,0 +1,161 @@
+# Footprinter Configuration (example)
+# Copy to config.yaml and fill in your values.
+
+# Directories to scan - only Work, Personal, and .claude
+directories:
+  - "~/Work"
+  - "~/Personal"
+  - "~/.claude"
+
+# Browser history sources
+browsers:
+  - safari
+  - chrome
+
+# Exclusion patterns (regex)
+# Patterns starting with ^~/ expand ~ to your home directory at runtime.
+#
+# Files matching these patterns are NEVER scanned — they don't appear in the
+# database at all. This is different from hidden files (dot-files/dot-directories),
+# which ARE scanned and indexed with status='hidden' so they appear in the catalog
+# but are excluded from Drive sync.
+#
+# Two tiers: always (all folders), sensitive (all folders)
+exclusions:
+  # Always skip - applies to all folders
+  # Files matching these patterns are skipped during scanning and never enter the database.
+  # Use for regeneratable dependencies and system noise that has no archival value.
+  always:
+    # Regeneratable dependencies
+    - ".*/node_modules/.*"       # npm dependencies
+    - ".*/__pycache__/.*"        # Python bytecode
+    - ".*/venv/.*"               # Python virtualenvs (non-hidden)
+    - ".*/\\.venv/.*"            # Python virtualenvs (hidden)
+    - ".*/site-packages/.*"      # Python packages
+    - ".*\\.pyc$"                # Python compiled files
+    - ".*/\\.vscode/extensions/.*"  # VS Code extensions (reinstallable)
+    - ".*/\\.npm/.*"             # npm cache
+    - ".*/\\.nvm/.*"             # Node version manager
+    - ".*/\\.cache/.*"           # Generic caches
+    - ".*/\\.sf/.*"              # Salesforce CLI
+    - ".*/\\.sfdx/.*"            # Salesforce DX CLI
+    - ".*/\\.docker/.*"          # Docker config
+    - ".*/\\.cumulusci/.*"       # CumulusCI cache
+    # Home-level Claude dirs only (keep .claude within Work/Personal)
+    - "^~/\\.claude/.*"           # Home-level .claude
+    - "^~/\\.claude-worktrees/.*" # Home-level .claude-worktrees
+    # Git internals (but NOT .gitignore, .gitattributes - those are kept)
+    - ".*/\\.git/.*"             # Git internal directory contents
+    # System noise
+    - ".*\\.tmp$"                # Temp files
+    - ".*/~\\$.*"                # Office lock files (~$document.docx)
+    - ".*/\\.Trash/.*"           # macOS Trash
+    - ".*/Library/.*"            # macOS system directory
+    - ".*\\.DS_Store$"           # macOS Finder metadata
+    # User directories not for archival
+    - "^~/Downloads/.*"      # Transient downloads
+    - "^~/Music/Audio Music Apps/.*"  # App databases
+    - "^~/Applications/.*"   # User-installed apps
+    # Hidden app data (not in Work/Personal)
+    - "^~/\\.local/.*"       # Local app data
+    - "^~/\\.cursor/.*"      # Cursor IDE
+    - "^~/\\.codex/.*"       # Codex CLI
+    - "^~/\\.zsh_sessions/.*"  # Shell history
+    - "^~/\\.bash_sessions/.*" # Shell history
+
+  # Always skip - sensitive data
+  # Credential files and key stores that should never be indexed for security.
+  # Matched files are skipped during scanning (never enter the database).
+  sensitive:
+    - ".*/Library/Keychains/.*"  # macOS keychains
+    - ".*/\\.aws/.*"             # AWS credentials
+    - ".*/\\.ssh/.*"             # SSH keys
+    - ".*/\\.kube/.*"            # Kubernetes config
+
+# Indexing configuration - INDEX ALL FILE TYPES
+indexing:
+  supported_extensions: []  # Empty = index ALL file types
+  max_file_size_mb: 0       # 0 = no size limit (index everything)
+  lookback_days: 14         # Browser history window (days back to index)
+  content_snippets: false   # Extract file/email content previews for keyword search
+
+# Semantic search — stores content as embeddings in a local ChromaDB database
+# Enables finding files and chats by meaning, not just keywords
+# Trade-off: additional disk space and longer indexing time
+semantic:
+  file_vectorization: false
+  chat_vectorization: false
+
+# Vectorization — controls what gets embedded for semantic search
+# Requires semantic.file_vectorization: true to take effect for files
+vectorization:
+  # File extensions to embed (omit = all types; empty list = disable vectorization)
+  # Default: prose documents that benefit from meaning-based search.
+  # Code files (.py, .js, etc.) are searchable via FTS — vectorization adds
+  # little value for structured code but significant noise to the vector space.
+  # NOTE: file_types filters among types the extractor can read (text, code,
+  # .pdf, .docx, .csv). Adding unsupported types (.xlsx, .pptx, etc.) has no
+  # effect — files pass the allowlist but produce no embeddings.
+  file_types:
+    - .md
+    - .txt
+    - .pdf
+    - .docx
+  # Chunk size in characters — tuned for MiniLM-L6-v2 (256-token input window).
+  # ~1000 chars ≈ 250 tokens. Larger chunks get silently truncated by the model,
+  # meaning content past the window is invisible to semantic search.
+  chunk_size: 1000
+  # Overlap between consecutive chunks (fraction of chunk_size, 0.0 to 1.0).
+  # Applies to file vectorization only. Chat message chunking uses a fixed
+  # character overlap defined in footprinter/semantic/chunking.py.
+  chunk_overlap: 0.15
+  # Patterns (fnmatch syntax) for file paths to skip during vectorization.
+  # Matched against the full absolute path (e.g. ~/Work/file.json expands at runtime).
+  # Files matching these patterns are still indexed (searchable via FTS)
+  # but not embedded in the vector store.
+  #
+  # Common exclusions — system caches, IDE output, and tool output that
+  # match text file extensions but contain no meaningful prose content:
+  #   - "**/Photos Library.photoslibrary/**"  # macOS Spotlight index cache (.txt)
+  #   - "**/.claude/debug/**"                 # Claude Code debug logs
+  #   - "**/.claude/paste-cache/**"           # Claude Code paste cache
+  #   - "**/.claude/cache/**"                 # Claude Code cache
+  #   - "**/.claude/projects/**"              # Claude Code session data
+  #   - "**/.claude/plans/**"                 # Claude Code auto-generated plans
+  #   - "**/.claude/plugins/**"              # Claude Code plugin cache
+  #   - "**/.cci/**"                          # CumulusCI cache
+  #   - "**/.context/**"                      # IDE context directories
+  exclude_patterns: []
+
+# Source registry seeds — loaded into the sources table on init
+# Connector sources added by: fp connect install <name>
+source_seeds:
+  - name: local
+    source_type: file
+    account: null
+    label: "Local Files"
+    icon: folder
+  - name: browser
+    source_type: browser
+    account: null
+    label: "Browser History"
+    icon: globe
+  - name: email
+    source_type: email
+    account: null
+    label: "Email"
+    icon: envelope
+  - name: chat
+    source_type: chat
+    account: null
+    label: "Chat"
+    icon: message
+
+# Display labels for the group/project hierarchy
+# Customize these to match your organization's terminology
+domain:
+  labels:
+    group_singular: "Client"
+    group_plural: "Clients"
+    project_singular: "Project"
+    project_plural: "Projects"

footprinter/bundled/patterns/context_patterns.yaml

@@ -0,0 +1,18 @@
+# Context and identity path patterns
+
+# Primary context detection by path
+# Keys must match Context enum names
+context_path_patterns:
+  CLIENT:
+    - '/Work/clients/'
+  INTERNAL: []  # Populated from config folder_classifications.internal_data
+  PERSONAL:
+    - '/Personal/'
+  CAREER:
+    - '/Work/admin/'
+    - '/Work/archive/'
+
+# Path patterns for identity document detection (matched against lowercased paths)
+identity_path_patterns:
+  - '/personal/identity/'
+  - '/identity/'