footprinter-cli 1.0.0rc1__py3-none-any.whl

footprinter/connectors/__init__.py

@@ -0,0 +1,171 @@
+"""Connector registry — metadata and discovery for optional integrations.
+
+Defines ConnectorSpec (the metadata dataclass) and discover_connectors()
+which finds installed connector plugins via importlib.metadata entry points.
+Helper functions check install status, config, and credentials.
+"""
+
+import importlib
+import importlib.metadata
+import importlib.util
+import logging
+import os
+from dataclasses import dataclass, field
+from enum import Enum
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+class AuthType(str, Enum):
+    """Authentication mechanism a connector uses — routing label for CLI dispatch."""
+
+    OAUTH2 = "oauth2"
+    BEARER = "bearer"
+    IMPORT = "import"
+    FILESYSTEM = "filesystem"
+
+
+@dataclass(frozen=True)
+class ConnectorSpec:
+    """Metadata for an optional integration."""
+
+    name: str
+    extra: str
+    description: str
+    pipes: tuple[str, ...]
+    probe_module: str
+    config_sections: tuple[str, ...]
+    setup_hook: str  # dotted path to callable
+    remove_packages: tuple[str, ...]
+    adapter_entries: dict[str, str] = field(default_factory=dict)  # pipe_name → "module:ClassName"
+    services: tuple[str, ...] = ()
+    seed_prefix: str = ""  # prefix for source_seed names (e.g. "gdrive")
+    schema_extensions: dict[str, list[tuple[str, str]]] = field(default_factory=dict)
+    auth_type: AuthType = AuthType.OAUTH2
+    check_auth: str = ""  # dotted path to callable(config) → auth status string
+    config_apply: str = ""  # dotted path to callable(config, result) → None
+    health_check: str = ""  # dotted path to callable(config) → list[dict]
+    read_file: str = ""  # dotted path to callable(external_id, account, mime_type) → bytes|None
+    seed_label_fn: str = ""  # dotted path to callable(display) → str
+    features: tuple[tuple[str, str, str, str], ...] = ()  # (name, probe_module, config_section, hint)
+    zero_result_checks: tuple[tuple[str, str], ...] = ()  # (pipe_name, count_key) for status warnings
+
+
+def discover_connectors() -> dict[str, ConnectorSpec]:
+    """Discover connector plugins via entry points.
+
+    Each entry point in the ``footprinter.connectors`` group should be
+    either a :class:`ConnectorSpec` instance or a callable that returns one.
+    The entry point name becomes the connector key.
+    """
+    connectors: dict[str, ConnectorSpec] = {}
+    eps = importlib.metadata.entry_points(group="footprinter.connectors")
+    for ep in eps:
+        try:
+            spec = ep.load()
+            if callable(spec):
+                spec = spec()
+            connectors[ep.name] = spec
+        except Exception:
+            logger.warning("Failed to load connector entry point: %s", ep.name)
+    return connectors
+
+
+def is_installed(spec: ConnectorSpec) -> bool:
+    """Check if the connector's pip extra is installed."""
+    try:
+        return importlib.util.find_spec(spec.probe_module) is not None
+    except (ValueError, ModuleNotFoundError):
+        return False
+
+
+def is_configured(spec: ConnectorSpec, config: dict) -> bool:
+    """Check if any of the connector's config sections are enabled."""
+    for section in spec.config_sections:
+        if section in config and config[section].get("enabled"):
+            return True
+    return False
+
+
+def has_credentials(spec: ConnectorSpec, config: dict) -> bool:
+    """Check if the credentials file exists for any config section."""
+    for section in spec.config_sections:
+        if section in config:
+            creds_path = config[section].get("credentials_path")
+            if creds_path and Path(os.path.expanduser(creds_path)).exists():
+                return True
+    return False
+
+
+def get_status(spec: ConnectorSpec, config: dict) -> str:
+    """Return user-facing status: 'not available', 'available', or 'installed'."""
+    if not is_installed(spec):
+        return "not available"
+    if is_configured(spec, config):
+        return "installed"
+    return "available"
+
+
+def resolve_hook(dotted_path: str):
+    """Resolve a dotted path to a callable.
+
+    Returns the callable, or None if *dotted_path* is empty.
+    Raises ValueError on malformed paths, ImportError/AttributeError on bad paths.
+    """
+    if not dotted_path:
+        return None
+    if "." not in dotted_path:
+        raise ValueError(f"Invalid hook path {dotted_path!r}: expected dotted path like 'module.func'")
+    module_path, func_name = dotted_path.rsplit(".", 1)
+    mod = importlib.import_module(module_path)
+    return getattr(mod, func_name)
+
+
+def resolve_check_auth(spec: ConnectorSpec, config: dict) -> str | None:
+    """Resolve and call a connector's check_auth callable.
+
+    Returns the auth status string (e.g. "authenticated", "expired"),
+    None if no check_auth is configured, or "error" on failure.
+    """
+    if not spec.check_auth:
+        return None
+    try:
+        module_path, func_name = spec.check_auth.rsplit(".", 1)
+        mod = importlib.import_module(module_path)
+        fn = getattr(mod, func_name)
+        return str(fn(config))
+    except Exception:
+        logger.warning("check_auth failed for connector %s", spec.name, exc_info=True)
+        return "error"
+
+
+def get_schema_specs(connectors: dict[str, ConnectorSpec] | None = None) -> list[ConnectorSpec]:
+    """Return installed connector specs that declare schema extensions."""
+    if connectors is None:
+        connectors = discover_connectors()
+    return [s for s in connectors.values() if is_installed(s) and s.schema_extensions]
+
+
+def get_connector_pipes(connectors: dict[str, ConnectorSpec] | None = None) -> dict[str, type]:
+    """Discover adapter classes from installed connectors.
+
+    If *connectors* is provided, uses that dict; otherwise calls
+    :func:`discover_connectors`. For installed connectors, imports their
+    adapter modules and collects adapter classes via the explicit
+    ``adapter_entries`` mapping. Returns ``{pipe_name: adapter_class}``.
+
+    Connectors whose pip extra is NOT installed are skipped entirely —
+    no import is attempted.
+    """
+    if connectors is None:
+        connectors = discover_connectors()
+    sources: dict[str, type] = {}
+    for spec in connectors.values():
+        if not is_installed(spec):
+            continue
+        for pipe_name, entry in spec.adapter_entries.items():
+            module_path, class_name = entry.rsplit(":", 1)
+            module = importlib.import_module(module_path)
+            sources[pipe_name] = getattr(module, class_name)
+    return sources

footprinter/connectors/config_utils.py

@@ -0,0 +1,141 @@
+"""Framework utilities for connector configuration.
+
+Convention-derived paths and seed entries shared across connectors.
+Implements decisions D4 and D5 from connector-configuration-lifecycle.md.
+"""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import logging
+import os
+import platform
+import secrets
+import uuid
+from pathlib import Path
+
+from cryptography.fernet import Fernet
+
+logger = logging.getLogger(__name__)
+
+# Magic header to distinguish encrypted files from legacy plaintext
+_HEADER = b"FP_ENC\x01"
+
+
+def credential_path(connector: str, account: str) -> Path:
+    """Convention: ~/.config/footprinter/{connector}_{account}_token.json
+
+    Returns the portable tilde-form path. Callers needing a filesystem path
+    should call ``.expanduser()`` on the result.
+    """
+    return Path(f"~/.config/footprinter/{connector}_{account}_token.json")
+
+
+def _salt_path() -> Path:
+    """Path to the salt file used for token encryption key derivation."""
+    return Path(os.path.expanduser("~/.config/footprinter/.token_salt"))
+
+
+def _get_or_create_salt(salt_file: Path | None = None) -> bytes:
+    """Read existing salt or generate a new 16-byte random salt."""
+    path = salt_file or _salt_path()
+    path = Path(os.path.expanduser(str(path)))
+    if path.exists():
+        return path.read_bytes()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    salt = secrets.token_bytes(16)
+    path.write_bytes(salt)
+    return salt
+
+
+def _derive_key(salt: bytes) -> bytes:
+    """Derive a Fernet key from machine identity and salt.
+
+    Uses PBKDF2-HMAC-SHA256 with machine identity (MAC address + hostname)
+    as the password material. Returns a URL-safe base64-encoded 32-byte key
+    suitable for Fernet.
+    """
+    machine_id = f"{uuid.getnode()}-{platform.node()}".encode()
+    raw = hashlib.pbkdf2_hmac("sha256", machine_id, salt, 480_000)
+    return base64.urlsafe_b64encode(raw)
+
+
+def save_token(path: Path, data: str | bytes) -> None:
+    """Encrypt *data* with Fernet and write the ciphertext to *path*."""
+    path = Path(os.path.expanduser(str(path)))
+    path.parent.mkdir(parents=True, exist_ok=True)
+
+    if isinstance(data, str):
+        raw = data.encode()
+    else:
+        raw = data
+
+    salt = _get_or_create_salt()
+    key = _derive_key(salt)
+    f = Fernet(key)
+    encrypted = f.encrypt(raw)
+    path.write_bytes(_HEADER + encrypted)
+
+
+def load_token(path: Path) -> str | bytes | None:
+    """Read and decrypt a token file.
+
+    Returns ``None`` if the file does not exist. Reads legacy plaintext files
+    transparently (files without the encryption header).
+    """
+    path = Path(os.path.expanduser(str(path)))
+    if not path.exists():
+        return None
+
+    raw = path.read_bytes()
+
+    if raw.startswith(_HEADER):
+        salt = _get_or_create_salt()
+        key = _derive_key(salt)
+        f = Fernet(key)
+        decrypted = f.decrypt(raw[len(_HEADER) :])
+        # Return same type that was saved: try UTF-8, fall back to bytes
+        try:
+            return decrypted.decode()
+        except UnicodeDecodeError:
+            return decrypted
+
+    # Legacy plaintext — return as string
+    try:
+        return raw.decode()
+    except UnicodeDecodeError:
+        return raw
+
+
+def source_seed_entry(
+    source_type: str,
+    account: str,
+    *,
+    name: str | None = None,
+    label: str | None = None,
+) -> dict:
+    """Build a source seed dict for config.
+
+    Args:
+        source_type: Seed source type (e.g. "remote").
+        account: Account name.
+        name: Override the default ``{source_type}_{account}`` name.
+        label: Override the default ``{Source_type} ({account})`` label.
+    """
+    return {
+        "name": name or f"{source_type}_{account}",
+        "source_type": source_type,
+        "account": account,
+        "label": label or f"{source_type.title()} ({account})",
+        "icon": "cloud",
+        "enabled": True,
+    }
+
+
+def account_label(account: dict) -> str:
+    """Return user-facing label for an account config entry.
+
+    Falls back to the internal name if label is missing or empty.
+    """
+    return account.get("label") or account["name"]

footprinter/db/__init__.py

@@ -0,0 +1,37 @@
+"""footprinter.db — public Python API for querying indexed data.
+
+Stable signatures, type hints, plain dict returns.
+All functions take sqlite3.Connection as their first parameter.
+"""
+
+from footprinter.db import (
+    browser,
+    chats,
+    clients,
+    emails,
+    files,
+    folders,
+    messages,
+    policies,
+    projects,
+    search,
+    sql_utils,
+    status,
+    uploads,
+)
+
+__all__ = [
+    "browser",
+    "chats",
+    "clients",
+    "emails",
+    "files",
+    "folders",
+    "messages",
+    "policies",
+    "projects",
+    "search",
+    "sql_utils",
+    "status",
+    "uploads",
+]

footprinter/db/browser.py

@@ -0,0 +1,198 @@
+"""Browser visit queries and write operations.
+
+Provides list, detail lookups, and insert functions for the visits table.
+"""
+
+import sqlite3
+from typing import Any, Dict, Optional, Union
+
+from footprinter.db.sql_utils import paginate, paginated_response
+
+
+def list_visits(conn: sqlite3.Connection, *, limit: int = 50, page: int = 1) -> dict:
+    """List browser visit entries ordered by visit_time descending.
+
+    Parameters
+    ----------
+    conn : sqlite3.Connection
+    limit : int
+        Maximum rows per page (default 50).
+    page : int
+        1-based page number (default 1).
+
+    Returns
+    -------
+    dict
+        ``{"visits": [...], "pagination": {page, limit, total, total_pages}}``
+    """
+    rows, pagination = paginate(
+        conn,
+        "SELECT COUNT(*) FROM visits bv WHERE bv.status != 'removed'",
+        """
+        SELECT bv.id, bv.url, bv.title, bv.visit_time, bv.browser, bv.visit_count,
+               bv.client_id, bv.project_id,
+               client.name AS client_name, project.project_name,
+               bv.mcp_view, bv.mcp_read
+        FROM visits bv
+        LEFT JOIN clients client ON bv.client_id = client.id
+        LEFT JOIN projects project ON bv.project_id = project.id
+        WHERE bv.status != 'removed'
+        ORDER BY bv.visit_time DESC
+        LIMIT ? OFFSET ?
+        """,
+        [],
+        page=page,
+        limit=limit,
+    )
+    visits = [
+        {
+            "id": r["id"],
+            "url": r["url"],
+            "title": r["title"],
+            "visit_time": r["visit_time"],
+            "browser": r["browser"],
+            "visit_count": r["visit_count"],
+            "client_id": r["client_id"],
+            "project_id": r["project_id"],
+            "client_name": r["client_name"],
+            "project_name": r["project_name"],
+            "mcp_view": r["mcp_view"],
+            "mcp_read": r["mcp_read"],
+        }
+        for r in rows
+    ]
+
+    return paginated_response("visits", visits, pagination)
+
+
+def get_visit(conn: sqlite3.Connection, entry_id: int) -> dict | None:
+    """Get a single browser history entry by ID.
+
+    Returns
+    -------
+    dict or None
+        Includes indexed_at. None if not found.
+    """
+    cursor = conn.execute(
+        """
+        SELECT bv.id, bv.url, bv.title, bv.visit_time, bv.browser, bv.visit_count,
+               bv.indexed_at, bv.status,
+               bv.client_id, bv.project_id,
+               client.name AS client_name, project.project_name,
+               bv.mcp_view, bv.mcp_read
+        FROM visits bv
+        LEFT JOIN clients client ON bv.client_id = client.id
+        LEFT JOIN projects project ON bv.project_id = project.id
+        WHERE bv.id = ? AND bv.status != 'removed'
+        """,
+        (entry_id,),
+    )
+    row = cursor.fetchone()
+    if row is None:
+        return None
+    return {
+        "id": row["id"],
+        "url": row["url"],
+        "title": row["title"],
+        "visit_time": row["visit_time"],
+        "browser": row["browser"],
+        "visit_count": row["visit_count"],
+        "indexed_at": row["indexed_at"],
+        "status": row["status"],
+        "client_id": row["client_id"],
+        "project_id": row["project_id"],
+        "client_name": row["client_name"],
+        "project_name": row["project_name"],
+        "mcp_view": row["mcp_view"],
+        "mcp_read": row["mcp_read"],
+    }
+
+
+def update_visit_relationships(
+    conn: sqlite3.Connection,
+    visit_id: int,
+    *,
+    project_id: Optional[int] = None,
+    client_id: Optional[int] = None,
+) -> Optional[bool]:
+    """Update project and/or client assignment on a visit.
+
+    Only updates fields that are passed (not None). Pass ``0`` to clear
+    a field (set to NULL). Stamps ``assignment_source = 'user'``
+    when the column exists (app-scope DBs only).
+    Returns True on success, None if visit not found.
+    """
+    cursor = conn.execute("SELECT id FROM visits WHERE id = ?", (visit_id,))
+    if cursor.fetchone() is None:
+        return None
+
+    if project_id is not None and project_id != 0:
+        proj = conn.execute("SELECT id FROM projects WHERE id = ?", (project_id,)).fetchone()
+        if not proj:
+            raise ValueError(f"No project with id {project_id}")
+    if client_id is not None and client_id != 0:
+        cli = conn.execute("SELECT id FROM clients WHERE id = ?", (client_id,)).fetchone()
+        if not cli:
+            raise ValueError(f"No client with id {client_id}")
+
+    sets: list[str] = []
+    params: list = []
+    if project_id is not None:
+        if project_id == 0:
+            sets.append("project_id = NULL")
+        else:
+            sets.append("project_id = ?")
+            params.append(project_id)
+    if client_id is not None:
+        if client_id == 0:
+            sets.append("client_id = NULL")
+        else:
+            sets.append("client_id = ?")
+            params.append(client_id)
+    if not sets:
+        return True
+
+    sets.append("assignment_source = 'user'")
+    params.append(visit_id)
+    try:
+        conn.execute(f"UPDATE visits SET {', '.join(sets)} WHERE id = ?", params)
+    except sqlite3.OperationalError as e:
+        if "no such column" not in str(e):
+            raise
+        # assignment_source not present (tool-only DB)
+        sets.pop()
+        conn.execute(f"UPDATE visits SET {', '.join(sets)} WHERE id = ?", params)
+    conn.commit()
+    return True
+
+
+# ---------------------------------------------------------------------------
+# Write operations
+# ---------------------------------------------------------------------------
+
+
+def insert_visit(conn: sqlite3.Connection, history_data: Dict[str, Any]) -> Union[int, bool]:
+    """Insert a browser visit record.
+
+    Returns the row ID on success, or False if the visit already exists
+    (duplicate on url + visit_time + browser).
+    """
+    cursor = conn.cursor()
+    cursor.execute(
+        """
+        INSERT OR IGNORE INTO visits
+        (url, title, visit_time, browser, visit_count,
+         indexed_at, updated_at)
+        VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
+    """,
+        (
+            history_data["url"],
+            history_data.get("title"),
+            history_data["visit_time"],
+            history_data["browser"],
+            history_data.get("visit_count", 1),
+        ),
+    )
+    if cursor.rowcount == 0:
+        return False
+    return cursor.lastrowid