code-context-engine 0.4.0__py3-none-any.whl

context_engine/dashboard/server.py

@@ -0,0 +1,429 @@
+"""FastAPI dashboard server for CCE index inspection."""
+from __future__ import annotations
+
+import hashlib
+import hmac
+import json
+import os
+from pathlib import Path
+from urllib.parse import quote
+
+from typing import Literal
+
+from fastapi import FastAPI, Request
+from fastapi.responses import HTMLResponse, JSONResponse, Response
+from pydantic import BaseModel
+
+from context_engine.config import Config
+from context_engine.dashboard._page import PAGE_HTML
+from context_engine.indexer.pipeline import PathOutsideProjectError, run_indexing
+from context_engine.storage.local_backend import LocalBackend
+
+# Mutating HTTP methods require a same-origin browser request OR a non-browser
+# client (Sec-Fetch-Site absent). This blocks CSRF from a malicious local page
+# without breaking the dashboard's own fetch() calls or curl/tests.
+_MUTATING_METHODS = {"POST", "PUT", "PATCH", "DELETE"}
+# Optional bearer token for mutating endpoints. When CCE_DASHBOARD_TOKEN is set
+# in the environment, mutating requests must include `Authorization: Bearer
+# <token>` (the dashboard JS picks the token up from a `?token=` URL param so
+# users can paste a single URL into a browser). When the env var is unset the
+# dashboard remains open like before — the CSRF check above is the only guard.
+_DASHBOARD_TOKEN_ENV = "CCE_DASHBOARD_TOKEN"
+
+
+class ReindexRequest(BaseModel):
+    full: bool = False
+
+
+class CompressionRequest(BaseModel):
+    level: Literal["off", "lite", "standard", "max"]
+
+
+def create_app(config: Config, project_dir: Path) -> FastAPI:
+    """Build and return the FastAPI application.
+
+    All route handlers close over `storage_base` and `project_dir` so the
+    app is self-contained and trivial to test with TestClient.
+    """
+    project_name = project_dir.name
+    storage_base = Path(config.storage_path) / project_name
+
+    app = FastAPI(title="CCE Dashboard", docs_url=None, redoc_url=None)
+
+    expected_token = (os.environ.get(_DASHBOARD_TOKEN_ENV) or "").strip() or None
+
+    @app.middleware("http")
+    async def csrf_and_auth(request: Request, call_next):
+        if request.method in _MUTATING_METHODS:
+            # CSRF: browser cross-origin requests are rejected. Non-browser
+            # clients (curl, tests) don't send Sec-Fetch-Site at all and are
+            # allowed to proceed to the auth check.
+            sfs = request.headers.get("sec-fetch-site")
+            if sfs is not None and sfs != "same-origin":
+                return JSONResponse(
+                    {"error": "cross-origin requests not allowed"},
+                    status_code=403,
+                )
+            # Auth: only enforced when CCE_DASHBOARD_TOKEN is set. Use
+            # constant-time comparison so a token-guessing attacker can't
+            # learn anything from response timing.
+            if expected_token is not None:
+                auth = request.headers.get("authorization", "")
+                presented = ""
+                if auth.startswith("Bearer "):
+                    presented = auth[len("Bearer "):]
+                if not presented or not hmac.compare_digest(presented, expected_token):
+                    return JSONResponse(
+                        {"error": "invalid or missing bearer token"},
+                        status_code=401,
+                    )
+        return await call_next(request)
+
+    backend = LocalBackend(base_path=str(storage_base))
+
+    # ── helpers ────────────────────────────────────────────────────────────
+
+    def _read_json(path: Path) -> dict:
+        if path.exists():
+            try:
+                return json.loads(path.read_text())
+            except (json.JSONDecodeError, OSError):
+                pass
+        return {}
+
+    def _read_manifest() -> dict[str, str]:
+        """Return {file_path: content_hash} regardless of on-disk schema.
+
+        Manifest.save() writes {"__schema_version": 2, "files": {...},
+        "last_git_sha": ...}. Older installs may have left the flat
+        {file_path: hash} form. Both shapes collapse to the same dict here.
+        """
+        raw = _read_json(storage_base / "manifest.json")
+        if isinstance(raw.get("files"), dict):
+            return raw["files"]
+        # Legacy flat manifest, or empty / unreadable file.
+        return raw if raw and "__schema_version" not in raw else {}
+
+    def _read_stats() -> dict:
+        return _read_json(storage_base / "stats.json")
+
+    def _read_state() -> dict:
+        return _read_json(storage_base / "state.json")
+
+    def _read_sessions(limit: int = 20) -> list[dict]:
+        sessions_dir = storage_base / "sessions"
+        if not sessions_dir.exists():
+            return []
+        files = sorted(sessions_dir.glob("*.json"), key=lambda p: p.stat().st_mtime, reverse=True)
+        result = []
+        for f in files[:limit]:
+            try:
+                result.append(json.loads(f.read_text()))
+            except (json.JSONDecodeError, OSError):
+                pass
+        return result
+
+    # ── routes ─────────────────────────────────────────────────────────────
+
+    @app.get("/", response_class=HTMLResponse)
+    async def serve_page() -> str:
+        return PAGE_HTML
+
+    @app.get("/api/status")
+    async def get_status() -> dict:
+        stats = _read_stats()
+        manifest = _read_manifest()
+        state = _read_state()
+
+        try:
+            chunks = backend.count_chunks()
+        except Exception:
+            chunks = 0
+
+        full_file = stats.get("full_file_tokens", 0)
+        served = stats.get("served_tokens", 0)
+        baseline = full_file if full_file > 0 else stats.get("raw_tokens", 0)
+        saved_pct = max(0, int((1 - served / baseline) * 100)) if baseline > 0 else 0
+
+        output_level = state.get("output_level", config.output_compression)
+
+        return {
+            "project": project_name,
+            "initialized": bool(manifest),
+            "chunks": chunks,
+            "files": len(manifest),
+            "queries": stats.get("queries", 0),
+            "tokens_saved_pct": saved_pct,
+            "output_level": output_level,
+        }
+
+    @app.get("/api/files")
+    async def get_files() -> list:
+        manifest = _read_manifest()
+        if not manifest:
+            return []
+
+        chunk_counts = backend.file_chunk_counts()
+
+        result = []
+        for rel_path, stored_hash in sorted(manifest.items()):
+            abs_path = project_dir / rel_path
+            if not abs_path.exists():
+                status = "missing"
+            else:
+                try:
+                    current = abs_path.read_text(encoding="utf-8", errors="strict")
+                    current_hash = hashlib.sha256(current.encode("utf-8")).hexdigest()
+                    status = "ok" if current_hash == stored_hash else "stale"
+                except (UnicodeDecodeError, OSError):
+                    status = "ok"  # binary file, trust the manifest
+            result.append({
+                "path": rel_path,
+                "chunks": chunk_counts.get(rel_path, 0),
+                "status": status,
+            })
+        return result
+
+    @app.get("/api/sessions")
+    async def get_sessions() -> list:
+        return _read_sessions()
+
+    # ── memory.db API (PR 5) ────────────────────────────────────────────────
+    # These endpoints expose the per-project memory.db introduced in PRs 1–4.
+    # All queries open a fresh connection (cheap on SQLite WAL) so the
+    # dashboard process never holds a long-lived handle that would block the
+    # MCP server's writes.
+
+    def _open_memory_conn():
+        from context_engine.memory import db as memory_db
+        path = memory_db.memory_db_path(storage_base)
+        if not path.exists():
+            return None
+        return memory_db.connect(path)
+
+    @app.get("/api/memory/sessions")
+    async def memory_sessions_list(limit: int = 50) -> list:
+        """Sessions list: most-recent first. Limited to `limit` rows.
+
+        Stored `rollup_summary` was passed through grammar.compress on write;
+        we expand here so the dashboard renders natural prose, not the
+        article-stripped storage form.
+        """
+        from context_engine.memory.grammar import expand as _grammar_expand
+        conn = _open_memory_conn()
+        if conn is None:
+            return []
+        try:
+            rows = list(conn.execute(
+                "SELECT id, project, started_at, ended_at, status, "
+                "prompt_count, rollup_summary FROM sessions "
+                "ORDER BY started_at_epoch DESC LIMIT ?",
+                (max(1, min(limit, 500)),),
+            ))
+        finally:
+            conn.close()
+        out = []
+        for r in rows:
+            d = dict(r)
+            if d.get("rollup_summary"):
+                d["rollup_summary"] = _grammar_expand(d["rollup_summary"])
+            out.append(d)
+        return out
+
+    @app.get("/api/memory/sessions/{session_id}/timeline")
+    async def memory_session_timeline(session_id: str) -> dict:
+        """A single session's turn summaries plus header metadata."""
+        from context_engine.memory.grammar import expand as _grammar_expand
+        conn = _open_memory_conn()
+        if conn is None:
+            return {"session": None, "turns": []}
+        try:
+            session_row = conn.execute(
+                "SELECT id, project, started_at, ended_at, status, "
+                "prompt_count, rollup_summary FROM sessions WHERE id = ?",
+                (session_id,),
+            ).fetchone()
+            if session_row is None:
+                return {"session": None, "turns": []}
+            turn_rows = list(conn.execute(
+                "SELECT prompt_number, summary, tier, created_at_epoch "
+                "FROM turn_summaries WHERE session_id = ? "
+                "ORDER BY prompt_number ASC",
+                (session_id,),
+            ))
+        finally:
+            conn.close()
+        session_d = dict(session_row)
+        if session_d.get("rollup_summary"):
+            session_d["rollup_summary"] = _grammar_expand(session_d["rollup_summary"])
+        turns = []
+        for r in turn_rows:
+            d = dict(r)
+            if d.get("summary"):
+                d["summary"] = _grammar_expand(d["summary"])
+            turns.append(d)
+        return {"session": session_d, "turns": turns}
+
+    @app.get("/api/memory/decisions")
+    async def memory_decisions_search(
+        q: str = "", source: str | None = None, limit: int = 50
+    ) -> list:
+        """FTS5 search over decisions, optionally filtered by source."""
+        conn = _open_memory_conn()
+        if conn is None:
+            return []
+        limit = max(1, min(limit, 500))
+        try:
+            params: list = []
+            sql = (
+                "SELECT d.id, d.session_id, d.decision, d.reason, d.source, "
+                "d.created_at FROM decisions d "
+            )
+            if q.strip():
+                sql += (
+                    "JOIN decisions_fts ON decisions_fts.rowid = d.id "
+                    "WHERE decisions_fts MATCH ? "
+                )
+                # Wrap in a phrase quote so FTS5's tokeniser treats user
+                # input as literal text — without this, characters like '-'
+                # parse as operators (`bge-small` becomes "bge AND NOT
+                # small") and queries silently miss real hits.
+                params.append('"' + q.strip().replace('"', '""') + '"')
+            else:
+                sql += "WHERE 1=1 "
+            if source in {"manual", "auto", "migrated"}:
+                sql += "AND d.source = ? "
+                params.append(source)
+            sql += "ORDER BY d.created_at_epoch DESC LIMIT ?"
+            params.append(limit)
+            try:
+                rows = list(conn.execute(sql, params))
+            except Exception:
+                # FTS5 syntax errors on weird queries — fall back to unranked.
+                rows = list(conn.execute(
+                    "SELECT id, session_id, decision, reason, source, "
+                    "created_at FROM decisions ORDER BY created_at_epoch "
+                    "DESC LIMIT ?",
+                    (limit,),
+                ))
+        finally:
+            conn.close()
+        # Stored decision/reason are compressed; expand for display.
+        from context_engine.memory.grammar import expand as _grammar_expand
+        out = []
+        for r in rows:
+            d = dict(r)
+            if d.get("decision"):
+                d["decision"] = _grammar_expand(d["decision"])
+            if d.get("reason"):
+                d["reason"] = _grammar_expand(d["reason"])
+            out.append(d)
+        return out
+
+    @app.get("/api/savings")
+    async def get_savings() -> dict:
+        stats = _read_stats()
+        full_file = stats.get("full_file_tokens", 0)
+        served = stats.get("served_tokens", 0)
+        raw = stats.get("raw_tokens", 0)
+        baseline = full_file if full_file > 0 else raw
+        saved = max(0, baseline - served)
+        pct = int(saved / baseline * 100) if baseline > 0 else 0
+        return {
+            "queries": stats.get("queries", 0),
+            "baseline_tokens": baseline,
+            "served_tokens": served,
+            "tokens_saved": saved,
+            "savings_pct": pct,
+        }
+
+    # ── action routes ──────────────────────────────────────────────────────
+
+    @app.post("/api/reindex")
+    async def reindex(req: ReindexRequest) -> dict:
+        result = await run_indexing(config, project_dir, full=req.full)
+        return {
+            "total_chunks": result.total_chunks,
+            "indexed_files": result.indexed_files,
+            "deleted_files": result.deleted_files,
+            "skipped_files": result.skipped_files,
+            "errors": result.errors,
+        }
+
+    @app.post("/api/reindex/{file_path:path}", response_model=None)
+    async def reindex_file(file_path: str) -> dict | JSONResponse:
+        try:
+            result = await run_indexing(config, project_dir, target_path=file_path)
+        except PathOutsideProjectError:
+            return JSONResponse({"error": "invalid file_path"}, status_code=400)
+        return {
+            "total_chunks": result.total_chunks,
+            "indexed_files": result.indexed_files,
+            "deleted_files": result.deleted_files,
+            "skipped_files": result.skipped_files,
+            "errors": result.errors,
+        }
+
+    @app.post("/api/clear")
+    async def clear_index() -> dict:
+        await backend.clear()
+        from context_engine.utils import atomic_write_text
+
+        atomic_write_text(
+            storage_base / "manifest.json",
+            json.dumps({"__schema_version": 2, "files": {}, "last_git_sha": None}),
+        )
+        atomic_write_text(
+            storage_base / "stats.json",
+            json.dumps({"queries": 0, "raw_tokens": 0, "served_tokens": 0, "full_file_tokens": 0}),
+        )
+        return {"ok": True}
+
+    @app.delete("/api/files/{file_path:path}", response_model=None)
+    async def delete_file(file_path: str) -> dict | JSONResponse:
+        # Reject absolute paths and traversal — the manifest stores project-relative
+        # paths, so anything else is either an attacker probe or a bug.
+        if file_path.startswith("/") or ".." in Path(file_path).parts:
+            return JSONResponse({"error": "invalid file_path"}, status_code=400)
+        files = _read_manifest()
+        if file_path not in files:
+            return JSONResponse({"error": "file not indexed"}, status_code=404)
+        await backend.delete_by_file(file_path)
+        files.pop(file_path, None)
+        # Preserve schema fields (last_git_sha) when rewriting.
+        raw = _read_json(storage_base / "manifest.json")
+        if isinstance(raw.get("files"), dict):
+            raw["files"] = files
+            payload = raw
+        else:
+            payload = {"__schema_version": 2, "files": files, "last_git_sha": None}
+        from context_engine.utils import atomic_write_text
+
+        atomic_write_text(storage_base / "manifest.json", json.dumps(payload))
+        return {"ok": True, "deleted": file_path}
+
+    @app.post("/api/compression")
+    async def set_compression(req: CompressionRequest) -> dict:
+        state = _read_state()
+        state["output_level"] = req.level
+        (storage_base / "state.json").write_text(json.dumps(state))
+        return {"level": req.level}
+
+    @app.get("/api/export")
+    async def export_data():
+        payload = {
+            "project": project_name,
+            "stats": _read_stats(),
+            "manifest": _read_manifest(),
+            "sessions": _read_sessions(),
+        }
+        safe_name = quote(project_name, safe="")
+        return Response(
+            content=json.dumps(payload, indent=2),
+            media_type="application/json",
+            headers={
+                "Content-Disposition": f"attachment; filename={safe_name}-cce-export.json"
+            },
+        )
+
+    return app

context_engine/editors.py

@@ -0,0 +1,265 @@
+"""Multi-editor MCP configuration.
+
+Detects installed editors and writes MCP server config in each editor's
+format. Supports Claude Code, VS Code/Copilot, Cursor, Gemini CLI, and
+OpenAI Codex CLI.
+"""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from context_engine.utils import atomic_write_text, resolve_cce_binary
+
+
+# ── Editor definitions ────────────────────────────────────────────────
+# format: "json" (default) or "toml" for Codex
+
+EDITORS: dict[str, dict] = {
+    "claude": {
+        "name": "Claude Code",
+        "config_path": ".mcp.json",
+        "servers_key": "mcpServers",
+        "format": "json",
+        "detect": [".mcp.json"],
+    },
+    "vscode": {
+        "name": "VS Code / Copilot",
+        "config_path": ".vscode/mcp.json",
+        "servers_key": "servers",
+        "format": "json",
+        "detect": [".vscode"],
+    },
+    "cursor": {
+        "name": "Cursor",
+        "config_path": ".cursor/mcp.json",
+        "servers_key": "mcpServers",
+        "format": "json",
+        "detect": [".cursor", ".cursorrules"],
+    },
+    "gemini": {
+        "name": "Gemini CLI",
+        "config_path": ".gemini/settings.json",
+        "servers_key": "mcpServers",
+        "format": "json",
+        "detect": [".gemini", "GEMINI.md"],
+    },
+    "codex": {
+        "name": "OpenAI Codex",
+        "config_path": ".codex/config.toml",
+        "format": "toml",
+        "detect": [".codex"],
+    },
+}
+
+# ── Instruction file definitions ──────────────────────────────────────
+
+# Editor-agnostic CCE instructions (no "Claude Code" references)
+_CCE_INSTRUCTIONS = """\
+## Context Engine (CCE)
+
+This project uses Code Context Engine for intelligent code retrieval and
+cross-session memory.
+
+### Searching the codebase
+
+**Use `context_search` instead of reading files directly** when exploring
+the codebase, answering questions about code, or understanding how things
+work. `context_search` returns the most relevant code chunks with
+confidence scores instead of whole files.
+
+When to use `context_search`:
+- Answering questions about the codebase ("how does X work?", "where is Y?")
+- Exploring structure or architecture
+- Finding related code, functions, or patterns
+
+Other tools:
+- `expand_chunk` for full source of a compressed result
+- `related_context` for what calls/imports a function
+- `session_recall` to recall past decisions
+
+### Cross-session memory
+
+Call `session_recall("topic phrase")` before answering non-trivial questions.
+Call `record_decision(decision="...", reason="...")` after making choices.
+Call `record_code_area(file_path="...", description="...")` after meaningful work.
+"""
+
+INSTRUCTION_FILES: dict[str, dict] = {
+    "cursorrules": {
+        "name": ".cursorrules",
+        "path": ".cursorrules",
+        "detect": [".cursor", ".cursorrules"],
+    },
+    "gemini": {
+        "name": "GEMINI.md",
+        "path": "GEMINI.md",
+        "detect": [".gemini", "GEMINI.md"],
+    },
+}
+
+
+# ── Public API ────────────────────────────────────────────────────────
+
+def detect_editors(project_dir: Path) -> list[str]:
+    """Return list of editor keys detected in the project directory."""
+    found = []
+    for key, editor in EDITORS.items():
+        for marker in editor["detect"]:
+            if (project_dir / marker).exists():
+                found.append(key)
+                break
+    return found
+
+
+def _codex_toml_block(command: str, project_dir: str) -> str:
+    """Generate the TOML block for Codex CLI's config.toml."""
+    args_toml = ", ".join(f'"{a}"' for a in ["serve", "--project-dir", project_dir])
+    return f'[mcp_servers.context-engine]\ncommand = "{command}"\nargs = [{args_toml}]\n'
+
+
+def configure_mcp(project_dir: Path, editor_key: str) -> bool:
+    """Write MCP config for a specific editor. Returns True if changed."""
+    editor = EDITORS[editor_key]
+    config_path = project_dir / editor["config_path"]
+    command = resolve_cce_binary()
+
+    config_path.parent.mkdir(parents=True, exist_ok=True)
+
+    if editor.get("format") == "toml":
+        return _configure_toml(config_path, command, str(project_dir))
+
+    servers_key = editor["servers_key"]
+    entry = {"command": command, "args": ["serve", "--project-dir", str(project_dir)]}
+
+    if config_path.exists():
+        try:
+            data = json.loads(config_path.read_text())
+        except (json.JSONDecodeError, OSError):
+            data = {}
+    else:
+        data = {}
+
+    servers = data.setdefault(servers_key, {})
+    if "context-engine" in servers:
+        existing = servers["context-engine"]
+        if existing.get("command") == command and existing.get("args") == entry["args"]:
+            return False
+        servers["context-engine"] = entry
+        atomic_write_text(config_path, json.dumps(data, indent=2) + "\n")
+        return True
+
+    servers["context-engine"] = entry
+    atomic_write_text(config_path, json.dumps(data, indent=2) + "\n")
+    return True
+
+
+def _configure_toml(config_path: Path, command: str, project_dir: str) -> bool:
+    """Add CCE to a TOML config file (Codex). Returns True if changed."""
+    block = _codex_toml_block(command, project_dir)
+    marker = "[mcp_servers.context-engine]"
+
+    if config_path.exists():
+        content = config_path.read_text()
+        if marker in content:
+            return False  # already configured
+        config_path.write_text(content.rstrip() + "\n\n" + block)
+    else:
+        config_path.write_text(block)
+    return True
+
+
+def remove_mcp(project_dir: Path, editor_key: str) -> str | None:
+    """Remove CCE from an editor's MCP config. Returns status message or None."""
+    editor = EDITORS[editor_key]
+    config_path = project_dir / editor["config_path"]
+
+    if not config_path.exists():
+        return None
+
+    if editor.get("format") == "toml":
+        return _remove_toml(config_path, editor["config_path"])
+
+    servers_key = editor["servers_key"]
+    try:
+        data = json.loads(config_path.read_text())
+        servers = data.get(servers_key, {})
+        if "context-engine" not in servers:
+            return None
+        del servers["context-engine"]
+        if servers:
+            config_path.write_text(json.dumps(data, indent=2) + "\n")
+            return f"Removed context-engine from {editor['config_path']}"
+        else:
+            config_path.unlink()
+            return f"Removed {editor['config_path']}"
+    except (json.JSONDecodeError, OSError):
+        return None
+
+
+def _remove_toml(config_path: Path, display_path: str) -> str | None:
+    """Remove CCE block from a TOML config file."""
+    import re
+    content = config_path.read_text()
+    marker = "[mcp_servers.context-engine]"
+    if marker not in content:
+        return None
+
+    # Remove the [mcp_servers.context-engine] block (until next section header or EOF)
+    pattern = r"\[mcp_servers\.context-engine\].*?(?=\n\[|$)"
+    new_content = re.sub(pattern, "", content, flags=re.DOTALL).strip()
+    if new_content:
+        config_path.write_text(new_content + "\n")
+        return f"Removed context-engine from {display_path}"
+    else:
+        config_path.unlink()
+        return f"Removed {display_path}"
+
+
+def write_instruction_file(project_dir: Path, file_key: str) -> bool:
+    """Write CCE instructions to an editor's instruction file. Returns True if written."""
+    info = INSTRUCTION_FILES[file_key]
+    path = project_dir / info["path"]
+    marker = "## Context Engine (CCE)"
+
+    if path.exists():
+        content = path.read_text()
+        if marker in content:
+            return False  # already has CCE block
+        # Append
+        path.write_text(content.rstrip() + "\n\n" + _CCE_INSTRUCTIONS)
+    else:
+        path.write_text(_CCE_INSTRUCTIONS)
+    return True
+
+
+def remove_instruction_file(project_dir: Path, file_key: str) -> str | None:
+    """Remove CCE block from an editor's instruction file. Returns status or None."""
+    info = INSTRUCTION_FILES[file_key]
+    path = project_dir / info["path"]
+    marker = "## Context Engine (CCE)"
+
+    if not path.exists():
+        return None
+
+    content = path.read_text()
+    if marker not in content:
+        return None
+
+    # Remove the CCE block
+    start = content.index(marker)
+    # Find the next ## heading or end of file
+    rest = content[start + len(marker):]
+    next_heading = rest.find("\n## ")
+    if next_heading >= 0:
+        end = start + len(marker) + next_heading
+    else:
+        end = len(content)
+
+    new_content = (content[:start] + content[end:]).strip()
+    if new_content:
+        path.write_text(new_content + "\n")
+        return f"Removed CCE block from {info['name']}"
+    else:
+        path.unlink()
+        return f"Removed {info['name']}"