bgts-context-engine 0.1.0__py3-none-any.whl

bce/__init__.py

@@ -0,0 +1,8 @@
+"""BGTS Context Engine (BCE).
+
+A deterministic, multi-language code-graph context engine. See README.md and the architecture
+specification for details. This package is the provider side of the deterministic line (P1): it
+contains no LLM and produces reproducible context packages.
+"""
+
+__version__ = "0.1.0"

bce/api/__init__.py

@@ -0,0 +1,6 @@
+"""Interface layer (thin adapters).
+
+MCP and REST call the same core/tools; logic is never duplicated here (spec section 7). These
+adapters only translate transport <-> tool calls and resolve the request locale for the i18n
+presentation layer.
+"""

bce/api/mcp/__init__.py

@@ -0,0 +1,11 @@
+"""MCP server adapter (spec section 7: agent-native surface).
+
+MCP is a *thin* adapter over the same core the REST API uses - no logic is duplicated. The tool
+catalog (:mod:`bce.api.mcp.tools`) is SDK-independent so it can be introspected and tested without
+the optional ``mcp`` package installed; :func:`bce.api.mcp.server.build_server` wires it into a real
+MCP server when the SDK is present.
+"""
+
+from bce.api.mcp.tools import TOOL_SPECS, dispatch_tool
+
+__all__ = ["TOOL_SPECS", "dispatch_tool"]

bce/api/mcp/server.py

@@ -0,0 +1,61 @@
+"""MCP server wiring (optional ``mcp`` SDK).
+
+Binds the SDK-independent tool catalog (:mod:`bce.api.mcp.tools`) to a real MCP server. The ``mcp``
+package is an optional dependency; importing this module without it raises a clear error only when
+you actually try to build/run the server, so the rest of the package (and tests) stay importable.
+
+Each tool call opens a fresh database connection (single DB: graph + vector, P5), dispatches to the
+core, and returns the deterministic envelope as JSON text content.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+
+def build_server(name: str = "bgts-context-engine") -> Any:
+    """Construct an MCP ``Server`` exposing every BCE tool. Requires the ``mcp`` package."""
+    try:
+        from mcp.server import Server
+        from mcp.types import TextContent, Tool
+    except ModuleNotFoundError as exc:  # pragma: no cover - depends on optional dep
+        raise RuntimeError(
+            "The 'mcp' package is required for the MCP server. Install it with: pip install mcp"
+        ) from exc
+
+    from bce.api.mcp.tools import TOOL_SPECS, dispatch_tool
+    from bce.storage.relational.db import connection
+
+    server = Server(name)
+
+    @server.list_tools()
+    async def _list_tools() -> list[Tool]:  # pragma: no cover - requires SDK runtime
+        return [
+            Tool(name=tool, description=spec["description"], inputSchema=spec["schema"])
+            for tool, spec in TOOL_SPECS.items()
+        ]
+
+    @server.call_tool()
+    async def _call_tool(
+        name: str, arguments: dict[str, Any]
+    ) -> list[TextContent]:  # pragma: no cover
+        with connection() as conn:
+            try:
+                result = dispatch_tool(conn, name, arguments)
+                conn.rollback()  # read-only tools never persist
+            except Exception:
+                conn.rollback()
+                raise
+        return [TextContent(type="text", text=json.dumps(result, ensure_ascii=False, default=str))]
+
+    return server
+
+
+async def run_stdio(name: str = "bgts-context-engine") -> None:  # pragma: no cover - SDK runtime
+    """Run the MCP server over stdio (the common agent transport)."""
+    from mcp.server.stdio import stdio_server
+
+    server = build_server(name)
+    async with stdio_server() as (read, write):
+        await server.run(read, write, server.create_initialization_options())

bce/api/mcp/tools.py

@@ -0,0 +1,262 @@
+"""MCP tool catalog + dispatcher (SDK-independent).
+
+Declares every Layer 1-2-3 tool as an MCP-style spec (name, description, JSON input schema) and
+maps a tool name + arguments to a call against the deterministic core, using the caller's connection.
+This is the single source of truth the SDK server binds to, and it is importable/testable without the
+optional ``mcp`` package.
+
+Every tool returns the standard envelope ``{tool, payload, message, locale}`` (P1: payload is
+language-neutral; only ``message`` is localized).
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import psycopg
+
+from bce.core.auth.scope import Principal, ScopeFilter
+from bce.storage.graph.client import GraphClient
+from bce.storage.graph.repository import GraphRepository
+from bce.storage.vector.store import VectorStore
+from bce.tools.layer1 import (
+    find_implementers,
+    find_references,
+    get_call_graph,
+    get_dependencies,
+    get_type_hierarchy,
+    resolve_symbol,
+)
+from bce.tools.layer2 import find_similar_code, hybrid_search, semantic_search
+from bce.tools.layer3 import (
+    assemble_context,
+    expand_blast_radius,
+    get_context_for_task,
+    select_repos,
+    suggest_change_sites,
+)
+
+_STR = {"type": "string"}
+_STR_LIST = {"type": "array", "items": {"type": "string"}}
+_INT = {"type": "integer"}
+_BOOL = {"type": "boolean"}
+
+
+def _schema(props: dict[str, Any], required: list[str]) -> dict[str, Any]:
+    return {"type": "object", "properties": props, "required": required}
+
+
+# Tool catalog: name -> (description, input schema). Mirrors the REST surface 1:1.
+TOOL_SPECS: dict[str, dict[str, Any]] = {
+    "resolve_symbol": {
+        "description": "Layer 1: resolve a symbol by name to its definition + symbol_id.",
+        "schema": _schema({"name": _STR, "repo_id": _STR, "locale": _STR}, ["name"]),
+    },
+    "find_references": {
+        "description": "Layer 1: find references (callsites) to a symbol_id.",
+        "schema": _schema({"symbol_id": _STR, "locale": _STR}, ["symbol_id"]),
+    },
+    "find_implementers": {
+        "description": "Layer 1: types that inherit/implement a type symbol.",
+        "schema": _schema({"symbol_id": _STR, "locale": _STR}, ["symbol_id"]),
+    },
+    "get_call_graph": {
+        "description": "Layer 1: callers/callees subgraph around a symbol.",
+        "schema": _schema(
+            {"symbol_id": _STR, "hops": _INT, "direction": _STR, "locale": _STR}, ["symbol_id"]
+        ),
+    },
+    "get_dependencies": {
+        "description": "Layer 1: IMPORTS edges out of a file (optionally transitive).",
+        "schema": _schema(
+            {"file_id": _STR, "transitive": {"type": "boolean"}, "locale": _STR}, ["file_id"]
+        ),
+    },
+    "get_type_hierarchy": {
+        "description": "Layer 1: super/sub types of a type symbol.",
+        "schema": _schema({"symbol_id": _STR, "locale": _STR}, ["symbol_id"]),
+    },
+    "semantic_search": {
+        "description": "Layer 2: nearest symbols for a natural-language query (anchor finding).",
+        "schema": _schema(
+            {"query": _STR, "repo_ids": _STR_LIST, "limit": _INT, "locale": _STR}, ["query"]
+        ),
+    },
+    "hybrid_search": {
+        "description": "Layer 2: keyword + semantic + structural blended search.",
+        "schema": _schema(
+            {"query": _STR, "repo_ids": _STR_LIST, "limit": _INT, "locale": _STR}, ["query"]
+        ),
+    },
+    "find_similar_code": {
+        "description": "Layer 2: nearest symbols to a code fragment.",
+        "schema": _schema(
+            {"code": _STR, "repo_ids": _STR_LIST, "limit": _INT, "locale": _STR}, ["code"]
+        ),
+    },
+    "get_context_for_task": {
+        "description": "Layer 3: assembled context package + coverage for a task.",
+        "schema": _schema(
+            {
+                "task_text": _STR,
+                "task_id": _STR,
+                "max_tokens": _INT,
+                "max_candidates": _INT,
+                "commit": _STR,
+                "repo_ids": _STR_LIST,
+                "explicit_symbols": _STR_LIST,
+                "route_paths": _STR_LIST,
+                "history_file_ids": _STR_LIST,
+                "component_repo_ids": _STR_LIST,
+                "semantic_candidates": _STR_LIST,
+                "auto_semantic": _BOOL,
+                "locale": _STR,
+            },
+            ["task_text"],
+        ),
+    },
+    "suggest_change_sites": {
+        "description": "Layer 3: scored change-site candidates for a task.",
+        "schema": _schema(
+            {
+                "task_text": _STR,
+                "max_candidates": _INT,
+                "commit": _STR,
+                "auto_semantic": _BOOL,
+                "locale": _STR,
+            },
+            ["task_text"],
+        ),
+    },
+    "expand_blast_radius": {
+        "description": "Layer 3: impacted files/repos/symbols for target symbols.",
+        "schema": _schema({"target_symbols": _STR_LIST, "locale": _STR}, ["target_symbols"]),
+    },
+    "select_repos": {
+        "description": "Layer 3: candidate repo set for a task.",
+        "schema": _schema({"task_text": _STR, "locale": _STR}, ["task_text"]),
+    },
+    "assemble_context": {
+        "description": "Layer 3: dedup a symbol list and fit it into a token budget.",
+        "schema": _schema(
+            {"symbol_ids": _STR_LIST, "max_tokens": _INT, "locale": _STR}, ["symbol_ids"]
+        ),
+    },
+}
+
+
+def dispatch_tool(
+    conn: psycopg.Connection,
+    name: str,
+    arguments: dict[str, Any],
+    *,
+    user_id: str | None = None,
+) -> dict[str, Any]:
+    """Route an MCP tool call to the core, using ``conn`` for graph + vector access (single DB)."""
+    if name not in TOOL_SPECS:
+        raise KeyError(f"unknown tool: {name}")
+
+    repository = GraphRepository(GraphClient(conn))
+    store = VectorStore(conn)
+    args = dict(arguments)
+    locale = args.pop("locale", None)
+    scope = (
+        ScopeFilter.from_scopes(conn, user_id)
+        if user_id is not None
+        else ScopeFilter(Principal.system())
+    )
+
+    if name == "resolve_symbol":
+        return resolve_symbol(repository, args["name"], repo_id=args.get("repo_id"), locale=locale)
+    if name == "find_references":
+        return find_references(repository, args["symbol_id"], locale=locale)
+    if name == "find_implementers":
+        return find_implementers(repository, args["symbol_id"], locale=locale)
+    if name == "get_call_graph":
+        return get_call_graph(
+            repository,
+            args["symbol_id"],
+            hops=args.get("hops", 1),
+            direction=args.get("direction", "both"),
+            locale=locale,
+        )
+    if name == "get_dependencies":
+        return get_dependencies(
+            repository, args["file_id"], transitive=args.get("transitive", False), locale=locale
+        )
+    if name == "get_type_hierarchy":
+        return get_type_hierarchy(repository, args["symbol_id"], locale=locale)
+    if name == "semantic_search":
+        return semantic_search(
+            store,
+            args["query"],
+            repo_ids=args.get("repo_ids"),
+            limit=args.get("limit", 10),
+            locale=locale,
+        )
+    if name == "hybrid_search":
+        return hybrid_search(
+            repository,
+            store,
+            args["query"],
+            repo_ids=args.get("repo_ids"),
+            limit=args.get("limit", 10),
+            locale=locale,
+        )
+    if name == "find_similar_code":
+        return find_similar_code(
+            store,
+            args["code"],
+            repo_ids=args.get("repo_ids"),
+            limit=args.get("limit", 10),
+            locale=locale,
+        )
+    if name == "get_context_for_task":
+        return get_context_for_task(
+            repository, store=store, scope=scope, locale=locale, **_l3_kwargs(args)
+        )
+    if name == "suggest_change_sites":
+        return suggest_change_sites(
+            repository, store=store, scope=scope, locale=locale, **_l3_kwargs(args)
+        )
+    if name == "expand_blast_radius":
+        return expand_blast_radius(
+            repository, target_symbols=args["target_symbols"], scope=scope, locale=locale
+        )
+    if name == "select_repos":
+        return select_repos(
+            repository,
+            task_text=args["task_text"],
+            component_repo_ids=args.get("component_repo_ids"),
+            history_file_ids=args.get("history_file_ids"),
+            scope=scope,
+            locale=locale,
+        )
+    if name == "assemble_context":
+        return assemble_context(
+            repository,
+            symbol_ids=args["symbol_ids"],
+            max_tokens=args.get("max_tokens", 4000),
+            scope=scope,
+            locale=locale,
+        )
+    raise KeyError(f"unhandled tool: {name}")  # pragma: no cover - guarded above
+
+
+def _l3_kwargs(args: dict[str, Any]) -> dict[str, Any]:
+    """Filter task-oriented Layer-3 kwargs (shared by context/change-sites)."""
+    allowed = {
+        "task_text",
+        "task_id",
+        "max_tokens",
+        "max_candidates",
+        "commit",
+        "repo_ids",
+        "explicit_symbols",
+        "route_paths",
+        "history_file_ids",
+        "component_repo_ids",
+        "semantic_candidates",
+        "auto_semantic",
+    }
+    return {k: v for k, v in args.items() if k in allowed}

bce/api/rest/__init__.py

@@ -0,0 +1,11 @@
+"""REST adapter (FastAPI).
+
+A thin transport over the Layer-1 tools. It does not implement retrieval/graph logic itself; it
+opens a per-request database connection, resolves the request locale (i18n presentation layer), and
+delegates to ``bce.tools``. Import :func:`create_app` (factory) or ``app`` (module-level instance,
+for ``uvicorn bce.api.rest.app:app``).
+"""
+
+from bce.api.rest.app import app, create_app
+
+__all__ = ["app", "create_app"]

bce/api/rest/app.py

@@ -0,0 +1,239 @@
+"""FastAPI application factory.
+
+``create_app`` wires the router, the request-logging middleware, and the lifespan that runs the
+background job workers; ``app`` is a module-level instance so the server can be started with
+``uvicorn bce.api.rest.app:app`` (or via ``bce serve``). Every request gets its own database
+connection through the ``get_repository`` dependency.
+
+Request logging has two levels:
+
+- INFO: one summary line per request (method, path, status, duration).
+- DEBUG (``BCE_LOG_LEVEL=DEBUG``): additionally logs the request start, query params, the JSON
+  request body (secrets masked, size-capped) and the response body (size-capped) for **every**
+  endpoint.
+
+Released distributions carry a compiled frontend which is served under ``/ui``. A source
+checkout has no bundle, so that mount is skipped and the Vite dev server is used instead.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import mimetypes
+import time
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+from pathlib import Path
+from typing import Any
+
+from fastapi import FastAPI, Request, Response
+from fastapi.responses import RedirectResponse
+from fastapi.staticfiles import StaticFiles
+
+from bce import __version__
+from bce.api.rest.routes import router
+from bce.config import get_settings
+from bce.core.logging import get_logger, setup_logging
+
+logger = get_logger("api.rest")
+
+#: Request/response bodies logged at DEBUG are truncated to this many characters.
+_BODY_LOG_LIMIT = 4000
+
+#: Body keys whose values are masked in DEBUG logs (credentials must never reach the logs).
+_SENSITIVE_KEYS = ("token", "password", "secret", "api_key", "username", "authorization")
+
+
+def _mask(value: Any) -> Any:
+    """Recursively mask sensitive fields in a decoded JSON structure."""
+    if isinstance(value, dict):
+        return {
+            key: ("***" if any(s in key.lower() for s in _SENSITIVE_KEYS) and val else _mask(val))
+            for key, val in value.items()
+        }
+    if isinstance(value, list):
+        return [_mask(item) for item in value]
+    return value
+
+
+def _body_for_log(raw: bytes) -> str:
+    """Masked, truncated representation of a request/response body for DEBUG logs."""
+    if not raw:
+        return ""
+    try:
+        text = json.dumps(_mask(json.loads(raw)), ensure_ascii=False)
+    except (ValueError, UnicodeDecodeError):
+        text = raw[:_BODY_LOG_LIMIT].decode("utf-8", errors="replace")
+    if len(text) > _BODY_LOG_LIMIT:
+        text = text[:_BODY_LOG_LIMIT] + f"... (+{len(text) - _BODY_LOG_LIMIT} chars)"
+    return text
+
+
+@asynccontextmanager
+async def _lifespan(app: FastAPI) -> AsyncIterator[None]:
+    """Start the job worker pool with the server; drain it on shutdown.
+
+    Shutdown is graceful: workers stop claiming new jobs and the in-flight job is awaited.
+    """
+    from bce.jobs.worker import JobWorkerPool
+
+    setup_logging()
+    pool = JobWorkerPool()
+    pool.start()
+    try:
+        yield
+    finally:
+        pool.stop()
+
+
+#: Directory the compiled frontend is vendored into at build time (scripts/build_ui.py).
+UI_DIST_DIR = Path(__file__).parent / "static"
+
+
+def ui_is_bundled() -> bool:
+    """True when a compiled frontend ships with this installation."""
+    return (UI_DIST_DIR / "index.html").is_file()
+
+
+#: StaticFiles resolves content types through :mod:`mimetypes`, which on Windows consults the
+#: registry -- where installed software commonly maps ``.js`` to ``text/plain``. Browsers apply
+#: strict MIME checking to ``<script type="module">``, which is what Vite emits, so that mapping
+#: makes the UI a blank page. Pin the types the bundle relies on instead of trusting the host.
+_UI_CONTENT_TYPES = {
+    ".js": "text/javascript",
+    ".mjs": "text/javascript",
+    ".css": "text/css",
+    ".svg": "image/svg+xml",
+    ".json": "application/json",
+    ".woff": "font/woff",
+    ".woff2": "font/woff2",
+}
+
+
+def _mount_ui(app: FastAPI) -> None:
+    """Serve the compiled frontend under ``/ui``.
+
+    Does nothing when the bundle is absent, which is the normal case for a source checkout:
+    developers run the Vite dev server instead (see web/README.md).
+    """
+    if not ui_is_bundled():
+        logger.debug("ui bundle not present; skipping /ui mount", extra={"path": str(UI_DIST_DIR)})
+        return
+
+    for suffix, content_type in _UI_CONTENT_TYPES.items():
+        mimetypes.add_type(content_type, suffix)
+
+    @app.get("/ui", include_in_schema=False)
+    async def ui_root() -> RedirectResponse:
+        return RedirectResponse(url="/ui/")
+
+    # html=True serves index.html for the directory root. There is deliberately no catch-all
+    # fallback: the UI is a single view with no client-side router, so a missing asset should
+    # surface as a 404 rather than be masked by an HTML response.
+    app.mount("/ui/", StaticFiles(directory=UI_DIST_DIR, html=True), name="ui")
+    logger.info("ui bundle mounted", extra={"path": str(UI_DIST_DIR), "url": "/ui/"})
+
+
+def create_app(*, serve_ui: bool = True) -> FastAPI:
+    setup_logging()
+    app = FastAPI(
+        title="BGTS Context Engine",
+        version=__version__,
+        summary="Deterministic, multi-language code-graph context engine (Layer 1-2-3 REST surface).",
+        lifespan=_lifespan,
+    )
+
+    @app.middleware("http")
+    async def log_requests(request: Request, call_next) -> Response:
+        debug = logger.isEnabledFor(logging.DEBUG)
+        start = time.perf_counter()
+
+        if debug:
+            extra: dict[str, Any] = {
+                "method": request.method,
+                "path": request.url.path,
+                "query": dict(request.query_params),
+                "client": request.client.host if request.client else None,
+            }
+            # Safe with BaseHTTPMiddleware: Starlette caches the body and replays it downstream.
+            raw = await request.body()
+            if raw:
+                extra["body"] = _body_for_log(raw)
+            logger.debug("request started", extra=extra)
+
+        try:
+            response = await call_next(request)
+        except Exception:
+            duration_ms = round((time.perf_counter() - start) * 1000, 2)
+            logger.exception(
+                "request failed",
+                extra={
+                    "method": request.method,
+                    "path": request.url.path,
+                    "duration_ms": duration_ms,
+                },
+            )
+            raise
+
+        duration_ms = round((time.perf_counter() - start) * 1000, 2)
+        logger.info(
+            "request",
+            extra={
+                "method": request.method,
+                "path": request.url.path,
+                "status_code": response.status_code,
+                "duration_ms": duration_ms,
+            },
+        )
+
+        if debug and hasattr(response, "body_iterator"):
+            chunks = [chunk async for chunk in response.body_iterator]
+            raw_body = b"".join(chunks)
+            logger.debug(
+                "response body",
+                extra={
+                    "method": request.method,
+                    "path": request.url.path,
+                    "status_code": response.status_code,
+                    "size_bytes": len(raw_body),
+                    "body": _body_for_log(raw_body),
+                },
+            )
+            response = Response(
+                content=raw_body,
+                status_code=response.status_code,
+                headers=dict(response.headers),
+                media_type=response.media_type,
+                background=response.background,
+            )
+        return response
+
+    app.include_router(router)
+
+    # --- UI layer (optional, self-contained; see bce/api/rest/ui/__init__.py) ---
+    # To remove the UI layer: delete the bce/api/rest/ui package and this block.
+    from fastapi.middleware.cors import CORSMiddleware
+
+    from bce.api.rest.ui import ui_router
+
+    app.include_router(ui_router)
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=list(get_settings().cors_origins),
+        allow_methods=["GET", "POST"],  # POST: /v1/ui/context-trace (read-only pipeline trace).
+        allow_headers=["*"],
+    )
+    if serve_ui:
+        _mount_ui(app)
+    # --- end UI layer ---
+
+    return app
+
+
+def create_api_only_app() -> FastAPI:
+    """Application without the ``/ui`` mount, for ``bce serve --no-ui``."""
+    return create_app(serve_ui=False)
+
+
+app = create_app()

bce/api/rest/deps.py

@@ -0,0 +1,67 @@
+"""FastAPI dependencies: per-request database repository, vector store, locale, and scope.
+
+These are deliberately tiny so they can be overridden in tests (``app.dependency_overrides``)
+without a live database.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+
+from fastapi import Depends, Header, Query
+
+from bce.core.auth.scope import Principal, ScopeFilter
+from bce.core.i18n import get_translator
+from bce.storage.graph.client import GraphClient
+from bce.storage.graph.repository import GraphRepository
+from bce.storage.relational.db import connection
+from bce.storage.vector.store import VectorStore
+
+
+def get_repository() -> Iterator[GraphRepository]:
+    """Yield a graph repository bound to a fresh connection (closed when the request ends).
+
+    Reads do not commit. Write endpoints (e.g. ``/index``) commit explicitly; on any error we roll
+    back so a half-written transaction never lingers on the pooled connection.
+    """
+    with connection() as conn:
+        try:
+            yield GraphRepository(GraphClient(conn))
+        except Exception:
+            conn.rollback()
+            raise
+
+
+def get_vector_store(
+    repository: GraphRepository = Depends(get_repository),
+) -> VectorStore:
+    """Vector store bound to the same request connection as the graph repository (single DB, P5)."""
+    return VectorStore(repository.client.conn)
+
+
+def get_scope(
+    repository: GraphRepository = Depends(get_repository),
+    user_id: str | None = Header(default=None, alias="X-BCE-User"),
+) -> ScopeFilter:
+    """Build the scope filter from the ``X-BCE-User`` header via the ``scopes`` table (section 9).
+
+    No header -> system principal (allow-all), matching pre-Phase-4 behaviour. On any lookup error we
+    fail closed to an empty scope so a misconfiguration cannot leak inaccessible repos.
+    """
+    if user_id is None:
+        return ScopeFilter(Principal.system())
+    try:
+        return ScopeFilter.from_scopes(repository.client.conn, user_id)
+    except Exception:
+        return ScopeFilter(Principal(user_id=user_id, allowed_repo_ids=[]))
+
+
+def get_locale(
+    locale: str | None = Query(default=None, description="Force a locale (en/tr)."),
+    accept_language: str | None = Header(default=None),
+) -> str:
+    """Resolve the response locale from ``?locale=`` (wins) then the ``Accept-Language`` header.
+
+    Returns a supported locale; the deterministic payload is unaffected (i18n is presentation only).
+    """
+    return get_translator().resolve(requested=locale, accept_language=accept_language)