opencontextengine-client 0.1.0__py3-none-any.whl

oce_client/ignore.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Iterable
+
+from pathspec.gitignore import GitIgnoreSpec
+
+
+DEFAULT_PATTERNS = (
+    ".git/",
+    ".oce-client/",
+    "__pycache__/",
+    ".pytest_cache/",
+    ".mypy_cache/",
+    ".ruff_cache/",
+    ".venv/",
+    "venv/",
+    "node_modules/",
+    "target/",
+    "dist/",
+    "build/",
+    "coverage/",
+    ".idea/",
+    ".vscode/",
+)
+
+
+class _RuleLayer:
+    def __init__(self, lines: Iterable[str]) -> None:
+        cleaned = []
+        for raw in lines:
+            line = raw.strip("\r\n")
+            if line and not line.startswith("#"):
+                cleaned.append(line)
+        self.patterns = list(GitIgnoreSpec.from_lines(cleaned).patterns)
+
+    def match(self, path: str, is_dir: bool) -> bool | None:
+        candidate = path.rstrip("/") + ("/" if is_dir else "")
+        result: bool | None = None
+        for pattern in self.patterns:
+            if pattern.match_file(candidate) is not None:
+                # GitWildMatchPattern.include=True means the path is ignored;
+                # a leading ! produces include=False and re-includes it.
+                result = bool(pattern.include)
+        return result
+
+
+class LayeredIgnoreMatcher:
+    """Merge runtime, project, git, and built-in ignore rules by precedence."""
+
+    def __init__(
+        self,
+        root: Path,
+        runtime_patterns: Iterable[str] = (),
+        *,
+        oceignore_name: str = ".oceignore",
+        gitignore_name: str = ".gitignore",
+    ) -> None:
+        self.root = root
+        self._hard = _RuleLayer((".git/", ".git/**", ".oce-client/", ".oce-client/**"))
+        self._runtime = _RuleLayer(runtime_patterns)
+        self._oce = _RuleLayer(self._read_lines(root / oceignore_name))
+        self._git = _RuleLayer(self._read_lines(root / gitignore_name))
+        self._defaults = _RuleLayer(DEFAULT_PATTERNS[2:])
+
+    @staticmethod
+    def _read_lines(path: Path) -> list[str]:
+        try:
+            return path.read_text(encoding="utf-8").splitlines()
+        except (FileNotFoundError, UnicodeDecodeError, OSError):
+            return []
+
+    def ignores(self, path: str, *, is_dir: bool = False) -> bool:
+        normalized = path.replace("\\", "/")
+        while normalized.startswith("./"):
+            normalized = normalized[2:]
+        # A hard rule cannot be undone by a higher-priority negation.
+        if self._hard.match(normalized, is_dir) is True:
+            return True
+        for layer in (self._runtime, self._oce, self._git, self._defaults):
+            decision = layer.match(normalized, is_dir)
+            if decision is not None:
+                return decision
+        return False

oce_client/indexer.py

@@ -0,0 +1,265 @@
+from __future__ import annotations
+
+import threading
+import time
+from collections.abc import Callable
+from pathlib import Path
+
+from .ignore import LayeredIgnoreMatcher
+from .runtime import ClientRuntime, ClientSettings
+from .watcher import WatchHandle
+
+
+class WorkspaceIndexer:
+    """Own one workspace's background synchronization and readiness barrier."""
+
+    def __init__(
+        self,
+        settings: ClientSettings,
+        *,
+        runtime_factory: Callable[[ClientSettings], ClientRuntime] = ClientRuntime,
+        debounce_ms: int = 500,
+    ) -> None:
+        self.settings = settings
+        self.debounce_ms = debounce_ms
+        self._runtime = runtime_factory(settings)
+        self._condition = threading.Condition()
+        self._context_lock = threading.Lock()
+        self._stop = False
+        self._started = False
+        self._initialized = False
+        self._recovery_required = False
+        self._full_pending = False
+        self._pending_paths: set[Path] = set()
+        self._requested_generation = 0
+        self._synced_generation = 0
+        self._state = "idle"
+        self._last_error: str | None = None
+        self._watch: WatchHandle | None = None
+        self._worker: threading.Thread | None = None
+
+    @property
+    def root(self) -> Path:
+        return self.settings.root.resolve()
+
+    def start(self, *, initial_sync: bool = True) -> None:
+        with self._condition:
+            if self._started:
+                if (
+                    initial_sync
+                    and not self._initialized
+                    and self._state in {"idle", "error"}
+                    and not self._full_pending
+                ):
+                    self._request_full_locked()
+                return
+            self._started = True
+            self._watch = WatchHandle(self.root, self.notify_changes, self.debounce_ms)
+            self._worker = threading.Thread(
+                target=self._run,
+                name=f"oce-indexer-{self.root.name}",
+                daemon=True,
+            )
+            self._worker.start()
+            if initial_sync:
+                self._request_full_locked()
+
+    def stop(self) -> None:
+        watch: WatchHandle | None
+        worker: threading.Thread | None
+        with self._condition:
+            if not self._started:
+                self._runtime.close()
+                return
+            self._stop = True
+            watch = self._watch
+            worker = self._worker
+            self._condition.notify_all()
+        if watch is not None:
+            watch.stop()
+            watch.join(2.0)
+        if worker is not None:
+            worker.join(5.0)
+            if worker.is_alive():
+                return
+        with self._context_lock:
+            self._runtime.close()
+
+    def _request_full_locked(self) -> None:
+        self._requested_generation += 1
+        self._full_pending = True
+        self._state = "indexing"
+        self._last_error = None
+        self._condition.notify_all()
+
+    def request_full_sync(self) -> None:
+        with self._condition:
+            self._request_full_locked()
+
+    def notify_changes(self, paths: set[Path]) -> None:
+        matcher = LayeredIgnoreMatcher(self.root, self.settings.runtime_patterns)
+        relevant: set[Path] = set()
+        for path in paths:
+            resolved = path.resolve()
+            try:
+                relative = resolved.relative_to(self.root).as_posix()
+            except ValueError:
+                continue
+            if resolved.name in {".gitignore", ".oceignore"} or not matcher.ignores(
+                relative,
+                is_dir=resolved.is_dir(),
+            ):
+                relevant.add(resolved)
+        if not relevant:
+            return
+        with self._condition:
+            if self._stop:
+                return
+            self._requested_generation += 1
+            if not self._initialized or self._recovery_required or any(
+                path.name in {".gitignore", ".oceignore"} for path in relevant
+            ):
+                self._full_pending = True
+            else:
+                self._pending_paths.update(relevant)
+            self._state = "indexing"
+            self._last_error = None
+            self._condition.notify_all()
+
+    def _next_batch(self) -> tuple[bool, set[Path], int] | None:
+        with self._condition:
+            self._condition.wait_for(
+                lambda: self._stop or self._full_pending or bool(self._pending_paths)
+            )
+            if self._stop:
+                return None
+            full = self._full_pending
+            paths = set(self._pending_paths)
+            generation = self._requested_generation
+            self._full_pending = False
+            self._pending_paths.clear()
+            self._state = "indexing"
+            return full, paths, generation
+
+    def _run(self) -> None:
+        while True:
+            batch = self._next_batch()
+            if batch is None:
+                return
+            full, paths, generation = batch
+            try:
+                with self._context_lock:
+                    context = self._runtime.context()
+                    if full:
+                        context.sync()
+                    elif paths:
+                        context.sync_paths(paths)
+            except Exception as exc:
+                with self._condition:
+                    self._recovery_required = True
+                    if self._full_pending or self._pending_paths:
+                        self._full_pending = True
+                        self._pending_paths.clear()
+                    self._state = "error"
+                    self._last_error = str(exc)
+                    self._condition.notify_all()
+                continue
+            with self._condition:
+                self._initialized = True
+                self._recovery_required = False
+                self._synced_generation = max(self._synced_generation, generation)
+                self._last_error = None
+                if self._full_pending or self._pending_paths:
+                    self._state = "indexing"
+                else:
+                    self._state = "ready"
+                self._condition.notify_all()
+
+    def _ready_locked(self) -> bool:
+        return (
+            self._initialized
+            and self._state == "ready"
+            and not self._full_pending
+            and not self._pending_paths
+            and self._synced_generation >= self._requested_generation
+        )
+
+    def wait_until_ready(self, timeout: float | None) -> str:
+        deadline = None if timeout is None else time.monotonic() + timeout
+        with self._condition:
+            if not self._started:
+                raise RuntimeError("workspace indexer has not been started")
+            if (
+                not self._initialized
+                and self._state in {"idle", "error"}
+                and not self._full_pending
+            ):
+                self._request_full_locked()
+            while not self._ready_locked():
+                if (
+                    self._state == "error"
+                    and not self._full_pending
+                    and not self._pending_paths
+                ):
+                    return "error"
+                remaining = (
+                    None if deadline is None else deadline - time.monotonic()
+                )
+                if remaining is not None and remaining <= 0:
+                    return "indexing"
+                self._condition.wait(remaining)
+            return "ready"
+
+    def retrieve(self, query: str, timeout: float) -> dict[str, object]:
+        self.start(initial_sync=True)
+        with self._condition:
+            if (
+                self._state == "error"
+                and not self._full_pending
+                and not self._pending_paths
+            ):
+                self._request_full_locked()
+        deadline = time.monotonic() + timeout
+        while True:
+            remaining = max(0.0, deadline - time.monotonic())
+            status = self.wait_until_ready(remaining)
+            if status != "ready":
+                with self._condition:
+                    payload: dict[str, object] = {
+                        "status": status,
+                        "workspace_folder": str(self.root),
+                    }
+                    if status == "error":
+                        payload["error"] = self._last_error or "workspace synchronization failed"
+                    else:
+                        payload["message"] = "Workspace indexing is still in progress; retry shortly."
+                    return payload
+
+            with self._context_lock:
+                with self._condition:
+                    if not self._ready_locked():
+                        continue
+                try:
+                    result = self._runtime.context().retrieve(query)
+                except Exception as exc:
+                    return {
+                        "status": "error",
+                        "workspace_folder": str(self.root),
+                        "error": str(exc),
+                    }
+                return {
+                    "status": "ready",
+                    "workspace_folder": str(self.root),
+                    "formatted_retrieval": result.formatted_retrieval,
+                    "elapsed_ms": result.elapsed_ms,
+                }
+
+    def status(self) -> dict[str, object]:
+        with self._condition:
+            return {
+                "status": self._state,
+                "workspace_folder": str(self.root),
+                "requested_generation": self._requested_generation,
+                "synced_generation": self._synced_generation,
+                "error": self._last_error,
+            }

oce_client/mcp_server.py

@@ -0,0 +1,309 @@
+from __future__ import annotations
+
+import argparse
+import asyncio
+import hashlib
+import os
+import sys
+from collections.abc import Callable, Sequence
+from contextlib import asynccontextmanager
+from pathlib import Path
+from typing import Annotated, Any
+
+from .indexer import WorkspaceIndexer
+from .runtime import (
+    ClientConfigurationError,
+    ClientRuntime,
+    ClientSettings,
+    McpConfiguration,
+    iter_runtime_patterns,
+)
+
+
+TOOL_DESCRIPTION = """This tool is Open Context Engine(oce), Open source codebase context engine. It:
+1. Takes in a natural language description of the code you are looking for;
+2. Uses a proprietary retrieval/embedding model suite that produces the highest-quality recall of relevant code snippets from across the codebase;
+3. Maintains a real-time index of the codebase, so the results are always up-to-date and reflects the current state of the codebase;
+4. Can retrieve across different programming languages;
+5. Only reflects the current state of the codebase on the disk, and has no information on version control or code history."""
+
+
+def _require_sdk() -> Any:
+    try:
+        from mcp.server.fastmcp import FastMCP
+    except ImportError as exc:
+        raise ClientConfigurationError(
+            "MCP support is not installed; install opencontextengine-client with the 'mcp' extra"
+        ) from exc
+    return FastMCP
+
+
+def add_mcp_arguments(parser: argparse.ArgumentParser) -> None:
+    """Add the MCP server launch options to the standalone entry point."""
+    parser.add_argument(
+        "--workspace",
+        action="append",
+        default=argparse.SUPPRESS,
+        metavar="PATH",
+        help="allowed workspace folder; repeat for multiple workspaces",
+    )
+    parser.add_argument(
+        "--api-url",
+        default=argparse.SUPPRESS,
+        help="OCE API URL (default: OCE_API_URL or the local default)",
+    )
+    parser.add_argument(
+        "--state-path",
+        default=argparse.SUPPRESS,
+        help="SQLite state file; only valid with one workspace",
+    )
+    parser.add_argument(
+        "--state-dir",
+        default=argparse.SUPPRESS,
+        help="directory for per-workspace SQLite state (OCE_STATE_DIR)",
+    )
+    parser.add_argument(
+        "--ignore",
+        action="append",
+        default=argparse.SUPPRESS,
+        metavar="PATTERN",
+        help="runtime ignore pattern; repeat or comma-separate (OCE_IGNORE)",
+    )
+    parser.add_argument(
+        "--debounce-ms",
+        type=int,
+        default=argparse.SUPPRESS,
+        help="filesystem watcher debounce interval",
+    )
+    parser.add_argument(
+        "--initial-sync",
+        choices=("background", "blocking", "off"),
+        default=argparse.SUPPRESS,
+        help="initial workspace synchronization strategy",
+    )
+    parser.add_argument(
+        "--ready-timeout",
+        type=float,
+        default=argparse.SUPPRESS,
+        help="seconds a retrieval waits for the latest index generation",
+    )
+    parser.add_argument(
+        "--log-level",
+        choices=("debug", "info", "warning", "error", "critical"),
+        default=argparse.SUPPRESS,
+        help="MCP server log level",
+    )
+
+
+def mcp_configuration_from_args(args: argparse.Namespace) -> McpConfiguration:
+    values = getattr(args, "workspace", None)
+    return McpConfiguration.from_environment(
+        workspace_roots=tuple(values) if values else None,
+        api_url=getattr(args, "api_url", None),
+        state_path=getattr(args, "state_path", None),
+        state_dir=getattr(args, "state_dir", None),
+        runtime_patterns=(
+            iter_runtime_patterns(args.ignore)
+            if getattr(args, "ignore", None) is not None
+            else None
+        ),
+        debounce_ms=getattr(args, "debounce_ms", None),
+        initial_sync=getattr(args, "initial_sync", None),
+        ready_timeout=getattr(args, "ready_timeout", None),
+        log_level=getattr(args, "log_level", None),
+    )
+
+
+def create_server(
+    settings: ClientSettings,
+    *,
+    workspace_roots: Sequence[Path] | None = None,
+    state_dir: Path | None = None,
+    debounce_ms: int = 500,
+    initial_sync: str = "background",
+    ready_timeout: float = 3.0,
+    log_level: str = "WARNING",
+    runtime_factory: Callable[[ClientSettings], ClientRuntime] = ClientRuntime,
+) -> Any:
+    if debounce_ms < 0:
+        raise ClientConfigurationError("debounce-ms must not be negative")
+    if ready_timeout < 0:
+        raise ClientConfigurationError("ready-timeout must not be negative")
+    if initial_sync not in {"background", "blocking", "off"}:
+        raise ClientConfigurationError(f"unknown initial sync mode: {initial_sync}")
+
+    roots = tuple(
+        dict.fromkeys(
+            root.expanduser().resolve()
+            for root in (workspace_roots or (settings.root,))
+        )
+    )
+    if not roots:
+        raise ClientConfigurationError("at least one workspace is required")
+    for root in roots:
+        if not root.is_dir():
+            raise ClientConfigurationError(f"workspace is not a directory: {root}")
+
+    FastMCP = _require_sdk()
+    indexers: dict[Path, WorkspaceIndexer] = {}
+    for root in roots:
+        if state_dir is not None:
+            state_path = _state_path(state_dir, root)
+        elif root == settings.root.resolve():
+            state_path = settings.state_path
+        else:
+            state_path = None
+        root_settings = ClientSettings(
+            root=root,
+            api_url=settings.api_url,
+            api_key=settings.api_key,
+            state_path=state_path,
+            runtime_patterns=settings.runtime_patterns,
+        )
+        indexers[root] = WorkspaceIndexer(
+            root_settings,
+            runtime_factory=runtime_factory,
+            debounce_ms=debounce_ms,
+        )
+
+    def indexer_for(workspace_folder: str | None) -> WorkspaceIndexer:
+        if workspace_folder is None:
+            if len(indexers) != 1:
+                raise ValueError(
+                    "workspace_folder is required when multiple workspaces are configured"
+                )
+            return next(iter(indexers.values()))
+        if not workspace_folder.strip():
+            raise ValueError("workspace_folder must not be empty")
+        requested = Path(workspace_folder).expanduser().resolve()
+        indexer = indexers.get(requested)
+        if indexer is None:
+            allowed = ", ".join(str(root) for root in roots)
+            raise ValueError(
+                f"workspace_folder is not configured: {requested}; allowed: {allowed}"
+            )
+        return indexer
+
+    @asynccontextmanager
+    async def lifespan(_server: Any):
+        try:
+            if initial_sync != "off":
+                for indexer in indexers.values():
+                    indexer.start(initial_sync=True)
+                if initial_sync == "blocking":
+                    for indexer in indexers.values():
+                        status = indexer.wait_until_ready(None)
+                        if status == "error":
+                            detail = indexer.status().get("error")
+                            raise ClientConfigurationError(
+                                f"initial workspace synchronization failed: {detail}"
+                            )
+            yield
+        finally:
+            for indexer in indexers.values():
+                indexer.stop()
+
+    server = FastMCP("oce-client", lifespan=lifespan, log_level=log_level.upper())
+
+    async def codebase_retrieval(
+        information_request: str,
+        workspace_folder: str | None = None,
+    ) -> dict[str, object]:
+        """Retrieve code context after the background index reaches the latest change."""
+        if not information_request.strip():
+            raise ValueError("information_request must not be empty")
+        return await asyncio.to_thread(
+            indexer_for(workspace_folder).retrieve,
+            information_request,
+            ready_timeout,
+        )
+
+    # FastMCP derives JSON Schema descriptions from Pydantic Field metadata. Keep
+    # pydantic behind the optional MCP extra so the base CLI remains dependency-free.
+    from pydantic import Field
+
+    codebase_retrieval.__annotations__ = {
+        "information_request": Annotated[
+            str,
+            Field(description="A description of the information you need."),
+        ],
+        "workspace_folder": Annotated[
+            str,
+            Field(
+                description=(
+                    "Path to the workspace folder to search. Required when multiple "
+                    "workspace folders are open. Use the folder paths shown in your "
+                    "system prompt."
+                )
+            ),
+        ],
+        "return": dict[str, object],
+    }
+    server.tool(name="codebase-retrieval", description=TOOL_DESCRIPTION)(
+        codebase_retrieval
+    )
+    server._oce_indexers = indexers
+    return server
+
+
+def _state_path(state_dir: Path, root: Path) -> Path:
+    identity = os.path.normcase(str(root.resolve())).encode("utf-8")
+    name = hashlib.sha256(identity).hexdigest()[:16]
+    return state_dir.expanduser().resolve() / f"{name}.sqlite3"
+
+
+def run_mcp(
+    settings: ClientSettings,
+    *,
+    workspace_roots: Sequence[Path] | None = None,
+    state_dir: Path | None = None,
+    debounce_ms: int = 500,
+    initial_sync: str = "background",
+    ready_timeout: float = 3.0,
+    log_level: str = "WARNING",
+) -> None:
+    server = create_server(
+        settings,
+        workspace_roots=workspace_roots,
+        state_dir=state_dir,
+        debounce_ms=debounce_ms,
+        initial_sync=initial_sync,
+        ready_timeout=ready_timeout,
+        log_level=log_level,
+    )
+    server.run(transport="stdio")
+
+
+def run_mcp_configuration(configuration: McpConfiguration) -> None:
+    run_mcp(
+        configuration.client,
+        workspace_roots=configuration.workspace_roots,
+        state_dir=configuration.state_dir,
+        debounce_ms=configuration.debounce_ms,
+        initial_sync=configuration.initial_sync,
+        ready_timeout=configuration.ready_timeout,
+        log_level=configuration.log_level,
+    )
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="oce-client-mcp",
+        description="Run the OpenContextEngine MCP server over stdio.",
+    )
+    add_mcp_arguments(parser)
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    try:
+        args = build_parser().parse_args(argv)
+        run_mcp_configuration(mcp_configuration_from_args(args))
+    except (ClientConfigurationError, OSError, ValueError) as exc:
+        print(f"oce-client-mcp: {exc}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())