codecortex 0.2.0__py3-none-any.whl

codeintel/providers/lsp.py

@@ -0,0 +1,407 @@
+from __future__ import annotations
+
+import asyncio
+import enum
+import json
+import re
+import shutil
+import threading
+import time
+from typing import Any, Optional
+
+from mcp import ClientSession
+from mcp.client.stdio import stdio_client
+
+from codeintel.provider import Result, safe_null_result
+
+_COOLDOWN_SECONDS = 60
+_DEFAULT_TIMEOUT_S = 5.0
+
+# Serena ships as the `serena-agent` package (executable name `serena`); `uvx serena` does NOT
+# work ("Package `serena` does not provide any executables"). The working invocation — verified
+# against the installed serena and the machine's own serena MCP config — pulls it straight from
+# the upstream repo and starts the stdio MCP server, binding the project via `--project`.
+_SERENA_GIT = "git+https://github.com/oraios/serena"
+
+
+def _serena_launch_args(cmd: str, project_root: str) -> list[str]:
+    """Build the real serena start-mcp-server argv. Kept pure + module-level so the exact
+    contract (the thing that had drifted) can be asserted without launching a subprocess."""
+    common = [
+        "start-mcp-server",
+        "--context", "ide-assistant",       # tool set tuned for a coding agent, no chat scaffolding
+        "--enable-web-dashboard", "false",  # headless: don't pop a browser from a background thread
+        "--project", project_root,          # bind the project at launch (tools take no project arg)
+    ]
+    if cmd == "uvx":
+        return ["uvx", "--from", _SERENA_GIT, "serena", *common]
+    # A directly-installed `serena` (or serena-mcp-server shim) on PATH.
+    return [cmd, *common]
+
+
+class _State(enum.Enum):
+    WARMING = "WARMING"
+    READY = "READY"
+    FAILED = "FAILED"
+
+
+class _LspSession:
+    def __init__(self, project_root: str, cmd: str) -> None:
+        self.state = _State.WARMING
+        self.cooldown_until: float = 0.0
+        self._lock = threading.Lock()
+        self._loop: asyncio.AbstractEventLoop = asyncio.new_event_loop()
+        self._mcp_session: Optional[ClientSession] = None
+        self._thread = threading.Thread(
+            target=self._run,
+            args=(project_root, cmd),
+            daemon=True,
+        )
+        self._thread.start()
+
+    def _run(self, project_root: str, cmd: str) -> None:
+        try:
+            self._loop.run_until_complete(self._warmup(project_root, cmd))
+        except Exception:
+            with self._lock:
+                self.state = _State.FAILED
+                self.cooldown_until = time.monotonic() + _COOLDOWN_SECONDS
+        finally:
+            try:
+                self._loop.close()
+            except Exception:
+                pass
+
+    async def _warmup(self, project_root: str, cmd: str) -> None:
+        from mcp import StdioServerParameters
+
+        launch_args = _serena_launch_args(cmd, project_root)
+        async with stdio_client(
+            StdioServerParameters(command=launch_args[0], args=launch_args[1:])
+        ) as (read, write):
+            async with ClientSession(read, write) as session:
+                await session.initialize()
+                with self._lock:
+                    self._mcp_session = session
+                    self.state = _State.READY
+                # Keep the loop (and the subprocess/session it owns) alive so _call_tool can
+                # schedule coroutines onto it. Resolves only when the thread/loop is torn down.
+                await asyncio.get_running_loop().create_future()
+
+
+class LspProvider:
+    """Wraps serena's LSP-over-MCP bridge. Never raises.
+
+    Serena tool contract (verified live, not assumed):
+      * ``find_symbol``             — arg ``name_path_pattern``; returns a JSON list of
+                                      ``{name_path, kind, relative_path, body_location, body?}``.
+      * ``find_referencing_symbols`` — args ``name_path`` AND ``relative_path`` (both required);
+                                      returns ``{file: {kind: [{name_path, content_around_reference}]}}``.
+      * ``get_symbols_overview``    — arg ``relative_path``; returns ``{kind: [names]}``.
+    No tool takes a ``project_root`` — the project is bound once at launch via ``--project``.
+    Finding references therefore needs two steps: locate the symbol, then query with its path.
+    """
+
+    def __init__(self) -> None:
+        self._sessions: dict[str, _LspSession] = {}
+        self._sessions_lock = threading.Lock()
+        self._detect_backend()
+
+    def _detect_backend(self) -> None:
+        # Prefer a directly-installed serena; otherwise drive it through uvx.
+        if shutil.which("serena"):
+            self.available = True
+            self._cmd: Optional[str] = "serena"
+        elif shutil.which("uvx"):
+            self.available = True
+            self._cmd = "uvx"
+        else:
+            self.available = False
+            self._cmd = None
+
+    def _get_or_create_session(self, root: str) -> _LspSession:
+        with self._sessions_lock:
+            existing = self._sessions.get(root)
+            if existing is not None:
+                with existing._lock:
+                    if existing.state == _State.FAILED:
+                        if time.monotonic() > existing.cooldown_until:
+                            del self._sessions[root]  # cooldown elapsed → allow one respawn
+                        else:
+                            return existing  # still cooling down — no per-request respawn
+                    else:
+                        return existing
+            session = _LspSession(root, self._cmd)  # type: ignore[arg-type]
+            self._sessions[root] = session
+            return session
+
+    def probe(self, project_root: str, deep: bool = False, timeout_s: float = 20.0) -> dict:
+        """Never-raise health check for the doctor. Shallow (default) is FREE — PATH presence
+        plus any existing session's live state. Deep boots serena and polls until READY/FAILED,
+        bounded by ``timeout_s`` (first boot pulls serena via uvx and is slow). ``repo_indexed``
+        is always None: serena keeps no persistent index, it warms per-root on demand."""
+        if not self.available:
+            return {
+                "installed": False, "runnable": False, "repo_indexed": None,
+                "detail": "neither `serena` nor `uvx` found on PATH",
+                "remediation": "install uv (provides uvx) — serena is fetched on first use",
+            }
+        cmd = self._cmd
+        if not deep:
+            existing = self._sessions.get(project_root)
+            if existing is None:
+                return {
+                    "installed": True, "runnable": None, "repo_indexed": None,
+                    "detail": f"serena via `{cmd}`; boot not verified (warms on 1st query; --deep to check now)",
+                    "remediation": None,
+                }
+            with existing._lock:
+                st = existing.state
+            if st == _State.READY:
+                return {"installed": True, "runnable": True, "repo_indexed": None,
+                        "detail": "serena session is READY for this repo", "remediation": None}
+            if st == _State.FAILED:
+                return {"installed": True, "runnable": False, "repo_indexed": None,
+                        "detail": "serena session failed to boot for this repo",
+                        "remediation": "re-run `codeintel doctor --deep` to see the boot error"}
+            return {"installed": True, "runnable": None, "repo_indexed": None,
+                    "detail": "serena session is warming for this repo", "remediation": None}
+
+        # deep: boot (or reuse) a session and poll to a hard deadline — never hangs.
+        session = self._get_or_create_session(project_root)
+        deadline = time.monotonic() + timeout_s
+        while time.monotonic() < deadline:
+            with session._lock:
+                st = session.state
+            if st == _State.READY:
+                return {"installed": True, "runnable": True, "repo_indexed": None,
+                        "detail": f"serena booted via `{cmd}` and reached READY", "remediation": None}
+            if st == _State.FAILED:
+                return {"installed": True, "runnable": False, "repo_indexed": None,
+                        "detail": "serena failed to boot",
+                        "remediation": "check uvx + network: `uvx --from git+https://github.com/oraios/serena serena start-mcp-server`"}
+            time.sleep(0.5)
+        return {"installed": True, "runnable": None, "repo_indexed": None,
+                "detail": f"serena did not reach READY within {int(timeout_s)}s (still warming)",
+                "remediation": "retry — first boot pulls serena via uvx and can be slow"}
+
+    def build_result(
+        self,
+        op: Any,
+        target: Any,
+        files: Any,
+        budget: Any,
+        project_root: Any,
+    ) -> Result:
+        try:
+            op_str = str(op or "")
+            target_str = str(target or "")
+            root_str = str(project_root or "")
+
+            if not self.available:
+                return safe_null_result(op_str, target_str, engine="lsp", reason="engine-unavailable")
+
+            try:
+                budget_ms = int(budget) if budget else 0
+            except Exception:
+                budget_ms = 0
+            timeout_s = (budget_ms / 1000) if budget_ms > 0 else _DEFAULT_TIMEOUT_S
+
+            session = self._get_or_create_session(root_str)
+
+            with session._lock:
+                state = session.state
+
+            if state == _State.WARMING:
+                return safe_null_result(op_str, target_str, engine="lsp", reason="warming")
+
+            if state == _State.FAILED:
+                return safe_null_result(op_str, target_str, engine="lsp", reason="boot-failed")
+
+            # READY
+            result_text = self._dispatch(session, op_str, target_str, root_str, timeout_s)
+            if result_text is None:
+                return safe_null_result(op_str, target_str, engine="lsp", reason="unsupported-op")
+
+            return {
+                "ok": True,
+                "op": op_str,
+                "target": target_str,
+                "result": result_text,
+                "engine": "lsp",
+                "cached": False,
+            }
+        except Exception:
+            return safe_null_result(op, target, engine="lsp", reason="error")
+
+    def _dispatch(
+        self,
+        session: _LspSession,
+        op: str,
+        target: str,
+        root: str,
+        timeout_s: float,
+    ) -> Optional[str]:
+        if op == "symbol" or op == "context":
+            # `context` (fan-out op) → the LSP's richest single-symbol view: definition + refs.
+            return self._op_symbol(session, target, root, timeout_s)
+        if op == "overview":
+            return self._op_overview(session, target, root, timeout_s)
+        return None
+
+    def _call_tool(
+        self,
+        session: _LspSession,
+        tool: str,
+        args: dict,
+        timeout_s: float,
+    ) -> Optional[Any]:
+        try:
+            mcp_session = session._mcp_session
+            if mcp_session is None:
+                return None
+            coro = mcp_session.call_tool(tool, args)
+            future = asyncio.run_coroutine_threadsafe(coro, session._loop)
+            return future.result(timeout=timeout_s)
+        except Exception:
+            return None
+
+    def _extract_text(self, raw: Any) -> Optional[str]:
+        if raw is None:
+            return None
+        if isinstance(raw, str):
+            return raw
+        # mcp CallToolResult has a .content list of TextContent
+        try:
+            parts = []
+            for item in raw.content:
+                if hasattr(item, "text"):
+                    parts.append(item.text)
+            return "\n".join(parts) if parts else None
+        except Exception:
+            return None
+
+    @staticmethod
+    def _loads(text: Optional[str]) -> Any:
+        if not text:
+            return None
+        try:
+            return json.loads(text)
+        except Exception:
+            return None
+
+    @staticmethod
+    def _ref_line(content: Any) -> Optional[str]:
+        """Pull the referenced line number out of serena's `content_around_reference` blob,
+        which marks the reference line with a leading `>` (e.g. `  >   7:from ...`)."""
+        if not isinstance(content, str):
+            return None
+        m = re.search(r">\s*(\d+):", content)
+        return m.group(1) if m else None
+
+    def _format_matches(self, target: str, matches: list) -> tuple[str, Optional[dict]]:
+        parts = [f"## Symbol: {target}"]
+        first: Optional[dict] = None
+        for m in matches:
+            if not isinstance(m, dict):
+                continue
+            if first is None:
+                first = m
+            kind = m.get("kind") or "symbol"
+            rel = m.get("relative_path") or "?"
+            loc = m.get("body_location") if isinstance(m.get("body_location"), dict) else {}
+            s, e = loc.get("start_line"), loc.get("end_line")
+            span = f":{s}-{e}" if s is not None else ""
+            parts.append(f"**{kind}** — {rel}{span}")
+            body = m.get("body")
+            if body:
+                parts.append(f"```\n{body}\n```")
+        return "\n".join(parts), first
+
+    def _format_refs(self, data: Any) -> list[str]:
+        lines: list[str] = []
+        if not isinstance(data, dict):
+            return lines
+        for file, kinds in data.items():
+            if not isinstance(kinds, dict):
+                continue
+            for _kind, entries in kinds.items():
+                if not isinstance(entries, list):
+                    continue
+                for ent in entries:
+                    if not isinstance(ent, dict):
+                        continue
+                    np = str(ent.get("name_path") or "").strip()
+                    line = self._ref_line(ent.get("content_around_reference"))
+                    loc = f"{file}:{line}" if line else str(file)
+                    suffix = f"  ({np})" if np else ""
+                    lines.append(f"- {loc}{suffix}")
+                    if len(lines) >= 50:
+                        return lines
+        return lines
+
+    def _op_symbol(
+        self, session: _LspSession, target: str, root: str, timeout_s: float
+    ) -> Optional[str]:
+        try:
+            def_raw = self._call_tool(
+                session,
+                "find_symbol",
+                {"name_path_pattern": target, "include_body": True, "max_matches": 5},
+                timeout_s,
+            )
+            def_text = self._extract_text(def_raw)
+            matches = self._loads(def_text)
+
+            first: Optional[dict] = None
+            if isinstance(matches, list) and matches:
+                def_section, first = self._format_matches(target, matches)
+            else:
+                # Non-JSON / degenerate response — surface whatever serena returned.
+                def_section = f"## Symbol: {target}\n{def_text or '(not found)'}"
+
+            # References require the located symbol's own path (two-step contract).
+            ref_section = "## References\n(none)"
+            if first and first.get("relative_path"):
+                ref_raw = self._call_tool(
+                    session,
+                    "find_referencing_symbols",
+                    {
+                        "name_path": first.get("name_path") or target,
+                        "relative_path": first.get("relative_path"),
+                    },
+                    timeout_s,
+                )
+                ref_lines = self._format_refs(self._loads(self._extract_text(ref_raw)))
+                if ref_lines:
+                    ref_section = f"## References ({len(ref_lines)})\n" + "\n".join(ref_lines)
+
+            return f"{def_section}\n\n{ref_section}"
+        except Exception:
+            return None
+
+    def _op_overview(
+        self, session: _LspSession, target: str, root: str, timeout_s: float
+    ) -> Optional[str]:
+        try:
+            raw = self._call_tool(
+                session,
+                "get_symbols_overview",
+                {"relative_path": target or ""},
+                timeout_s,
+            )
+            text = self._extract_text(raw)
+            if not text:
+                return None
+            parsed = self._loads(text)
+            if isinstance(parsed, dict):
+                parts = [f"## Overview: {target}"]
+                for kind, names in parsed.items():
+                    if isinstance(names, list):
+                        parts.append(f"**{kind}**: " + ", ".join(str(n) for n in names))
+                    else:
+                        parts.append(f"**{kind}**: {names}")
+                return "\n".join(parts)
+            return text
+        except Exception:
+            return None

codeintel/providers/none.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from codeintel.provider import Result, safe_null_result
+
+
+class NoneProvider:
+    """Always returns a safe-null Result. Never raises."""
+
+    def build_result(
+        self,
+        op,
+        target,
+        files,
+        budget,
+        project_root,
+    ) -> Result:
+        try:
+            op = str(op or "")
+            target = str(target or "")
+            return safe_null_result(op, target, engine="none", reason="no-engine")
+        except Exception:
+            return {
+                "ok": True,
+                "op": str(op or ""),
+                "target": str(target or ""),
+                "result": None,
+                "engine": "none",
+                "cached": False,
+                "reason": "no-engine",
+            }

codeintel/providers/semantic.py

@@ -0,0 +1,139 @@
+from __future__ import annotations
+
+import pathlib
+
+from codeintel.provider import Result, safe_null_result
+
+try:
+    import fastembed  # noqa: F401
+    import sqlite_vec  # noqa: F401
+    _DEPS_OK = True
+except ImportError:
+    _DEPS_OK = False
+
+_DB_PATH = pathlib.Path.home() / ".codeintel" / "semantic.db"
+
+
+class SemanticProvider:
+    """Real semantic search provider backed by SemanticDb and Searcher."""
+
+    @property
+    def available(self) -> bool:
+        return _DEPS_OK
+
+    def probe(self, project_root: str) -> dict:
+        """Never-raise health check for the doctor. READ-ONLY and MODEL-FREE: it opens the db
+        read-only and counts this repo's chunks — it must NOT call SemanticDb.init() (a schema
+        write) or load fastembed. ``repo_indexed`` is project-scoped (mirrors Searcher.has_index)."""
+        if not self.available:
+            return {
+                "installed": False, "runnable": False, "repo_indexed": False,
+                "detail": "fastembed / sqlite-vec not importable",
+                "remediation": "pip install fastembed sqlite-vec  (or: pip install -e .)",
+            }
+        import os
+        import sqlite3
+
+        try:
+            from codeintel.semantic_db import default_db_path
+            db_path = default_db_path()
+        except Exception:
+            db_path = ""
+        if not db_path or not os.path.exists(db_path):
+            return {
+                "installed": True, "runnable": True, "repo_indexed": False,
+                "detail": "no semantic index database yet",
+                "remediation": f"codeintel index {project_root}",
+            }
+        try:
+            conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
+            try:
+                real = os.path.realpath(project_root) if project_root else ""
+                row = conn.execute(
+                    "SELECT COUNT(*) FROM chunk_hashes WHERE project_root = ?", (real,)
+                ).fetchone()
+            finally:
+                conn.close()
+            count = int(row[0]) if row else 0
+        except Exception as exc:
+            return {
+                "installed": True, "runnable": False, "repo_indexed": False,
+                "detail": f"semantic.db present but unreadable ({type(exc).__name__})",
+                "remediation": "rm ~/.codeintel/semantic.db && codeintel index <root>",
+            }
+        if count > 0:
+            return {
+                "installed": True, "runnable": True, "repo_indexed": True,
+                "detail": f"{count} indexed chunks for this repo", "remediation": None,
+            }
+        return {
+            "installed": True, "runnable": True, "repo_indexed": False,
+            "detail": "semantic.db present but 0 chunks for this repo",
+            "remediation": f"codeintel index {project_root}",
+        }
+
+    def build_result(
+        self,
+        op: str,
+        target: str,
+        files: list[str],
+        budget: int,
+        project_root: str,
+    ) -> Result:
+        # `context` (fan-out op) → semantic's contribution is a similarity search on the target.
+        if op not in ("search", "context"):
+            return safe_null_result(op, target, engine="semantic", reason="op-not-supported")
+        if not self.available:
+            return safe_null_result(op, target, engine="semantic", reason="engine-unavailable")
+        if not project_root:
+            return safe_null_result(op, target, engine="semantic", reason="no-project-root")
+
+        try:
+            from codeintel.config import load_config
+            from codeintel.semantic_db import SemanticDb
+            from codeintel.indexer import Indexer
+            from codeintel.searcher import Searcher
+
+            cfg = load_config(project_root)
+            model = str(cfg.get("model") or "BAAI/bge-small-en-v1.5")
+
+            _DB_PATH.parent.mkdir(parents=True, exist_ok=True)
+            db = SemanticDb(str(_DB_PATH))
+            db.init()
+
+            Indexer(
+                db,
+                model_name=model,
+                window=int(cfg.get("window", 20)),
+                stride=int(cfg.get("stride", 10)),
+                max_chunks=int(cfg.get("max_chunks", 500)),
+            ).index(project_root)
+
+            searcher = Searcher(db, model_name=model)
+            if not searcher.has_index(project_root):
+                return safe_null_result(
+                    op, target, engine="semantic", reason="no-index",
+                    hint=f"run: codeintel index {project_root}  (or: codeintel doctor)",
+                )
+
+            matches = searcher.search(
+                target, project_root, cosine_floor=float(cfg.get("cosine_floor", 0.25))
+            )
+            if not matches:
+                return safe_null_result(op, target, engine="semantic", reason="below-floor")
+
+            lines = [
+                f"{m['path']}:{m['line']} | {m['snippet'].splitlines()[0] if m['snippet'].splitlines() else m['snippet']}"
+                for m in matches
+            ]
+            result: Result = {
+                "ok": True,
+                "op": op,
+                "target": target,
+                "result": "\n".join(lines),
+                "engine": "semantic",
+                "cached": False,
+            }
+            return result
+        except Exception:
+            return safe_null_result(op, target, engine="semantic", reason="provider-error")

codeintel/reindexer.py

@@ -0,0 +1,112 @@
+from __future__ import annotations
+
+import logging
+import os
+import threading
+import time
+
+logger = logging.getLogger(__name__)
+
+
+class _DaemonPool:
+    """Runs background work on DAEMON threads so it can never block interpreter shutdown.
+
+    A one-shot ``codeintel query`` must return immediately — with a non-daemon pool, Python's
+    shutdown joins the workers, so a first query on a large repo hung for minutes while the
+    repo-wide reindex finished (and, with buffered/piped stdout, the result never even flushed
+    until the join completed). Daemon threads exit with the process; on the persistent server
+    they run to completion normally. Rate is bounded by the Reindexer's debounce, so a
+    thread-per-task is fine. ``shutdown(wait=)`` is kept for deterministic test draining."""
+
+    def __init__(self, max_workers: int = 2) -> None:
+        self._max_workers = max_workers  # advisory; the Reindexer debounce is the real rate limit
+        self._threads: list[threading.Thread] = []
+        self._lock = threading.Lock()
+
+    def submit(self, fn, *args) -> None:
+        t = threading.Thread(target=fn, args=args, daemon=True)
+        with self._lock:
+            self._threads = [x for x in self._threads if x.is_alive()]  # drop finished
+            self._threads.append(t)
+        t.start()
+
+    def shutdown(self, wait: bool = True) -> None:
+        if not wait:
+            return
+        with self._lock:
+            threads = list(self._threads)
+        for t in threads:
+            t.join()
+
+
+class Reindexer:
+    def __init__(self, debounce_seconds: float = 30, enabled: bool = True) -> None:
+        self._debounce_seconds = debounce_seconds
+        self._enabled = (
+            os.environ.get("CODEINTEL_REINDEX", "on").strip().lower() != "off"
+            and enabled
+        )
+        self._lock = threading.Lock()
+        self._last_fired: dict[str, float] = {}
+        # Per-project index generation — bumped when a reindex completes. The gateway
+        # folds it into the cache key so a structural answer is invalidated once the
+        # index actually moves (a symbol/free-text target has no file content to hash).
+        self._generation: dict[str, int] = {}
+        self._executor = _DaemonPool(max_workers=2)
+
+    def generation(self, project_root: str) -> int:
+        with self._lock:
+            return self._generation.get(project_root, 0)
+
+    def maybe_reindex(self, project_root: str) -> None:
+        if not self._enabled:
+            return
+        if not project_root:
+            return
+
+        now = time.monotonic()
+        with self._lock:
+            last = self._last_fired.get(project_root, 0.0)
+            if now - last < self._debounce_seconds:
+                return
+            self._last_fired[project_root] = now
+
+        self._executor.submit(self._do_reindex, project_root)
+
+    def _do_reindex(self, project_root: str) -> None:
+        try:
+            self._semantic_reindex(project_root)
+            self._graph_reindex(project_root)
+            with self._lock:
+                self._generation[project_root] = self._generation.get(project_root, 0) + 1
+        except Exception as exc:
+            logger.warning("Reindexer._do_reindex failed for %s: %s", project_root, exc)
+
+    def _semantic_reindex(self, project_root: str) -> None:
+        import pathlib
+
+        from codeintel.semantic_db import SemanticDb, default_db_path
+        from codeintel.indexer import Indexer
+
+        # Same per-machine cache the SemanticProvider reads — index and search must
+        # never diverge onto different files.
+        db_path = default_db_path()
+        pathlib.Path(db_path).parent.mkdir(parents=True, exist_ok=True)
+        db = SemanticDb(db_path)
+        try:
+            db.init()
+            Indexer(db).index(project_root)
+        finally:
+            db.close()
+
+    def _graph_reindex(self, project_root: str) -> None:
+        # Route through the graph provider's single subprocess/JSON seam (piped stdin, with a
+        # deprecated raw-JSON fallback) rather than duplicating the deprecated raw-JSON call here.
+        try:
+            from codeintel.providers.graph import GraphProvider
+            gp = GraphProvider()
+            if not gp.available:
+                return
+            gp._run("detect_changes", {"project_root": project_root}, 120_000)
+        except Exception as exc:
+            logger.warning("graph detect_changes failed: %s", exc)