codecortex 0.2.0__py3-none-any.whl

codeintel/indexer.py

@@ -0,0 +1,250 @@
+from __future__ import annotations
+
+import hashlib
+import logging
+import os
+import struct
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from codeintel.semantic_db import SemanticDb
+
+logger = logging.getLogger(__name__)
+
+_INDEXED_EXTS = frozenset({
+    ".py", ".ts", ".js", ".go", ".rs", ".java", ".c", ".cpp", ".h", ".md"
+})
+_SKIP_DIRS = frozenset({"__pycache__", ".git", "node_modules"})
+# Vendored / regenerable dirs skipped even without a .gitignore entry.
+_DEFAULT_IGNORES = frozenset({
+    ".venv", "venv", "env", "dist", "build", "target",
+    ".mypy_cache", ".pytest_cache", ".tox", ".idea", ".vscode", ".cache",
+})
+
+
+def _project_key(project_root_real: str) -> str:
+    """A short, stable id for a project root — prefixes every chunk_id so two repos
+    with an identically-named file never collide in the shared cache."""
+    return hashlib.sha256(project_root_real.encode()).hexdigest()[:12]
+
+
+class Indexer:
+    def __init__(
+        self,
+        db: SemanticDb,
+        model_name: str = "BAAI/bge-small-en-v1.5",
+        window: int = 20,
+        stride: int = 10,
+        max_chunks: int = 500,
+    ) -> None:
+        self.db = db
+        self.model_name = model_name
+        self.window = window
+        self.stride = stride
+        self.max_chunks = max_chunks
+        self._embedder = None
+
+    def _get_embedder(self):
+        if self._embedder is None:
+            from fastembed import TextEmbedding
+            self._embedder = TextEmbedding(model_name=self.model_name)
+        return self._embedder
+
+    def index(self, project_root: str) -> int:
+        """Return count of newly embedded chunks, or -1 on unrecoverable failure."""
+        try:
+            return self._index(project_root)
+        except Exception as exc:
+            logger.error("Indexer.index() unrecoverable failure: %s", exc)
+            return -1
+
+    def _load_gitignore(self, root: Path) -> set[str]:
+        """Best-effort ``.gitignore``: collect simple name/dir patterns to skip. This is
+        NOT full gitignore semantics (no globs, negations, or nesting) — just enough to
+        avoid indexing vendored/build output the user already told git to ignore."""
+        patterns: set[str] = set()
+        gi = root / ".gitignore"
+        try:
+            if gi.is_file():
+                for line in gi.read_text(encoding="utf-8", errors="replace").splitlines():
+                    line = line.strip()
+                    if not line or line.startswith("#") or line.startswith("!"):
+                        continue
+                    name = line.rstrip("/").lstrip("/")
+                    if name and "*" not in name and "/" not in name:
+                        patterns.add(name)
+        except Exception:
+            pass
+        return patterns
+
+    def _cleanup_deleted(self, root: Path, project_root_real: str) -> None:
+        """Drop rows for THIS project whose file no longer exists — scoped by
+        project_root so touching one repo can never purge another's index."""
+        conn = self.db.conn()
+        try:
+            rows = conn.execute(
+                "SELECT DISTINCT file_path FROM chunk_hashes WHERE project_root = ?",
+                (project_root_real,),
+            ).fetchall()
+            deleted_paths = [
+                row[0] for row in rows if not (root / row[0]).exists()
+            ]
+            for fp in deleted_paths:
+                chunk_ids = [
+                    r[0]
+                    for r in conn.execute(
+                        "SELECT chunk_id FROM chunk_hashes"
+                        " WHERE project_root = ? AND file_path = ?",
+                        (project_root_real, fp),
+                    ).fetchall()
+                ]
+                for cid in chunk_ids:
+                    conn.execute(
+                        "DELETE FROM code_embeddings WHERE chunk_id = ?", (cid,)
+                    )
+                    conn.execute(
+                        "DELETE FROM chunk_hashes WHERE chunk_id = ?", (cid,)
+                    )
+            conn.commit()
+        except Exception as exc:
+            logger.warning("Cleanup pass failed: %s", exc)
+
+    def _walk_files(self, root: Path):
+        ignores = set(_SKIP_DIRS) | set(_DEFAULT_IGNORES) | self._load_gitignore(root)
+        for dirpath, dirnames, filenames in os.walk(root):
+            dirnames[:] = [
+                d for d in dirnames
+                if d not in ignores and not d.endswith(".egg-info")
+            ]
+            for fname in filenames:
+                if fname in ignores:
+                    continue
+                if Path(fname).suffix.lower() in _INDEXED_EXTS:
+                    yield Path(dirpath) / fname
+
+    def _collect_new_chunks(
+        self, root: Path, project_key: str, project_root_real: str
+    ) -> list[tuple[str, str, str, int, str]]:
+        """Walk files; return (chunk_id, text, rel_path, start, hash) for new/changed chunks."""
+        conn = self.db.conn()
+        new_chunks: list[tuple[str, str, str, int, str]] = []
+
+        for filepath in self._walk_files(root):
+            try:
+                with open(filepath, encoding="utf-8", errors="replace") as f:
+                    lines = f.readlines()
+            except FileNotFoundError:
+                logger.debug("file disappeared: %s", filepath)
+                continue
+            except Exception as exc:
+                logger.debug("skipping %s: %s", filepath, exc)
+                continue
+
+            rel_path = str(filepath.relative_to(root))
+            chunk_count = 0
+
+            for chunk_start in range(0, len(lines), self.stride):
+                if chunk_count >= self.max_chunks:
+                    logger.debug(
+                        "chunk cap hit for %s, truncating at %d",
+                        rel_path,
+                        self.max_chunks,
+                    )
+                    break
+
+                chunk_lines = lines[chunk_start: chunk_start + self.window]
+                if not chunk_lines:
+                    break
+
+                chunk_text = "".join(chunk_lines)
+                if not chunk_text.strip():
+                    # EC3.4: never embed empty/whitespace-only chunks (zero vectors pollute results).
+                    chunk_count += 1
+                    continue
+
+                chunk_id = f"{project_key}:{rel_path}:{chunk_start}"
+                content_hash = hashlib.sha256(chunk_text.encode()).hexdigest()[:16]
+
+                try:
+                    row = conn.execute(
+                        "SELECT content_hash FROM chunk_hashes WHERE chunk_id = ?",
+                        (chunk_id,),
+                    ).fetchone()
+                    if row and row[0] == content_hash:
+                        chunk_count += 1
+                        continue
+                except Exception as exc:
+                    logger.debug("hash check failed for %s: %s", chunk_id, exc)
+
+                new_chunks.append(
+                    (chunk_id, chunk_text, rel_path, chunk_start, content_hash)
+                )
+                chunk_count += 1
+
+        return new_chunks
+
+    def _embed_and_write(
+        self, new_chunks: list[tuple[str, str, str, int, str]], project_root_real: str
+    ) -> int:
+        embedder = self._get_embedder()  # may raise → propagates to index() → returns -1
+        conn = self.db.conn()
+        embedded_count = 0
+        batch_size = 32
+
+        for i in range(0, len(new_chunks), batch_size):
+            batch = new_chunks[i: i + batch_size]
+            texts = [c[1] for c in batch]
+
+            try:
+                embeddings = list(embedder.embed(texts))
+            except Exception as exc:
+                logger.warning("embedding batch %d failed: %s", i // batch_size, exc)
+                continue
+
+            for j, (chunk_id, _, rel_path, chunk_start, content_hash) in enumerate(batch):
+                if j >= len(embeddings):
+                    break
+                try:
+                    vec = embeddings[j]
+                    vec_bytes = struct.pack(f"{len(vec)}f", *vec)
+                    conn.execute(
+                        "INSERT OR REPLACE INTO code_embeddings(chunk_id, embedding)"
+                        " VALUES (?, ?)",
+                        (chunk_id, vec_bytes),
+                    )
+                    conn.execute(
+                        "INSERT OR REPLACE INTO chunk_hashes"
+                        "(chunk_id, project_root, file_path, chunk_start, content_hash)"
+                        " VALUES (?, ?, ?, ?, ?)",
+                        (chunk_id, project_root_real, rel_path, chunk_start, content_hash),
+                    )
+                    embedded_count += 1
+                except Exception as exc:
+                    logger.warning("writing chunk %s failed: %s", chunk_id, exc)
+
+            try:
+                conn.commit()
+            except Exception as exc:
+                logger.warning("commit failed after batch %d: %s", i // batch_size, exc)
+
+        return embedded_count
+
+    def _index(self, project_root: str) -> int:
+        if not project_root:
+            return 0
+
+        root = Path(project_root)
+        if not root.exists():
+            return 0
+
+        project_root_real = os.path.realpath(project_root)
+        project_key = _project_key(project_root_real)
+
+        self._cleanup_deleted(root, project_root_real)
+
+        new_chunks = self._collect_new_chunks(root, project_key, project_root_real)
+        if not new_chunks:
+            return 0
+
+        return self._embed_and_write(new_chunks, project_root_real)

codeintel/injector.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+import logging
+import os
+
+_START_MARKER = "<!-- codeintel-map-start -->"
+_END_MARKER = "<!-- codeintel-map-end -->"
+_CONTEXT_FILES = ["CLAUDE.md", "AGENTS.md"]
+
+_BLOCK_CONTENT = (
+    "\n## codeintel orientation map\n\n"
+    "See [CODE_INTEL.md](CODE_INTEL.md) for a ranked overview of this codebase "
+    "modules, key symbols (by call frequency), and entry points. "
+    "Refresh with: `codeintel map`."
+    "\n"
+)
+
+_logger = logging.getLogger(__name__)
+
+
+class Injector:
+    """Idempotently injects a CODE_INTEL.md reference block into CLAUDE.md or AGENTS.md."""
+
+    def inject(self, project_root: str) -> tuple[str | None, str]:
+        try:
+            path = _find_context_file(project_root)
+            if path is None:
+                return (None, "no-context-file")
+
+            content = _read_file(path)
+            if content is None:
+                return (None, "error")
+
+            new_content, action = _update_block(content)
+            _write_file(path, new_content)
+            return (path, action)
+        except Exception as exc:
+            _logger.warning("Injector.inject failed: %s", exc)
+            return (None, "error")
+
+
+def _find_context_file(project_root: str) -> str | None:
+    for name in _CONTEXT_FILES:
+        candidate = os.path.join(project_root, name)
+        if os.path.isfile(candidate):
+            return candidate
+    return None
+
+
+def _read_file(path: str) -> str | None:
+    try:
+        with open(path, encoding="utf-8", errors="strict") as f:
+            return f.read()
+    except Exception as exc:
+        _logger.warning("Injector: could not read %s: %s", path, exc)
+        return None
+
+
+def _write_file(path: str, content: str) -> None:
+    with open(path, "w", encoding="utf-8") as f:
+        f.write(content)
+
+
+def _update_block(content: str) -> tuple[str, str]:
+    has_start = _START_MARKER in content
+    has_end = _END_MARKER in content
+
+    block = _START_MARKER + _BLOCK_CONTENT + _END_MARKER
+
+    if has_start and has_end:
+        start_idx = content.index(_START_MARKER)
+        end_idx = content.index(_END_MARKER) + len(_END_MARKER)
+        new_content = content[:start_idx] + block + content[end_idx:]
+        return (new_content, "updated")
+
+    # Missing or corrupted (only one marker): append a fresh block
+    separator = "\n\n" if not content.endswith("\n\n") else ""
+    if content.endswith("\n"):
+        separator = "\n"
+    new_content = content + separator + block
+    return (new_content, "appended")

codeintel/installer.py

@@ -0,0 +1,103 @@
+from __future__ import annotations
+
+import json
+import pathlib
+
+_AGENTS = ["claude", "codex", "gemini", "zed"]
+
+_CONFIG: dict[str, dict] = {
+    "claude": {
+        "path": "~/.claude/settings.json",
+        "key": ["mcpServers", "codeintel"],
+        "value": {"command": "codeintel", "args": ["serve"]},
+    },
+    "codex": {
+        "path": "~/.codex/config.json",
+        "key": ["mcpServers", "codeintel"],
+        "value": {"command": "codeintel", "args": ["serve"]},
+    },
+    "gemini": {
+        "path": "~/.gemini/settings.json",
+        "key": ["mcpServers", "codeintel"],
+        "value": {"command": "codeintel", "args": ["serve"]},
+    },
+    "zed": {
+        "path": "~/.config/zed/settings.json",
+        "key": ["context_servers", "codeintel"],
+        "value": {"command": {"path": "codeintel", "args": ["serve"]}},
+    },
+}
+
+
+def _get_nested(data: dict, keys: list[str]):
+    node = data
+    for k in keys:
+        if not isinstance(node, dict) or k not in node:
+            return None
+        node = node[k]
+    return node
+
+
+def _set_nested(data: dict, keys: list[str], value) -> None:
+    node = data
+    for k in keys[:-1]:
+        if k not in node or not isinstance(node[k], dict):
+            node[k] = {}
+        node = node[k]
+    node[keys[-1]] = value
+
+
+class Installer:
+    def register(self, agent: str) -> dict:
+        spec = _CONFIG.get(agent)
+        if spec is None:
+            return {
+                "agent": agent,
+                "path": "",
+                "ok": False,
+                "action": "failed",
+                "reason": f"unknown agent '{agent}'",
+            }
+
+        config_path = pathlib.Path(spec["path"]).expanduser()
+        try:
+            if config_path.exists():
+                data = json.loads(config_path.read_text(encoding="utf-8"))
+                if not isinstance(data, dict):
+                    data = {}
+            else:
+                data = {}
+
+            current = _get_nested(data, spec["key"])
+            if current == spec["value"]:
+                return {
+                    "agent": agent,
+                    "path": str(config_path),
+                    "ok": True,
+                    "action": "already",
+                    "reason": "",
+                }
+
+            config_path.parent.mkdir(parents=True, exist_ok=True)
+            _set_nested(data, spec["key"], spec["value"])
+            config_path.write_text(json.dumps(data, indent=2), encoding="utf-8")
+
+            return {
+                "agent": agent,
+                "path": str(config_path),
+                "ok": True,
+                "action": "registered",
+                "reason": "",
+            }
+
+        except Exception as exc:
+            return {
+                "agent": agent,
+                "path": str(config_path),
+                "ok": False,
+                "action": "failed",
+                "reason": str(exc),
+            }
+
+    def register_all(self) -> list[dict]:
+        return [self.register(agent) for agent in _AGENTS]

codeintel/mapper.py

@@ -0,0 +1,192 @@
+from __future__ import annotations
+
+import os
+from typing import Optional, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from codeintel.providers.graph import GraphProvider
+
+# Real codebase-memory-mcp contract (verified live): query_graph returns {columns, rows}
+# value-arrays, module-level calls are USAGE (not only CALLS), and nodes carry `is_entry_point`
+# (there is no `fn.type` property — that filter matched nothing, so the old queries were dead).
+_FAN_IN_CYPHER = (
+    "MATCH (caller)-[:CALLS|USAGE]->(fn) WHERE fn.name IS NOT NULL "
+    "RETURN fn.name, fn.qualified_name, fn.file_path, count(caller) AS in_degree "
+    "ORDER BY in_degree DESC LIMIT 40"
+)
+
+_ENTRY_CYPHER = (
+    "MATCH (fn) WHERE fn.is_entry_point = true "
+    "RETURN fn.name, fn.file_path LIMIT 10"
+)
+
+# Builtins and the synthetic project/config node are high-fan-in but useless as "symbols".
+_RANK_SKIP_PATHS = frozenset({"<python-builtins>", "pyproject.toml", ""})
+
+
+def _rows_from(raw: object) -> list[dict]:
+    """Normalize a query_graph response to a list of column->value dicts.
+
+    Accepts the real ``{"columns": [...], "rows": [[...]]}`` shape AND the legacy list-of-dicts
+    shape (used by older mocks); anything else yields ``[]``. Mirrors GraphProvider._query_rows
+    so the map generator reads the same reality every other graph op now does."""
+    if isinstance(raw, list):
+        return [r for r in raw if isinstance(r, dict)]
+    if not isinstance(raw, dict):
+        return []
+    cols, rows = raw.get("columns"), raw.get("rows")
+    if not isinstance(cols, list) or not isinstance(rows, list):
+        return []
+    out: list[dict] = []
+    for row in rows:
+        if isinstance(row, list):
+            out.append({str(cols[i]): row[i] for i in range(min(len(cols), len(row)))})
+        elif isinstance(row, dict):
+            out.append(row)
+    return out
+
+
+class MapGenerator:
+    """Generates a ranked, byte-bounded CODE_INTEL.md from the graph index."""
+
+    def __init__(self, provider: Optional[GraphProvider] = None) -> None:
+        self._provider = provider
+
+    def generate(self, project_root: str, budget_bytes: int = 32768) -> str:
+        provider = self._provider
+        if not provider or not getattr(provider, "available", False):
+            return _minimal_map(project_root, note="graph engine not available — install codebase-memory-mcp and run `codeintel index`")
+
+        # Top-level guard: the map is best-effort orientation — a malformed backend
+        # response must degrade to a minimal map, never crash the caller (never-raise).
+        try:
+            project = provider._resolve_project(project_root)
+            if project is None:
+                return _minimal_map(project_root, note="project not yet indexed — run `codeintel index` first")
+
+            # Query 1: architecture overview
+            arch_result = provider.build_result("overview", "", [], 5000, project_root)
+            arch_text: Optional[str] = None
+            if arch_result and arch_result.get("ok") and arch_result.get("result"):
+                arch_text = str(arch_result["result"])
+
+            # Query 2: ranked symbols by fan-in
+            ranked_rows = _query_ranked_symbols(provider, project)
+
+            # Query 3: entry points
+            entry_rows = _query_entry_points(provider, project)
+
+            content = _render(project_root, arch_text, ranked_rows, entry_rows)
+            content = _enforce_budget(content, budget_bytes, project_root, arch_text, ranked_rows, entry_rows)
+            return content
+        except Exception as exc:
+            return _minimal_map(project_root, note=f"map generation failed: {exc}")
+
+    def write(self, project_root: str, content: str) -> str:
+        path = os.path.join(project_root, "CODE_INTEL.md")
+        try:
+            with open(path, "w", encoding="utf-8") as f:
+                f.write(content)
+        except Exception as exc:
+            import logging
+            logging.getLogger(__name__).warning("CODE_INTEL.md write failed: %s", exc)
+        return path
+
+
+def _query_ranked_symbols(provider: GraphProvider, project: str) -> list[dict]:
+    raw = provider._run("query_graph", {"project": project, "query": _FAN_IN_CYPHER}, 8000)
+    rows = []
+    for row in _rows_from(raw):
+        name = row.get("fn.name") or row.get("name") or "?"
+        path = str(row.get("fn.file_path") or row.get("file_path") or "")
+        deg = row.get("in_degree", 0)
+        if path in _RANK_SKIP_PATHS:  # drop builtins / project node noise
+            continue
+        rows.append({"name": name, "file_path": path, "in_degree": deg})
+        if len(rows) >= 30:
+            break
+    return rows
+
+
+def _query_entry_points(provider: GraphProvider, project: str) -> list[dict]:
+    raw = provider._run("query_graph", {"project": project, "query": _ENTRY_CYPHER}, 5000)
+    rows = []
+    for row in _rows_from(raw):
+        name = row.get("fn.name") or row.get("name") or "?"
+        path = str(row.get("fn.file_path") or row.get("file_path") or "")
+        rows.append({"name": name, "file_path": path})
+    return rows
+
+
+def _render(
+    project_root: str,
+    arch_text: Optional[str],
+    ranked_rows: list[dict],
+    entry_rows: list[dict],
+) -> str:
+    repo_name = os.path.basename(os.path.abspath(project_root)) if project_root else "repo"
+    parts: list[str] = [f"# CODE_INTEL.md — {repo_name}\n"]
+    parts.append(
+        "> Auto-generated by `codeintel map`. Re-run after `codeintel index` to refresh.\n"
+    )
+
+    if arch_text:
+        parts.append("## Architecture Overview\n")
+        parts.append(arch_text.strip())
+        parts.append("")
+
+    if ranked_rows:
+        parts.append("## Ranked Symbols (by caller count)\n")
+        parts.append("| Symbol | File | Callers |")
+        parts.append("|--------|------|---------|")
+        for row in ranked_rows:
+            parts.append(f"| `{row['name']}` | {row['file_path']} | {row['in_degree']} |")
+        parts.append("")
+    else:
+        parts.append("## Ranked Symbols\n")
+        parts.append("_(no symbols found — run `codeintel index` first)_\n")
+
+    if entry_rows:
+        parts.append("## Entry Points\n")
+        for row in entry_rows:
+            parts.append(f"- `{row['name']}` ({row['file_path']})")
+        parts.append("")
+
+    return "\n".join(parts)
+
+
+def _enforce_budget(
+    content: str,
+    budget_bytes: int,
+    project_root: str,
+    arch_text: Optional[str],
+    ranked_rows: list[dict],
+    entry_rows: list[dict],
+) -> str:
+    if budget_bytes <= 0:
+        budget_bytes = 32768
+
+    if len(content.encode("utf-8")) <= budget_bytes:
+        return content
+
+    # Binary-search-style: drop rows from ranked table until we fit, then append notice
+    rows = list(ranked_rows)
+    notice = f"\n> Content truncated to fit {budget_bytes} byte budget.\n"
+    while rows:
+        candidate = _render(project_root, arch_text, rows, entry_rows) + notice
+        if len(candidate.encode("utf-8")) <= budget_bytes:
+            return candidate
+        rows = rows[:-1]
+
+    # Even with no ranked rows the header+notice must still be returned
+    candidate = _render(project_root, arch_text, [], entry_rows) + notice
+    return candidate
+
+
+def _minimal_map(project_root: str, note: str) -> str:
+    repo_name = os.path.basename(os.path.abspath(project_root)) if project_root else "repo"
+    return (
+        f"# CODE_INTEL.md — {repo_name}\n\n"
+        f"> Auto-generated by `codeintel map`.\n\n"
+        f"**Note**: {note}\n"
+    )