code-context-mcp 1.0.0__py3-none-any.whl

code_context/adapters/driven/git_source_cli.py

@@ -0,0 +1,211 @@
+"""GitCliSource — subprocess to `git` with ASCII unit-separator parsing."""
+
+from __future__ import annotations
+
+import logging
+import re
+import subprocess
+from datetime import datetime
+from pathlib import Path
+
+from code_context.domain.models import Change, DiffFile
+
+log = logging.getLogger(__name__)
+
+_FS = "\x1f"  # ASCII unit separator
+_PRETTY = f"%H{_FS}%aI{_FS}%an{_FS}%s"
+
+
+class GitCliSource:
+    def is_repo(self, root: Path) -> bool:
+        return (root / ".git").exists()
+
+    def head_sha(self, root: Path) -> str:
+        if not self.is_repo(root):
+            return ""
+        try:
+            out = subprocess.run(
+                ["git", "rev-parse", "HEAD"],
+                cwd=str(root),
+                capture_output=True,
+                text=True,
+                encoding="utf-8",
+                errors="replace",
+                check=True,
+            )
+            return (out.stdout or "").strip()
+        except subprocess.CalledProcessError as exc:
+            log.warning("git rev-parse HEAD failed: %s", exc)
+            return ""
+
+    def commits(
+        self,
+        root: Path,
+        since: datetime | None = None,
+        paths: list[str] | None = None,
+        max_count: int = 20,
+    ) -> list[Change]:
+        if not self.is_repo(root):
+            return []
+
+        cmd = ["git", "log", f"--pretty=format:{_PRETTY}", "--name-only", f"-{max_count}"]
+        if since is not None:
+            cmd.append(f"--since={since.isoformat()}")
+        if paths:
+            cmd.append("--")
+            cmd.extend(paths)
+
+        try:
+            res = subprocess.run(
+                cmd,
+                cwd=str(root),
+                capture_output=True,
+                text=True,
+                encoding="utf-8",
+                errors="replace",
+                check=True,
+            )
+        except subprocess.CalledProcessError as exc:
+            log.warning("git log failed: %s", exc)
+            return []
+
+        return _parse(res.stdout or "")
+
+    def diff_files(self, root: Path, ref: str) -> list[DiffFile]:
+        """Use git diff-tree + numstat-like parsing to get hunks per file.
+
+        Strategy: `git diff <ref>^! --unified=0 --no-color` gives a unified
+        diff with zero context lines. Each hunk header line is:
+            @@ -<old_start>,<old_count> +<new_start>,<new_count> @@
+        We parse those into (new_start, new_start + new_count - 1) pairs.
+
+        For ref == HEAD, the worktree diff (uncommitted changes) is excluded;
+        we always show the committed diff. To diff worktree, the caller would
+        pass an explicit ref like "HEAD" with a different strategy — out of
+        scope for v0.7.0.
+        """
+        if not self.is_repo(root):
+            return []
+
+        # ^! syntax means "this commit's changes vs its parent". Equivalent to
+        # `git diff <ref>~1 <ref>` for non-merge commits. For the initial
+        # commit, ^! is invalid; fall back to `git diff --root <ref>`.
+        #
+        # Critical Windows note: text=True alone uses Python's default
+        # locale encoding (cp1252 on Windows), which CANNOT decode many
+        # bytes that legitimately appear in git diff output (binary chunks,
+        # mixed-encoding source files). When the reader thread fails to
+        # decode, `res.stdout` becomes None even though the subprocess
+        # exited successfully. We force UTF-8 + errors="replace" to ensure
+        # we always get a string back, and we defensively guard against
+        # None in case future git versions change the behavior again.
+        try:
+            res = subprocess.run(
+                ["git", "diff", f"{ref}^!", "--unified=0", "--no-color"],
+                cwd=str(root),
+                capture_output=True,
+                text=True,
+                encoding="utf-8",
+                errors="replace",
+                check=True,
+            )
+            diff_text = res.stdout
+        except subprocess.CalledProcessError:
+            # Probably the initial commit. Try --root.
+            try:
+                res = subprocess.run(
+                    ["git", "diff", "--root", "--unified=0", "--no-color", ref],
+                    cwd=str(root),
+                    capture_output=True,
+                    text=True,
+                    encoding="utf-8",
+                    errors="replace",
+                    check=True,
+                )
+                diff_text = res.stdout
+            except subprocess.CalledProcessError as exc:
+                log.warning("git diff failed for ref %r: %s", ref, exc)
+                return []
+
+        if diff_text is None:
+            log.warning("git diff returned None stdout for ref %r — empty []", ref)
+            return []
+        return _parse_diff(diff_text)
+
+
+def _parse(stdout: str) -> list[Change]:
+    """Parse the formatted output into Change objects.
+
+    Each commit is:
+        <sha>\\x1f<iso_date>\\x1f<author>\\x1f<subject>\\n
+        <path1>\\n
+        <path2>\\n
+        ...
+        \\n  (blank separator)
+    """
+    commits: list[Change] = []
+    blocks = [b for b in stdout.split("\n\n") if b.strip()]
+    for block in blocks:
+        lines = block.splitlines()
+        if not lines:
+            continue
+        header = lines[0]
+        parts = header.split(_FS)
+        if len(parts) < 4:
+            continue
+        sha, iso_date, author, summary = parts[0], parts[1], parts[2], parts[3]
+        path_lines = [p.strip() for p in lines[1:] if p.strip()]
+        try:
+            date = datetime.fromisoformat(iso_date)
+        except ValueError:
+            continue
+        commits.append(
+            Change(
+                sha=sha,
+                date=date,
+                author=author,
+                paths=path_lines,
+                summary=summary,
+            )
+        )
+    return commits
+
+
+def _parse_diff(diff_text: str) -> list[DiffFile]:
+    """Parse a unified diff into a list of (path, hunks) pairs.
+
+    Hunk headers look like:
+        @@ -<old>,<oc> +<new>,<nc> @@
+
+    File headers look like:
+        diff --git a/<path> b/<path>
+        +++ b/<path>
+
+    We use the +++ header for the "new file" path; a/<path> would point
+    at the old name in renames.
+    """
+    files_to_hunks: dict[str, list[tuple[int, int]]] = {}
+    current_path: str | None = None
+    hunk_re = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@")
+    plus_path_re = re.compile(r"^\+\+\+ b/(.+)$")
+    null_path_re = re.compile(r"^\+\+\+ /dev/null")
+
+    for line in diff_text.splitlines():
+        m = plus_path_re.match(line)
+        if m:
+            current_path = m.group(1)
+            files_to_hunks.setdefault(current_path, [])
+            continue
+        if null_path_re.match(line):
+            current_path = None  # File deletion — no new-file hunks.
+            continue
+        m = hunk_re.match(line)
+        if m and current_path:
+            new_start = int(m.group(1))
+            new_count = int(m.group(2)) if m.group(2) else 1
+            # new_count == 0 means pure deletion — use the surrounding line
+            # as a single-line range.
+            end_line = new_start if new_count == 0 else new_start + new_count - 1
+            files_to_hunks[current_path].append((new_start, end_line))
+
+    return [DiffFile(path=p, hunks=tuple(h)) for p, h in files_to_hunks.items()]

code_context/adapters/driven/introspector_fs.py

@@ -0,0 +1,224 @@
+"""FilesystemIntrospector — extracts a ProjectSummary from filesystem heuristics."""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import tomllib
+from collections import Counter
+from pathlib import Path
+
+import pathspec
+
+from code_context.domain.models import ProjectSummary
+
+# Universally-noisy directories that mean "compiled output / vendored deps /
+# editor scratch", not source. Skipped even if .gitignore is missing —
+# every language ecosystem has at least one of these and they bloat
+# stats by 10-1000x (e.g. Sprint 5 smoke against WinServiceScheduler
+# reported 2179 files / 6.5M LOC because bin/obj/.dll were walked).
+_DENYLIST_DIRS = frozenset(
+    {
+        ".git",
+        ".hg",
+        ".svn",
+        ".venv",
+        "venv",
+        "node_modules",
+        "__pycache__",
+        ".pytest_cache",
+        ".mypy_cache",
+        ".ruff_cache",
+        ".tox",
+        "dist",
+        "build",
+        "bin",
+        "obj",
+        "out",
+        "publish",
+        "target",
+        "coverage",
+        ".idea",
+        ".vscode",
+        ".vs",
+    }
+)
+
+
+class FilesystemIntrospector:
+    def summary(
+        self, root: Path, scope: str = "project", path: Path | None = None
+    ) -> ProjectSummary:
+        target = path if (scope == "module" and path is not None) else root
+        gitignore = self._load_gitignore(root)
+        name = self._project_name(target)
+        purpose = self._readme_first_paragraph(target)
+        stack = self._detect_stack(target)
+        key_modules = self._key_modules(target, root, gitignore)
+        stats = self._stats(target, root, gitignore)
+        entry_points = self._entry_points(target)
+        return ProjectSummary(
+            name=name,
+            purpose=purpose,
+            stack=stack,
+            entry_points=entry_points,
+            key_modules=key_modules,
+            stats=stats,
+        )
+
+    @staticmethod
+    def _load_gitignore(root: Path) -> pathspec.PathSpec:
+        """Return a pathspec covering .gitignore + .git/ + the denylist.
+
+        Mirrors FilesystemSource._load_gitignore (Sprint 1). Adds a
+        baseline `.git/` line so even repos without a .gitignore skip
+        version-control internals; denylist dirs are appended as
+        gitignore-style patterns so the same matcher handles both.
+        """
+        lines = [".git/", *(f"{d}/" for d in sorted(_DENYLIST_DIRS))]
+        gi = root / ".gitignore"
+        if gi.exists():
+            with contextlib.suppress(OSError):
+                lines.extend(gi.read_text(encoding="utf-8", errors="replace").splitlines())
+        return pathspec.PathSpec.from_lines("gitignore", lines)
+
+    @staticmethod
+    def _project_name(root: Path) -> str:
+        py = root / "pyproject.toml"
+        if py.exists():
+            try:
+                data = tomllib.loads(py.read_text())
+                name = data.get("project", {}).get("name")
+                if isinstance(name, str):
+                    return name
+            except (tomllib.TOMLDecodeError, OSError):
+                pass
+        pkg = root / "package.json"
+        if pkg.exists():
+            try:
+                data = json.loads(pkg.read_text())
+                if isinstance(data.get("name"), str):
+                    return data["name"]
+            except (json.JSONDecodeError, OSError):
+                pass
+        return root.name
+
+    @staticmethod
+    def _readme_first_paragraph(root: Path) -> str:
+        for candidate in ("README.md", "readme.md", "README.rst", "README"):
+            f = root / candidate
+            if f.exists():
+                text = f.read_text(encoding="utf-8", errors="replace")
+                # Find the first non-heading non-blank paragraph.
+                for chunk in text.split("\n\n"):
+                    stripped = chunk.strip()
+                    if not stripped:
+                        continue
+                    if stripped.startswith("#"):
+                        continue
+                    return stripped
+        return ""
+
+    @staticmethod
+    def _detect_stack(root: Path) -> list[str]:
+        stack: list[str] = []
+        if (root / "pyproject.toml").exists() or (root / "setup.py").exists():
+            stack.append("Python")
+        if (root / "package.json").exists():
+            stack.append("Node")
+        if (root / "Cargo.toml").exists():
+            stack.append("Rust")
+        if (root / "go.mod").exists():
+            stack.append("Go")
+        if (root / "pom.xml").exists() or (root / "build.gradle").exists():
+            stack.append("Java")
+        return stack
+
+    @staticmethod
+    def _entry_points(root: Path) -> list[str]:
+        candidates = [
+            "src/main.py",
+            "src/index.js",
+            "src/index.ts",
+            "src/main.go",
+            "src/main.rs",
+            "main.py",
+            "index.js",
+            "main.go",
+        ]
+        return [c for c in candidates if (root / c).exists()]
+
+    @staticmethod
+    def _key_modules(
+        target: Path,
+        root: Path,
+        gitignore: pathspec.PathSpec,
+    ) -> list[dict[str, str]]:
+        out: list[dict[str, str]] = []
+        try:
+            entries = sorted(target.iterdir())
+        except OSError:
+            return out
+        for child in entries:
+            if not child.is_dir():
+                continue
+            name = child.name
+            if name.startswith(".") or name in _DENYLIST_DIRS:
+                continue
+            try:
+                rel_dir = child.resolve().relative_to(root.resolve()).as_posix()
+            except ValueError:
+                rel_dir = name  # target is outside root; don't gitignore-filter
+            # gitignore patterns expect dir entries with trailing slash.
+            if gitignore.match_file(rel_dir + "/") or gitignore.match_file(rel_dir):
+                continue
+            out.append({"path": name, "purpose": ""})
+        return out
+
+    @staticmethod
+    def _stats(
+        target: Path,
+        root: Path,
+        gitignore: pathspec.PathSpec,
+    ) -> dict[str, object]:
+        files = 0
+        loc = 0
+        langs: Counter[str] = Counter()
+        root_resolved = root
+        with contextlib.suppress(OSError):
+            root_resolved = root.resolve()
+        for f in target.rglob("*"):
+            if not f.is_file():
+                continue
+            # Filter against the denylist anywhere in the path so a nested
+            # `bin/`/`node_modules/` is excluded even if .gitignore is silent.
+            try:
+                rel_target = f.relative_to(target).parts
+            except ValueError:
+                continue
+            if any(part in _DENYLIST_DIRS for part in rel_target):
+                continue
+            if any(part.startswith(".") for part in rel_target):
+                continue
+            # Cross-check against .gitignore (which is anchored at repo root,
+            # so use the path relative to root, not target).
+            try:
+                rel_root = f.resolve().relative_to(root_resolved).as_posix()
+            except ValueError:
+                rel_root = "/".join(rel_target)
+            if gitignore.match_file(rel_root):
+                continue
+            files += 1
+            try:
+                content = f.read_text(encoding="utf-8", errors="replace")
+                loc += content.count("\n")
+            except OSError:
+                continue
+            ext = f.suffix.lstrip(".")
+            if ext:
+                langs[ext] += 1
+        return {
+            "files": files,
+            "loc": loc,
+            "languages": [ext for ext, _ in langs.most_common(10)],
+        }

code_context/adapters/driven/keyword_index_sqlite.py

@@ -0,0 +1,206 @@
+"""SqliteFTS5Index — BM25 keyword index using SQLite's FTS5 module.
+
+Each chunk is stored as a row in an FTS5 virtual table. SQLite's BM25
+ranking is exposed as a function in FTS5; we use it directly in the
+ORDER BY. The vector field is NOT stored here — only metadata + snippet
+text — so this index is much smaller than the vector store on disk.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import sqlite3
+from collections.abc import Iterable
+from pathlib import Path
+
+import numpy as np
+
+from code_context.domain.models import Chunk, IndexEntry
+
+log = logging.getLogger(__name__)
+
+_FILE = "keyword.sqlite"
+_FTS_TABLE = "chunks_fts"
+
+# FTS5 has a small set of reserved tokens (AND/OR/NOT/NEAR) AND treats
+# punctuation in queries as syntax (a `.` is a column separator, a `-`
+# starts an exclusion clause, `:` is column-qualified term). The default
+# unicode61 tokenizer handles punctuation INSIDE indexed text fine, but
+# in the QUERY the parser sees punctuation before tokenization. Strip
+# everything that isn't a word char / whitespace; the resulting token
+# list still matches the indexed tokens because the tokenizer would
+# have split them at the same boundaries on the way in.
+_FTS_KEEP_RE = re.compile(r"[^\w\s]", flags=re.UNICODE)
+_FTS_BOOLEAN_RE = re.compile(r"\b(AND|OR|NOT|NEAR)\b", re.IGNORECASE)
+
+
+class SqliteFTS5Index:
+    @property
+    def version(self) -> str:
+        return f"sqlite-fts5-{sqlite3.sqlite_version}-v1"
+
+    def __init__(self) -> None:
+        self._conn: sqlite3.Connection | None = None
+        self._db_path: Path | None = None
+        self._open_inmem()
+
+    def _open_inmem(self) -> None:
+        # check_same_thread=False: the MCP server runs query handlers via
+        # asyncio.to_thread, which uses a thread pool. Without this flag, a
+        # connection opened on the main thread cannot be used from worker
+        # threads (sqlite3.ProgrammingError). SQLite's library is built in
+        # serialized threading mode by default, so a single connection is
+        # safe across threads as long as we don't have concurrent writes —
+        # which we don't (writes happen at indexer.run() time, queries are
+        # read-only).
+        self._conn = sqlite3.connect(":memory:", check_same_thread=False)
+        self._init_schema()
+
+    def _init_schema(self) -> None:
+        assert self._conn is not None
+        self._conn.executescript(
+            f"""
+            CREATE VIRTUAL TABLE IF NOT EXISTS {_FTS_TABLE} USING fts5(
+                path, line_start UNINDEXED, line_end UNINDEXED,
+                content_hash UNINDEXED, snippet,
+                tokenize='unicode61 remove_diacritics 2'
+            );
+            -- vector storage is intentionally absent — vectors live in NumPyParquetStore.
+            """
+        )
+
+    def add(self, entries: Iterable[IndexEntry]) -> None:
+        assert self._conn is not None
+        rows = []
+        for e in entries:
+            c = e.chunk
+            rows.append((c.path, c.line_start, c.line_end, c.content_hash, c.snippet))
+        if not rows:
+            return
+        self._conn.executemany(
+            f"INSERT INTO {_FTS_TABLE} (path, line_start, line_end, content_hash, snippet) "
+            "VALUES (?, ?, ?, ?, ?)",
+            rows,
+        )
+        self._conn.commit()
+
+    def delete_by_path(self, path: str) -> int:
+        """Remove every row whose path == `path` from the FTS5 table.
+        Returns the rowcount. Used by Sprint 6 incremental reindex."""
+        assert self._conn is not None
+        cur = self._conn.execute(f"DELETE FROM {_FTS_TABLE} WHERE path = ?", (path,))
+        self._conn.commit()
+        return cur.rowcount
+
+    def search(self, query: str, k: int) -> list[tuple[IndexEntry, float]]:
+        assert self._conn is not None
+        sanitised = _sanitise(query)
+        if not sanitised.strip():
+            return []
+        try:
+            cur = self._conn.execute(
+                f"""
+                SELECT path, line_start, line_end, content_hash, snippet,
+                       bm25({_FTS_TABLE}) AS score
+                FROM {_FTS_TABLE}
+                WHERE {_FTS_TABLE} MATCH ?
+                ORDER BY score
+                LIMIT ?;
+                """,
+                (sanitised, k),
+            )
+        except sqlite3.OperationalError as exc:
+            log.warning("fts5 query failed (%s) for %r -> returning []", exc, query)
+            return []
+        return [
+            (
+                IndexEntry(
+                    chunk=Chunk(
+                        path=row[0],
+                        line_start=row[1],
+                        line_end=row[2],
+                        content_hash=row[3],
+                        snippet=row[4],
+                    ),
+                    vector=np.zeros(0, dtype=np.float32),  # Vector unused on this path.
+                ),
+                # bm25() returns negative scores; flip sign for "higher is better".
+                -float(row[5]),
+            )
+            for row in cur.fetchall()
+        ]
+
+    def persist(self, path: Path) -> None:
+        assert self._conn is not None
+        path.mkdir(parents=True, exist_ok=True)
+        target = path / _FILE
+        # Commit any open implicit transaction first — backup() blocks on
+        # uncommitted writes in the source connection.
+        self._conn.commit()
+        # Backup the in-memory DB to disk. sqlite3.Connection's context manager
+        # only commits on exit; it does NOT close. We close explicitly so
+        # Windows releases the file lock (otherwise tmp_path cleanup hangs).
+        # Backup target only used inside this method, no thread-safety concerns.
+        disk = sqlite3.connect(target, check_same_thread=False)
+        try:
+            self._conn.backup(disk)
+        finally:
+            disk.close()
+        self._db_path = target
+
+    def load(self, path: Path) -> None:
+        """Restore the index from `<path>/keyword.sqlite` into a fresh
+        in-memory connection.
+
+        Pre-Sprint-6 versions opened the on-disk file directly — fast,
+        zero RAM, but mutations (Sprint 6's incremental reindex calls
+        delete_by_path / add after load) wrote directly to the active
+        index file, breaking atomicity, AND a subsequent persist(same_dir)
+        deadlocked on SQLite's backup-to-itself constraint. The fix is
+        to load disk→memory: subsequent mutations stay in RAM and a
+        later persist() does the standard memory→fresh-disk backup. RAM
+        cost on the WinServiceScheduler smoke is ~5 MB; trivial.
+        """
+        target = path / _FILE
+        if not target.exists():
+            raise FileNotFoundError(f"keyword index missing at {target}")
+        if self._conn is not None:
+            self._conn.close()
+        # check_same_thread=False — see _open_inmem rationale.
+        self._conn = sqlite3.connect(":memory:", check_same_thread=False)
+        disk = sqlite3.connect(target, check_same_thread=False)
+        try:
+            disk.backup(self._conn)
+        finally:
+            disk.close()
+        self._db_path = target
+
+
+def _sanitise(query: str) -> str:
+    """Strip FTS5 syntax so user input never reaches the query parser
+    as anything other than bare whitespace-separated tokens.
+
+    Caught by Sprint 8's eval suite: 3/35 queries with periods or
+    hyphens silently returned [] from the sanitiser-as-was — `.`,
+    `-`, `:` are FTS5 query syntax even though they're tokenized
+    away in indexed text by unicode61.
+
+    Steps:
+    1. Drop every non-word, non-whitespace char.
+    2. Drop the boolean operators (AND/OR/NOT/NEAR) so e.g.
+       "tracking changes and merges" doesn't accidentally parse as
+       `tracking changes AND merges`.
+    3. Collapse whitespace.
+
+    The result is space-joined; FTS5 combines bare tokens with
+    implicit AND. We deliberately keep AND semantics: short queries
+    (1-3 tokens) get tight, high-precision matches; long
+    natural-language queries (5+ tokens) effectively return [] from
+    the keyword leg, leaving the vector leg to drive the result.
+    Sprint 8 eval confirmed that ORing tokens makes long-query
+    BM25 too noisy and hurts NDCG@10 by ~0.13.
+    """
+    cleaned = _FTS_KEEP_RE.sub(" ", query)
+    cleaned = _FTS_BOOLEAN_RE.sub(" ", cleaned)
+    return " ".join(cleaned.split())

code_context/adapters/driven/reranker_crossencoder.py

@@ -0,0 +1,61 @@
+"""CrossEncoderReranker — re-scores candidates using a sentence-transformers CrossEncoder.
+
+Lazy-loads the model on first use; constructing the adapter doesn't
+trigger torch loading. Empty candidate list short-circuits and never
+loads the model.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from code_context.domain.models import IndexEntry
+
+log = logging.getLogger(__name__)
+
+
+def _load_model(model_name: str) -> Any:  # pragma: no cover - integration-tested
+    from sentence_transformers import CrossEncoder
+
+    log.info("loading cross-encoder model: %s", model_name)
+    return CrossEncoder(model_name)
+
+
+def _lib_version() -> str:
+    try:
+        from importlib.metadata import PackageNotFoundError, version
+
+        return version("sentence-transformers")
+    except PackageNotFoundError:  # pragma: no cover
+        return "unknown"
+
+
+class CrossEncoderReranker:
+    def __init__(self, model_name: str = "cross-encoder/ms-marco-MiniLM-L-6-v2") -> None:
+        self.model_name = model_name
+        self._model: Any = None
+
+    @property
+    def version(self) -> str:
+        return "crossencoder-v1"
+
+    @property
+    def model_id(self) -> str:
+        return f"crossencoder:{self.model_name}@v{_lib_version()}"
+
+    def rerank(
+        self,
+        query: str,
+        candidates: list[tuple[IndexEntry, float]],
+        k: int,
+    ) -> list[tuple[IndexEntry, float]]:
+        if not candidates:
+            return []
+        if self._model is None:
+            self._model = _load_model(self.model_name)
+        pairs = [(query, e.chunk.snippet[:2048]) for e, _ in candidates]
+        scores = self._model.predict(pairs)
+        scored = [(c[0], float(s)) for c, s in zip(candidates, scores, strict=True)]
+        scored.sort(key=lambda x: x[1], reverse=True)
+        return scored[:k]