code-context-mcp 1.0.0__py3-none-any.whl

code_context/adapters/driven/chunker_line.py

@@ -0,0 +1,54 @@
+"""LineChunker — splits text into N-line windows with overlap."""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass
+
+from code_context.domain.models import Chunk
+
+_MIN_LINES = 5
+
+
+@dataclass
+class LineChunker:
+    """Splits content into `chunk_lines`-line chunks with `overlap` between consecutive chunks."""
+
+    chunk_lines: int = 50
+    overlap: int = 10
+
+    @property
+    def version(self) -> str:
+        return f"line-{self.chunk_lines}-{self.overlap}-v1"
+
+    def chunk(self, content: str, path: str) -> list[Chunk]:
+        if not content:
+            return []
+        lines = content.splitlines()
+        if len(lines) < _MIN_LINES:
+            return []
+
+        step = self.chunk_lines - self.overlap
+        if step <= 0:
+            raise ValueError(
+                f"overlap ({self.overlap}) must be less than chunk_lines ({self.chunk_lines})"
+            )
+
+        chunks: list[Chunk] = []
+        i = 0
+        while i < len(lines):
+            j = min(i + self.chunk_lines, len(lines))
+            snippet = "\n".join(lines[i:j])
+            chunks.append(
+                Chunk(
+                    path=path,
+                    line_start=i + 1,
+                    line_end=j,
+                    content_hash=hashlib.sha256(snippet.encode("utf-8")).hexdigest(),
+                    snippet=snippet,
+                )
+            )
+            if j >= len(lines):
+                break
+            i += step
+        return chunks

code_context/adapters/driven/chunker_treesitter.py

@@ -0,0 +1,215 @@
+"""TreeSitterChunker — AST-aware chunking via tree-sitter.
+
+Lazy-loads parsers per language. Returns whole-function / whole-class
+chunks. On unsupported language or parse failure, returns []. Caller
+(usually ChunkerDispatcher) is responsible for routing unsupported
+files to LineChunker.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from code_context.adapters.driven.chunker_treesitter_queries import QUERIES_BY_LANG
+from code_context.domain.models import Chunk, SymbolDef
+
+log = logging.getLogger(__name__)
+
+_EXT_TO_LANG: dict[str, str] = {
+    ".py": "python",
+    ".js": "javascript",
+    ".jsx": "javascript",
+    ".ts": "typescript",
+    ".tsx": "typescript",
+    ".go": "go",
+    ".rs": "rust",
+    ".cs": "csharp",
+}
+
+
+def _load_language(lang: str) -> tuple[Any, Any]:  # pragma: no cover - exercised in tests
+    """Lazy import + load. Patched in unit tests where needed."""
+    from tree_sitter_language_pack import get_language, get_parser
+
+    return get_language(lang), get_parser(lang)
+
+
+def _make_query_cursor(language: Any, source: str) -> Any:  # pragma: no cover
+    """Lazy import of tree-sitter's Query + QueryCursor."""
+    from tree_sitter import Query, QueryCursor
+
+    return QueryCursor(Query(language, source))
+
+
+@dataclass
+class TreeSitterChunker:
+    """Splits source code into chunks aligned to AST node boundaries."""
+
+    @property
+    def version(self) -> str:
+        # Bump the trailing -vN when query semantics change — invalidates the index cache.
+        return "treesitter-v2"
+
+    def chunk(self, content: str, path: str) -> list[Chunk]:
+        if not content:
+            return []
+        lang = _detect_language(path)
+        if lang is None or lang not in QUERIES_BY_LANG:
+            return []
+        try:
+            return _chunk_via_treesitter(content, path, lang)
+        except Exception as exc:  # parse errors are rare; LineChunker fallback handles them
+            log.warning("treesitter parse failed for %s (%s); returning []", path, exc)
+            return []
+
+    def extract_definitions(self, content: str, path: str) -> list[SymbolDef]:
+        """Walk the AST and emit a SymbolDef per @chunk node paired with its @name."""
+        if not content:
+            return []
+        lang = _detect_language(path)
+        if lang is None or lang not in QUERIES_BY_LANG:
+            return []
+        try:
+            return _extract_via_treesitter(content, path, lang)
+        except Exception as exc:
+            log.warning("treesitter extract_definitions failed for %s (%s)", path, exc)
+            return []
+
+
+def _detect_language(path: str) -> str | None:
+    suffix = Path(path).suffix.lower()
+    return _EXT_TO_LANG.get(suffix)
+
+
+def _chunk_via_treesitter(content: str, path: str, lang: str) -> list[Chunk]:
+    language, parser = _load_language(lang)
+    tree = parser.parse(content.encode("utf-8"))
+    cursor = _make_query_cursor(language, QUERIES_BY_LANG[lang])
+    captures = cursor.captures(tree.root_node)
+    # QueryCursor.captures returns dict[capture_name, list[Node]] in tree-sitter ≥0.24.
+    # Older fallback: list of (Node, capture_name) tuples.
+    chunk_nodes = _flatten_chunk_nodes(captures)
+    # Sort by start line for stable, document-order output.
+    chunk_nodes.sort(key=lambda n: (n.start_point[0], n.start_point[1]))
+    source_lines = content.splitlines()
+    chunks: list[Chunk] = []
+    for node in chunk_nodes:
+        start_line = node.start_point[0] + 1
+        end_line = node.end_point[0] + 1
+        # Use the source-line slice (not node.text) so leading indentation is
+        # preserved — matters for indented methods whose tree-sitter node.text
+        # starts at the column where the keyword sits.
+        snippet = "\n".join(source_lines[start_line - 1 : end_line])
+        chunks.append(
+            Chunk(
+                path=path,
+                line_start=start_line,
+                line_end=end_line,
+                content_hash=hashlib.sha256(snippet.encode("utf-8")).hexdigest(),
+                snippet=snippet,
+            )
+        )
+    return chunks
+
+
+def _extract_via_treesitter(content: str, path: str, lang: str) -> list[SymbolDef]:
+    """Pair @chunk nodes with their @name children to produce SymbolDef list."""
+    language, parser = _load_language(lang)
+    tree = parser.parse(content.encode("utf-8"))
+    cursor = _make_query_cursor(language, QUERIES_BY_LANG[lang])
+    captures = cursor.captures(tree.root_node)
+    # captures is dict[capture_name, list[Node]] in tree-sitter ≥0.24.
+
+    chunk_nodes = list(captures.get("chunk", [])) if isinstance(captures, dict) else []
+    name_nodes = list(captures.get("name", [])) if isinstance(captures, dict) else []
+    if not chunk_nodes or not name_nodes:
+        return []
+
+    # Pair @name with the closest enclosing @chunk by walking up parents.
+    # Use node.id (the underlying tree-sitter AST node identity) rather than
+    # id(node) (Python wrapper identity) — re-fetched nodes get fresh wrappers
+    # but the same underlying tree-sitter id.
+    chunk_set = {n.id: n for n in chunk_nodes}
+    pairs: list[tuple[Any, Any]] = []
+    for name_node in name_nodes:
+        cur = name_node.parent
+        while cur is not None and cur.id not in chunk_set:
+            cur = cur.parent
+        if cur is not None:
+            pairs.append((chunk_set[cur.id], name_node))
+
+    defs: list[SymbolDef] = []
+    seen_keys: set[tuple[str, int, int]] = set()
+    for chunk_node, name_node in pairs:
+        try:
+            symbol_name = name_node.text.decode("utf-8", errors="replace")
+        except (AttributeError, UnicodeDecodeError):
+            continue
+        start_line = chunk_node.start_point[0] + 1
+        end_line = chunk_node.end_point[0] + 1
+        # Skip duplicates (same chunk node with multiple matched names).
+        key = (symbol_name, start_line, end_line)
+        if key in seen_keys:
+            continue
+        seen_keys.add(key)
+        defs.append(
+            SymbolDef(
+                name=symbol_name,
+                path=path,
+                lines=(start_line, end_line),
+                kind=_kind_from_node(chunk_node, lang),
+                language=lang,
+            )
+        )
+    # Sort for stable output (by start_line, then name).
+    defs.sort(key=lambda d: (d.lines[0], d.name))
+    return defs
+
+
+def _kind_from_node(node: Any, lang: str) -> str:
+    """Map tree-sitter node types to our SymbolDef.kind vocabulary."""
+    kind_map = {
+        # Python
+        "function_definition": "function",
+        "class_definition": "class",
+        # JS / TS
+        "function_declaration": "function",
+        "class_declaration": "class",
+        "method_definition": "method",
+        "interface_declaration": "interface",
+        "type_alias_declaration": "type",
+        "variable_declarator": "function",  # arrow function bound to const
+        # Go
+        "method_declaration": "method",
+        "type_declaration": "type",
+        # Rust
+        "function_item": "function",
+        "struct_item": "struct",
+        "enum_item": "enum",
+        "impl_item": "impl",
+        "trait_item": "trait",
+        # C# (some overlap with the above; latest hit wins, ordering matters
+        # because dicts preserve insertion order — listed here last on purpose
+        # so e.g. C# class_declaration produces "class" not "class" via the JS path).
+        "constructor_declaration": "constructor",
+        "struct_declaration": "struct",
+        "record_declaration": "record",
+        "enum_declaration": "enum",
+    }
+    return kind_map.get(node.type, "unknown")
+
+
+def _flatten_chunk_nodes(captures: Any) -> list[Any]:
+    """Return the @chunk-tagged nodes regardless of which API shape we got."""
+    if isinstance(captures, dict):
+        return list(captures.get("chunk", []))
+    out: list[Any] = []
+    for item in captures:
+        # item is (node, name) in older bindings.
+        if isinstance(item, tuple) and len(item) == 2 and item[1] == "chunk":
+            out.append(item[0])
+    return out

code_context/adapters/driven/chunker_treesitter_queries.py

@@ -0,0 +1,111 @@
+"""Tree-sitter S-expression queries: one per supported language.
+
+Each query captures the AST nodes we want to emit as chunks. A node is
+"chunk-worthy" when it represents a complete top-level semantic unit:
+
+- Python: function_definition, class_definition.
+- JavaScript / TypeScript: function_declaration, class_declaration,
+  method_definition, arrow function assigned at module scope.
+- Go: function_declaration, method_declaration, type_declaration.
+- Rust: function_item, struct_item, enum_item, impl_item, trait_item.
+- C#: method_declaration, constructor_declaration, class_declaration,
+  interface_declaration, struct_declaration, record_declaration,
+  enum_declaration.
+
+Each query also captures the symbol's identifier as ``@name`` so callers
+that want to mine ``SymbolDef`` objects (extract_definitions) can pair
+the chunk node with its name child. Existing chunk-only consumers that
+filter ``captures["chunk"]`` continue to work unchanged.
+
+Smaller nodes (assignments, single statements) are NOT captured — they
+are rolled up into a synthetic "module-prelude" chunk in the chunker
+(future work; v0.2.0 simply skips them).
+"""
+
+from __future__ import annotations
+
+PYTHON = """
+(function_definition
+  name: (identifier) @name) @chunk
+(class_definition
+  name: (identifier) @name) @chunk
+"""
+
+JAVASCRIPT = """
+(function_declaration
+  name: (identifier) @name) @chunk
+(class_declaration
+  name: (identifier) @name) @chunk
+(method_definition
+  name: (property_identifier) @name) @chunk
+(variable_declarator
+  name: (identifier) @name
+  value: (arrow_function)) @chunk
+"""
+
+# TypeScript shares JS's overall shape but the grammar names the class with
+# (type_identifier) rather than (identifier), so we cannot reuse the JS
+# string verbatim — the TS class_declaration pattern needs its own form.
+TYPESCRIPT = """
+(function_declaration
+  name: (identifier) @name) @chunk
+(class_declaration
+  name: (type_identifier) @name) @chunk
+(method_definition
+  name: (property_identifier) @name) @chunk
+(variable_declarator
+  name: (identifier) @name
+  value: (arrow_function)) @chunk
+(interface_declaration
+  name: (type_identifier) @name) @chunk
+(type_alias_declaration
+  name: (type_identifier) @name) @chunk
+"""
+
+GO = """
+(function_declaration
+  name: (identifier) @name) @chunk
+(method_declaration
+  name: (field_identifier) @name) @chunk
+(type_declaration
+  (type_spec name: (type_identifier) @name)) @chunk
+"""
+
+RUST = """
+(function_item
+  name: (identifier) @name) @chunk
+(struct_item
+  name: (type_identifier) @name) @chunk
+(enum_item
+  name: (type_identifier) @name) @chunk
+(impl_item
+  type: (type_identifier) @name) @chunk
+(trait_item
+  name: (type_identifier) @name) @chunk
+"""
+
+CSHARP = """
+(method_declaration
+  name: (identifier) @name) @chunk
+(constructor_declaration
+  name: (identifier) @name) @chunk
+(class_declaration
+  name: (identifier) @name) @chunk
+(interface_declaration
+  name: (identifier) @name) @chunk
+(struct_declaration
+  name: (identifier) @name) @chunk
+(record_declaration
+  name: (identifier) @name) @chunk
+(enum_declaration
+  name: (identifier) @name) @chunk
+"""
+
+QUERIES_BY_LANG: dict[str, str] = {
+    "python": PYTHON,
+    "javascript": JAVASCRIPT,
+    "typescript": TYPESCRIPT,
+    "go": GO,
+    "rust": RUST,
+    "csharp": CSHARP,
+}

code_context/adapters/driven/code_source_fs.py

@@ -0,0 +1,122 @@
+"""FilesystemSource — gitignore-aware walk + binary detection."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pathspec
+
+from code_context.domain.models import FileTreeNode
+
+_BINARY_PROBE_BYTES = 4096
+
+
+class FilesystemSource:
+    def list_files(self, root: Path, include_exts: list[str], max_bytes: int) -> list[Path]:
+        gitignore = self._load_gitignore(root)
+        results: list[Path] = []
+        ext_set = {e.lower() for e in include_exts}
+
+        for path in sorted(root.rglob("*")):
+            if not path.is_file():
+                continue
+            rel = path.relative_to(root).as_posix()
+            if gitignore.match_file(rel):
+                continue
+            if path.suffix.lower() not in ext_set:
+                continue
+            try:
+                size = path.stat().st_size
+            except OSError:
+                continue
+            if size > max_bytes:
+                continue
+            if self._looks_binary(path):
+                continue
+            results.append(path)
+        return results
+
+    def read(self, path: Path) -> str:
+        return path.read_text(encoding="utf-8", errors="replace")
+
+    def walk_tree(
+        self,
+        root: Path,
+        max_depth: int = 4,
+        include_hidden: bool = False,
+        subpath: Path | None = None,
+    ) -> FileTreeNode:
+        root_resolved = root.resolve()
+        target = root_resolved if subpath is None else (root / subpath).resolve()
+        # Refuse to walk outside the root.
+        try:
+            target.relative_to(root_resolved)
+        except ValueError as exc:
+            raise ValueError(f"subpath {subpath!r} escapes root {root!r}") from exc
+
+        gitignore = self._load_gitignore(root)
+        return self._walk_node(target, root_resolved, gitignore, max_depth, include_hidden, 0)
+
+    def _walk_node(
+        self,
+        node: Path,
+        root: Path,
+        gitignore: pathspec.PathSpec,
+        max_depth: int,
+        include_hidden: bool,
+        current_depth: int,
+    ) -> FileTreeNode:
+        rel = node.relative_to(root).as_posix() if node != root else ""
+        rel_display = rel if rel else "."
+
+        if node.is_file():
+            try:
+                size = node.stat().st_size
+            except OSError:
+                size = None
+            return FileTreeNode(path=rel_display, kind="file", children=(), size=size)
+
+        # Directory.
+        if current_depth >= max_depth:
+            # Cap reached — empty dir node (signals depth cap to caller).
+            return FileTreeNode(path=rel_display, kind="dir", children=(), size=None)
+
+        children: list[FileTreeNode] = []
+        try:
+            entries = sorted(node.iterdir(), key=lambda p: (p.is_file(), p.name.lower()))
+        except OSError:
+            return FileTreeNode(path=rel_display, kind="dir", children=(), size=None)
+
+        for child in entries:
+            name = child.name
+            if not include_hidden and name.startswith("."):
+                continue
+            child_rel = child.relative_to(root).as_posix()
+            # gitignore matching: dirs need trailing slash to match dir patterns.
+            match_path = child_rel + ("/" if child.is_dir() else "")
+            if gitignore.match_file(match_path) or gitignore.match_file(child_rel):
+                continue
+            children.append(
+                self._walk_node(
+                    child, root, gitignore, max_depth, include_hidden, current_depth + 1
+                )
+            )
+
+        return FileTreeNode(path=rel_display, kind="dir", children=tuple(children), size=None)
+
+    @staticmethod
+    def _load_gitignore(root: Path) -> pathspec.PathSpec:
+        lines = [".git/"]
+        gi = root / ".gitignore"
+        if gi.exists():
+            lines.extend(gi.read_text().splitlines())
+        return pathspec.PathSpec.from_lines("gitignore", lines)
+
+    @staticmethod
+    def _looks_binary(path: Path) -> bool:
+        try:
+            with path.open("rb") as fh:
+                probe = fh.read(_BINARY_PROBE_BYTES)
+            return b"\x00" in probe
+        except OSError:
+            return True

code_context/adapters/driven/embeddings_local.py

@@ -0,0 +1,111 @@
+"""LocalST — sentence-transformers wrapped as an EmbeddingsProvider.
+
+The sentence-transformers import is lazy: constructing this adapter doesn't
+trigger torch loading. The model is loaded on first `embed()` call.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+import numpy as np
+
+log = logging.getLogger(__name__)
+
+
+# MODEL_REGISTRY enumerates models we have verified and characterised. Models
+# missing from here still work, but staleness, dimension hints, and benchmarks
+# won't recognise them and the adapter will warn at construction time.
+#
+# v0.3.3 trimmed this list to verified entries only. v0.3.0-v0.3.2 listed
+# `BAAI/bge-code-v1.5` which never existed on Hugging Face — a planning error
+# corrected here. Other code-tuned candidates (`jinaai/jina-embeddings-v2-base-code`,
+# `BAAI/bge-code-v1`) work via `CC_EMBEDDINGS_MODEL` override but are not yet
+# pre-characterised here because their embedding dims have not been independently
+# verified. v0.4 will re-introduce a verified code-tuned default after benchmark
+# validation and a CI check that pings the HF API for each registered name.
+MODEL_REGISTRY: dict[str, dict[str, int | str]] = {
+    "sentence-transformers/all-MiniLM-L6-v2": {"dimension": 384, "kind": "general"},
+    "all-MiniLM-L6-v2": {"dimension": 384, "kind": "general"},  # short alias
+    # Code-tuned (opt-in via CC_EMBEDDINGS_MODEL + CC_TRUST_REMOTE_CODE=true).
+    # 161M params (~640 MB FP32), Apache-2.0, English + 30 programming languages.
+    # Verified existing on HF as of v0.6.0 release; CI's hf-guard job re-checks
+    # on every push.
+    "jinaai/jina-embeddings-v2-base-code": {"dimension": 768, "kind": "code"},
+}
+
+
+# Whole-function chunks from tree-sitter can run 5K+ chars and overflow the
+# 512-token context of BERT-family encoders. We embed the truncated head; the
+# full snippet is preserved in the chunk for the search response payload, so
+# users still see the complete code. 2048 chars ~= 512 tokens for code-heavy
+# text.
+_MAX_EMBED_CHARS = 2048
+
+
+def _load_model(
+    model_name: str, *, trust_remote_code: bool = False
+) -> Any:  # pragma: no cover - integration-tested
+    """Lazy import + load. Patched in unit tests."""
+    from sentence_transformers import SentenceTransformer
+
+    log.info(
+        "loading sentence-transformers model: %s (trust_remote_code=%s)",
+        model_name,
+        trust_remote_code,
+    )
+    return SentenceTransformer(model_name, trust_remote_code=trust_remote_code)
+
+
+def _lib_version() -> str:
+    try:
+        from importlib.metadata import version
+
+        return version("sentence-transformers")
+    except Exception:  # pragma: no cover
+        return "unknown"
+
+
+class LocalST:
+    def __init__(
+        self,
+        model_name: str = "all-MiniLM-L6-v2",
+        *,
+        trust_remote_code: bool = False,
+    ) -> None:
+        if model_name not in MODEL_REGISTRY:
+            log.warning(
+                "embeddings model %r not in MODEL_REGISTRY; staleness, "
+                "dimension hints, and benchmarks won't recognise it",
+                model_name,
+            )
+        self.model_name = model_name
+        self.trust_remote_code = trust_remote_code
+        self._model: Any = None
+
+    @property
+    def dimension(self) -> int:
+        self._ensure_loaded()
+        # sentence-transformers >= 5 renamed the method; fall back to the old name
+        # so we work across both lines without a hard pin.
+        getter = getattr(self._model, "get_embedding_dimension", None) or (
+            self._model.get_sentence_embedding_dimension
+        )
+        return int(getter())
+
+    @property
+    def model_id(self) -> str:
+        return f"local:{self.model_name}@v{_lib_version()}"
+
+    def embed(self, texts: list[str]) -> np.ndarray:
+        self._ensure_loaded()
+        if not texts:
+            return np.empty((0, self.dimension), dtype=np.float32)
+        truncated = [t[:_MAX_EMBED_CHARS] for t in texts]
+        out = self._model.encode(truncated, convert_to_numpy=True, show_progress_bar=False)
+        return out.astype(np.float32, copy=False)
+
+    def _ensure_loaded(self) -> None:
+        if self._model is None:
+            self._model = _load_model(self.model_name, trust_remote_code=self.trust_remote_code)

code_context/adapters/driven/embeddings_openai.py

@@ -0,0 +1,58 @@
+"""OpenAIProvider — OpenAI embeddings via the openai SDK."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+import numpy as np
+
+log = logging.getLogger(__name__)
+
+_DIMENSION_BY_MODEL = {
+    "text-embedding-3-small": 1536,
+    "text-embedding-3-large": 3072,
+    "text-embedding-ada-002": 1536,
+}
+
+
+def _load_client(api_key: str) -> Any:  # pragma: no cover - patched in tests
+    """Lazy import + construct."""
+    from openai import OpenAI
+
+    return OpenAI(api_key=api_key)
+
+
+def _lib_version() -> str:
+    try:
+        from importlib.metadata import version
+
+        return version("openai")
+    except Exception:  # pragma: no cover
+        return "unknown"
+
+
+class OpenAIProvider:
+    def __init__(self, model: str, api_key: str) -> None:
+        if not api_key:
+            raise ValueError("api_key is required for OpenAIProvider")
+        self.model = model
+        self._api_key = api_key
+        self._client: Any = None
+
+    @property
+    def dimension(self) -> int:
+        return _DIMENSION_BY_MODEL.get(self.model, 1536)
+
+    @property
+    def model_id(self) -> str:
+        return f"openai:{self.model}@v{_lib_version()}"
+
+    def embed(self, texts: list[str]) -> np.ndarray:
+        if self._client is None:
+            self._client = _load_client(self._api_key)
+        if not texts:
+            return np.empty((0, self.dimension), dtype=np.float32)
+        resp = self._client.embeddings.create(model=self.model, input=texts)
+        out = np.array([d.embedding for d in resp.data], dtype=np.float32)
+        return out