codecortex 0.7.0__tar.gz → 0.8.0__tar.gz

{codecortex-0.7.0/src/codecortex.egg-info → codecortex-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: codecortex
-Version: 0.7.0
+Version: 0.8.0
 Summary: Local-first, MCP-native code-intelligence server — graph, LSP, and semantic search behind one safe code.query tool for coding agents.
 Author: Shammai Hamilton
 License-Expression: MIT
@@ -26,6 +26,7 @@ License-File: LICENSE
 Requires-Dist: mcp>=1.0
 Requires-Dist: sqlite-vec>=0.1
 Requires-Dist: fastembed>=0.3
+Requires-Dist: tree-sitter-language-pack>=1.0
 Provides-Extra: dev
 Requires-Dist: pytest>=8; extra == "dev"
 Requires-Dist: numpy>=1.24; extra == "dev"

{codecortex-0.7.0 → codecortex-0.8.0}/pyproject.toml

@@ -31,7 +31,12 @@ classifiers = [
     "Topic :: Software Development :: Quality Assurance",
     "Typing :: Typed",
 ]
-dependencies = ["mcp>=1.0", "sqlite-vec>=0.1", "fastembed>=0.3"]
+dependencies = [
+    "mcp>=1.0",
+    "sqlite-vec>=0.1",
+    "fastembed>=0.3",
+    "tree-sitter-language-pack>=1.0",  # def-aligned chunking for non-Python languages (P3)
+]
 
 [project.urls]
 Homepage = "https://github.com/hamilton-sky/codeintel"

{codecortex-0.7.0 → codecortex-0.8.0/src/codecortex.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: codecortex
-Version: 0.7.0
+Version: 0.8.0
 Summary: Local-first, MCP-native code-intelligence server — graph, LSP, and semantic search behind one safe code.query tool for coding agents.
 Author: Shammai Hamilton
 License-Expression: MIT
@@ -26,6 +26,7 @@ License-File: LICENSE
 Requires-Dist: mcp>=1.0
 Requires-Dist: sqlite-vec>=0.1
 Requires-Dist: fastembed>=0.3
+Requires-Dist: tree-sitter-language-pack>=1.0
 Provides-Extra: dev
 Requires-Dist: pytest>=8; extra == "dev"
 Requires-Dist: numpy>=1.24; extra == "dev"

{codecortex-0.7.0 → codecortex-0.8.0}/src/codecortex.egg-info/SOURCES.txt

@@ -59,4 +59,5 @@ tests/test_reindexer.py
 tests/test_rerank.py
 tests/test_reset.py
 tests/test_semantic_provider.py
-tests/test_term.py
+tests/test_term.py
+tests/test_treesitter.py

{codecortex-0.7.0 → codecortex-0.8.0}/src/codecortex.egg-info/requires.txt

@@ -1,6 +1,7 @@
 mcp>=1.0
 sqlite-vec>=0.1
 fastembed>=0.3
+tree-sitter-language-pack>=1.0
 
 [dev]
 pytest>=8

codecortex-0.8.0/src/codeintel/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.8.0"

{codecortex-0.7.0 → codecortex-0.8.0}/src/codeintel/indexer.py

@@ -14,7 +14,10 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 _INDEXED_EXTS = frozenset({
-    ".py", ".ts", ".js", ".go", ".rs", ".java", ".c", ".cpp", ".h", ".md"
+    ".py", ".md",
+    ".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs",       # TS/JS variants
+    ".go", ".rs", ".java",
+    ".c", ".h", ".cpp", ".cc", ".cxx", ".hpp", ".hh",    # C/C++ variants
 })
 _SKIP_DIRS = frozenset({"__pycache__", ".git", "node_modules"})
 # Vendored / regenerable dirs skipped even without a .gitignore entry.
@@ -41,6 +44,49 @@ def _pos_int(val: object, default: int) -> int:
     return n if n > 0 else default
 
 
+# ---- tree-sitter chunking (P3): def-aligned chunks for non-Python languages -----------------
+# Extends the P1 chunker interface to TS/JS/Go/Rust/Java/C/C++ via tree-sitter, behind the same
+# _primary_spans → _cover pipeline. The grammar pack is a normal dependency but never a hard
+# requirement: if it (or a grammar) is unavailable, or a file's language config is missing, the
+# file falls back to line windowing — exactly like a Python parse failure.
+
+# file extension -> tree_sitter_language_pack language name. Only the *code* exts in
+# _INDEXED_EXTS; .md (and anything unmapped) keeps line-windowing.
+_TS_LANG_BY_EXT = {
+    ".ts": "typescript", ".tsx": "tsx", ".js": "javascript", ".jsx": "javascript",
+    ".mjs": "javascript", ".cjs": "javascript",
+    ".go": "go", ".rs": "rust", ".java": "java",
+    ".c": "c", ".h": "cpp", ".cpp": "cpp", ".cc": "cpp", ".cxx": "cpp",
+    ".hpp": "cpp", ".hh": "cpp",
+}
+# "Leaf" definition node types — each emitted as one whole chunk (like a top-level Python def).
+_TS_FUNC_TYPES = {
+    "typescript": {"function_declaration", "generator_function_declaration", "method_definition",
+                   "method_signature", "abstract_method_signature", "function_signature"},
+    "tsx": {"function_declaration", "generator_function_declaration", "method_definition",
+            "method_signature", "abstract_method_signature", "function_signature"},
+    "javascript": {"function_declaration", "generator_function_declaration", "method_definition"},
+    "go": {"function_declaration", "method_declaration", "type_declaration"},
+    "rust": {"function_item", "function_signature_item", "struct_item", "enum_item",
+             "union_item", "type_item", "macro_definition"},
+    "java": {"method_declaration", "constructor_declaration"},
+    "c": {"function_definition", "struct_specifier", "enum_specifier", "union_specifier"},
+    "cpp": {"function_definition"},
+}
+# "Container" definition types — emitted as a header chunk + one chunk per nested member, exactly
+# like a Python class (so per-method chunks are never shadowed by a whole-class chunk).
+_TS_CONTAINER_TYPES = {
+    "typescript": {"class_declaration", "abstract_class_declaration", "interface_declaration"},
+    "tsx": {"class_declaration", "abstract_class_declaration", "interface_declaration"},
+    "javascript": {"class_declaration"},
+    "go": set(),
+    "rust": {"impl_item", "trait_item", "mod_item"},
+    "java": {"class_declaration", "interface_declaration", "enum_declaration", "record_declaration"},
+    "c": set(),
+    "cpp": {"class_specifier", "struct_specifier", "namespace_definition"},
+}
+
+
 class Indexer:
     def __init__(
         self,
@@ -76,6 +122,7 @@ class Indexer:
             else 2 * self.window
         )
         self._embedder = None
+        self._ts_parsers: dict = {}  # per-instance tree-sitter parser cache (lang -> parser|None)
 
     def _get_embedder(self):
         if self._embedder is None:
@@ -249,16 +296,109 @@ class Indexer:
         n = len(lines)
         return self._cover(self._primary_spans(tree, n), n)
 
+    def _get_ts_parser(self, lang: str):
+        """Lazily load + cache (per instance) the tree-sitter parser for a language. Returns None
+        when ``tree-sitter-language-pack`` isn't installed or the grammar is unavailable — the
+        caller then windows (tree-sitter is a normal dep, never a hard requirement)."""
+        cache = self._ts_parsers
+        if lang in cache:
+            return cache[lang]
+        parser = None
+        try:
+            from tree_sitter_language_pack import get_parser
+            parser = get_parser(lang)
+        except Exception as exc:
+            logger.debug("tree-sitter parser for %s unavailable: %s", lang, exc)
+        cache[lang] = parser
+        return parser
+
+    @staticmethod
+    def _ts_end_line(node, n: int) -> int:
+        """0-based half-open end line of a tree-sitter node. ``end_point`` is (row, col); a col of 0
+        means the node ends at the start of that row (so the row isn't included). Clamped to
+        ``[start+1, n]``."""
+        row, col = node.end_point
+        end = row if col == 0 else row + 1
+        return min(n, max(node.start_point[0] + 1, end))
+
+    def _primary_spans_ts(self, root, lang: str, n: int) -> list[tuple[int, int]]:
+        """Def-aligned 'primary' spans from a tree-sitter tree, mirroring ``_primary_spans``: each
+        function/method → one span; each container (class/impl/trait/…) → a header span plus one
+        span per nested member. Node types the language config doesn't list are simply not treated
+        as defs — they fall into ``_cover``'s window-filled gaps, so an incomplete config degrades
+        precision, never correctness."""
+        func_types = _TS_FUNC_TYPES.get(lang, set())
+        cont_types = _TS_CONTAINER_TYPES.get(lang, set())
+        all_types = func_types | cont_types
+        if not all_types:
+            return []  # unknown language -> caller windows the whole file
+        spans: list[tuple[int, int]] = []
+
+        def nearest_defs(node):
+            """Def-type nodes reachable without crossing another def-type — top-level defs from the
+            root, a container's direct members from the container. Iterative, so deep nesting can't
+            blow the stack."""
+            found = []
+            stack = [c for c in reversed(node.children) if c.is_named]
+            while stack:
+                c = stack.pop()
+                if c.type in all_types:
+                    found.append(c)
+                else:
+                    stack.extend(g for g in reversed(c.children) if g.is_named)
+            return found
+
+        def emit(node):
+            s = node.start_point[0]
+            e = self._ts_end_line(node, n)
+            if node.type in cont_types:
+                members = sorted(nearest_defs(node), key=lambda m: m.start_point[0])
+                if members:
+                    header_end = max(s + 1, min(members[0].start_point[0], e))
+                    spans.append((s, header_end))  # header: class/impl line -> first member
+                    for m in members:
+                        emit(m)
+                    return
+            spans.append((s, e))
+
+        for node in sorted(nearest_defs(root), key=lambda x: x.start_point[0]):
+            emit(node)
+        return spans
+
+    def _chunk_treesitter(self, lines: list[str], source: str, lang: str):
+        """Parse ``source`` with tree-sitter → a complete, def-aligned cover, or ``None`` when the
+        parser is unavailable (caller windows). tree-sitter is error-tolerant, so a syntactically
+        broken file still yields a partial tree (and thus useful spans) rather than raising."""
+        parser = self._get_ts_parser(lang)
+        if parser is None:
+            return None
+        tree = parser.parse(source.encode("utf-8", errors="replace"))
+        n = len(lines)
+        return self._cover(self._primary_spans_ts(tree.root_node, lang, n), n)
+
     def _spans_for_file(
         self, filepath: Path, lines: list[str], rel_path: str
     ) -> list[tuple[int, int]]:
-        """Choose spans for one file: syntax-aware for ``.py`` under the syntax strategy (falling
-        back to windowing on any parse failure), fixed windows for everything else."""
-        if self.chunk_strategy == "syntax" and filepath.suffix.lower() == ".py":
-            try:
-                return self._chunk_python_ast(lines, "".join(lines))
-            except Exception as exc:
-                logger.debug("syntax chunking failed for %s (%s) — windowing", rel_path, exc)
+        """Choose spans for one file under the syntax strategy: ``ast`` for ``.py``, tree-sitter for
+        the mapped languages (TS/JS/Go/Rust/Java/C/C++), fixed windows for everything else and on
+        any parse/grammar failure or when tree-sitter isn't installed."""
+        if self.chunk_strategy == "syntax":
+            suffix = filepath.suffix.lower()
+            if suffix == ".py":
+                try:
+                    return self._chunk_python_ast(lines, "".join(lines))
+                except Exception as exc:
+                    logger.debug("syntax chunking failed for %s (%s) — windowing", rel_path, exc)
+            else:
+                lang = _TS_LANG_BY_EXT.get(suffix)
+                if lang is not None:
+                    try:
+                        spans = self._chunk_treesitter(lines, "".join(lines), lang)
+                        if spans is not None:
+                            return spans
+                    except Exception as exc:
+                        logger.debug("tree-sitter chunking failed for %s (%s) — windowing",
+                                     rel_path, exc)
         return self._window_spans(0, len(lines))
 
     # ---- materialisation -------------------------------------------------------------------

{codecortex-0.7.0 → codecortex-0.8.0}/tests/test_chunking.py

@@ -195,12 +195,19 @@ def test_nul_byte_source_falls_back_without_raising():
     assert spans == idx._window_spans(0, len(lines))
 
 
-def test_non_python_file_always_windows():
-    src = "".join(f"col{i},val{i}\n" for i in range(30))
-    lines = src.splitlines(keepends=True)
-    idx = Indexer(_mem_db())  # syntax strategy, but a .md/.ts/.csv is never AST-parsed
-    for name in ("data.md", "app.ts", "notes.txt"):
-        assert idx._spans_for_file(Path(name), lines, name) == idx._window_spans(0, len(lines))
+def test_non_code_exts_window_but_treesitter_langs_do_not():
+    # .md / unmapped exts are never parsed → line windows (as of 0.8.0 tree-sitter handles the
+    # code langs; see tests/test_treesitter.py for the def-aligned behavior)
+    csv = "".join(f"col{i},val{i}\n" for i in range(30))
+    csv_lines = csv.splitlines(keepends=True)
+    idx = Indexer(_mem_db())
+    for name in ("data.md", "notes.txt"):
+        assert idx._spans_for_file(Path(name), csv_lines, name) == idx._window_spans(0, len(csv_lines))
+    # a real .ts file with definitions is def-aligned, NOT windowed
+    ts = "function a() {\n  return 1;\n}\nfunction b() {\n  return 2;\n}\n".splitlines(keepends=True)
+    ts_spans = idx._spans_for_file(Path("app.ts"), ts, "app.ts")
+    assert ts_spans != idx._window_spans(0, len(ts))
+    assert (0, 3) in ts_spans and (3, 6) in ts_spans  # each function its own chunk
 
 
 # --------------------------------------------------------------------------- 5: "lines" parity

codecortex-0.8.0/tests/test_treesitter.py

@@ -0,0 +1,185 @@
+"""Tree-sitter chunking (0.8.0, Phase 3 of docs/roadmap-semantic.md).
+
+Extends 0.6.0 syntax-aware chunking to non-Python languages (TS/JS/Go/Rust/Java/C/C++) via
+tree-sitter, behind the same `_primary_spans → _cover` pipeline. `tree-sitter-language-pack` is a
+normal dependency; these tests exercise real grammars, plus the graceful fallback to line-windowing
+when a parser is unavailable or a file fails to yield defs.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+from codeintel.indexer import Indexer, _INDEXED_EXTS, _TS_LANG_BY_EXT
+from codeintel.semantic_db import SemanticDb
+
+# A normal dependency — but skip cleanly rather than error if a platform lacks a wheel.
+pytest.importorskip("tree_sitter_language_pack")
+
+
+def _idx(**kw) -> Indexer:
+    db = SemanticDb(":memory:")
+    db.init()
+    return Indexer(db, **kw)
+
+
+def _spans(name: str, src: str, **kw):
+    lines = src.splitlines(keepends=True)
+    return _idx(**kw)._spans_for_file(Path(name), lines, name), len(lines)
+
+
+def _assert_gapless_cover(spans, n):
+    assert spans and spans[0][0] == 0, f"must start at 0: {spans}"
+    assert len({s for s, _ in spans}) == len(spans), "duplicate start → chunk_id collision"
+    frontier = 0
+    for s, e in sorted(spans):
+        assert s < e, f"empty/negative span {(s, e)}"
+        assert s <= frontier, f"gap before {(s, e)} (covered to {frontier})"
+        frontier = max(frontier, e)
+    assert frontier == n, f"must cover to {n}, reached {frontier}"
+
+
+# --------------------------------------------------------------------------- per-language def-alignment
+
+def test_typescript_class_and_function_are_def_aligned():
+    src = (
+        "export class Widget {\n"      # 0  class header
+        "  greet(name: string) {\n"    # 1  method
+        "    return name;\n"           # 2
+        "  }\n"                         # 3
+        "}\n"                           # 4
+        "function top() {\n"           # 5  top-level function
+        "  return 1;\n"                # 6
+        "}\n"                           # 7
+    )
+    spans, n = _spans("a.ts", src)
+    _assert_gapless_cover(spans, n)
+    starts = {s for s, _ in spans}
+    assert 0 in starts   # class header chunk begins on the class line
+    assert 1 in starts   # the method is its own chunk (not shadowed by a whole-class chunk)
+    assert 5 in starts   # the top-level function is its own chunk
+
+
+def test_go_functions_and_methods():
+    src = ("package p\n\n"
+           "func Free() int {\n\treturn 1\n}\n\n"
+           "func (t T) Method() int {\n\treturn 2\n}\n")
+    spans, n = _spans("b.go", src)
+    _assert_gapless_cover(spans, n)
+    starts = {s for s, _ in spans}
+    assert 2 in starts   # func Free at line 2 (0-based)
+    assert 6 in starts   # method at line 6
+
+
+def test_rust_impl_method_is_its_own_chunk():
+    src = ("struct S { x: i32 }\n\n"
+           "impl S {\n"
+           "    fn method(&self) -> i32 {\n"
+           "        self.x\n"
+           "    }\n"
+           "}\n")
+    spans, n = _spans("c.rs", src)
+    _assert_gapless_cover(spans, n)
+    assert 3 in {s for s, _ in spans}   # fn method at line 3, carved out of the impl block
+
+
+def test_java_class_method():
+    src = ("class A {\n"
+           "    void m() {\n"
+           "        return;\n"
+           "    }\n"
+           "}\n")
+    spans, n = _spans("d.java", src)
+    _assert_gapless_cover(spans, n)
+    starts = {s for s, _ in spans}
+    assert 0 in starts and 1 in starts   # class header + method m
+
+
+def test_cpp_class_and_free_function():
+    src = ("class C {\npublic:\n  int m() { return 1; }\n};\nint free_fn() { return 2; }\n")
+    spans, n = _spans("e.cpp", src)
+    _assert_gapless_cover(spans, n)
+
+
+def test_multiple_functions_are_separate_chunks():
+    src = "function a() {\n  return 1;\n}\nfunction b() {\n  return 2;\n}\n"
+    spans, n = _spans("m.ts", src)
+    _assert_gapless_cover(spans, n)
+    assert (0, 3) in spans and (3, 6) in spans   # each function whole and distinct
+
+
+# --------------------------------------------------------------------------- fallbacks / never-raise
+
+def test_error_tolerant_source_still_covers_and_never_raises():
+    spans, n = _spans("bad.js", "this is @#$ not valid javascript !!!\n")
+    _assert_gapless_cover(spans, n)   # tree-sitter is error-tolerant → partial tree, no raise
+
+
+def test_lines_strategy_bypasses_treesitter():
+    src = "function foo() {\n  return 1;\n}\n"
+    lines = src.splitlines(keepends=True)
+    idx = _idx(chunk_strategy="lines")
+    assert idx._spans_for_file(Path("x.ts"), lines, "x.ts") == idx._window_spans(0, len(lines))
+
+
+def test_unmapped_exts_window():
+    src = "".join(f"line {i}\n" for i in range(25))
+    lines = src.splitlines(keepends=True)
+    idx = _idx()
+    for name in ("readme.md", "notes.txt"):
+        assert idx._spans_for_file(Path(name), lines, name) == idx._window_spans(0, len(lines))
+
+
+def test_falls_back_to_windowing_when_parser_unavailable():
+    src = "function foo() {\n  return 1;\n}\n"
+    lines = src.splitlines(keepends=True)
+    idx = _idx()
+    with patch.object(idx, "_get_ts_parser", return_value=None):   # simulate the dep not installed
+        assert idx._spans_for_file(Path("x.ts"), lines, "x.ts") == idx._window_spans(0, len(lines))
+
+
+def test_never_raises_on_parser_exception():
+    src = "function foo() {\n  return 1;\n}\n"
+    lines = src.splitlines(keepends=True)
+    idx = _idx()
+
+    class _Boom:
+        def parse(self, *a, **k):
+            raise RuntimeError("grammar exploded")
+
+    with patch.object(idx, "_get_ts_parser", return_value=_Boom()):
+        assert idx._spans_for_file(Path("x.ts"), lines, "x.ts") == idx._window_spans(0, len(lines))
+
+
+def test_parser_cache_loads_each_language_once():
+    idx = _idx()
+    with patch("tree_sitter_language_pack.get_parser") as gp:
+        gp.return_value = object()
+        idx._get_ts_parser("go")
+        idx._get_ts_parser("go")
+        idx._get_ts_parser("rust")
+    assert gp.call_count == 2   # one load per distinct language, then cached
+
+
+def test_typescript_interface_decomposes_into_members():
+    # interface method members are `method_signature` (distinct from abstract-class members); the
+    # interface must decompose into a header + per-method chunks, not one whole-interface chunk
+    src = ("interface Repo {\n"        # 0  header
+           "  get(id: string): Item;\n"   # 1  method_signature
+           "  put(item: Item): void;\n"   # 2  method_signature
+           "}\n")                          # 3
+    spans, n = _spans("r.ts", src)
+    _assert_gapless_cover(spans, n)
+    starts = {s for s, _ in spans}
+    assert 1 in starts and 2 in starts, f"interface methods must be their own chunks: {spans}"
+
+
+def test_lang_map_and_indexed_exts_agree_both_ways():
+    code_exts = _INDEXED_EXTS - {".py", ".md"}
+    # every walked code ext maps to a tree-sitter language...
+    assert code_exts <= set(_TS_LANG_BY_EXT), f"walked but unmapped: {code_exts - set(_TS_LANG_BY_EXT)}"
+    # ...AND every mapped ext is actually walked (else it's indexed by nothing — silent omission)
+    assert set(_TS_LANG_BY_EXT) <= _INDEXED_EXTS, \
+        f"mapped but never walked: {set(_TS_LANG_BY_EXT) - _INDEXED_EXTS}"

codecortex-0.7.0/src/codeintel/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.7.0"