codecortex 0.5.0__tar.gz → 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {codecortex-0.5.0/src/codecortex.egg-info → codecortex-0.7.0}/PKG-INFO +1 -1
- {codecortex-0.5.0 → codecortex-0.7.0/src/codecortex.egg-info}/PKG-INFO +1 -1
- {codecortex-0.5.0 → codecortex-0.7.0}/src/codecortex.egg-info/SOURCES.txt +2 -0
- codecortex-0.7.0/src/codeintel/__init__.py +1 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/__main__.py +1 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/config.py +6 -1
- {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/doctor.py +5 -0
- codecortex-0.7.0/src/codeintel/indexer.py +430 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/onboarding.py +1 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/providers/graph.py +3 -1
- {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/providers/lsp.py +3 -1
- {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/providers/semantic.py +5 -1
- {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/reindexer.py +1 -0
- codecortex-0.7.0/src/codeintel/searcher.py +311 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/semantic_db.py +47 -2
- codecortex-0.7.0/tests/test_chunking.py +357 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_config.py +18 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_doctor.py +12 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_integration.py +11 -5
- codecortex-0.7.0/tests/test_rerank.py +256 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_semantic_provider.py +48 -0
- codecortex-0.5.0/src/codeintel/__init__.py +0 -1
- codecortex-0.5.0/src/codeintel/indexer.py +0 -261
- codecortex-0.5.0/src/codeintel/searcher.py +0 -143
- {codecortex-0.5.0 → codecortex-0.7.0}/LICENSE +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/README.md +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/pyproject.toml +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/setup.cfg +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/src/codecortex.egg-info/dependency_links.txt +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/src/codecortex.egg-info/entry_points.txt +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/src/codecortex.egg-info/requires.txt +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/src/codecortex.egg-info/top_level.txt +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/auth.py +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/cache.py +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/gateway.py +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/http_server.py +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/injector.py +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/installer.py +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/logconfig.py +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/mapper.py +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/metrics.py +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/policy.py +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/provider.py +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/providers/__init__.py +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/providers/none.py +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/reset.py +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/server.py +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/term.py +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_cache.py +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_e2e.py +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_enterprise.py +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_gateway.py +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_graph_provider.py +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_graph_real.py +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_graph_stdin.py +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_hardening.py +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_http_auth.py +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_http_server.py +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_lsp_provider.py +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_lsp_real.py +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_mapper.py +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_never_raise.py +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_onboarding.py +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_rbac.py +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_reindexer.py +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_reset.py +0 -0
- {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_term.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: codecortex
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0
|
|
4
4
|
Summary: Local-first, MCP-native code-intelligence server — graph, LSP, and semantic search behind one safe code.query tool for coding agents.
|
|
5
5
|
Author: Shammai Hamilton
|
|
6
6
|
License-Expression: MIT
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: codecortex
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0
|
|
4
4
|
Summary: Local-first, MCP-native code-intelligence server — graph, LSP, and semantic search behind one safe code.query tool for coding agents.
|
|
5
5
|
Author: Shammai Hamilton
|
|
6
6
|
License-Expression: MIT
|
|
@@ -36,6 +36,7 @@ src/codeintel/providers/lsp.py
|
|
|
36
36
|
src/codeintel/providers/none.py
|
|
37
37
|
src/codeintel/providers/semantic.py
|
|
38
38
|
tests/test_cache.py
|
|
39
|
+
tests/test_chunking.py
|
|
39
40
|
tests/test_config.py
|
|
40
41
|
tests/test_doctor.py
|
|
41
42
|
tests/test_e2e.py
|
|
@@ -55,6 +56,7 @@ tests/test_never_raise.py
|
|
|
55
56
|
tests/test_onboarding.py
|
|
56
57
|
tests/test_rbac.py
|
|
57
58
|
tests/test_reindexer.py
|
|
59
|
+
tests/test_rerank.py
|
|
58
60
|
tests/test_reset.py
|
|
59
61
|
tests/test_semantic_provider.py
|
|
60
62
|
tests/test_term.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.7.0"
|
|
@@ -124,6 +124,7 @@ def main() -> None:
|
|
|
124
124
|
stride=int(cfg.get("stride", 10)),
|
|
125
125
|
max_chunks=int(cfg.get("max_chunks", 500)),
|
|
126
126
|
max_total_chunks=int(cfg.get("max_total_chunks", 100000)),
|
|
127
|
+
chunk_strategy=str(cfg.get("chunk_strategy", "syntax")),
|
|
127
128
|
).index(project_root)
|
|
128
129
|
if count > 0:
|
|
129
130
|
print(f"Indexed {count} chunks")
|
|
@@ -24,6 +24,9 @@ _DEFAULTS: dict = {
|
|
|
24
24
|
"max_total_chunks": 100000, # safety ceiling on chunks embedded in one index pass
|
|
25
25
|
"cosine_floor": 0.25,
|
|
26
26
|
"model": "BAAI/bge-small-en-v1.5",
|
|
27
|
+
"chunk_strategy": "syntax", # syntax-aware (def/class boundaries) vs fixed line windows
|
|
28
|
+
"rerank": "on", # hybrid lexical+semantic rerank of search results
|
|
29
|
+
"rerank_candidates": 30, # cosine candidates fused/re-ranked before returning top-k
|
|
27
30
|
}
|
|
28
31
|
|
|
29
32
|
# Values restricted to a fixed set — anything else falls back to the default.
|
|
@@ -31,8 +34,10 @@ _ENUMS: dict = {
|
|
|
31
34
|
"backend": {"auto", "graph", "lsp", "semantic"},
|
|
32
35
|
"semantic": {"on", "off"},
|
|
33
36
|
"reindex": {"on-demand", "never"},
|
|
37
|
+
"chunk_strategy": {"syntax", "lines"},
|
|
38
|
+
"rerank": {"on", "off"},
|
|
34
39
|
}
|
|
35
|
-
_POSITIVE_INTS = ("window", "stride", "max_chunks", "max_total_chunks")
|
|
40
|
+
_POSITIVE_INTS = ("window", "stride", "max_chunks", "max_total_chunks", "rerank_candidates")
|
|
36
41
|
|
|
37
42
|
|
|
38
43
|
def _read_toml(path: pathlib.Path) -> dict:
|
|
@@ -158,4 +158,9 @@ def render_doctor_text(report: dict) -> str:
|
|
|
158
158
|
tail = "" if report.get("deep") else c.dim(" (run with --deep to boot-check serena)")
|
|
159
159
|
out.append("")
|
|
160
160
|
out.append(f" {count} engines ready for this repo.{tail}")
|
|
161
|
+
if healthy is False:
|
|
162
|
+
out.append(" " + c.dim(
|
|
163
|
+
"tip: `codeintel setup --install-uv --install-deps --index` bootstraps the "
|
|
164
|
+
"pip-installable backends; each fix: line above has the per-engine command."
|
|
165
|
+
))
|
|
161
166
|
return "\n".join(out)
|
|
@@ -0,0 +1,430 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import ast
|
|
4
|
+
import hashlib
|
|
5
|
+
import logging
|
|
6
|
+
import os
|
|
7
|
+
import struct
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import TYPE_CHECKING
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
from codeintel.semantic_db import SemanticDb
|
|
13
|
+
|
|
14
|
+
logger = logging.getLogger(__name__)
|
|
15
|
+
|
|
16
|
+
_INDEXED_EXTS = frozenset({
|
|
17
|
+
".py", ".ts", ".js", ".go", ".rs", ".java", ".c", ".cpp", ".h", ".md"
|
|
18
|
+
})
|
|
19
|
+
_SKIP_DIRS = frozenset({"__pycache__", ".git", "node_modules"})
|
|
20
|
+
# Vendored / regenerable dirs skipped even without a .gitignore entry.
|
|
21
|
+
_DEFAULT_IGNORES = frozenset({
|
|
22
|
+
".venv", "venv", "env", "dist", "build", "target",
|
|
23
|
+
".mypy_cache", ".pytest_cache", ".tox", ".idea", ".vscode", ".cache",
|
|
24
|
+
})
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _project_key(project_root_real: str) -> str:
|
|
28
|
+
"""A short, stable id for a project root — prefixes every chunk_id so two repos
|
|
29
|
+
with an identically-named file never collide in the shared cache."""
|
|
30
|
+
return hashlib.sha256(project_root_real.encode()).hexdigest()[:12]
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _pos_int(val: object, default: int) -> int:
|
|
34
|
+
"""A usable positive int or the default — mirrors ``config._coerce`` so a direct
|
|
35
|
+
``Indexer(...)`` caller (which bypasses config validation) can't set a zero/negative/non-int
|
|
36
|
+
``stride`` (would raise inside ``range()``) or ``window`` (would silently drop every region)."""
|
|
37
|
+
try:
|
|
38
|
+
n = int(val) # type: ignore[arg-type]
|
|
39
|
+
except (TypeError, ValueError, OverflowError):
|
|
40
|
+
return default
|
|
41
|
+
return n if n > 0 else default
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class Indexer:
|
|
45
|
+
def __init__(
|
|
46
|
+
self,
|
|
47
|
+
db: SemanticDb,
|
|
48
|
+
model_name: str = "BAAI/bge-small-en-v1.5",
|
|
49
|
+
window: int = 20,
|
|
50
|
+
stride: int = 10,
|
|
51
|
+
max_chunks: int = 500,
|
|
52
|
+
max_total_chunks: int = 100000,
|
|
53
|
+
chunk_strategy: str = "syntax",
|
|
54
|
+
max_chunk_lines: int | None = None,
|
|
55
|
+
) -> None:
|
|
56
|
+
self.db = db
|
|
57
|
+
self.model_name = model_name
|
|
58
|
+
# Coerce the numeric knobs defensively: the 4 production call sites pass config-validated
|
|
59
|
+
# values, but a direct caller must not be able to set a stride/window that raises in
|
|
60
|
+
# range() or silently drops regions (mirrors config._coerce's _POSITIVE_INTS clamp).
|
|
61
|
+
self.window = _pos_int(window, 20)
|
|
62
|
+
self.stride = _pos_int(stride, 10)
|
|
63
|
+
self.max_chunks = _pos_int(max_chunks, 500) # per file
|
|
64
|
+
self.max_total_chunks = _pos_int(max_total_chunks, 100000) # ceiling per pass (mem backstop)
|
|
65
|
+
# "syntax" chunks Python on def/class boundaries (ast); "lines" is the fixed-window
|
|
66
|
+
# fallback used for every non-.py file, on any parse failure, and as a runtime escape
|
|
67
|
+
# hatch. Case-normalized then range-checked so an unknown value degrades to "syntax"
|
|
68
|
+
# (config already validates; this keeps a direct caller from silently disabling — or
|
|
69
|
+
# accidentally case-swapping — the strategy with a typo).
|
|
70
|
+
strategy = str(chunk_strategy).strip().lower()
|
|
71
|
+
self.chunk_strategy = strategy if strategy in ("syntax", "lines") else "syntax"
|
|
72
|
+
# A def longer than this is window-chunked internally so no single chunk overflows the
|
|
73
|
+
# embedder (~512 tokens). Defaults to 2*window; never <= 0 (would loop / never split).
|
|
74
|
+
self.max_chunk_lines = (
|
|
75
|
+
max_chunk_lines if isinstance(max_chunk_lines, int) and max_chunk_lines > 0
|
|
76
|
+
else 2 * self.window
|
|
77
|
+
)
|
|
78
|
+
self._embedder = None
|
|
79
|
+
|
|
80
|
+
def _get_embedder(self):
|
|
81
|
+
if self._embedder is None:
|
|
82
|
+
from fastembed import TextEmbedding
|
|
83
|
+
self._embedder = TextEmbedding(model_name=self.model_name)
|
|
84
|
+
return self._embedder
|
|
85
|
+
|
|
86
|
+
def index(self, project_root: str) -> int:
|
|
87
|
+
"""Return count of newly embedded chunks, or -1 on unrecoverable failure."""
|
|
88
|
+
try:
|
|
89
|
+
return self._index(project_root)
|
|
90
|
+
except Exception as exc:
|
|
91
|
+
logger.error("Indexer.index() unrecoverable failure: %s", exc)
|
|
92
|
+
return -1
|
|
93
|
+
|
|
94
|
+
def _load_gitignore(self, root: Path) -> set[str]:
|
|
95
|
+
"""Best-effort ``.gitignore``: collect simple name/dir patterns to skip. This is
|
|
96
|
+
NOT full gitignore semantics (no globs, negations, or nesting) — just enough to
|
|
97
|
+
avoid indexing vendored/build output the user already told git to ignore."""
|
|
98
|
+
patterns: set[str] = set()
|
|
99
|
+
gi = root / ".gitignore"
|
|
100
|
+
try:
|
|
101
|
+
if gi.is_file():
|
|
102
|
+
for line in gi.read_text(encoding="utf-8", errors="replace").splitlines():
|
|
103
|
+
line = line.strip()
|
|
104
|
+
if not line or line.startswith("#") or line.startswith("!"):
|
|
105
|
+
continue
|
|
106
|
+
name = line.rstrip("/").lstrip("/")
|
|
107
|
+
if name and "*" not in name and "/" not in name:
|
|
108
|
+
patterns.add(name)
|
|
109
|
+
except Exception:
|
|
110
|
+
pass
|
|
111
|
+
return patterns
|
|
112
|
+
|
|
113
|
+
def _cleanup_deleted(self, root: Path, project_root_real: str) -> None:
|
|
114
|
+
"""Drop rows for THIS project whose file no longer exists — scoped by
|
|
115
|
+
project_root so touching one repo can never purge another's index."""
|
|
116
|
+
conn = self.db.conn()
|
|
117
|
+
try:
|
|
118
|
+
rows = conn.execute(
|
|
119
|
+
"SELECT DISTINCT file_path FROM chunk_hashes WHERE project_root = ?",
|
|
120
|
+
(project_root_real,),
|
|
121
|
+
).fetchall()
|
|
122
|
+
deleted_paths = [
|
|
123
|
+
row[0] for row in rows if not (root / row[0]).exists()
|
|
124
|
+
]
|
|
125
|
+
for fp in deleted_paths:
|
|
126
|
+
chunk_ids = [
|
|
127
|
+
r[0]
|
|
128
|
+
for r in conn.execute(
|
|
129
|
+
"SELECT chunk_id FROM chunk_hashes"
|
|
130
|
+
" WHERE project_root = ? AND file_path = ?",
|
|
131
|
+
(project_root_real, fp),
|
|
132
|
+
).fetchall()
|
|
133
|
+
]
|
|
134
|
+
for cid in chunk_ids:
|
|
135
|
+
conn.execute(
|
|
136
|
+
"DELETE FROM code_embeddings WHERE chunk_id = ?", (cid,)
|
|
137
|
+
)
|
|
138
|
+
conn.execute(
|
|
139
|
+
"DELETE FROM chunk_hashes WHERE chunk_id = ?", (cid,)
|
|
140
|
+
)
|
|
141
|
+
conn.commit()
|
|
142
|
+
except Exception as exc:
|
|
143
|
+
logger.warning("Cleanup pass failed: %s", exc)
|
|
144
|
+
|
|
145
|
+
def _walk_files(self, root: Path):
|
|
146
|
+
ignores = set(_SKIP_DIRS) | set(_DEFAULT_IGNORES) | self._load_gitignore(root)
|
|
147
|
+
for dirpath, dirnames, filenames in os.walk(root):
|
|
148
|
+
dirnames[:] = [
|
|
149
|
+
d for d in dirnames
|
|
150
|
+
if d not in ignores and not d.endswith(".egg-info")
|
|
151
|
+
]
|
|
152
|
+
for fname in filenames:
|
|
153
|
+
if fname in ignores:
|
|
154
|
+
continue
|
|
155
|
+
if Path(fname).suffix.lower() in _INDEXED_EXTS:
|
|
156
|
+
yield Path(dirpath) / fname
|
|
157
|
+
|
|
158
|
+
# ---- chunk-span computation ------------------------------------------------------------
|
|
159
|
+
# A file is turned into a list of 0-based, half-open ``(start, end)`` line spans; every
|
|
160
|
+
# strategy funnels through the same span list so downstream materialisation (whitespace
|
|
161
|
+
# skip, hash-dedup, caps, orphan reconcile) is shared and identical.
|
|
162
|
+
|
|
163
|
+
def _window_spans(self, start: int, end: int) -> list[tuple[int, int]]:
|
|
164
|
+
"""Fixed overlapping line windows over ``[start, end)`` — the original chunking, reused to
|
|
165
|
+
fill inter-def gaps and split oversized defs. ``_window_spans(0, len(lines))`` reproduces
|
|
166
|
+
the old ``range(0, n, stride)`` + ``lines[s:s+window]`` output exactly."""
|
|
167
|
+
spans: list[tuple[int, int]] = []
|
|
168
|
+
if end <= start:
|
|
169
|
+
return spans
|
|
170
|
+
for s in range(start, end, self.stride):
|
|
171
|
+
spans.append((s, min(s + self.window, end)))
|
|
172
|
+
return spans
|
|
173
|
+
|
|
174
|
+
def _maybe_split(self, start: int, end: int) -> list[tuple[int, int]]:
|
|
175
|
+
"""A def span kept whole, or window-split when it exceeds ``max_chunk_lines`` so no single
|
|
176
|
+
chunk overflows the embedder."""
|
|
177
|
+
if end - start <= self.max_chunk_lines:
|
|
178
|
+
return [(start, end)]
|
|
179
|
+
return self._window_spans(start, end)
|
|
180
|
+
|
|
181
|
+
@staticmethod
|
|
182
|
+
def _node_span(node: ast.AST, n: int) -> tuple[int, int]:
|
|
183
|
+
"""0-based half-open ``[start, end)`` span of a def/class node, decorators included
|
|
184
|
+
(``min(decorator linenos, node.lineno)`` … ``end_lineno``), clamped into ``[0, n]``."""
|
|
185
|
+
start = node.lineno # type: ignore[attr-defined]
|
|
186
|
+
for dec in getattr(node, "decorator_list", None) or []:
|
|
187
|
+
dline = getattr(dec, "lineno", None)
|
|
188
|
+
if isinstance(dline, int):
|
|
189
|
+
start = min(start, dline)
|
|
190
|
+
end = getattr(node, "end_lineno", None)
|
|
191
|
+
if not isinstance(end, int):
|
|
192
|
+
end = node.lineno # type: ignore[attr-defined]
|
|
193
|
+
start0 = max(0, start - 1)
|
|
194
|
+
end0 = min(n, max(start0 + 1, end))
|
|
195
|
+
return (start0, end0)
|
|
196
|
+
|
|
197
|
+
def _primary_spans(self, tree: ast.Module, n: int) -> list[tuple[int, int]]:
|
|
198
|
+
"""Def-aligned 'primary' spans: each top-level function, and for each top-level class a
|
|
199
|
+
header span (class line → just before its first method/nested def) plus one span per
|
|
200
|
+
method/nested def. Inter-method and module-level runs are intentionally left uncovered
|
|
201
|
+
here — ``_cover`` window-fills them — so per-method chunks are never double-embedded."""
|
|
202
|
+
spans: list[tuple[int, int]] = []
|
|
203
|
+
for node in tree.body:
|
|
204
|
+
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
205
|
+
spans.append(self._node_span(node, n))
|
|
206
|
+
elif isinstance(node, ast.ClassDef):
|
|
207
|
+
cstart, cend = self._node_span(node, n)
|
|
208
|
+
members = [
|
|
209
|
+
self._node_span(c, n)
|
|
210
|
+
for c in node.body
|
|
211
|
+
if isinstance(c, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef))
|
|
212
|
+
]
|
|
213
|
+
if not members:
|
|
214
|
+
spans.append((cstart, cend)) # no methods → the class is one unit
|
|
215
|
+
continue
|
|
216
|
+
members.sort()
|
|
217
|
+
header_end = max(cstart + 1, min(members[0][0], cend))
|
|
218
|
+
spans.append((cstart, header_end)) # header: bases + class docstring
|
|
219
|
+
spans.extend(members)
|
|
220
|
+
return spans
|
|
221
|
+
|
|
222
|
+
def _cover(self, primary: list[tuple[int, int]], n: int) -> list[tuple[int, int]]:
|
|
223
|
+
"""Gapless cover of ``[0, n)`` in file order: window-fill every gap between primary spans
|
|
224
|
+
and window-split any oversized def. The def-aligned *primary* spans are whole and mutually
|
|
225
|
+
non-overlapping (no whole-class chunk shadowing its per-method chunks); the window-filled
|
|
226
|
+
gaps and oversized-def splits reuse the existing ``window``/``stride``, so — exactly like
|
|
227
|
+
the legacy line windower — adjacent windows inside one filled run *do* overlap when
|
|
228
|
+
``stride < window``. Coverage is always complete; chunk starts are always unique."""
|
|
229
|
+
result: list[tuple[int, int]] = []
|
|
230
|
+
cursor = 0
|
|
231
|
+
for s0, e0 in sorted(primary):
|
|
232
|
+
s = max(s0, cursor) # clamp any (pathological) overlap so nothing is double-covered
|
|
233
|
+
e = min(e0, n)
|
|
234
|
+
if s >= e:
|
|
235
|
+
continue
|
|
236
|
+
if s > cursor:
|
|
237
|
+
result.extend(self._window_spans(cursor, s)) # module-level / inter-def run
|
|
238
|
+
result.extend(self._maybe_split(s, e))
|
|
239
|
+
cursor = e
|
|
240
|
+
if cursor < n:
|
|
241
|
+
result.extend(self._window_spans(cursor, n))
|
|
242
|
+
return result
|
|
243
|
+
|
|
244
|
+
def _chunk_python_ast(self, lines: list[str], source: str) -> list[tuple[int, int]]:
|
|
245
|
+
"""Parse ``source`` → a complete, non-overlapping, def-aligned cover of the file as
|
|
246
|
+
0-based half-open ``(start, end)`` spans. Raises on parse failure (``SyntaxError`` /
|
|
247
|
+
``ValueError`` on NUL bytes / ``RecursionError`` / …) so the caller falls back to windows."""
|
|
248
|
+
tree = ast.parse(source)
|
|
249
|
+
n = len(lines)
|
|
250
|
+
return self._cover(self._primary_spans(tree, n), n)
|
|
251
|
+
|
|
252
|
+
def _spans_for_file(
|
|
253
|
+
self, filepath: Path, lines: list[str], rel_path: str
|
|
254
|
+
) -> list[tuple[int, int]]:
|
|
255
|
+
"""Choose spans for one file: syntax-aware for ``.py`` under the syntax strategy (falling
|
|
256
|
+
back to windowing on any parse failure), fixed windows for everything else."""
|
|
257
|
+
if self.chunk_strategy == "syntax" and filepath.suffix.lower() == ".py":
|
|
258
|
+
try:
|
|
259
|
+
return self._chunk_python_ast(lines, "".join(lines))
|
|
260
|
+
except Exception as exc:
|
|
261
|
+
logger.debug("syntax chunking failed for %s (%s) — windowing", rel_path, exc)
|
|
262
|
+
return self._window_spans(0, len(lines))
|
|
263
|
+
|
|
264
|
+
# ---- materialisation -------------------------------------------------------------------
|
|
265
|
+
|
|
266
|
+
def _emit_spans(
|
|
267
|
+
self,
|
|
268
|
+
spans: list[tuple[int, int]],
|
|
269
|
+
lines: list[str],
|
|
270
|
+
rel_path: str,
|
|
271
|
+
project_key: str,
|
|
272
|
+
conn,
|
|
273
|
+
new_chunks: list[tuple[str, str, str, int, str]],
|
|
274
|
+
) -> tuple[set[str], bool]:
|
|
275
|
+
"""Materialise spans into new/changed chunk records — shared by both strategies, so the
|
|
276
|
+
whitespace-skip, hash-dedup, and per-file cap behave identically. Returns
|
|
277
|
+
``(keep_ids, complete)``: ``keep_ids`` is every chunk_id this file legitimately produces
|
|
278
|
+
(drives orphan reconciliation); ``complete`` is False iff the *global* ceiling cut the
|
|
279
|
+
file short, in which case ``keep_ids`` is partial and MUST NOT delete anything."""
|
|
280
|
+
keep_ids: set[str] = set()
|
|
281
|
+
chunk_count = 0
|
|
282
|
+
for start, end in spans:
|
|
283
|
+
if len(new_chunks) >= self.max_total_chunks:
|
|
284
|
+
return keep_ids, False # global ceiling mid-file — keep_ids is partial
|
|
285
|
+
if chunk_count >= self.max_chunks:
|
|
286
|
+
logger.debug("chunk cap hit for %s, truncating at %d", rel_path, self.max_chunks)
|
|
287
|
+
break # per-file cap is deterministic (same first-N each pass) → reconcile is safe
|
|
288
|
+
chunk_lines = lines[start:end]
|
|
289
|
+
if not chunk_lines:
|
|
290
|
+
continue
|
|
291
|
+
chunk_text = "".join(chunk_lines)
|
|
292
|
+
if not chunk_text.strip():
|
|
293
|
+
# EC3.4: never embed empty/whitespace-only chunks (zero vectors pollute results).
|
|
294
|
+
chunk_count += 1
|
|
295
|
+
continue
|
|
296
|
+
chunk_id = f"{project_key}:{rel_path}:{start}"
|
|
297
|
+
content_hash = hashlib.sha256(chunk_text.encode()).hexdigest()[:16]
|
|
298
|
+
keep_ids.add(chunk_id) # produced this pass — keep even when dedup skips re-embed
|
|
299
|
+
try:
|
|
300
|
+
row = conn.execute(
|
|
301
|
+
"SELECT content_hash FROM chunk_hashes WHERE chunk_id = ?",
|
|
302
|
+
(chunk_id,),
|
|
303
|
+
).fetchone()
|
|
304
|
+
if row and row[0] == content_hash:
|
|
305
|
+
chunk_count += 1
|
|
306
|
+
continue
|
|
307
|
+
except Exception as exc:
|
|
308
|
+
logger.debug("hash check failed for %s: %s", chunk_id, exc)
|
|
309
|
+
new_chunks.append((chunk_id, chunk_text, rel_path, start, content_hash))
|
|
310
|
+
chunk_count += 1
|
|
311
|
+
return keep_ids, True
|
|
312
|
+
|
|
313
|
+
def _collect_new_chunks(
|
|
314
|
+
self, root: Path, project_key: str, project_root_real: str
|
|
315
|
+
) -> list[tuple[str, str, str, int, str]]:
|
|
316
|
+
"""Walk files; return (chunk_id, text, rel_path, start, hash) for new/changed chunks, and
|
|
317
|
+
reconcile each fully-processed file (dropping rows for chunks it no longer produces)."""
|
|
318
|
+
conn = self.db.conn()
|
|
319
|
+
new_chunks: list[tuple[str, str, str, int, str]] = []
|
|
320
|
+
|
|
321
|
+
for filepath in self._walk_files(root):
|
|
322
|
+
if len(new_chunks) >= self.max_total_chunks:
|
|
323
|
+
logger.warning(
|
|
324
|
+
"index: reached max_total_chunks=%d this pass — stopping "
|
|
325
|
+
"(raise it in .codeintel.toml to embed more of a very large repo)",
|
|
326
|
+
self.max_total_chunks,
|
|
327
|
+
)
|
|
328
|
+
break
|
|
329
|
+
try:
|
|
330
|
+
with open(filepath, encoding="utf-8", errors="replace") as f:
|
|
331
|
+
lines = f.readlines()
|
|
332
|
+
except FileNotFoundError:
|
|
333
|
+
logger.debug("file disappeared: %s", filepath)
|
|
334
|
+
continue
|
|
335
|
+
except Exception as exc:
|
|
336
|
+
logger.debug("skipping %s: %s", filepath, exc)
|
|
337
|
+
continue
|
|
338
|
+
|
|
339
|
+
rel_path = str(filepath.relative_to(root))
|
|
340
|
+
spans = self._spans_for_file(filepath, lines, rel_path)
|
|
341
|
+
keep_ids, complete = self._emit_spans(
|
|
342
|
+
spans, lines, rel_path, project_key, conn, new_chunks
|
|
343
|
+
)
|
|
344
|
+
if complete:
|
|
345
|
+
# Reconcile only a fully-processed file: drop rows for defs/windows it no longer
|
|
346
|
+
# produces (a moved/deleted function, or a strategy switch). Skipped when the
|
|
347
|
+
# global ceiling truncated the file — its partial keep_ids would delete rows past
|
|
348
|
+
# the cut that are still valid. Scoped by (project_root, file_path) in the db layer.
|
|
349
|
+
# Caveat: two *concurrent* index passes over the same project computed from
|
|
350
|
+
# different point-in-time reads can transiently resurrect a just-deleted stale row
|
|
351
|
+
# (the later pass's INSERT OR REPLACE re-adds what the earlier reconcile removed).
|
|
352
|
+
# This self-heals on the next single-reader pass, and is strictly better than the
|
|
353
|
+
# pre-0.6 behaviour (the stale row persisted forever); cross-pass serialization is
|
|
354
|
+
# out of scope for the indexer.
|
|
355
|
+
self.db.delete_file_orphans(project_root_real, rel_path, keep_ids)
|
|
356
|
+
|
|
357
|
+
return new_chunks
|
|
358
|
+
|
|
359
|
+
def _embed_and_write(
|
|
360
|
+
self, new_chunks: list[tuple[str, str, str, int, str]], project_root_real: str
|
|
361
|
+
) -> int:
|
|
362
|
+
embedder = self._get_embedder() # may raise → propagates to index() → returns -1
|
|
363
|
+
conn = self.db.conn()
|
|
364
|
+
embedded_count = 0
|
|
365
|
+
batch_size = 32
|
|
366
|
+
|
|
367
|
+
for i in range(0, len(new_chunks), batch_size):
|
|
368
|
+
batch = new_chunks[i: i + batch_size]
|
|
369
|
+
texts = [c[1] for c in batch]
|
|
370
|
+
|
|
371
|
+
try:
|
|
372
|
+
embeddings = list(embedder.embed(texts))
|
|
373
|
+
except Exception as exc:
|
|
374
|
+
logger.warning("embedding batch %d failed: %s", i // batch_size, exc)
|
|
375
|
+
continue
|
|
376
|
+
|
|
377
|
+
for j, (chunk_id, _, rel_path, chunk_start, content_hash) in enumerate(batch):
|
|
378
|
+
if j >= len(embeddings):
|
|
379
|
+
break
|
|
380
|
+
try:
|
|
381
|
+
vec = embeddings[j]
|
|
382
|
+
vec_bytes = struct.pack(f"{len(vec)}f", *vec)
|
|
383
|
+
# sqlite-vec's vec0 virtual table does NOT honor INSERT OR REPLACE — it raises
|
|
384
|
+
# UNIQUE on an existing chunk_id instead of replacing. So re-embedding a chunk
|
|
385
|
+
# whose content changed but whose chunk_id (start line) is stable — the common
|
|
386
|
+
# case under syntax chunking, where a def's chunk_id is its def line — would
|
|
387
|
+
# silently fail and keep the STALE vector. DELETE-then-INSERT is the supported
|
|
388
|
+
# upsert for vec0. (chunk_hashes below is a normal table, where REPLACE works.)
|
|
389
|
+
conn.execute(
|
|
390
|
+
"DELETE FROM code_embeddings WHERE chunk_id = ?", (chunk_id,)
|
|
391
|
+
)
|
|
392
|
+
conn.execute(
|
|
393
|
+
"INSERT INTO code_embeddings(chunk_id, embedding) VALUES (?, ?)",
|
|
394
|
+
(chunk_id, vec_bytes),
|
|
395
|
+
)
|
|
396
|
+
conn.execute(
|
|
397
|
+
"INSERT OR REPLACE INTO chunk_hashes"
|
|
398
|
+
"(chunk_id, project_root, file_path, chunk_start, content_hash)"
|
|
399
|
+
" VALUES (?, ?, ?, ?, ?)",
|
|
400
|
+
(chunk_id, project_root_real, rel_path, chunk_start, content_hash),
|
|
401
|
+
)
|
|
402
|
+
embedded_count += 1
|
|
403
|
+
except Exception as exc:
|
|
404
|
+
logger.warning("writing chunk %s failed: %s", chunk_id, exc)
|
|
405
|
+
|
|
406
|
+
try:
|
|
407
|
+
conn.commit()
|
|
408
|
+
except Exception as exc:
|
|
409
|
+
logger.warning("commit failed after batch %d: %s", i // batch_size, exc)
|
|
410
|
+
|
|
411
|
+
return embedded_count
|
|
412
|
+
|
|
413
|
+
def _index(self, project_root: str) -> int:
|
|
414
|
+
if not project_root:
|
|
415
|
+
return 0
|
|
416
|
+
|
|
417
|
+
root = Path(project_root)
|
|
418
|
+
if not root.exists():
|
|
419
|
+
return 0
|
|
420
|
+
|
|
421
|
+
project_root_real = os.path.realpath(project_root)
|
|
422
|
+
project_key = _project_key(project_root_real)
|
|
423
|
+
|
|
424
|
+
self._cleanup_deleted(root, project_root_real)
|
|
425
|
+
|
|
426
|
+
new_chunks = self._collect_new_chunks(root, project_key, project_root_real)
|
|
427
|
+
if not new_chunks:
|
|
428
|
+
return 0
|
|
429
|
+
|
|
430
|
+
return self._embed_and_write(new_chunks, project_root_real)
|
|
@@ -73,6 +73,7 @@ def _bounded_index(project_root: str, *, timeout_s: float, out) -> dict:
|
|
|
73
73
|
window=int(cfg.get("window", 20)), stride=int(cfg.get("stride", 10)),
|
|
74
74
|
max_chunks=int(cfg.get("max_chunks", 500)),
|
|
75
75
|
max_total_chunks=int(cfg.get("max_total_chunks", 100000)),
|
|
76
|
+
chunk_strategy=str(cfg.get("chunk_strategy", "syntax")),
|
|
76
77
|
).index(project_root)
|
|
77
78
|
finally:
|
|
78
79
|
db.close()
|
|
@@ -157,7 +157,9 @@ class GraphProvider:
|
|
|
157
157
|
return {
|
|
158
158
|
"installed": False, "runnable": False, "repo_indexed": False, "project": None,
|
|
159
159
|
"detail": "codebase-memory-mcp not found on PATH",
|
|
160
|
-
"remediation": "
|
|
160
|
+
"remediation": "put the codebase-memory-mcp binary on PATH — it's an external "
|
|
161
|
+
"native backend (see docs/graph.md); once present it self-updates "
|
|
162
|
+
"via `codebase-memory-mcp update`",
|
|
161
163
|
}
|
|
162
164
|
raw = self._run("list_projects", {}, timeout_ms)
|
|
163
165
|
if raw is None:
|
|
@@ -144,7 +144,9 @@ class LspProvider:
|
|
|
144
144
|
return {
|
|
145
145
|
"installed": False, "runnable": False, "repo_indexed": None,
|
|
146
146
|
"detail": "neither `serena` nor `uvx` found on PATH",
|
|
147
|
-
"remediation": "install uv (provides uvx)
|
|
147
|
+
"remediation": "install uv (provides uvx): `codeintel setup --install-uv` "
|
|
148
|
+
"(or `brew install uv` / `pip install uv`) — serena is then "
|
|
149
|
+
"fetched on first use",
|
|
148
150
|
}
|
|
149
151
|
cmd = self._cmd
|
|
150
152
|
if not deep:
|
|
@@ -120,6 +120,7 @@ class SemanticProvider:
|
|
|
120
120
|
stride=int(cfg.get("stride", 10)),
|
|
121
121
|
max_chunks=int(cfg.get("max_chunks", 500)),
|
|
122
122
|
max_total_chunks=int(cfg.get("max_total_chunks", 100000)),
|
|
123
|
+
chunk_strategy=str(cfg.get("chunk_strategy", "syntax")),
|
|
123
124
|
).index(project_root)
|
|
124
125
|
|
|
125
126
|
if not searcher.has_index(project_root):
|
|
@@ -129,7 +130,10 @@ class SemanticProvider:
|
|
|
129
130
|
)
|
|
130
131
|
|
|
131
132
|
matches = searcher.search(
|
|
132
|
-
target, project_root,
|
|
133
|
+
target, project_root,
|
|
134
|
+
cosine_floor=float(cfg.get("cosine_floor", 0.25)),
|
|
135
|
+
rerank=str(cfg.get("rerank", "on")),
|
|
136
|
+
rerank_candidates=int(cfg.get("rerank_candidates", 30)),
|
|
133
137
|
)
|
|
134
138
|
if not matches:
|
|
135
139
|
return safe_null_result(op, target, engine="semantic", reason="below-floor")
|
|
@@ -118,6 +118,7 @@ class Reindexer:
|
|
|
118
118
|
stride=int(cfg.get("stride", 10)),
|
|
119
119
|
max_chunks=int(cfg.get("max_chunks", 500)),
|
|
120
120
|
max_total_chunks=int(cfg.get("max_total_chunks", 100000)),
|
|
121
|
+
chunk_strategy=str(cfg.get("chunk_strategy", "syntax")),
|
|
121
122
|
).index(project_root)
|
|
122
123
|
finally:
|
|
123
124
|
db.close()
|