codecortex 0.5.0__tar.gz → 0.7.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. {codecortex-0.5.0/src/codecortex.egg-info → codecortex-0.7.0}/PKG-INFO +1 -1
  2. {codecortex-0.5.0 → codecortex-0.7.0/src/codecortex.egg-info}/PKG-INFO +1 -1
  3. {codecortex-0.5.0 → codecortex-0.7.0}/src/codecortex.egg-info/SOURCES.txt +2 -0
  4. codecortex-0.7.0/src/codeintel/__init__.py +1 -0
  5. {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/__main__.py +1 -0
  6. {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/config.py +6 -1
  7. {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/doctor.py +5 -0
  8. codecortex-0.7.0/src/codeintel/indexer.py +430 -0
  9. {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/onboarding.py +1 -0
  10. {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/providers/graph.py +3 -1
  11. {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/providers/lsp.py +3 -1
  12. {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/providers/semantic.py +5 -1
  13. {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/reindexer.py +1 -0
  14. codecortex-0.7.0/src/codeintel/searcher.py +311 -0
  15. {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/semantic_db.py +47 -2
  16. codecortex-0.7.0/tests/test_chunking.py +357 -0
  17. {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_config.py +18 -0
  18. {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_doctor.py +12 -0
  19. {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_integration.py +11 -5
  20. codecortex-0.7.0/tests/test_rerank.py +256 -0
  21. {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_semantic_provider.py +48 -0
  22. codecortex-0.5.0/src/codeintel/__init__.py +0 -1
  23. codecortex-0.5.0/src/codeintel/indexer.py +0 -261
  24. codecortex-0.5.0/src/codeintel/searcher.py +0 -143
  25. {codecortex-0.5.0 → codecortex-0.7.0}/LICENSE +0 -0
  26. {codecortex-0.5.0 → codecortex-0.7.0}/README.md +0 -0
  27. {codecortex-0.5.0 → codecortex-0.7.0}/pyproject.toml +0 -0
  28. {codecortex-0.5.0 → codecortex-0.7.0}/setup.cfg +0 -0
  29. {codecortex-0.5.0 → codecortex-0.7.0}/src/codecortex.egg-info/dependency_links.txt +0 -0
  30. {codecortex-0.5.0 → codecortex-0.7.0}/src/codecortex.egg-info/entry_points.txt +0 -0
  31. {codecortex-0.5.0 → codecortex-0.7.0}/src/codecortex.egg-info/requires.txt +0 -0
  32. {codecortex-0.5.0 → codecortex-0.7.0}/src/codecortex.egg-info/top_level.txt +0 -0
  33. {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/auth.py +0 -0
  34. {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/cache.py +0 -0
  35. {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/gateway.py +0 -0
  36. {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/http_server.py +0 -0
  37. {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/injector.py +0 -0
  38. {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/installer.py +0 -0
  39. {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/logconfig.py +0 -0
  40. {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/mapper.py +0 -0
  41. {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/metrics.py +0 -0
  42. {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/policy.py +0 -0
  43. {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/provider.py +0 -0
  44. {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/providers/__init__.py +0 -0
  45. {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/providers/none.py +0 -0
  46. {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/reset.py +0 -0
  47. {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/server.py +0 -0
  48. {codecortex-0.5.0 → codecortex-0.7.0}/src/codeintel/term.py +0 -0
  49. {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_cache.py +0 -0
  50. {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_e2e.py +0 -0
  51. {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_enterprise.py +0 -0
  52. {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_gateway.py +0 -0
  53. {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_graph_provider.py +0 -0
  54. {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_graph_real.py +0 -0
  55. {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_graph_stdin.py +0 -0
  56. {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_hardening.py +0 -0
  57. {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_http_auth.py +0 -0
  58. {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_http_server.py +0 -0
  59. {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_lsp_provider.py +0 -0
  60. {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_lsp_real.py +0 -0
  61. {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_mapper.py +0 -0
  62. {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_never_raise.py +0 -0
  63. {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_onboarding.py +0 -0
  64. {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_rbac.py +0 -0
  65. {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_reindexer.py +0 -0
  66. {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_reset.py +0 -0
  67. {codecortex-0.5.0 → codecortex-0.7.0}/tests/test_term.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: codecortex
3
- Version: 0.5.0
3
+ Version: 0.7.0
4
4
  Summary: Local-first, MCP-native code-intelligence server — graph, LSP, and semantic search behind one safe code.query tool for coding agents.
5
5
  Author: Shammai Hamilton
6
6
  License-Expression: MIT
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: codecortex
3
- Version: 0.5.0
3
+ Version: 0.7.0
4
4
  Summary: Local-first, MCP-native code-intelligence server — graph, LSP, and semantic search behind one safe code.query tool for coding agents.
5
5
  Author: Shammai Hamilton
6
6
  License-Expression: MIT
@@ -36,6 +36,7 @@ src/codeintel/providers/lsp.py
36
36
  src/codeintel/providers/none.py
37
37
  src/codeintel/providers/semantic.py
38
38
  tests/test_cache.py
39
+ tests/test_chunking.py
39
40
  tests/test_config.py
40
41
  tests/test_doctor.py
41
42
  tests/test_e2e.py
@@ -55,6 +56,7 @@ tests/test_never_raise.py
55
56
  tests/test_onboarding.py
56
57
  tests/test_rbac.py
57
58
  tests/test_reindexer.py
59
+ tests/test_rerank.py
58
60
  tests/test_reset.py
59
61
  tests/test_semantic_provider.py
60
62
  tests/test_term.py
@@ -0,0 +1 @@
1
+ __version__ = "0.7.0"
@@ -124,6 +124,7 @@ def main() -> None:
124
124
  stride=int(cfg.get("stride", 10)),
125
125
  max_chunks=int(cfg.get("max_chunks", 500)),
126
126
  max_total_chunks=int(cfg.get("max_total_chunks", 100000)),
127
+ chunk_strategy=str(cfg.get("chunk_strategy", "syntax")),
127
128
  ).index(project_root)
128
129
  if count > 0:
129
130
  print(f"Indexed {count} chunks")
@@ -24,6 +24,9 @@ _DEFAULTS: dict = {
24
24
  "max_total_chunks": 100000, # safety ceiling on chunks embedded in one index pass
25
25
  "cosine_floor": 0.25,
26
26
  "model": "BAAI/bge-small-en-v1.5",
27
+ "chunk_strategy": "syntax", # syntax-aware (def/class boundaries) vs fixed line windows
28
+ "rerank": "on", # hybrid lexical+semantic rerank of search results
29
+ "rerank_candidates": 30, # cosine candidates fused/re-ranked before returning top-k
27
30
  }
28
31
 
29
32
  # Values restricted to a fixed set — anything else falls back to the default.
@@ -31,8 +34,10 @@ _ENUMS: dict = {
31
34
  "backend": {"auto", "graph", "lsp", "semantic"},
32
35
  "semantic": {"on", "off"},
33
36
  "reindex": {"on-demand", "never"},
37
+ "chunk_strategy": {"syntax", "lines"},
38
+ "rerank": {"on", "off"},
34
39
  }
35
- _POSITIVE_INTS = ("window", "stride", "max_chunks", "max_total_chunks")
40
+ _POSITIVE_INTS = ("window", "stride", "max_chunks", "max_total_chunks", "rerank_candidates")
36
41
 
37
42
 
38
43
  def _read_toml(path: pathlib.Path) -> dict:
@@ -158,4 +158,9 @@ def render_doctor_text(report: dict) -> str:
158
158
  tail = "" if report.get("deep") else c.dim(" (run with --deep to boot-check serena)")
159
159
  out.append("")
160
160
  out.append(f" {count} engines ready for this repo.{tail}")
161
+ if healthy is False:
162
+ out.append(" " + c.dim(
163
+ "tip: `codeintel setup --install-uv --install-deps --index` bootstraps the "
164
+ "pip-installable backends; each fix: line above has the per-engine command."
165
+ ))
161
166
  return "\n".join(out)
@@ -0,0 +1,430 @@
1
+ from __future__ import annotations
2
+
3
+ import ast
4
+ import hashlib
5
+ import logging
6
+ import os
7
+ import struct
8
+ from pathlib import Path
9
+ from typing import TYPE_CHECKING
10
+
11
+ if TYPE_CHECKING:
12
+ from codeintel.semantic_db import SemanticDb
13
+
14
+ logger = logging.getLogger(__name__)
15
+
16
+ _INDEXED_EXTS = frozenset({
17
+ ".py", ".ts", ".js", ".go", ".rs", ".java", ".c", ".cpp", ".h", ".md"
18
+ })
19
+ _SKIP_DIRS = frozenset({"__pycache__", ".git", "node_modules"})
20
+ # Vendored / regenerable dirs skipped even without a .gitignore entry.
21
+ _DEFAULT_IGNORES = frozenset({
22
+ ".venv", "venv", "env", "dist", "build", "target",
23
+ ".mypy_cache", ".pytest_cache", ".tox", ".idea", ".vscode", ".cache",
24
+ })
25
+
26
+
27
+ def _project_key(project_root_real: str) -> str:
28
+ """A short, stable id for a project root — prefixes every chunk_id so two repos
29
+ with an identically-named file never collide in the shared cache."""
30
+ return hashlib.sha256(project_root_real.encode()).hexdigest()[:12]
31
+
32
+
33
+ def _pos_int(val: object, default: int) -> int:
34
+ """A usable positive int or the default — mirrors ``config._coerce`` so a direct
35
+ ``Indexer(...)`` caller (which bypasses config validation) can't set a zero/negative/non-int
36
+ ``stride`` (would raise inside ``range()``) or ``window`` (would silently drop every region)."""
37
+ try:
38
+ n = int(val) # type: ignore[arg-type]
39
+ except (TypeError, ValueError, OverflowError):
40
+ return default
41
+ return n if n > 0 else default
42
+
43
+
44
+ class Indexer:
45
+ def __init__(
46
+ self,
47
+ db: SemanticDb,
48
+ model_name: str = "BAAI/bge-small-en-v1.5",
49
+ window: int = 20,
50
+ stride: int = 10,
51
+ max_chunks: int = 500,
52
+ max_total_chunks: int = 100000,
53
+ chunk_strategy: str = "syntax",
54
+ max_chunk_lines: int | None = None,
55
+ ) -> None:
56
+ self.db = db
57
+ self.model_name = model_name
58
+ # Coerce the numeric knobs defensively: the 4 production call sites pass config-validated
59
+ # values, but a direct caller must not be able to set a stride/window that raises in
60
+ # range() or silently drops regions (mirrors config._coerce's _POSITIVE_INTS clamp).
61
+ self.window = _pos_int(window, 20)
62
+ self.stride = _pos_int(stride, 10)
63
+ self.max_chunks = _pos_int(max_chunks, 500) # per file
64
+ self.max_total_chunks = _pos_int(max_total_chunks, 100000) # ceiling per pass (mem backstop)
65
+ # "syntax" chunks Python on def/class boundaries (ast); "lines" is the fixed-window
66
+ # fallback used for every non-.py file, on any parse failure, and as a runtime escape
67
+ # hatch. Case-normalized then range-checked so an unknown value degrades to "syntax"
68
+ # (config already validates; this keeps a direct caller from silently disabling — or
69
+ # accidentally case-swapping — the strategy with a typo).
70
+ strategy = str(chunk_strategy).strip().lower()
71
+ self.chunk_strategy = strategy if strategy in ("syntax", "lines") else "syntax"
72
+ # A def longer than this is window-chunked internally so no single chunk overflows the
73
+ # embedder (~512 tokens). Defaults to 2*window; never <= 0 (would loop / never split).
74
+ self.max_chunk_lines = (
75
+ max_chunk_lines if isinstance(max_chunk_lines, int) and max_chunk_lines > 0
76
+ else 2 * self.window
77
+ )
78
+ self._embedder = None
79
+
80
+ def _get_embedder(self):
81
+ if self._embedder is None:
82
+ from fastembed import TextEmbedding
83
+ self._embedder = TextEmbedding(model_name=self.model_name)
84
+ return self._embedder
85
+
86
+ def index(self, project_root: str) -> int:
87
+ """Return count of newly embedded chunks, or -1 on unrecoverable failure."""
88
+ try:
89
+ return self._index(project_root)
90
+ except Exception as exc:
91
+ logger.error("Indexer.index() unrecoverable failure: %s", exc)
92
+ return -1
93
+
94
+ def _load_gitignore(self, root: Path) -> set[str]:
95
+ """Best-effort ``.gitignore``: collect simple name/dir patterns to skip. This is
96
+ NOT full gitignore semantics (no globs, negations, or nesting) — just enough to
97
+ avoid indexing vendored/build output the user already told git to ignore."""
98
+ patterns: set[str] = set()
99
+ gi = root / ".gitignore"
100
+ try:
101
+ if gi.is_file():
102
+ for line in gi.read_text(encoding="utf-8", errors="replace").splitlines():
103
+ line = line.strip()
104
+ if not line or line.startswith("#") or line.startswith("!"):
105
+ continue
106
+ name = line.rstrip("/").lstrip("/")
107
+ if name and "*" not in name and "/" not in name:
108
+ patterns.add(name)
109
+ except Exception:
110
+ pass
111
+ return patterns
112
+
113
+ def _cleanup_deleted(self, root: Path, project_root_real: str) -> None:
114
+ """Drop rows for THIS project whose file no longer exists — scoped by
115
+ project_root so touching one repo can never purge another's index."""
116
+ conn = self.db.conn()
117
+ try:
118
+ rows = conn.execute(
119
+ "SELECT DISTINCT file_path FROM chunk_hashes WHERE project_root = ?",
120
+ (project_root_real,),
121
+ ).fetchall()
122
+ deleted_paths = [
123
+ row[0] for row in rows if not (root / row[0]).exists()
124
+ ]
125
+ for fp in deleted_paths:
126
+ chunk_ids = [
127
+ r[0]
128
+ for r in conn.execute(
129
+ "SELECT chunk_id FROM chunk_hashes"
130
+ " WHERE project_root = ? AND file_path = ?",
131
+ (project_root_real, fp),
132
+ ).fetchall()
133
+ ]
134
+ for cid in chunk_ids:
135
+ conn.execute(
136
+ "DELETE FROM code_embeddings WHERE chunk_id = ?", (cid,)
137
+ )
138
+ conn.execute(
139
+ "DELETE FROM chunk_hashes WHERE chunk_id = ?", (cid,)
140
+ )
141
+ conn.commit()
142
+ except Exception as exc:
143
+ logger.warning("Cleanup pass failed: %s", exc)
144
+
145
+ def _walk_files(self, root: Path):
146
+ ignores = set(_SKIP_DIRS) | set(_DEFAULT_IGNORES) | self._load_gitignore(root)
147
+ for dirpath, dirnames, filenames in os.walk(root):
148
+ dirnames[:] = [
149
+ d for d in dirnames
150
+ if d not in ignores and not d.endswith(".egg-info")
151
+ ]
152
+ for fname in filenames:
153
+ if fname in ignores:
154
+ continue
155
+ if Path(fname).suffix.lower() in _INDEXED_EXTS:
156
+ yield Path(dirpath) / fname
157
+
158
+ # ---- chunk-span computation ------------------------------------------------------------
159
+ # A file is turned into a list of 0-based, half-open ``(start, end)`` line spans; every
160
+ # strategy funnels through the same span list so downstream materialisation (whitespace
161
+ # skip, hash-dedup, caps, orphan reconcile) is shared and identical.
162
+
163
+ def _window_spans(self, start: int, end: int) -> list[tuple[int, int]]:
164
+ """Fixed overlapping line windows over ``[start, end)`` — the original chunking, reused to
165
+ fill inter-def gaps and split oversized defs. ``_window_spans(0, len(lines))`` reproduces
166
+ the old ``range(0, n, stride)`` + ``lines[s:s+window]`` output exactly."""
167
+ spans: list[tuple[int, int]] = []
168
+ if end <= start:
169
+ return spans
170
+ for s in range(start, end, self.stride):
171
+ spans.append((s, min(s + self.window, end)))
172
+ return spans
173
+
174
+ def _maybe_split(self, start: int, end: int) -> list[tuple[int, int]]:
175
+ """A def span kept whole, or window-split when it exceeds ``max_chunk_lines`` so no single
176
+ chunk overflows the embedder."""
177
+ if end - start <= self.max_chunk_lines:
178
+ return [(start, end)]
179
+ return self._window_spans(start, end)
180
+
181
+ @staticmethod
182
+ def _node_span(node: ast.AST, n: int) -> tuple[int, int]:
183
+ """0-based half-open ``[start, end)`` span of a def/class node, decorators included
184
+ (``min(decorator linenos, node.lineno)`` … ``end_lineno``), clamped into ``[0, n]``."""
185
+ start = node.lineno # type: ignore[attr-defined]
186
+ for dec in getattr(node, "decorator_list", None) or []:
187
+ dline = getattr(dec, "lineno", None)
188
+ if isinstance(dline, int):
189
+ start = min(start, dline)
190
+ end = getattr(node, "end_lineno", None)
191
+ if not isinstance(end, int):
192
+ end = node.lineno # type: ignore[attr-defined]
193
+ start0 = max(0, start - 1)
194
+ end0 = min(n, max(start0 + 1, end))
195
+ return (start0, end0)
196
+
197
+ def _primary_spans(self, tree: ast.Module, n: int) -> list[tuple[int, int]]:
198
+ """Def-aligned 'primary' spans: each top-level function, and for each top-level class a
199
+ header span (class line → just before its first method/nested def) plus one span per
200
+ method/nested def. Inter-method and module-level runs are intentionally left uncovered
201
+ here — ``_cover`` window-fills them — so per-method chunks are never double-embedded."""
202
+ spans: list[tuple[int, int]] = []
203
+ for node in tree.body:
204
+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
205
+ spans.append(self._node_span(node, n))
206
+ elif isinstance(node, ast.ClassDef):
207
+ cstart, cend = self._node_span(node, n)
208
+ members = [
209
+ self._node_span(c, n)
210
+ for c in node.body
211
+ if isinstance(c, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef))
212
+ ]
213
+ if not members:
214
+ spans.append((cstart, cend)) # no methods → the class is one unit
215
+ continue
216
+ members.sort()
217
+ header_end = max(cstart + 1, min(members[0][0], cend))
218
+ spans.append((cstart, header_end)) # header: bases + class docstring
219
+ spans.extend(members)
220
+ return spans
221
+
222
+ def _cover(self, primary: list[tuple[int, int]], n: int) -> list[tuple[int, int]]:
223
+ """Gapless cover of ``[0, n)`` in file order: window-fill every gap between primary spans
224
+ and window-split any oversized def. The def-aligned *primary* spans are whole and mutually
225
+ non-overlapping (no whole-class chunk shadowing its per-method chunks); the window-filled
226
+ gaps and oversized-def splits reuse the existing ``window``/``stride``, so — exactly like
227
+ the legacy line windower — adjacent windows inside one filled run *do* overlap when
228
+ ``stride < window``. Coverage is always complete; chunk starts are always unique."""
229
+ result: list[tuple[int, int]] = []
230
+ cursor = 0
231
+ for s0, e0 in sorted(primary):
232
+ s = max(s0, cursor) # clamp any (pathological) overlap so nothing is double-covered
233
+ e = min(e0, n)
234
+ if s >= e:
235
+ continue
236
+ if s > cursor:
237
+ result.extend(self._window_spans(cursor, s)) # module-level / inter-def run
238
+ result.extend(self._maybe_split(s, e))
239
+ cursor = e
240
+ if cursor < n:
241
+ result.extend(self._window_spans(cursor, n))
242
+ return result
243
+
244
+ def _chunk_python_ast(self, lines: list[str], source: str) -> list[tuple[int, int]]:
245
+ """Parse ``source`` → a complete, non-overlapping, def-aligned cover of the file as
246
+ 0-based half-open ``(start, end)`` spans. Raises on parse failure (``SyntaxError`` /
247
+ ``ValueError`` on NUL bytes / ``RecursionError`` / …) so the caller falls back to windows."""
248
+ tree = ast.parse(source)
249
+ n = len(lines)
250
+ return self._cover(self._primary_spans(tree, n), n)
251
+
252
+ def _spans_for_file(
253
+ self, filepath: Path, lines: list[str], rel_path: str
254
+ ) -> list[tuple[int, int]]:
255
+ """Choose spans for one file: syntax-aware for ``.py`` under the syntax strategy (falling
256
+ back to windowing on any parse failure), fixed windows for everything else."""
257
+ if self.chunk_strategy == "syntax" and filepath.suffix.lower() == ".py":
258
+ try:
259
+ return self._chunk_python_ast(lines, "".join(lines))
260
+ except Exception as exc:
261
+ logger.debug("syntax chunking failed for %s (%s) — windowing", rel_path, exc)
262
+ return self._window_spans(0, len(lines))
263
+
264
+ # ---- materialisation -------------------------------------------------------------------
265
+
266
+ def _emit_spans(
267
+ self,
268
+ spans: list[tuple[int, int]],
269
+ lines: list[str],
270
+ rel_path: str,
271
+ project_key: str,
272
+ conn,
273
+ new_chunks: list[tuple[str, str, str, int, str]],
274
+ ) -> tuple[set[str], bool]:
275
+ """Materialise spans into new/changed chunk records — shared by both strategies, so the
276
+ whitespace-skip, hash-dedup, and per-file cap behave identically. Returns
277
+ ``(keep_ids, complete)``: ``keep_ids`` is every chunk_id this file legitimately produces
278
+ (drives orphan reconciliation); ``complete`` is False iff the *global* ceiling cut the
279
+ file short, in which case ``keep_ids`` is partial and MUST NOT delete anything."""
280
+ keep_ids: set[str] = set()
281
+ chunk_count = 0
282
+ for start, end in spans:
283
+ if len(new_chunks) >= self.max_total_chunks:
284
+ return keep_ids, False # global ceiling mid-file — keep_ids is partial
285
+ if chunk_count >= self.max_chunks:
286
+ logger.debug("chunk cap hit for %s, truncating at %d", rel_path, self.max_chunks)
287
+ break # per-file cap is deterministic (same first-N each pass) → reconcile is safe
288
+ chunk_lines = lines[start:end]
289
+ if not chunk_lines:
290
+ continue
291
+ chunk_text = "".join(chunk_lines)
292
+ if not chunk_text.strip():
293
+ # EC3.4: never embed empty/whitespace-only chunks (zero vectors pollute results).
294
+ chunk_count += 1
295
+ continue
296
+ chunk_id = f"{project_key}:{rel_path}:{start}"
297
+ content_hash = hashlib.sha256(chunk_text.encode()).hexdigest()[:16]
298
+ keep_ids.add(chunk_id) # produced this pass — keep even when dedup skips re-embed
299
+ try:
300
+ row = conn.execute(
301
+ "SELECT content_hash FROM chunk_hashes WHERE chunk_id = ?",
302
+ (chunk_id,),
303
+ ).fetchone()
304
+ if row and row[0] == content_hash:
305
+ chunk_count += 1
306
+ continue
307
+ except Exception as exc:
308
+ logger.debug("hash check failed for %s: %s", chunk_id, exc)
309
+ new_chunks.append((chunk_id, chunk_text, rel_path, start, content_hash))
310
+ chunk_count += 1
311
+ return keep_ids, True
312
+
313
+ def _collect_new_chunks(
314
+ self, root: Path, project_key: str, project_root_real: str
315
+ ) -> list[tuple[str, str, str, int, str]]:
316
+ """Walk files; return (chunk_id, text, rel_path, start, hash) for new/changed chunks, and
317
+ reconcile each fully-processed file (dropping rows for chunks it no longer produces)."""
318
+ conn = self.db.conn()
319
+ new_chunks: list[tuple[str, str, str, int, str]] = []
320
+
321
+ for filepath in self._walk_files(root):
322
+ if len(new_chunks) >= self.max_total_chunks:
323
+ logger.warning(
324
+ "index: reached max_total_chunks=%d this pass — stopping "
325
+ "(raise it in .codeintel.toml to embed more of a very large repo)",
326
+ self.max_total_chunks,
327
+ )
328
+ break
329
+ try:
330
+ with open(filepath, encoding="utf-8", errors="replace") as f:
331
+ lines = f.readlines()
332
+ except FileNotFoundError:
333
+ logger.debug("file disappeared: %s", filepath)
334
+ continue
335
+ except Exception as exc:
336
+ logger.debug("skipping %s: %s", filepath, exc)
337
+ continue
338
+
339
+ rel_path = str(filepath.relative_to(root))
340
+ spans = self._spans_for_file(filepath, lines, rel_path)
341
+ keep_ids, complete = self._emit_spans(
342
+ spans, lines, rel_path, project_key, conn, new_chunks
343
+ )
344
+ if complete:
345
+ # Reconcile only a fully-processed file: drop rows for defs/windows it no longer
346
+ # produces (a moved/deleted function, or a strategy switch). Skipped when the
347
+ # global ceiling truncated the file — its partial keep_ids would delete rows past
348
+ # the cut that are still valid. Scoped by (project_root, file_path) in the db layer.
349
+ # Caveat: two *concurrent* index passes over the same project computed from
350
+ # different point-in-time reads can transiently resurrect a just-deleted stale row
351
+ # (the later pass's INSERT OR REPLACE re-adds what the earlier reconcile removed).
352
+ # This self-heals on the next single-reader pass, and is strictly better than the
353
+ # pre-0.6 behaviour (the stale row persisted forever); cross-pass serialization is
354
+ # out of scope for the indexer.
355
+ self.db.delete_file_orphans(project_root_real, rel_path, keep_ids)
356
+
357
+ return new_chunks
358
+
359
+ def _embed_and_write(
360
+ self, new_chunks: list[tuple[str, str, str, int, str]], project_root_real: str
361
+ ) -> int:
362
+ embedder = self._get_embedder() # may raise → propagates to index() → returns -1
363
+ conn = self.db.conn()
364
+ embedded_count = 0
365
+ batch_size = 32
366
+
367
+ for i in range(0, len(new_chunks), batch_size):
368
+ batch = new_chunks[i: i + batch_size]
369
+ texts = [c[1] for c in batch]
370
+
371
+ try:
372
+ embeddings = list(embedder.embed(texts))
373
+ except Exception as exc:
374
+ logger.warning("embedding batch %d failed: %s", i // batch_size, exc)
375
+ continue
376
+
377
+ for j, (chunk_id, _, rel_path, chunk_start, content_hash) in enumerate(batch):
378
+ if j >= len(embeddings):
379
+ break
380
+ try:
381
+ vec = embeddings[j]
382
+ vec_bytes = struct.pack(f"{len(vec)}f", *vec)
383
+ # sqlite-vec's vec0 virtual table does NOT honor INSERT OR REPLACE — it raises
384
+ # UNIQUE on an existing chunk_id instead of replacing. So re-embedding a chunk
385
+ # whose content changed but whose chunk_id (start line) is stable — the common
386
+ # case under syntax chunking, where a def's chunk_id is its def line — would
387
+ # silently fail and keep the STALE vector. DELETE-then-INSERT is the supported
388
+ # upsert for vec0. (chunk_hashes below is a normal table, where REPLACE works.)
389
+ conn.execute(
390
+ "DELETE FROM code_embeddings WHERE chunk_id = ?", (chunk_id,)
391
+ )
392
+ conn.execute(
393
+ "INSERT INTO code_embeddings(chunk_id, embedding) VALUES (?, ?)",
394
+ (chunk_id, vec_bytes),
395
+ )
396
+ conn.execute(
397
+ "INSERT OR REPLACE INTO chunk_hashes"
398
+ "(chunk_id, project_root, file_path, chunk_start, content_hash)"
399
+ " VALUES (?, ?, ?, ?, ?)",
400
+ (chunk_id, project_root_real, rel_path, chunk_start, content_hash),
401
+ )
402
+ embedded_count += 1
403
+ except Exception as exc:
404
+ logger.warning("writing chunk %s failed: %s", chunk_id, exc)
405
+
406
+ try:
407
+ conn.commit()
408
+ except Exception as exc:
409
+ logger.warning("commit failed after batch %d: %s", i // batch_size, exc)
410
+
411
+ return embedded_count
412
+
413
+ def _index(self, project_root: str) -> int:
414
+ if not project_root:
415
+ return 0
416
+
417
+ root = Path(project_root)
418
+ if not root.exists():
419
+ return 0
420
+
421
+ project_root_real = os.path.realpath(project_root)
422
+ project_key = _project_key(project_root_real)
423
+
424
+ self._cleanup_deleted(root, project_root_real)
425
+
426
+ new_chunks = self._collect_new_chunks(root, project_key, project_root_real)
427
+ if not new_chunks:
428
+ return 0
429
+
430
+ return self._embed_and_write(new_chunks, project_root_real)
@@ -73,6 +73,7 @@ def _bounded_index(project_root: str, *, timeout_s: float, out) -> dict:
73
73
  window=int(cfg.get("window", 20)), stride=int(cfg.get("stride", 10)),
74
74
  max_chunks=int(cfg.get("max_chunks", 500)),
75
75
  max_total_chunks=int(cfg.get("max_total_chunks", 100000)),
76
+ chunk_strategy=str(cfg.get("chunk_strategy", "syntax")),
76
77
  ).index(project_root)
77
78
  finally:
78
79
  db.close()
@@ -157,7 +157,9 @@ class GraphProvider:
157
157
  return {
158
158
  "installed": False, "runnable": False, "repo_indexed": False, "project": None,
159
159
  "detail": "codebase-memory-mcp not found on PATH",
160
- "remediation": "install codebase-memory-mcp (the graph backend)",
160
+ "remediation": "put the codebase-memory-mcp binary on PATH — it's an external "
161
+ "native backend (see docs/graph.md); once present it self-updates "
162
+ "via `codebase-memory-mcp update`",
161
163
  }
162
164
  raw = self._run("list_projects", {}, timeout_ms)
163
165
  if raw is None:
@@ -144,7 +144,9 @@ class LspProvider:
144
144
  return {
145
145
  "installed": False, "runnable": False, "repo_indexed": None,
146
146
  "detail": "neither `serena` nor `uvx` found on PATH",
147
- "remediation": "install uv (provides uvx) serena is fetched on first use",
147
+ "remediation": "install uv (provides uvx): `codeintel setup --install-uv` "
148
+ "(or `brew install uv` / `pip install uv`) — serena is then "
149
+ "fetched on first use",
148
150
  }
149
151
  cmd = self._cmd
150
152
  if not deep:
@@ -120,6 +120,7 @@ class SemanticProvider:
120
120
  stride=int(cfg.get("stride", 10)),
121
121
  max_chunks=int(cfg.get("max_chunks", 500)),
122
122
  max_total_chunks=int(cfg.get("max_total_chunks", 100000)),
123
+ chunk_strategy=str(cfg.get("chunk_strategy", "syntax")),
123
124
  ).index(project_root)
124
125
 
125
126
  if not searcher.has_index(project_root):
@@ -129,7 +130,10 @@ class SemanticProvider:
129
130
  )
130
131
 
131
132
  matches = searcher.search(
132
- target, project_root, cosine_floor=float(cfg.get("cosine_floor", 0.25))
133
+ target, project_root,
134
+ cosine_floor=float(cfg.get("cosine_floor", 0.25)),
135
+ rerank=str(cfg.get("rerank", "on")),
136
+ rerank_candidates=int(cfg.get("rerank_candidates", 30)),
133
137
  )
134
138
  if not matches:
135
139
  return safe_null_result(op, target, engine="semantic", reason="below-floor")
@@ -118,6 +118,7 @@ class Reindexer:
118
118
  stride=int(cfg.get("stride", 10)),
119
119
  max_chunks=int(cfg.get("max_chunks", 500)),
120
120
  max_total_chunks=int(cfg.get("max_total_chunks", 100000)),
121
+ chunk_strategy=str(cfg.get("chunk_strategy", "syntax")),
121
122
  ).index(project_root)
122
123
  finally:
123
124
  db.close()