codecortex 0.6.0__tar.gz → 0.7.0__tar.gz

{codecortex-0.6.0/src/codecortex.egg-info → codecortex-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: codecortex
-Version: 0.6.0
+Version: 0.7.0
 Summary: Local-first, MCP-native code-intelligence server — graph, LSP, and semantic search behind one safe code.query tool for coding agents.
 Author: Shammai Hamilton
 License-Expression: MIT

{codecortex-0.6.0 → codecortex-0.7.0/src/codecortex.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: codecortex
-Version: 0.6.0
+Version: 0.7.0
 Summary: Local-first, MCP-native code-intelligence server — graph, LSP, and semantic search behind one safe code.query tool for coding agents.
 Author: Shammai Hamilton
 License-Expression: MIT

{codecortex-0.6.0 → codecortex-0.7.0}/src/codecortex.egg-info/SOURCES.txt

@@ -56,6 +56,7 @@ tests/test_never_raise.py
 tests/test_onboarding.py
 tests/test_rbac.py
 tests/test_reindexer.py
+tests/test_rerank.py
 tests/test_reset.py
 tests/test_semantic_provider.py
 tests/test_term.py

codecortex-0.7.0/src/codeintel/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.7.0"

{codecortex-0.6.0 → codecortex-0.7.0}/src/codeintel/config.py

@@ -25,6 +25,8 @@ _DEFAULTS: dict = {
     "cosine_floor": 0.25,
     "model": "BAAI/bge-small-en-v1.5",
     "chunk_strategy": "syntax",  # syntax-aware (def/class boundaries) vs fixed line windows
+    "rerank": "on",              # hybrid lexical+semantic rerank of search results
+    "rerank_candidates": 30,     # cosine candidates fused/re-ranked before returning top-k
 }
 
 # Values restricted to a fixed set — anything else falls back to the default.
@@ -33,8 +35,9 @@ _ENUMS: dict = {
     "semantic": {"on", "off"},
     "reindex": {"on-demand", "never"},
     "chunk_strategy": {"syntax", "lines"},
+    "rerank": {"on", "off"},
 }
-_POSITIVE_INTS = ("window", "stride", "max_chunks", "max_total_chunks")
+_POSITIVE_INTS = ("window", "stride", "max_chunks", "max_total_chunks", "rerank_candidates")
 
 
 def _read_toml(path: pathlib.Path) -> dict:

{codecortex-0.6.0 → codecortex-0.7.0}/src/codeintel/doctor.py

@@ -158,4 +158,9 @@ def render_doctor_text(report: dict) -> str:
     tail = "" if report.get("deep") else c.dim("  (run with --deep to boot-check serena)")
     out.append("")
     out.append(f"  {count} engines ready for this repo.{tail}")
+    if healthy is False:
+        out.append("  " + c.dim(
+            "tip: `codeintel setup --install-uv --install-deps --index` bootstraps the "
+            "pip-installable backends; each fix: line above has the per-engine command."
+        ))
     return "\n".join(out)

{codecortex-0.6.0 → codecortex-0.7.0}/src/codeintel/indexer.py

@@ -380,9 +380,17 @@ class Indexer:
                 try:
                     vec = embeddings[j]
                     vec_bytes = struct.pack(f"{len(vec)}f", *vec)
+                    # sqlite-vec's vec0 virtual table does NOT honor INSERT OR REPLACE — it raises
+                    # UNIQUE on an existing chunk_id instead of replacing. So re-embedding a chunk
+                    # whose content changed but whose chunk_id (start line) is stable — the common
+                    # case under syntax chunking, where a def's chunk_id is its def line — would
+                    # silently fail and keep the STALE vector. DELETE-then-INSERT is the supported
+                    # upsert for vec0. (chunk_hashes below is a normal table, where REPLACE works.)
                     conn.execute(
-                        "INSERT OR REPLACE INTO code_embeddings(chunk_id, embedding)"
-                        " VALUES (?, ?)",
+                        "DELETE FROM code_embeddings WHERE chunk_id = ?", (chunk_id,)
+                    )
+                    conn.execute(
+                        "INSERT INTO code_embeddings(chunk_id, embedding) VALUES (?, ?)",
                         (chunk_id, vec_bytes),
                     )
                     conn.execute(

{codecortex-0.6.0 → codecortex-0.7.0}/src/codeintel/providers/graph.py

@@ -157,7 +157,9 @@ class GraphProvider:
             return {
                 "installed": False, "runnable": False, "repo_indexed": False, "project": None,
                 "detail": "codebase-memory-mcp not found on PATH",
-                "remediation": "install codebase-memory-mcp (the graph backend)",
+                "remediation": "put the codebase-memory-mcp binary on PATH — it's an external "
+                               "native backend (see docs/graph.md); once present it self-updates "
+                               "via `codebase-memory-mcp update`",
             }
         raw = self._run("list_projects", {}, timeout_ms)
         if raw is None:

{codecortex-0.6.0 → codecortex-0.7.0}/src/codeintel/providers/lsp.py

@@ -144,7 +144,9 @@ class LspProvider:
             return {
                 "installed": False, "runnable": False, "repo_indexed": None,
                 "detail": "neither `serena` nor `uvx` found on PATH",
-                "remediation": "install uv (provides uvx) — serena is fetched on first use",
+                "remediation": "install uv (provides uvx): `codeintel setup --install-uv` "
+                               "(or `brew install uv` / `pip install uv`) — serena is then "
+                               "fetched on first use",
             }
         cmd = self._cmd
         if not deep:

{codecortex-0.6.0 → codecortex-0.7.0}/src/codeintel/providers/semantic.py

@@ -130,7 +130,10 @@ class SemanticProvider:
                 )
 
             matches = searcher.search(
-                target, project_root, cosine_floor=float(cfg.get("cosine_floor", 0.25))
+                target, project_root,
+                cosine_floor=float(cfg.get("cosine_floor", 0.25)),
+                rerank=str(cfg.get("rerank", "on")),
+                rerank_candidates=int(cfg.get("rerank_candidates", 30)),
             )
             if not matches:
                 return safe_null_result(op, target, engine="semantic", reason="below-floor")

codecortex-0.7.0/src/codeintel/searcher.py

@@ -0,0 +1,311 @@
+from __future__ import annotations
+
+import itertools
+import logging
+import os
+import re
+import struct
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from codeintel.semantic_db import SemanticDb
+
+logger = logging.getLogger(__name__)
+
+_SNIPPET_LINES = 5
+# Hybrid rerank (0.7.0): re-read up to this many lines per candidate for lexical scoring — a hard
+# per-candidate cap so one huge file can't blow up a query. The read is further bounded at the next
+# stored chunk start (see _rerank); the returned snippet still uses only the first _SNIPPET_LINES.
+_RERANK_READ_LINES = 40
+# Hard ceiling on the candidate set regardless of rerank_candidates, so a misconfigured value can't
+# turn one query into thousands of file reads on the interactive hot path.
+_RERANK_CANDIDATES_CAP = 200
+_RRF_K = 60          # Reciprocal Rank Fusion constant (standard ≈ 60; damps rank differences)
+_SYMBOL_BOOST = 0.1  # additive fusion bonus for an exact def/class-name match (≫ one RRF term)
+
+# `\w` is Unicode-aware in Python 3, so a non-ASCII identifier is kept whole; the sub-splitter is
+# Latin-cased on purpose (camelCase only exists there) and just adds bonus pieces for ASCII names.
+_IDENT_RE = re.compile(r"\w+")
+_SUBTOKEN_RE = re.compile(r"[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+|[0-9]+")
+_SYMBOL_RE = re.compile(r"[^\W\d]\w*")  # a query that is a single identifier (letter/_ then word)
+
+
+def _pos_int(val: object, default: int) -> int:
+    """A usable positive int or the default — guards the public Searcher.search against a caller
+    passing a non-int/zero/negative k or rerank_candidates (mirrors indexer._pos_int)."""
+    try:
+        n = int(val)  # type: ignore[arg-type]
+    except (TypeError, ValueError, OverflowError):
+        return default
+    return n if n > 0 else default
+
+
+def _tokenize(text: str) -> set[str]:
+    """Lexical tokens for overlap scoring: each whole identifier plus its camel/snake sub-pieces,
+    lowercased. Deliberately set-based (presence, not frequency) — cheap and enough for ranking."""
+    toks: set[str] = set()
+    for ident in _IDENT_RE.findall(text):
+        toks.add(ident.lower())
+        for sub in _SUBTOKEN_RE.findall(ident):
+            toks.add(sub.lower())
+    return toks
+
+
+class Searcher:
+    def __init__(
+        self,
+        db: SemanticDb,
+        model_name: str = "BAAI/bge-small-en-v1.5",
+    ) -> None:
+        self.db = db
+        self.model_name = model_name
+        self._embedder = None
+
+    def _get_embedder(self):
+        if self._embedder is None:
+            from fastembed import TextEmbedding
+            self._embedder = TextEmbedding(model_name=self.model_name)
+        return self._embedder
+
+    def _embed_query(self, query: str) -> bytes | None:
+        try:
+            embedder = self._get_embedder()
+            vecs = list(embedder.embed([query]))
+            if not vecs:
+                return None
+            vec = vecs[0]
+            return struct.pack(f"{len(vec)}f", *vec)
+        except Exception as exc:
+            logger.warning("query embedding failed: %s", exc)
+            return None
+
+    def _row_count(self, project_root_real: str) -> int:
+        try:
+            conn = self.db.conn()
+            row = conn.execute(
+                "SELECT COUNT(*) FROM chunk_hashes WHERE project_root = ?",
+                (project_root_real,),
+            ).fetchone()
+            return row[0] if row else 0
+        except Exception as exc:
+            logger.warning("rowcount check failed: %s", exc)
+            return 0
+
+    def has_index(self, project_root: str) -> bool:
+        """True when this project has at least one indexed chunk — lets the provider
+        distinguish 'nothing indexed yet' (no-index) from 'matches below floor'."""
+        return self._row_count(os.path.realpath(project_root)) > 0
+
+    def _read_snippet(self, file_path: Path, chunk_start: int) -> str:
+        try:
+            with open(file_path, encoding="utf-8", errors="replace") as f:
+                lines = f.readlines()
+            snippet_lines = lines[chunk_start: chunk_start + _SNIPPET_LINES]
+            return "".join(snippet_lines).rstrip()
+        except FileNotFoundError:
+            return "[file not found]"
+        except Exception as exc:
+            logger.debug("snippet read failed for %s:%d: %s", file_path, chunk_start, exc)
+            return "[file not found]"
+
+    def _read_chunk(self, file_path: Path, chunk_start: int) -> list[str] | None:
+        """Bounded re-read from ``chunk_start`` for rerank lexical scoring (and the snippet in one
+        read). Uses ``islice`` so a huge multi-line file is not fully materialised just to take a
+        40-line window. ``None`` on any failure — the caller scores that candidate 0 and shows a
+        not-found snippet, never crashing (a missing/edited file must degrade, per never-raise)."""
+        try:
+            with open(file_path, encoding="utf-8", errors="replace") as f:
+                return list(itertools.islice(f, chunk_start, chunk_start + _RERANK_READ_LINES))
+        except Exception as exc:
+            logger.debug("chunk re-read failed for %s:%d: %s", file_path, chunk_start, exc)
+            return None
+
+    def _chunk_starts(self, project_root_real: str, file_path: str) -> list[int]:
+        """All stored chunk start lines for one file, sorted. Used to bound each candidate's rerank
+        read at the *next* chunk — the chunk's true end for tiling syntax chunks, its owned region
+        for overlapping line windows — so lexical/boost text never bleeds into an unrelated def.
+        Uses the (project_root, file_path) composite index. ``[]`` on any error → no bound applied."""
+        try:
+            rows = self.db.conn().execute(
+                "SELECT chunk_start FROM chunk_hashes"
+                " WHERE project_root = ? AND file_path = ? ORDER BY chunk_start",
+                (project_root_real, file_path),
+            ).fetchall()
+            return [int(r[0]) for r in rows]
+        except Exception as exc:
+            logger.debug("chunk-start lookup failed for %s: %s", file_path, exc)
+            return []
+
+    @staticmethod
+    def _lexical_score(query_tokens: set[str], chunk_text: str) -> float:
+        """Token overlap in [0, 1]: the fraction of query (sub)tokens present in the chunk."""
+        if not query_tokens:
+            return 0.0
+        return len(query_tokens & _tokenize(chunk_text)) / len(query_tokens)
+
+    @staticmethod
+    def _symbol_boost(query: str, chunk_text: str) -> float:
+        """Additive fusion bonus for the 'exact symbol' case: when the query is a single identifier
+        that appears in the chunk as a ``def``/``class`` name (full boost) or as a standalone word
+        (half). Case-insensitive, to match the lexical score. Cosine alone under-ranks these literal
+        matches; this is what pulls them to the top. The ``_SYMBOL_RE`` guard runs before any
+        ``re.escape`` interpolation, so no query character can ever reach the built pattern."""
+        q = (query or "").strip()
+        if not _SYMBOL_RE.fullmatch(q):
+            return 0.0  # multi-word / non-identifier query → no structural signal
+        if re.search(rf"\b(?:def|class)\s+{re.escape(q)}\b", chunk_text, re.IGNORECASE):
+            return _SYMBOL_BOOST
+        if re.search(rf"\b{re.escape(q)}\b", chunk_text, re.IGNORECASE):
+            return _SYMBOL_BOOST * 0.5
+        return 0.0
+
+    def _rerank(
+        self, query: str, root: Path, project_root_real: str, candidates: list[dict]
+    ) -> list[dict]:
+        """Reorder floor-gated candidates (given in cosine order) by Reciprocal Rank Fusion over
+        the semantic rank and a lexical rank, plus a symbol boost. Reads each candidate's chunk
+        once (bounded) — caching its 5-line snippet — so the whole rerank costs ≤ len(candidates)
+        reads. When no candidate has any lexical overlap the lexical rank mirrors the semantic
+        rank, so the cosine order is returned unchanged (rerank only *reorders* on real signal)."""
+        query_tokens = _tokenize(query)
+        n = len(candidates)
+        # Bound each candidate's lexical/boost text at the NEXT stored chunk start in its file — the
+        # chunk's true end for tiling syntax chunks, its owned region for overlapping line windows.
+        # A chunk's end line isn't stored, so a fixed 40-line read would otherwise (a) bleed into an
+        # unrelated later def, handing this chunk a symbol boost that isn't its own, and/or (b) — if
+        # we capped at neighbouring *candidates* — truncate a chunk below its own overlapping span.
+        # Bounding at the next real chunk gives each source line to exactly one chunk: the one whose
+        # 5-line snippet will actually show it.
+        starts_by_file: dict[str, list[int]] = {}
+        for path in {c["path"] for c in candidates}:
+            starts_by_file[path] = self._chunk_starts(project_root_real, path)
+
+        lex = [0.0] * n
+        boost = [0.0] * n
+        for i, c in enumerate(candidates):
+            lines = self._read_chunk(root / c["path"], c["line"])
+            if lines is None:
+                c["snippet"] = "[file not found]"
+                text = ""
+            else:
+                # snippet keeps the existing 5-line preview (bleed-tolerant, unchanged); the
+                # lexical/boost text is capped at the next stored chunk start in this file.
+                c["snippet"] = "".join(lines[:_SNIPPET_LINES]).rstrip()
+                nxt = next((s for s in starts_by_file.get(c["path"], ()) if s > c["line"]), None)
+                text = "".join(lines if nxt is None else lines[: max(1, nxt - c["line"])])
+            lex[i] = self._lexical_score(query_tokens, text)
+            boost[i] = self._symbol_boost(query, text)
+
+        # lexical rank: highest lexical score first, ties broken by the semantic rank (index i)
+        order_by_lex = sorted(range(n), key=lambda i: (-lex[i], i))
+        rank_lex = [0] * n
+        for pos, i in enumerate(order_by_lex):
+            rank_lex[i] = pos
+
+        # sem rank is the candidate's position i (they arrive in cosine order)
+        fused = [1.0 / (_RRF_K + i) + 1.0 / (_RRF_K + rank_lex[i]) + boost[i] for i in range(n)]
+        order = sorted(range(n), key=lambda i: (-fused[i], i))  # fused desc, tie → better cosine
+        return [candidates[i] for i in order]
+
+    def search(
+        self,
+        query: str,
+        project_root: str,
+        k: int = 10,
+        cosine_floor: float = 0.25,
+        rerank: str = "on",
+        rerank_candidates: int = 30,
+    ) -> list[dict]:
+        if not query or not query.strip():
+            return []
+
+        # Guard the public API: a direct caller passing a non-int/zero/negative k or
+        # rerank_candidates must degrade, not raise (never-raise). Config callers already pre-cast.
+        k = _pos_int(k, 10)
+        rerank_candidates = _pos_int(rerank_candidates, 30)
+        # Accept the documented "off" plus the obvious falsy spellings (incl. the Python bool
+        # ``False`` → ``"false"``), so a direct caller isn't silently left with rerank on.
+        do_rerank = str(rerank).strip().lower() not in ("off", "false", "0", "no", "none")
+        # With rerank on, retrieve a wider candidate set by cosine, then re-order it; still return
+        # the top-k. Cap only the *extra* rerank breadth (rerank_candidates) — the DoS guard — while
+        # always honoring k, so enabling rerank never returns fewer results than the pure-cosine
+        # path would for the same k (even a large k > cap).
+        candidate_limit = max(k, min(rerank_candidates, _RERANK_CANDIDATES_CAP)) if do_rerank else k
+
+        project_root_real = os.path.realpath(project_root)
+
+        # Scope the KNN to THIS project — a search in repo B must never surface repo A's
+        # chunks (wrong-file, wrong-content hits) from the shared cache.
+        if self._row_count(project_root_real) == 0:
+            return []
+
+        query_vec = self._embed_query(query)
+        if query_vec is None:
+            return []
+
+        try:
+            conn = self.db.conn()
+            rows = conn.execute(
+                """
+                SELECT
+                    ce.chunk_id,
+                    ch.chunk_start,
+                    ch.file_path,
+                    vec_distance_cosine(ce.embedding, ?) AS dist
+                FROM code_embeddings ce
+                JOIN chunk_hashes ch ON ce.chunk_id = ch.chunk_id
+                WHERE ch.project_root = ?
+                ORDER BY dist
+                LIMIT ?
+                """,
+                (query_vec, project_root_real, candidate_limit),
+            ).fetchall()
+        except Exception as exc:
+            logger.warning("KNN query failed: %s", exc)
+            return []
+
+        root = Path(project_root)
+
+        # Floor-gated candidate set, in cosine order. The cosine_floor stays on the *semantic*
+        # candidates (not the fused score), so rerank can only re-order what pure cosine already
+        # judged good enough — quality can't regress below the pre-0.7 path.
+        candidates: list[dict] = []
+        for row in rows:
+            try:
+                score = 1.0 - float(row["dist"])
+                if score < cosine_floor:
+                    continue
+                candidates.append({
+                    "path": str(row["file_path"]),
+                    "line": int(row["chunk_start"]),
+                    "score": round(score, 6),
+                })
+            except Exception as exc:
+                logger.debug("candidate row processing failed: %s", exc)
+                continue
+
+        if not candidates:
+            return []
+
+        # Rerank re-reads chunk text; if anything goes wrong, fall back to the cosine order so a
+        # rerank fault can never do worse than today (and never raises).
+        if do_rerank and len(candidates) > 1:
+            try:
+                candidates = self._rerank(query, root, project_root_real, candidates)
+            except Exception as exc:
+                logger.warning("rerank failed, using cosine order: %s", exc)
+
+        results: list[dict] = []
+        for c in candidates[:k]:
+            # _rerank caches the snippet on each candidate it read; fill it in otherwise.
+            snippet = c.get("snippet")
+            if snippet is None:
+                snippet = self._read_snippet(root / c["path"], c["line"])
+            results.append({
+                "path": c["path"],
+                "line": c["line"],
+                "snippet": snippet,
+                "score": c["score"],
+            })
+        return results

{codecortex-0.6.0 → codecortex-0.7.0}/tests/test_config.py

@@ -36,6 +36,17 @@ def test_chunk_strategy_enum_validation():
     assert _coerce({"chunk_strategy": "ast"})["chunk_strategy"] == "syntax"   # unknown → default
 
 
+def test_rerank_config_validation():
+    assert _coerce({})["rerank"] == "on"                                      # default
+    assert _coerce({})["rerank_candidates"] == 30                            # default
+    assert _coerce({"rerank": "off"})["rerank"] == "off"                     # escape hatch kept
+    assert _coerce({"rerank": "ON"})["rerank"] == "on"                       # case-normalized
+    assert _coerce({"rerank": "maybe"})["rerank"] == "on"                    # unknown → default
+    assert _coerce({"rerank_candidates": 50})["rerank_candidates"] == 50     # positive kept
+    assert _coerce({"rerank_candidates": 0})["rerank_candidates"] == 30      # non-positive → default
+    assert _coerce({"rerank_candidates": "x"})["rerank_candidates"] == 30    # non-int → default
+
+
 def test_valid_values_are_preserved():
     c = _coerce({"backend": "graph", "cosine_floor": 0.5, "window": 40, "reindex": "never"})
     assert c["backend"] == "graph"

{codecortex-0.6.0 → codecortex-0.7.0}/tests/test_doctor.py

@@ -50,6 +50,18 @@ def test_render_shows_marks_and_remediation():
     assert "n/a" in text  # lsp repo-indexed
     assert "codeintel index /repo" in text  # remediation surfaced (two-line fix:)
     assert "1 / 3 engines ready" in text
+    assert "codeintel setup" in text  # actionable tip footer surfaces when not healthy
+
+
+def test_setup_tip_footer_only_when_unhealthy():
+    base = {"project_root": "/repo", "deep": False,
+            "engines": {"semantic": {"engine": "semantic", "status": "ok", "installed": True,
+                                     "runnable": True, "repo_indexed": True, "detail": "ok",
+                                     "remediation": None}}}
+    healthy = doctor.render_doctor_text({**base, "summary": {"ready": 1, "total": 1, "healthy": True}})
+    unhealthy = doctor.render_doctor_text({**base, "summary": {"ready": 0, "total": 1, "healthy": False}})
+    assert "codeintel setup" not in healthy   # no noise when everything is ready
+    assert "codeintel setup" in unhealthy      # actionable guidance only when something is missing
 
 
 # --------------------------------------------------------------------------- #

{codecortex-0.6.0 → codecortex-0.7.0}/tests/test_integration.py

@@ -146,7 +146,9 @@ def test_empty_project_reports_no_index(tmp_path, monkeypatch):
 
 def test_config_cosine_floor_reaches_searcher(tmp_path, monkeypatch):
     import codeintel.providers.semantic as sem
-    (tmp_path / ".codeintel.toml").write_text("cosine_floor = 0.99\n")
+    (tmp_path / ".codeintel.toml").write_text(
+        'cosine_floor = 0.99\nrerank = "off"\nrerank_candidates = 7\n'
+    )
     (tmp_path / "code.py").write_text("def f():\n    return 1\n")
     monkeypatch.setattr(sem, "_DB_PATH", tmp_path / "s.db")
     monkeypatch.setattr(sem, "_DEPS_OK", True)
@@ -154,14 +156,18 @@ def test_config_cosine_floor_reaches_searcher(tmp_path, monkeypatch):
     captured = {}
     real_search = Searcher.search
 
-    def spy(self, query, project_root, k=10, cosine_floor=0.25):
-        captured["floor"] = cosine_floor
-        return real_search(self, query, project_root, k=k, cosine_floor=cosine_floor)
+    def spy(self, query, project_root, k=10, cosine_floor=0.25, rerank="on", rerank_candidates=30):
+        captured.update(floor=cosine_floor, rerank=rerank, rerank_candidates=rerank_candidates)
+        return real_search(self, query, project_root, k=k, cosine_floor=cosine_floor,
+                           rerank=rerank, rerank_candidates=rerank_candidates)
 
     monkeypatch.setattr(Searcher, "search", spy)
     with patch("fastembed.TextEmbedding", _FakeTextEmbedding):
         SemanticProvider().build_result("search", "f", [], 0, str(tmp_path))
-    assert captured.get("floor") == 0.99   # the config value reached the searcher
+    # every semantic knob from .codeintel.toml must reach the searcher, not just cosine_floor
+    assert captured.get("floor") == 0.99
+    assert captured.get("rerank") == "off"
+    assert captured.get("rerank_candidates") == 7
 
 
 # --- Bug #8: HTTP transport 400s a malformed Content-Length (never crashes) --

codecortex-0.7.0/tests/test_rerank.py

@@ -0,0 +1,256 @@
+"""Hybrid reranking (0.7.0, Phase 2 of docs/roadmap-semantic.md).
+
+Cosine alone under-ranks exact lexical/symbol matches; a lexical + RRF fusion rerank fixes the
+ordering with no model dependency. These tests use a content-addressed fake embedder that places
+each fixture chunk at a known angle, so the *cosine* order is controlled and we can prove the
+rerank flips a literal-symbol match above a closer-but-lexically-distant one — and that
+``rerank="off"`` restores the pure-cosine order.
+"""
+from __future__ import annotations
+
+import math
+from unittest.mock import patch
+
+import numpy as np
+
+from codeintel.indexer import Indexer
+from codeintel.searcher import Searcher, _SYMBOL_BOOST, _tokenize
+from codeintel.semantic_db import SemanticDb
+
+
+def _angle_vec(theta_deg: float) -> np.ndarray:
+    v = np.zeros(384, dtype=np.float32)
+    r = math.radians(theta_deg)
+    v[0], v[1] = math.cos(r), math.sin(r)
+    return v
+
+
+class _AngleEmbedding:
+    """Deterministic, content-addressed: each fixture chunk / the query is placed at a fixed angle,
+    so cosine similarity between any two is a known cos(Δθ). Lets a test dictate the cosine order."""
+
+    def __init__(self, model_name=None):
+        pass
+
+    def embed(self, texts):
+        out = []
+        for t in texts:
+            if "def parse_config" in t:
+                out.append(_angle_vec(60))   # literal-symbol chunk — FARTHER from the query
+            elif "def load_settings" in t:
+                out.append(_angle_vec(20))    # semantic chunk — NEARER the query
+            else:
+                out.append(_angle_vec(0))     # the query (and anything else) at the origin
+        return out
+
+
+def _mem_db() -> SemanticDb:
+    db = SemanticDb(":memory:")
+    db.init()
+    return db
+
+
+_FIXTURE = (
+    "def parse_config(path):\n"                                       # line 0  (literal symbol)
+    "    return open(path).read()\n"                                  # line 1
+    "\n"                                                              # line 2
+    "def load_settings():\n"                                         # line 3  (semantic match)
+    '    """Read and parse configuration values from disk."""\n'      # line 4
+    "    return {}\n"                                                 # line 5
+)
+
+
+# --------------------------------------------------------------------------- core acceptance
+
+def test_exact_symbol_ranks_literal_match_above_semantic(tmp_path):
+    (tmp_path / "mod.py").write_text(_FIXTURE)
+    with patch("fastembed.TextEmbedding", _AngleEmbedding):
+        db = _mem_db()
+        Indexer(db).index(str(tmp_path))
+        s = Searcher(db)
+        on = s.search("parse_config", str(tmp_path), rerank="on")
+        off = s.search("parse_config", str(tmp_path), rerank="off")
+
+    # pure cosine puts the nearer (semantic) chunk first; rerank pulls the literal symbol up
+    assert [r["line"] for r in off][0] == 3, "cosine order should lead with the semantic chunk"
+    assert [r["line"] for r in on][0] == 0, "rerank should lead with the literal parse_config"
+    assert {r["line"] for r in on} == {0, 3}, "rerank reorders, it does not drop candidates"
+
+
+def test_rerank_off_matches_pure_cosine(tmp_path):
+    (tmp_path / "mod.py").write_text(_FIXTURE)
+    with patch("fastembed.TextEmbedding", _AngleEmbedding):
+        db = _mem_db()
+        Indexer(db).index(str(tmp_path))
+        s = Searcher(db)
+        off = s.search("parse_config", str(tmp_path), rerank="off")
+    # cosine order: load_settings (20°, score .94) then parse_config (60°, score .5), scores intact
+    assert [(r["line"], r["score"]) for r in off] == [
+        (3, round(1 - (1 - math.cos(math.radians(20))), 6)),
+        (0, round(1 - (1 - math.cos(math.radians(60))), 6)),
+    ]
+
+
+def test_no_lexical_signal_preserves_cosine_order(tmp_path):
+    # a query with zero token overlap must leave the cosine order untouched (rerank reorders only
+    # on real lexical signal — otherwise the lexical rank mirrors the semantic rank)
+    (tmp_path / "mod.py").write_text(_FIXTURE)
+    with patch("fastembed.TextEmbedding", _AngleEmbedding):
+        db = _mem_db()
+        Indexer(db).index(str(tmp_path))
+        s = Searcher(db)
+        on = s.search("zzz_totally_unrelated_qqq", str(tmp_path), rerank="on")
+        off = s.search("zzz_totally_unrelated_qqq", str(tmp_path), rerank="off")
+    assert [r["line"] for r in on] == [r["line"] for r in off]
+
+
+# --------------------------------------------------------------------------- never-raise / bounds
+
+def test_rerank_never_raises_on_missing_file(tmp_path):
+    (tmp_path / "gone.py").write_text("def target():\n    return 1\n")
+    (tmp_path / "keep.py").write_text("def other():\n    return 2\n")
+    with patch("fastembed.TextEmbedding", _AngleEmbedding):
+        db = _mem_db()
+        Indexer(db).index(str(tmp_path))
+        (tmp_path / "gone.py").unlink()  # deleted AFTER indexing → DB row survives, file is gone
+        s = Searcher(db)
+        res = s.search("target", str(tmp_path), rerank="on", cosine_floor=-1.0)
+
+    assert isinstance(res, list) and res  # no crash; still returns
+    gone = [r for r in res if r["path"] == "gone.py"]
+    assert gone and gone[0]["snippet"] == "[file not found]"  # missing file → sentinel, scored 0
+
+
+def test_rerank_does_not_bleed_boost_into_neighbouring_def(tmp_path):
+    # load_settings (nearer cosine) sits right above parse_config; a naive 40-line read from
+    # load_settings' start bleeds into `def parse_config` and would steal its symbol boost. The
+    # next-stored-chunk bound must confine each candidate to its own span, so the chunk that
+    # actually DEFINES the queried symbol wins despite its worse cosine.
+    (tmp_path / "mod.py").write_text(
+        "def load_settings():\n"           # 0  near cosine, does NOT define the query symbol
+        "    return 1\n"                   # 1
+        "\n"                               # 2
+        "def parse_config(path):\n"        # 3  far cosine, but DEFINES parse_config
+        "    return open(path).read()\n"   # 4
+    )
+    with patch("fastembed.TextEmbedding", _AngleEmbedding):
+        db = _mem_db()
+        Indexer(db).index(str(tmp_path))
+        res = Searcher(db).search("parse_config", str(tmp_path), rerank="on")
+    assert [r["line"] for r in res][0] == 3, "the chunk that actually defines the symbol must win"
+
+
+def test_search_survives_bad_param_types(tmp_path):
+    (tmp_path / "mod.py").write_text(_FIXTURE)
+    with patch("fastembed.TextEmbedding", _AngleEmbedding):
+        db = _mem_db()
+        Indexer(db).index(str(tmp_path))
+        # non-int k / rerank_candidates must degrade to defaults, never raise (public API)
+        res = Searcher(db).search("parse_config", str(tmp_path), k="oops", rerank_candidates="lots")
+    assert isinstance(res, list) and {r["line"] for r in res} == {0, 3}
+
+
+def test_rerank_falsey_values_treated_as_off(tmp_path):
+    (tmp_path / "mod.py").write_text(_FIXTURE)
+    with patch("fastembed.TextEmbedding", _AngleEmbedding):
+        db = _mem_db()
+        Indexer(db).index(str(tmp_path))
+        s = Searcher(db)
+        off = s.search("parse_config", str(tmp_path), rerank="off")
+        false_bool = s.search("parse_config", str(tmp_path), rerank=False)
+    assert [r["line"] for r in false_bool] == [r["line"] for r in off] == [3, 0]
+
+
+def test_candidate_limit_is_capped(tmp_path, monkeypatch):
+    import codeintel.searcher as sm
+    for i in range(6):
+        (tmp_path / f"f{i}.py").write_text(f"def fn{i}():\n    return {i}\n")
+    monkeypatch.setattr(sm, "_RERANK_CANDIDATES_CAP", 2)  # tiny cap for the test
+    with patch("fastembed.TextEmbedding", _AngleEmbedding):
+        db = _mem_db()
+        Indexer(db).index(str(tmp_path))
+        s = Searcher(db)
+        calls = {"n": 0}
+        orig = s._read_chunk
+
+        def counting(fp, cs):
+            calls["n"] += 1
+            return orig(fp, cs)
+
+        monkeypatch.setattr(s, "_read_chunk", counting)
+        s.search("fn1", str(tmp_path), k=1, rerank="on", rerank_candidates=1000)
+    assert calls["n"] <= 2, "candidate set must be clamped to the cap regardless of config"
+
+
+def test_large_k_is_not_shrunk_by_cap(tmp_path, monkeypatch):
+    # the cap bounds only the extra rerank breadth (rerank_candidates); a large k must still be
+    # honored, so rerank-on never returns fewer results than pure cosine would for that k
+    import codeintel.searcher as sm
+    for i in range(6):
+        (tmp_path / f"f{i}.py").write_text(f"def fn{i}():\n    return {i}\n")
+    monkeypatch.setattr(sm, "_RERANK_CANDIDATES_CAP", 2)  # tiny cap, but k below must still win
+    with patch("fastembed.TextEmbedding", _AngleEmbedding):
+        db = _mem_db()
+        Indexer(db).index(str(tmp_path))
+        res = Searcher(db).search(
+            "fn1", str(tmp_path), k=5, rerank="on", rerank_candidates=1, cosine_floor=-1.0
+        )
+    assert len(res) == 5, "k must be honored despite a smaller cap/rerank_candidates"
+
+
+def test_rerank_reads_are_bounded(tmp_path, monkeypatch):
+    for i in range(40):
+        (tmp_path / f"f{i}.py").write_text(f"def fn{i}():\n    return {i}\n")
+    with patch("fastembed.TextEmbedding", _AngleEmbedding):
+        db = _mem_db()
+        Indexer(db).index(str(tmp_path))
+        s = Searcher(db)
+        calls = {"n": 0}
+        orig = s._read_chunk
+
+        def counting(fp, cs):
+            calls["n"] += 1
+            return orig(fp, cs)
+
+        monkeypatch.setattr(s, "_read_chunk", counting)
+        s.search("fn1", str(tmp_path), k=3, rerank="on", rerank_candidates=5)
+
+    assert calls["n"] <= 5, "rerank must read at most rerank_candidates chunks, not the whole repo"
+
+
+# --------------------------------------------------------------------------- scoring units
+
+def test_tokenize_splits_camel_and_snake():
+    t = _tokenize("parseConfig load_settings HTTPServer")
+    assert {"parseconfig", "parse", "config",
+            "load_settings", "load", "settings",
+            "httpserver", "http", "server"} <= t
+
+
+def test_lexical_score_is_query_term_fraction():
+    q = _tokenize("parse_config")  # {parse_config, parse, config}
+    assert Searcher._lexical_score(q, "def parse_config(): pass") == 1.0
+    assert Searcher._lexical_score(q, "def unrelated(): pass") == 0.0
+    assert Searcher._lexical_score(q, "please parse the text") == 1 / 3  # only 'parse' overlaps
+    assert Searcher._lexical_score(set(), "anything") == 0.0
+
+
+def test_symbol_boost_prefers_def_name():
+    assert Searcher._symbol_boost("parse_config", "def parse_config(x): ...") == _SYMBOL_BOOST
+    assert Searcher._symbol_boost("parse_config", "y = parse_config()") == _SYMBOL_BOOST * 0.5
+    assert Searcher._symbol_boost("parse_config", "unrelated text") == 0.0
+    assert Searcher._symbol_boost("two words", "def two(): ...") == 0.0     # multi-word → no boost
+    assert Searcher._symbol_boost("", "def f(): ...") == 0.0
+    # case-insensitive, to match the lexical score's lowercasing
+    assert Searcher._symbol_boost("Parse_Config", "def parse_config(x): ...") == _SYMBOL_BOOST
+
+
+def test_rerank_default_is_on(tmp_path):
+    # the method default must match the config default so a direct caller gets reranking too
+    (tmp_path / "mod.py").write_text(_FIXTURE)
+    with patch("fastembed.TextEmbedding", _AngleEmbedding):
+        db = _mem_db()
+        Indexer(db).index(str(tmp_path))
+        default = Searcher(db).search("parse_config", str(tmp_path))          # no rerank kwarg
+        explicit = Searcher(db).search("parse_config", str(tmp_path), rerank="on")
+    assert [r["line"] for r in default] == [r["line"] for r in explicit] == [0, 3]

{codecortex-0.6.0 → codecortex-0.7.0}/tests/test_semantic_provider.py

@@ -1,6 +1,7 @@
 """Tests for SemanticProvider — covers all 4 USER_STORIES (7 test cases)."""
 from __future__ import annotations
 
+import hashlib
 import struct
 from unittest.mock import MagicMock, patch
 
@@ -33,6 +34,53 @@ def _mem_db() -> SemanticDb:
     return db
 
 
+class _ContentEmbedding:
+    """Content-addressed stub: the vector depends on the text, so a chunk whose content changed
+    yields a different embedding — lets a test detect whether a re-embed actually persisted
+    (the constant-vector _FakeTextEmbedding above cannot)."""
+    def __init__(self, model_name=None):
+        pass
+
+    def embed(self, texts):
+        out = []
+        for t in list(texts):
+            h = int(hashlib.sha256(t.encode()).hexdigest()[:8], 16)
+            v = np.zeros(384, dtype=np.float32)
+            v[0] = (h % 100000) / 100000.0
+            v[1] = 1.0
+            out.append(v)
+        return out
+
+
+def test_changed_chunk_reembeds_at_stable_chunk_id(tmp_path):
+    # Regression: sqlite-vec's vec0 ignores INSERT OR REPLACE and raises UNIQUE, so re-embedding a
+    # chunk whose content changed but whose chunk_id (def start line) is stable must go
+    # DELETE-then-INSERT — else the stale vector is kept forever. Syntax chunking exposes this on
+    # every function-body edit (the def line, hence the chunk_id, doesn't move).
+    mod = tmp_path / "mod.py"
+    mod.write_text("def f():\n    return 1\n")
+
+    def _vec(db):
+        row = db.conn().execute(
+            "SELECT embedding FROM code_embeddings WHERE chunk_id LIKE ?", ("%:mod.py:0",)
+        ).fetchone()
+        return row[0] if row else None
+
+    with patch("fastembed.TextEmbedding", _ContentEmbedding):
+        db = _mem_db()
+        idx = Indexer(db)
+        assert idx.index(str(tmp_path)) > 0
+        before = _vec(db)
+        assert before is not None
+
+        mod.write_text("def f():\n    return 999999\n")  # body changed, def line (chunk_id) stable
+        assert idx.index(str(tmp_path)) > 0, "the changed chunk must re-embed, not silently fail"
+        after = _vec(db)
+
+    assert after is not None
+    assert before != after, "the stored embedding must UPDATE for changed content on a vec0 table"
+
+
 # ---------------------------------------------------------------------------
 # Story 4: availability check
 # ---------------------------------------------------------------------------

codecortex-0.6.0/src/codeintel/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.6.0"

codecortex-0.6.0/src/codeintel/searcher.py

@@ -1,143 +0,0 @@
-from __future__ import annotations
-
-import logging
-import os
-import struct
-from pathlib import Path
-from typing import TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from codeintel.semantic_db import SemanticDb
-
-logger = logging.getLogger(__name__)
-
-_SNIPPET_LINES = 5
-
-
-class Searcher:
-    def __init__(
-        self,
-        db: SemanticDb,
-        model_name: str = "BAAI/bge-small-en-v1.5",
-    ) -> None:
-        self.db = db
-        self.model_name = model_name
-        self._embedder = None
-
-    def _get_embedder(self):
-        if self._embedder is None:
-            from fastembed import TextEmbedding
-            self._embedder = TextEmbedding(model_name=self.model_name)
-        return self._embedder
-
-    def _embed_query(self, query: str) -> bytes | None:
-        try:
-            embedder = self._get_embedder()
-            vecs = list(embedder.embed([query]))
-            if not vecs:
-                return None
-            vec = vecs[0]
-            return struct.pack(f"{len(vec)}f", *vec)
-        except Exception as exc:
-            logger.warning("query embedding failed: %s", exc)
-            return None
-
-    def _row_count(self, project_root_real: str) -> int:
-        try:
-            conn = self.db.conn()
-            row = conn.execute(
-                "SELECT COUNT(*) FROM chunk_hashes WHERE project_root = ?",
-                (project_root_real,),
-            ).fetchone()
-            return row[0] if row else 0
-        except Exception as exc:
-            logger.warning("rowcount check failed: %s", exc)
-            return 0
-
-    def has_index(self, project_root: str) -> bool:
-        """True when this project has at least one indexed chunk — lets the provider
-        distinguish 'nothing indexed yet' (no-index) from 'matches below floor'."""
-        return self._row_count(os.path.realpath(project_root)) > 0
-
-    def _read_snippet(self, file_path: Path, chunk_start: int) -> str:
-        try:
-            with open(file_path, encoding="utf-8", errors="replace") as f:
-                lines = f.readlines()
-            snippet_lines = lines[chunk_start: chunk_start + _SNIPPET_LINES]
-            return "".join(snippet_lines).rstrip()
-        except FileNotFoundError:
-            return "[file not found]"
-        except Exception as exc:
-            logger.debug("snippet read failed for %s:%d: %s", file_path, chunk_start, exc)
-            return "[file not found]"
-
-    def search(
-        self,
-        query: str,
-        project_root: str,
-        k: int = 10,
-        cosine_floor: float = 0.25,
-    ) -> list[dict]:
-        if not query or not query.strip():
-            return []
-
-        k = max(1, k)
-        project_root_real = os.path.realpath(project_root)
-
-        # Scope the KNN to THIS project — a search in repo B must never surface repo A's
-        # chunks (wrong-file, wrong-content hits) from the shared cache.
-        if self._row_count(project_root_real) == 0:
-            return []
-
-        query_vec = self._embed_query(query)
-        if query_vec is None:
-            return []
-
-        try:
-            conn = self.db.conn()
-            rows = conn.execute(
-                """
-                SELECT
-                    ce.chunk_id,
-                    ch.chunk_start,
-                    ch.file_path,
-                    vec_distance_cosine(ce.embedding, ?) AS dist
-                FROM code_embeddings ce
-                JOIN chunk_hashes ch ON ce.chunk_id = ch.chunk_id
-                WHERE ch.project_root = ?
-                ORDER BY dist
-                LIMIT ?
-                """,
-                (query_vec, project_root_real, k),
-            ).fetchall()
-        except Exception as exc:
-            logger.warning("KNN query failed: %s", exc)
-            return []
-
-        root = Path(project_root)
-        results: list[dict] = []
-
-        for row in rows:
-            try:
-                dist = float(row["dist"])
-                score = 1.0 - dist
-                if score < cosine_floor:
-                    continue
-
-                chunk_start = int(row["chunk_start"])
-                rel_path = str(row["file_path"])
-                abs_path = root / rel_path
-
-                snippet = self._read_snippet(abs_path, chunk_start)
-
-                results.append({
-                    "path": rel_path,
-                    "line": chunk_start,
-                    "snippet": snippet,
-                    "score": round(score, 6),
-                })
-            except Exception as exc:
-                logger.debug("result row processing failed: %s", exc)
-                continue
-
-        return results