codebase-navigator 0.1.0__py3-none-any.whl

codebase_navigator/config.py

@@ -0,0 +1,102 @@
+"""Configuration, schema definitions, and shared utilities for codebase-navigator."""
+
+from __future__ import annotations
+
+import contextlib
+import hashlib
+import os
+from pathlib import Path
+from typing import Iterator
+
+# Ensure portable user caching & offline environment variables before importing ML libraries
+_cache_base = Path(os.environ.get("XDG_CACHE_HOME") or (Path.home() / ".cache"))
+if "TRITON_CACHE_DIR" not in os.environ or "/root/" in os.environ["TRITON_CACHE_DIR"]:
+    os.environ["TRITON_CACHE_DIR"] = str(_cache_base / "triton")
+if "TORCH_HOME" not in os.environ or "/root/" in os.environ["TORCH_HOME"]:
+    os.environ["TORCH_HOME"] = str(_cache_base / "torch")
+if "HF_HOME" not in os.environ or "/root/" in os.environ["HF_HOME"]:
+    os.environ["HF_HOME"] = str(_cache_base / "huggingface")
+
+os.environ.setdefault("HF_HUB_OFFLINE", "1")
+os.environ.setdefault("TRANSFORMERS_OFFLINE", "1")
+os.environ.setdefault("HF_HUB_DISABLE_TELEMETRY", "1")
+os.environ.setdefault("HF_HUB_ENABLE_HF_TRANSFER", "0")
+os.environ.setdefault("TRANSFORMERS_VERBOSITY", "error")
+os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
+
+import pyarrow as pa
+
+EMBEDDING_MODEL_NAME = "all-MiniLM-L6-v2"
+VECTOR_DIM = 384
+
+DOC_SCHEMA = pa.schema([
+    pa.field("id", pa.string()),
+    pa.field("path", pa.string()),
+    pa.field("abs_path", pa.string()),
+    pa.field("doc_type", pa.string()),  # "markdown" or "code_doc"
+    pa.field("title", pa.string()),
+    pa.field("start_line", pa.int32()),
+    pa.field("end_line", pa.int32()),
+    pa.field("content", pa.string()),
+    pa.field("vector", pa.list_(pa.float32(), VECTOR_DIM)),
+])
+
+CODE_EXTENSIONS = {
+    ".py", ".rs", ".go", ".ts", ".tsx", ".js", ".jsx",
+    ".c", ".cpp", ".cc", ".cxx", ".h", ".hpp", ".hh",
+    ".nix", ".sh", ".bash", ".zsh", ".sql", ".java",
+    ".kt", ".scala", ".rb", ".php", ".cs", ".swift",
+    ".lua", ".zig", ".nim", ".elm", ".ex", ".exs",
+    ".erl", ".hrl", ".hs", ".ml", ".mli", ".pl", ".pm",
+    ".r", ".jl", ".clj", ".cljs", ".lisp", ".scm",
+}
+
+DOC_EXTENSIONS = {
+    ".md", ".markdown", ".rst", ".adoc", ".org",
+}
+
+IGNORE_DIR_NAMES = {
+    ".git", ".hg", ".svn",
+    "node_modules", "target", "build", "dist",
+    ".venv", "venv", "env", ".direnv",
+    "__pycache__", ".pytest_cache", ".mypy_cache", ".ruff_cache",
+    ".cache", ".devel-index", ".devel-tools", ".codebase-navigator", ".dagster_home", "pipeline-cache",
+}
+
+
+@contextlib.contextmanager
+def silence_stdio() -> Iterator[None]:
+    """Silence low-level stdout/stderr (e.g. from C-level libraries and model loaders)."""
+    try:
+        null_fd = os.open(os.devnull, os.O_RDWR)
+        save_stdout = os.dup(1)
+        save_stderr = os.dup(2)
+        os.dup2(null_fd, 1)
+        os.dup2(null_fd, 2)
+        try:
+            yield
+        finally:
+            os.dup2(save_stdout, 1)
+            os.dup2(save_stderr, 2)
+            os.close(null_fd)
+            os.close(save_stdout)
+            os.close(save_stderr)
+    except Exception:
+        yield
+
+
+def get_cache_dir(folder: Path, custom_index_dir: str | None = None) -> Path:
+    """Determine the persistence directory for vector indexes and tools metadata."""
+    if custom_index_dir:
+        cdir = Path(custom_index_dir).resolve()
+        cdir.mkdir(parents=True, exist_ok=True)
+        return cdir
+
+    target = folder / ".codebase-navigator"
+    target.mkdir(parents=True, exist_ok=True)
+    return target
+
+
+def get_socket_path(folder: Path, custom_index_dir: str | None = None) -> Path:
+    """Return the Unix Domain Socket path used for IPC with cn watch."""
+    return get_cache_dir(folder, custom_index_dir) / "watch.sock"

codebase_navigator/extractor.py

@@ -0,0 +1,281 @@
+"""Semantic content extraction from Markdown files and source code comments/docstrings."""
+
+from __future__ import annotations
+
+import ast
+import hashlib
+from pathlib import Path
+import re
+from typing import Any
+
+
+class DocExtractor:
+    """Extracts semantic chunks from Markdown documentation and code docstrings/comments."""
+
+    def __init__(self, base_folder: Path):
+        self.base_folder = base_folder
+
+    def extract_markdown(self, path: Path) -> list[dict[str, Any]]:
+        """Extract markdown sections and term definitions preserving headers and line ranges."""
+        try:
+            text = path.read_text(encoding="utf-8", errors="replace")
+        except Exception:
+            return []
+
+        try:
+            rel_path = str(path.relative_to(self.base_folder))
+        except ValueError:
+            rel_path = str(path)
+
+        lines = text.splitlines()
+        chunks: list[dict[str, Any]] = []
+
+        header_re = re.compile(r"^(#{1,6})\s+(.+)$")
+        term_re = re.compile(r"^(?:-\s+)?(?:\*\*([^*]+)\*\*|__([^_]+)__)\s*[-—:]\s*(.+)$")
+
+        current_headers: list[str] = []
+        chunk_start = 1
+        chunk_lines: list[str] = []
+
+        def flush_chunk(end_line: int):
+            nonlocal chunk_start, chunk_lines
+            content = "\n".join(chunk_lines).strip()
+            if len(content) >= 20:
+                title = " > ".join(current_headers) if current_headers else f"{path.name} (top)"
+                chunk_id = hashlib.sha256(f"{rel_path}:{chunk_start}:{title}".encode("utf-8")).hexdigest()[:16]
+                chunks.append({
+                    "id": chunk_id,
+                    "path": rel_path,
+                    "abs_path": str(path.resolve()),
+                    "doc_type": "markdown",
+                    "title": title,
+                    "start_line": chunk_start,
+                    "end_line": end_line,
+                    "content": f"# {title}\n\n{content}",
+                })
+            chunk_lines = []
+
+        # 1. Section-level chunking
+        for idx, line in enumerate(lines, start=1):
+            match = header_re.match(line)
+            if match:
+                level = len(match.group(1))
+                heading_text = match.group(2).strip()
+
+                if chunk_lines:
+                    flush_chunk(idx - 1)
+
+                while len(current_headers) >= level:
+                    current_headers.pop()
+                current_headers.append(heading_text)
+                chunk_start = idx
+                chunk_lines.append(line)
+            else:
+                chunk_lines.append(line)
+
+        if chunk_lines:
+            flush_chunk(len(lines))
+
+        # 2. Granular term/definition chunking (for glossaries, bullet rules, specifications)
+        t_start = 0
+        t_name = ""
+        t_lines: list[str] = []
+        for idx, line in enumerate(lines, start=1):
+            s = line.strip()
+            match = term_re.match(s)
+            if match:
+                if t_lines and t_name and len("\n".join(t_lines)) >= 30:
+                    c_id = hashlib.sha256(f"{rel_path}:{t_start}:{t_name}".encode("utf-8")).hexdigest()[:16]
+                    chunks.append({
+                        "id": c_id,
+                        "path": rel_path,
+                        "abs_path": str(path.resolve()),
+                        "doc_type": "markdown",
+                        "title": f"{path.name} > {t_name}",
+                        "start_line": t_start,
+                        "end_line": idx - 1,
+                        "content": f"# {path.name} > {t_name}\n\n" + "\n".join(t_lines),
+                    })
+                t_name = match.group(1) or match.group(2)
+                t_start = idx
+                t_lines = [line]
+            elif t_lines:
+                if s == "" and len(t_lines) >= 2 and not any(
+                    lines[min(len(lines) - 1, idx)].strip().startswith(p) for p in ["-", "*", ">"]
+                ):
+                    if len("\n".join(t_lines)) >= 30:
+                        c_id = hashlib.sha256(f"{rel_path}:{t_start}:{t_name}".encode("utf-8")).hexdigest()[:16]
+                        chunks.append({
+                            "id": c_id,
+                            "path": rel_path,
+                            "abs_path": str(path.resolve()),
+                            "doc_type": "markdown",
+                            "title": f"{path.name} > {t_name}",
+                            "start_line": t_start,
+                            "end_line": idx,
+                            "content": f"# {path.name} > {t_name}\n\n" + "\n".join(t_lines),
+                        })
+                    t_name = ""
+                    t_lines = []
+                else:
+                    t_lines.append(line)
+
+        if t_lines and t_name and len("\n".join(t_lines)) >= 30:
+            c_id = hashlib.sha256(f"{rel_path}:{t_start}:{t_name}".encode("utf-8")).hexdigest()[:16]
+            chunks.append({
+                "id": c_id,
+                "path": rel_path,
+                "abs_path": str(path.resolve()),
+                "doc_type": "markdown",
+                "title": f"{path.name} > {t_name}",
+                "start_line": t_start,
+                "end_line": len(lines),
+                "content": f"# {path.name} > {t_name}\n\n" + "\n".join(t_lines),
+            })
+
+        return chunks
+
+    def extract_code_doc(self, path: Path) -> list[dict[str, Any]]:
+        """Extract docstrings and comment blocks from source files."""
+        if path.suffix == ".py":
+            return self._extract_python(path)
+        return self._extract_generic_comments(path)
+
+    def _extract_python(self, path: Path) -> list[dict[str, Any]]:
+        try:
+            source = path.read_text(encoding="utf-8", errors="replace")
+        except Exception:
+            return []
+
+        try:
+            rel_path = str(path.relative_to(self.base_folder))
+        except ValueError:
+            rel_path = str(path)
+
+        chunks: list[dict[str, Any]] = []
+        try:
+            tree = ast.parse(source, filename=str(path))
+        except (SyntaxError, ValueError):
+            return self._extract_generic_comments(path)
+
+        lines = source.splitlines()
+
+        # Module docstring
+        module_doc = ast.get_docstring(tree)
+        if module_doc and len(module_doc.strip()) > 10:
+            doc_lines = len(module_doc.splitlines())
+            chunk_id = hashlib.sha256(f"{rel_path}:1:module".encode("utf-8")).hexdigest()[:16]
+            chunks.append({
+                "id": chunk_id,
+                "path": rel_path,
+                "abs_path": str(path.resolve()),
+                "doc_type": "code_doc",
+                "title": f"{path.name} (module docstring)",
+                "start_line": 1,
+                "end_line": min(len(lines), doc_lines + 5),
+                "content": f"Module {rel_path}:\n{module_doc.strip()}",
+            })
+
+        # Functions & Classes
+        for node in ast.walk(tree):
+            if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
+                doc = ast.get_docstring(node)
+                start_l = getattr(node, "lineno", 1)
+                end_l = getattr(node, "end_lineno", start_l + 10)
+                def_line = lines[start_l - 1].strip() if start_l <= len(lines) else ""
+
+                decorators = []
+                for dec in getattr(node, "decorator_list", []):
+                    d_line = getattr(dec, "lineno", None)
+                    if d_line and d_line <= len(lines):
+                        decorators.append(lines[d_line - 1].strip())
+
+                kind = "class" if isinstance(node, ast.ClassDef) else "function"
+                node_name = node.name
+
+                text_parts = []
+                if decorators:
+                    text_parts.append("\n".join(decorators))
+                text_parts.append(def_line)
+                if doc:
+                    text_parts.append(doc.strip())
+
+                text = "\n".join(text_parts).strip()
+                if len(text) > 30:
+                    chunk_id = hashlib.sha256(f"{rel_path}:{start_l}:{node_name}".encode("utf-8")).hexdigest()[:16]
+                    chunks.append({
+                        "id": chunk_id,
+                        "path": rel_path,
+                        "abs_path": str(path.resolve()),
+                        "doc_type": "code_doc",
+                        "title": f"{path.name} > {node_name} ({kind})",
+                        "start_line": start_l,
+                        "end_line": end_l,
+                        "content": f"{rel_path} ({kind} {node_name}):\n{text}",
+                    })
+
+        return chunks
+
+    def _extract_generic_comments(self, path: Path) -> list[dict[str, Any]]:
+        try:
+            source = path.read_text(encoding="utf-8", errors="replace")
+        except Exception:
+            return []
+
+        try:
+            rel_path = str(path.relative_to(self.base_folder))
+        except ValueError:
+            rel_path = str(path)
+
+        lines = source.splitlines()
+        chunks: list[dict[str, Any]] = []
+
+        comment_block: list[str] = []
+        block_start = 1
+
+        for idx, line in enumerate(lines, start=1):
+            s = line.strip()
+            is_comment = False
+            comment_content = ""
+
+            if s.startswith(("//", "#", "--", ";", "/*", "*")):
+                is_comment = True
+                comment_content = re.sub(r"^(\/\/|#|--|;|\/\*|\*)\s*", "", s)
+
+            if is_comment and comment_content:
+                if not comment_block:
+                    block_start = idx
+                comment_block.append(comment_content)
+            else:
+                if len(comment_block) >= 3:
+                    text = "\n".join(comment_block).strip()
+                    if len(text) > 40:
+                        chunk_id = hashlib.sha256(f"{rel_path}:{block_start}:comment".encode("utf-8")).hexdigest()[:16]
+                        chunks.append({
+                            "id": chunk_id,
+                            "path": rel_path,
+                            "abs_path": str(path.resolve()),
+                            "doc_type": "code_doc",
+                            "title": f"{path.name}: comment (L{block_start}-{idx-1})",
+                            "start_line": block_start,
+                            "end_line": idx - 1,
+                            "content": f"{rel_path} (L{block_start}-{idx-1}):\n{text}",
+                        })
+                comment_block = []
+
+        if len(comment_block) >= 3:
+            text = "\n".join(comment_block).strip()
+            if len(text) > 40:
+                chunk_id = hashlib.sha256(f"{rel_path}:{block_start}:comment".encode("utf-8")).hexdigest()[:16]
+                chunks.append({
+                    "id": chunk_id,
+                    "path": rel_path,
+                    "abs_path": str(path.resolve()),
+                    "doc_type": "code_doc",
+                    "title": f"{path.name}: comment (L{block_start}-{len(lines)})",
+                    "start_line": block_start,
+                    "end_line": len(lines),
+                    "content": f"{rel_path} (L{block_start}-{len(lines)}):\n{text}",
+                })
+
+        return chunks

codebase_navigator/index.py

@@ -0,0 +1,279 @@
+"""LanceDB vector index management and hybrid semantic search."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+import re
+from typing import TYPE_CHECKING, Any
+
+import lancedb
+
+if TYPE_CHECKING:
+    from sentence_transformers import SentenceTransformer
+
+from .config import (
+    DOC_SCHEMA,
+    EMBEDDING_MODEL_NAME,
+    get_cache_dir,
+    silence_stdio,
+)
+from .extractor import DocExtractor
+from .tags import get_available_files
+
+COMMON_STOPWORDS = {
+    "what", "is", "a", "an", "the", "in", "on", "of", "for", "to",
+    "and", "or", "how", "why", "where", "which", "does", "do", "can",
+}
+
+
+class VectorIndex:
+    """LanceDB index manager with SentenceTransformer embeddings and hybrid re-ranking."""
+
+    def __init__(self, folder: Path, custom_index_dir: str | None = None):
+        self.folder = folder
+        self.cache_dir = get_cache_dir(folder, custom_index_dir)
+        self.db_dir = self.cache_dir / "lancedb"
+        self.meta_file = self.cache_dir / "files_meta.json"
+        self.db = lancedb.connect(str(self.db_dir))
+        self._model: SentenceTransformer | None = None
+        self._ensure_table()
+
+    @property
+    def model(self) -> SentenceTransformer:
+        if self._model is None:
+            from sentence_transformers import SentenceTransformer
+            with silence_stdio():
+                try:
+                    self._model = SentenceTransformer(EMBEDDING_MODEL_NAME, local_files_only=True)
+                except Exception:
+                    self._model = SentenceTransformer(EMBEDDING_MODEL_NAME)
+        return self._model
+
+    def _ensure_table(self):
+        try:
+            self.table = self.db.open_table("documents")
+        except Exception:
+            self.table = self.db.create_table("documents", schema=DOC_SCHEMA, mode="create")
+
+    def load_meta(self) -> dict[str, dict[str, Any]]:
+        if self.meta_file.exists():
+            try:
+                return json.loads(self.meta_file.read_text(encoding="utf-8"))
+            except Exception:
+                return {}
+        return {}
+
+    def save_meta(self, meta: dict[str, dict[str, Any]]):
+        self.meta_file.write_text(json.dumps(meta, indent=2), encoding="utf-8")
+
+    def sync(self, force: bool = False) -> tuple[int, int, int]:
+        """Incrementally sync all available files into LanceDB.
+
+        Returns (files_updated, chunks_indexed, files_pruned).
+        """
+        extractor = DocExtractor(self.folder)
+        code_files, doc_files = get_available_files(self.folder)
+        all_files = code_files + doc_files
+
+        meta = {} if force else self.load_meta()
+        current_rel_paths: set[str] = set()
+
+        files_to_index: list[tuple[Path, list[dict[str, Any]]]] = []
+        all_chunks_to_embed: list[dict[str, Any]] = []
+
+        for fpath in all_files:
+            try:
+                rel_p = str(fpath.relative_to(self.folder))
+            except ValueError:
+                rel_p = str(fpath)
+            current_rel_paths.add(rel_p)
+
+            stat = fpath.stat()
+            last_meta = meta.get(rel_p)
+            if (
+                not force
+                and last_meta
+                and last_meta.get("mtime") == stat.st_mtime
+                and last_meta.get("size") == stat.st_size
+            ):
+                continue
+
+            if fpath.suffix.lower() in {".md", ".markdown", ".rst", ".adoc", ".org"}:
+                chunks = extractor.extract_markdown(fpath)
+            else:
+                chunks = extractor.extract_code_doc(fpath)
+
+            if chunks:
+                files_to_index.append((fpath, chunks))
+                all_chunks_to_embed.extend(chunks)
+
+        # Prune deleted files
+        deleted_paths = [p for p in meta.keys() if p not in current_rel_paths]
+        if deleted_paths:
+            for p in deleted_paths:
+                try:
+                    escaped_p = p.replace('"', '\\"')
+                    self.table.delete(f'path = "{escaped_p}"')
+                except Exception:
+                    pass
+                meta.pop(p, None)
+
+        if force:
+            try:
+                self.db.drop_table("documents")
+            except Exception:
+                pass
+            self.table = self.db.create_table("documents", schema=DOC_SCHEMA, mode="create")
+        else:
+            for fpath, _ in files_to_index:
+                try:
+                    rel_p = str(fpath.relative_to(self.folder))
+                except ValueError:
+                    rel_p = str(fpath)
+                escaped_p = rel_p.replace('"', '\\"')
+                try:
+                    self.table.delete(f'path = "{escaped_p}"')
+                except Exception:
+                    pass
+
+        if all_chunks_to_embed:
+            texts = [c["content"] for c in all_chunks_to_embed]
+            embeddings = self.model.encode(
+                texts,
+                batch_size=256,
+                show_progress_bar=False,
+                normalize_embeddings=True,
+            )
+            for chunk, vec in zip(all_chunks_to_embed, embeddings):
+                chunk["vector"] = vec.tolist()
+
+            self.table.add(all_chunks_to_embed)
+
+        for fpath, chunks in files_to_index:
+            try:
+                rel_p = str(fpath.relative_to(self.folder))
+            except ValueError:
+                rel_p = str(fpath)
+            stat = fpath.stat()
+            meta[rel_p] = {
+                "mtime": stat.st_mtime,
+                "size": stat.st_size,
+                "chunks": len(chunks),
+            }
+
+        self.save_meta(meta)
+        return (len(files_to_index), len(all_chunks_to_embed), len(deleted_paths))
+
+    def update_single_file(self, fpath: Path) -> int:
+        """Incrementally update one file in LanceDB."""
+        try:
+            rel_p = str(fpath.relative_to(self.folder))
+        except ValueError:
+            rel_p = str(fpath)
+
+        escaped_p = rel_p.replace('"', '\\"')
+        try:
+            self.table.delete(f'path = "{escaped_p}"')
+        except Exception:
+            pass
+
+        if not fpath.exists():
+            meta = self.load_meta()
+            meta.pop(rel_p, None)
+            self.save_meta(meta)
+            return 0
+
+        extractor = DocExtractor(self.folder)
+        if fpath.suffix.lower() in {".md", ".markdown", ".rst", ".adoc", ".org"}:
+            chunks = extractor.extract_markdown(fpath)
+        else:
+            chunks = extractor.extract_code_doc(fpath)
+
+        if chunks:
+            texts = [c["content"] for c in chunks]
+            embeddings = self.model.encode(texts, batch_size=64, show_progress_bar=False, normalize_embeddings=True)
+            for chunk, vec in zip(chunks, embeddings):
+                chunk["vector"] = vec.tolist()
+            self.table.add(chunks)
+
+        meta = self.load_meta()
+        stat = fpath.stat()
+        meta[rel_p] = {
+            "mtime": stat.st_mtime,
+            "size": stat.st_size,
+            "chunks": len(chunks),
+        }
+        self.save_meta(meta)
+        return len(chunks)
+
+    def search(
+        self,
+        query: str,
+        limit: int = 5,
+        doc_type: str | None = None,
+    ) -> list[dict[str, Any]]:
+        """Hybrid semantic vector search with keyword & title match re-ranking."""
+        q_vec = self.model.encode(query, normalize_embeddings=True).tolist()
+        fetch_limit = max(limit * 4, 20)
+        search_query = self.table.search(q_vec).metric("cosine").limit(fetch_limit)
+
+        if doc_type and doc_type != "all":
+            norm_type = "markdown" if doc_type in ["md", "markdown"] else "code_doc" if doc_type in ["code", "code_doc"] else doc_type
+            search_query = search_query.where(f'doc_type = "{norm_type}"')
+
+        try:
+            raw_results = search_query.to_list()
+        except Exception:
+            raw_results = []
+
+        if not raw_results:
+            return []
+
+        # Extract significant query terms
+        clean_terms = [
+            w.lower()
+            for w in re.findall(r"[A-Za-z0-9_]+", query)
+            if len(w) >= 2 and w.lower() not in COMMON_STOPWORDS
+        ]
+        clean_phrase = " ".join(clean_terms)
+
+        scored_results: list[dict[str, Any]] = []
+        for r in raw_results:
+            dist = r.get("_distance", 0.0)
+            base_score = max(0.0, min(1.0, 1.0 - (dist / 2.0)))
+            score = base_score
+
+            title_lower = r.get("title", "").lower()
+            content_lower = r.get("content", "").lower()
+            dtype = r.get("doc_type", "")
+
+            # 1. Exact phrase match in title
+            if clean_phrase and clean_phrase in title_lower:
+                score += 0.12
+            # 2. Individual term matches in title
+            for term in clean_terms:
+                if term in title_lower:
+                    score += 0.04
+
+            # 3. Term definitions or documentation boost
+            if dtype == "markdown":
+                score += 0.04
+
+            score = min(0.99, score)
+
+            scored_results.append({
+                "score": round(score, 3),
+                "base_score": round(base_score, 3),
+                "path": r["path"],
+                "abs_path": r["abs_path"],
+                "doc_type": r["doc_type"],
+                "title": r["title"],
+                "start_line": r["start_line"],
+                "end_line": r["end_line"],
+                "content": r["content"],
+            })
+
+        # Re-sort by boosted hybrid score
+        scored_results.sort(key=lambda x: x["score"], reverse=True)
+        return scored_results[:limit]