codebase-navigator 0.1.0__py3-none-any.whl

codebase_navigator/ipc.py

@@ -0,0 +1,221 @@
+"""Unix domain socket IPC server and client for fast semantic querying via cn watch."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+import socket
+import socketserver
+import threading
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from .index import VectorIndex
+
+
+class _IPCRequestHandler(socketserver.StreamRequestHandler):
+    """Handles single client connection requests over Unix Domain Socket."""
+
+    server: _IPCUnixStreamServer
+
+    def handle(self):
+        for line in self.rfile:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                req = json.loads(line.decode("utf-8"))
+                action = req.get("action")
+                if action == "ping":
+                    resp = {"status": "ok", "pong": True}
+                elif action == "search":
+                    query = req.get("query", "")
+                    limit = int(req.get("limit", 5))
+                    doc_type = req.get("type", "all")
+                    with self.server.lock:
+                        results = self.server.index.search(query, limit=limit, doc_type=doc_type)
+                    resp = {"status": "ok", "results": results}
+                elif action == "status":
+                    with self.server.lock:
+                        meta = self.server.index.load_meta()
+                    chunk_count = sum(m.get("chunks", 0) for m in meta.values())
+                    resp = {
+                        "status": "ok",
+                        "files_count": len(meta),
+                        "chunk_count": chunk_count,
+                        "cache_dir": str(self.server.index.cache_dir),
+                    }
+                else:
+                    resp = {"status": "error", "error": f"Unknown action: {action}"}
+            except Exception as e:
+                resp = {"status": "error", "error": str(e)}
+
+            response_bytes = json.dumps(resp).encode("utf-8") + b"\n"
+            self.wfile.write(response_bytes)
+            self.wfile.flush()
+
+
+class _IPCUnixStreamServer(socketserver.ThreadingUnixStreamServer):
+    daemon_threads = True
+    allow_reuse_address = True
+
+    def __init__(self, server_address: str, RequestHandlerClass, index: VectorIndex, lock: threading.Lock):
+        self.index = index
+        self.lock = lock
+        super().__init__(server_address, RequestHandlerClass)
+
+
+class IPCServer:
+    """Unix Domain Socket server hosting in-memory index for fast search queries."""
+
+    def __init__(self, socket_path: Path, index: VectorIndex, lock: threading.Lock | None = None):
+        self.socket_path = socket_path
+        self.index = index
+        self.lock = lock or threading.Lock()
+        self._server: _IPCUnixStreamServer | None = None
+        self._thread: threading.Thread | None = None
+
+    def start(self):
+        """Start socket server in a background thread.
+
+        Raises RuntimeError if another active daemon is already listening on this socket.
+        Cleans up stale socket files from prior crashes automatically.
+        """
+        self.socket_path.parent.mkdir(parents=True, exist_ok=True)
+        if self.socket_path.exists():
+            # Check if another process is actively listening on this socket
+            status = ping_socket(self.socket_path, timeout=0.5)
+            if status is not None:
+                raise RuntimeError(
+                    f"Another cn watch instance is already running on {self.socket_path}"
+                )
+            # Socket file exists but no process is listening -> stale socket from prior crash
+            try:
+                self.socket_path.unlink()
+            except OSError:
+                pass
+
+        self._server = _IPCUnixStreamServer(
+            str(self.socket_path),
+            _IPCRequestHandler,
+            index=self.index,
+            lock=self.lock,
+        )
+        self._thread = threading.Thread(target=self._server.serve_forever, daemon=True)
+        self._thread.start()
+
+    def stop(self):
+        """Stop socket server and clean up socket file."""
+        if self._server:
+            try:
+                self._server.shutdown()
+                self._server.server_close()
+            except Exception:
+                pass
+            self._server = None
+        if self.socket_path.exists():
+            try:
+                self.socket_path.unlink()
+            except OSError:
+                pass
+
+
+def query_socket(
+    socket_path: Path,
+    query: str,
+    limit: int = 5,
+    doc_type: str = "all",
+    timeout: float = 3.0,
+) -> list[dict[str, Any]] | None:
+    """Query the running cn watch daemon via Unix Domain Socket.
+
+    Returns search results if successful, or None if socket is unavailable/unresponsive.
+    Automatically unlinks stale socket files from dead daemons.
+    """
+    if not socket_path.exists():
+        return None
+
+    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    sock.settimeout(timeout)
+    try:
+        sock.connect(str(socket_path))
+        req = {
+            "action": "search",
+            "query": query,
+            "limit": limit,
+            "type": doc_type,
+        }
+        payload = json.dumps(req).encode("utf-8") + b"\n"
+        sock.sendall(payload)
+
+        # Read response line
+        buffer = b""
+        while b"\n" not in buffer:
+            chunk = sock.recv(4096)
+            if not chunk:
+                break
+            buffer += chunk
+
+        if not buffer:
+            return None
+
+        line = buffer.split(b"\n")[0]
+        resp = json.loads(line.decode("utf-8"))
+        if resp.get("status") == "ok":
+            return resp.get("results")
+        return None
+    except ConnectionRefusedError:
+        # Socket file exists but no process is listening -> stale socket from prior crash
+        try:
+            socket_path.unlink()
+        except OSError:
+            pass
+        return None
+    except (OSError, socket.error, json.JSONDecodeError, TimeoutError):
+        return None
+    finally:
+        sock.close()
+
+
+def ping_socket(socket_path: Path, timeout: float = 0.5) -> dict[str, Any] | None:
+    """Check if cn watch daemon is active and return its status info.
+
+    Automatically unlinks stale socket files from dead daemons.
+    """
+    if not socket_path.exists():
+        return None
+
+    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    sock.settimeout(timeout)
+    try:
+        sock.connect(str(socket_path))
+        req = {"action": "status"}
+        payload = json.dumps(req).encode("utf-8") + b"\n"
+        sock.sendall(payload)
+
+        buffer = b""
+        while b"\n" not in buffer:
+            chunk = sock.recv(4096)
+            if not chunk:
+                break
+            buffer += chunk
+
+        if not buffer:
+            return None
+
+        line = buffer.split(b"\n")[0]
+        resp = json.loads(line.decode("utf-8"))
+        if resp.get("status") == "ok":
+            return resp
+        return None
+    except ConnectionRefusedError:
+        # Socket file exists but no process is listening -> stale socket from prior crash
+        try:
+            socket_path.unlink()
+        except OSError:
+            pass
+        return None
+    except (OSError, socket.error, json.JSONDecodeError, TimeoutError):
+        return None
+    finally:
+        sock.close()

codebase_navigator/tags.py

@@ -0,0 +1,198 @@
+"""Git-aware ctags indexing and symbol lookup."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+import re
+import subprocess
+from typing import Any
+
+from .config import CODE_EXTENSIONS, DOC_EXTENSIONS, IGNORE_DIR_NAMES
+
+
+def get_available_files(folder: Path) -> tuple[list[Path], list[Path]]:
+    """Discover all Git-tracked and unignored source and documentation files."""
+    code_files: list[Path] = []
+    doc_files: list[Path] = []
+
+    # 1. Try Git-based discovery
+    try:
+        res = subprocess.run(
+            ["git", "ls-files", "--cached", "--others", "--exclude-standard"],
+            cwd=folder,
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        for line in res.stdout.splitlines():
+            fpath = folder / line.strip()
+            if fpath.is_file():
+                ext = fpath.suffix.lower()
+                if ext in CODE_EXTENSIONS:
+                    code_files.append(fpath)
+                elif ext in DOC_EXTENSIONS:
+                    doc_files.append(fpath)
+        return sorted(code_files), sorted(doc_files)
+    except Exception:
+        pass
+
+    # 2. Fallback to filesystem walk
+    for root, dirs, files in os.walk(folder):
+        dirs[:] = [d for d in dirs if d not in IGNORE_DIR_NAMES and not d.startswith(".")]
+        for file in files:
+            if file.startswith("."):
+                continue
+            fpath = Path(root) / file
+            ext = fpath.suffix.lower()
+            if ext in CODE_EXTENSIONS:
+                code_files.append(fpath)
+            elif ext in DOC_EXTENSIONS:
+                doc_files.append(fpath)
+
+    return sorted(code_files), sorted(doc_files)
+
+
+class TagsManager:
+    """Manages generation, updates, and symbol lookups from .tags files."""
+
+    def __init__(self, folder: Path):
+        self.folder = folder
+        self.tag_file = folder / ".tags"
+
+    def generate(self) -> tuple[bool, str]:
+        """Generate or regenerate .tags for all source files in the folder."""
+        code_files, _ = get_available_files(self.folder)
+        if not code_files:
+            return False, "No source files found to index"
+
+        rel_paths = []
+        for p in code_files:
+            try:
+                rel_paths.append(str(p.relative_to(self.folder)))
+            except ValueError:
+                rel_paths.append(str(p))
+
+        input_data = "\n".join(rel_paths) + "\n"
+        cmd = [
+            "ctags",
+            "-L", "-",
+            "-f", str(self.tag_file),
+            "--fields=+n+K",
+            "--sort=yes",
+        ]
+
+        try:
+            res = subprocess.run(
+                cmd,
+                input=input_data,
+                cwd=self.folder,
+                capture_output=True,
+                text=True,
+                check=False,
+            )
+            if res.returncode != 0:
+                return False, f"ctags error ({res.returncode}): {res.stderr.strip()}"
+            if self.tag_file.exists():
+                size_mb = self.tag_file.stat().st_size / (1024 * 1024)
+                return True, f"Indexed {len(code_files)} source files ({size_mb:.2f} MB)"
+            return False, "Tag file was not created"
+        except FileNotFoundError:
+            return False, "ctags binary not found on PATH"
+        except Exception as e:
+            return False, f"Error generating tags: {e}"
+
+    def find_tag_file(self) -> Path | None:
+        """Find .tags in folder or climb parent directories."""
+        curr = self.folder
+        for parent in [curr, *curr.parents]:
+            tf = parent / ".tags"
+            if tf.exists():
+                return tf
+        return self.tag_file if self.tag_file.exists() else None
+
+    def lookup_symbol(
+        self,
+        pattern: str,
+        exact: bool = False,
+        limit: int = 20,
+    ) -> list[dict[str, Any]]:
+        """Look up symbols matching a regex or exact string."""
+        tag_files: list[Path] = []
+        tf = self.find_tag_file()
+        if tf:
+            tag_files.append(tf)
+
+        # Also search child folders' .tags
+        for p in self.folder.rglob(".tags"):
+            if p not in tag_files:
+                tag_files.append(p)
+
+        if not tag_files:
+            return []
+
+        results: list[dict[str, Any]] = []
+        seen_keys: set[tuple[str, str, int]] = set()
+
+        for tag_file in tag_files:
+            try:
+                regex = re.compile(pattern if not exact else f"^{re.escape(pattern)}$", re.IGNORECASE)
+                with open(tag_file, "r", encoding="utf-8", errors="replace") as f:
+                    for line in f:
+                        if line.startswith("!_"):
+                            continue
+                        parts = line.split("\t")
+                        if not parts:
+                            continue
+                        sym = parts[0]
+                        if regex.search(sym):
+                            parsed = self._parse_tag_line(line, tag_file.parent)
+                            if parsed:
+                                key = (parsed["symbol"], parsed["path"], parsed["line"])
+                                if key not in seen_keys:
+                                    seen_keys.add(key)
+                                    results.append(parsed)
+                                    if len(results) >= limit:
+                                        return results
+            except Exception:
+                pass
+
+        return results
+
+    def _parse_tag_line(self, line: str, base_dir: Path) -> dict[str, Any] | None:
+        if not line or line.startswith("!_"):
+            return None
+        parts = line.rstrip("\r\n").split("\t")
+        if len(parts) < 3:
+            return None
+        sym = parts[0]
+        fpath = parts[1]
+        pattern_or_line = parts[2]
+
+        kind = "symbol"
+        line_no = 1
+        for field in parts[3:]:
+            if field.startswith("line:"):
+                try:
+                    line_no = int(field[5:])
+                except ValueError:
+                    pass
+            elif field.startswith("kind:"):
+                kind = field[5:]
+            elif len(field) == 1:
+                kind = field
+
+        abs_p = (base_dir / fpath).resolve()
+        try:
+            rel_p = str(abs_p.relative_to(self.folder))
+        except ValueError:
+            rel_p = fpath
+
+        return {
+            "symbol": sym,
+            "path": rel_p,
+            "abs_path": str(abs_p),
+            "line": line_no,
+            "kind": kind,
+            "preview": pattern_or_line.strip("/^$"),
+        }

codebase_navigator/watcher.py

@@ -0,0 +1,134 @@
+"""Filesystem watcher for live ctags and LanceDB vector re-indexing."""
+
+from __future__ import annotations
+
+from pathlib import Path
+import threading
+import time
+
+from watchfiles import Change, DefaultFilter, watch
+
+from .config import CODE_EXTENSIONS, DOC_EXTENSIONS, IGNORE_DIR_NAMES, get_socket_path
+from .index import VectorIndex
+from .ipc import IPCServer, ping_socket
+from .tags import TagsManager
+
+
+class SourceFilter(DefaultFilter):
+    """Filter that includes only recognized source/doc extensions and ignores build/git artifacts."""
+
+    def __init__(self, folder: Path):
+        super().__init__(
+            ignore_dirs=tuple(IGNORE_DIR_NAMES),
+            ignore_entity_patterns=(r"^\..*", r".*\.tags$", r".*tags$", r".*\.sock$"),
+        )
+        self.folder = folder
+
+    def __call__(self, change: Change, path: str) -> bool:
+        if not super().__call__(change, path):
+            return False
+
+        p = Path(path)
+        if p.name.startswith("."):
+            return False
+
+        for part in p.parts:
+            if part in IGNORE_DIR_NAMES or part.startswith("."):
+                return False
+
+        ext = p.suffix.lower()
+        return ext in CODE_EXTENSIONS or ext in DOC_EXTENSIONS
+
+
+class DirectoryWatcher:
+    """Watches folder, serves IPC Unix socket, and keeps .tags and LanceDB vector embeddings up to date."""
+
+    def __init__(
+        self,
+        folder: Path,
+        debounce_ms: int = 1000,
+        custom_index_dir: str | None = None,
+    ):
+        self.folder = folder
+        self.debounce_ms = debounce_ms
+        self.tags_mgr = TagsManager(folder)
+        self.index = VectorIndex(folder, custom_index_dir)
+        self.socket_path = get_socket_path(folder, custom_index_dir)
+        self.index_lock = threading.Lock()
+        self.ipc_server = IPCServer(self.socket_path, self.index, lock=self.index_lock)
+
+    def start(self):
+        """Run blocking live watcher loop and IPC server."""
+        # Early check: if an active instance is already watching, report and exit gracefully
+        if self.socket_path.exists():
+            active = ping_socket(self.socket_path, timeout=0.5)
+            if active is not None:
+                print(f"⚠️  Another cn watch instance is already running for: {self.folder}")
+                print(f"   Active socket: {self.socket_path}")
+                return
+
+        print(f"🚀 Starting cn watch for: {self.folder}")
+        print("  Performing initial sync...")
+        ok, msg = self.tags_mgr.generate()
+        print(f"  .tags: {msg}")
+
+        with self.index_lock:
+            u_files, u_chunks, p_files = self.index.sync()
+        print(f"  LanceDB: {u_files} files updated ({u_chunks} chunks), {p_files} pruned.")
+        print(f"  Index location: {self.index.cache_dir}")
+
+        self.ipc_server.start()
+        print(f"  🔌 IPC Socket: {self.socket_path}")
+        print("👀 Watching for file changes (Ctrl+C to stop)...\n")
+
+        source_filter = SourceFilter(self.folder)
+
+        try:
+            for changes in watch(
+                self.folder,
+                watch_filter=source_filter,
+                debounce=self.debounce_ms,
+                step=50,
+            ):
+                code_changed = False
+                doc_changed = False
+                affected_files: list[Path] = []
+
+                for change_type, change_path in changes:
+                    p = Path(change_path)
+                    ext = p.suffix.lower()
+                    if ext in CODE_EXTENSIONS:
+                        code_changed = True
+                        affected_files.append(p)
+                    elif ext in DOC_EXTENSIONS:
+                        doc_changed = True
+                        affected_files.append(p)
+
+                if not affected_files:
+                    continue
+
+                t0 = time.time()
+                ts_str = time.strftime("%H:%M:%S")
+
+                # Update tags if source code changed
+                if code_changed:
+                    ok, msg = self.tags_mgr.generate()
+                    print(f"[{ts_str}] 🏷️  Tags updated: {msg}")
+
+                # Update LanceDB embeddings incrementally
+                total_chunks = 0
+                for fpath in affected_files:
+                    try:
+                        with self.index_lock:
+                            n_chunks = self.index.update_single_file(fpath)
+                        total_chunks += n_chunks
+                    except Exception as e:
+                        print(f"[{ts_str}] ⚠️ Error updating embeddings for {fpath.name}: {e}")
+
+                dt = (time.time() - t0) * 1000
+                print(f"[{ts_str}] ⚡ Synced {len(affected_files)} file(s) ({total_chunks} chunks) in {dt:.0f}ms")
+
+        except KeyboardInterrupt:
+            print("\n👋 cn watch stopped.")
+        finally:
+            self.ipc_server.stop()

codebase_navigator-0.1.0.dist-info/METADATA

@@ -0,0 +1,114 @@
+Metadata-Version: 2.5
+Name: codebase-navigator
+Version: 0.1.0
+Summary: Fast Git-aware ctags indexing, live watchers, and LanceDB semantic search for developers
+Author: Nigel Choi
+License: MIT
+Requires-Python: >=3.11
+Requires-Dist: lancedb>=0.17.0
+Requires-Dist: numpy>=1.24.0
+Requires-Dist: pyarrow>=14.0.0
+Requires-Dist: sentence-transformers>=3.0.0
+Requires-Dist: torch
+Requires-Dist: watchfiles>=0.24.0
+Description-Content-Type: text/markdown
+
+# codebase-navigator
+
+Developer tools for ultra-fast codebase navigation, Git-aware ctags indexing, live watchers, and LanceDB semantic search.
+
+## Quick Start
+
+### Using nix
+
+Run `cn` instantly without installing:
+
+```bash
+# Run directly from GitHub
+nix run github:9gel/codebase-navigator -- sync
+
+# Run help or any command
+nix run github:9gel/codebase-navigator -- --help
+nix run github:9gel/codebase-navigator -- search "authentication flow"
+```
+
+### Using uvx
+
+You can run `cn` using `uvx` (the tool runner from [uv](https://docs.astral.sh/uv/)):
+
+```bash
+# Directly from the Git repository:
+uvx --from git+https://github.com/9gel/codebase-navigator.git cn --help
+
+# Once published to PyPI:
+uvx codebase-navigator --help
+```
+
+> **Note:** `cn tags` requires `universal-ctags` and `git` to be installed on your system.
+
+## Features
+
+- 🏷️ **Git-Aware `.tags` Generation**: Uses `universal-ctags` to index genuine source code while completely ignoring huge data dumps, JSON caches, `.git`, `node_modules`, and build artifacts.
+- 🧠 **LanceDB Semantic & Hybrid Search**: Vector search powered by `sentence-transformers/all-MiniLM-L6-v2` with hybrid phrase/title match boosting for markdown documentation, glossary terms, and code comments.
+- ⚡ **Strict Offline Mode**: Runs 100% locally from disk cache with zero HuggingFace network requests or unauthenticated token warnings.
+- 👀 **Live File Watcher**: Automatically re-indexes `.tags` and incrementally updates LanceDB embeddings on every save with sub-second debounce.
+- 🔗 **Clickable GitHub Markdown Links**: Returns results formatted as `[file:Lstart-Lend](file:///abs_path#Lstart-Lend)`.
+
+## Installation
+
+### Nix Flakes
+
+Add `codebase-navigator` to your `flake.nix`:
+
+```nix
+{
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+    codebase-navigator = {
+      url = "github:9gel/codebase-navigator";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+  };
+
+  outputs = { self, nixpkgs, codebase-navigator, ... }:
+    let
+      system = "x86_64-linux"; # or "aarch64-darwin", etc.
+      pkgs = nixpkgs.legacyPackages.''${system};
+    in
+    {
+      # Add to environment packages or devShells:
+      devShells.''${system}.default = pkgs.mkShell {
+        packages = [
+          codebase-navigator.packages.''${system}.default
+        ];
+      };
+    };
+}
+```
+
+Or install it to your user profile:
+
+```bash
+nix profile install github:9gel/codebase-navigator
+```
+
+## CLI Commands
+
+The unified `cn` command provides all indexing and search tools:
+
+| Command | Purpose |
+|---|---|
+| `cn search <query> [folder]` | Semantic & hybrid search in markdown docs and code comments |
+| `cn tags <symbol> [folder]` | Fast symbol definition lookup in `.tags` |
+| `cn sync [folder] [--force]` | Synchronize `.tags` and LanceDB vector embeddings |
+| `cn watch [folder]` | Live filesystem watcher for automatic re-indexing |
+| `cn status [folder]` | Inspect index and `.tags` status |
+
+## Development
+
+Requires Nix and `direnv`:
+
+```bash
+direnv allow
+uv run pytest
+```

codebase_navigator-0.1.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+codebase_navigator/__init__.py,sha256=e7d60tG-SgwP_05N7qBz7iyG2OSr0KyOUUaZ7P9SWLI,181
+codebase_navigator/ask.py,sha256=1MKE-Hk92xw3kEXlDS_rSAZbfP7J0ona0i6EvozaXOw,14115
+codebase_navigator/cli.py,sha256=UASkNs7RKf_6D89bgtGBy3mvIgureE6daQ9Bc55vlYs,9610
+codebase_navigator/config.py,sha256=UkXpAvyWUgnlCHTYPgIhVBupNX1zHcdHJPuiGT21nIw,3629
+codebase_navigator/extractor.py,sha256=kGy13gpDltum5n1rW_xuXF6Sw7lSoTYpD_-VxAOlJTM,11282
+codebase_navigator/index.py,sha256=rVl9ZVsdCmMhSCIw3ttvZy5-11tzPWdtlUM7Wgnx7PQ,9582
+codebase_navigator/ipc.py,sha256=EmlxaZ7KabIUMSXrONVfME3BB4sFJHXEYwVloa9iurM,7346
+codebase_navigator/tags.py,sha256=HH-Vk3DLKIPFukAD1daagNUg8I4HS28MydG3dA-rzyg,6620
+codebase_navigator/watcher.py,sha256=6ukf8qVUXqbeku51uykeg2SdNjFd8Z-F-jTEHDUgKYc,4882
+codebase_navigator-0.1.0.dist-info/METADATA,sha256=HbreUNr6yu-mw5V2KGDqIP8X8Ujg1XYNLx-mcsWYjKA,3485
+codebase_navigator-0.1.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+codebase_navigator-0.1.0.dist-info/entry_points.txt,sha256=K0dpqCSx-geADaO6s8rUvlZ76bVhqDA8YdBDUPX-_LA,51
+codebase_navigator-0.1.0.dist-info/RECORD,,

codebase_navigator-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.0
+Root-Is-Purelib: true
+Tag: py3-none-any

codebase_navigator-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+cn = codebase_navigator.cli:main