codespine 0.5.9__tar.gz → 0.6.0__tar.gz

{codespine-0.5.9 → codespine-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: codespine
-Version: 0.5.9
+Version: 0.6.0
 Summary: Local Java code intelligence indexer backed by a graph database
 Author: CodeSpine contributors
 License: MIT License

{codespine-0.5.9 → codespine-0.6.0}/codespine/__init__.py

@@ -1,4 +1,4 @@
 """CodeSpine package."""
 
 __all__ = ["__version__"]
-__version__ = "0.5.9"
+__version__ = "0.6.0"

{codespine-0.5.9 → codespine-0.6.0}/codespine/cli.py

@@ -77,6 +77,19 @@ def _dead_result_count(dead_result: list[dict] | None) -> int:
     return sum(1 for item in dead_result if isinstance(item, dict) and "_stats" not in item)
 
 
+def _bar(done: int, total: int, width: int = 20) -> str:
+    """Return an ASCII progress bar like [████████░░░░]  40%."""
+    if total <= 0:
+        return f"[{'░' * width}]  ---%"
+    frac = min(done / total, 1.0)
+    filled = int(width * frac)
+    return f"[{'█' * filled}{'░' * (width - filled)}] {int(frac * 100):3d}%"
+
+
+def _spinner_char() -> str:
+    return "⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏"[int(time.perf_counter() * 8) % 10]
+
+
 @click.group()
 def main() -> None:
     """CodeSpine CLI."""
@@ -88,16 +101,17 @@ def main() -> None:
 @click.option("--deep/--no-deep", default=False, show_default=True, help="Run expensive global analyses.")
 @click.option(
     "--embed/--no-embed",
-    default=False,
+    default=True,
     show_default=True,
-    help="Generate vector embeddings (slow if sentence-transformers installed; enables semantic search).",
+    help="Generate vector embeddings. Uses sentence-transformers if installed (pip install codespine[ml]), otherwise falls back to hash-based vectors.",
 )
 @click.option("--allow-running", is_flag=True, hidden=True, help="Skip MCP running check (used by MCP analyse_project tool).")
 def analyse(path: str, full: bool, deep: bool, embed: bool, allow_running: bool) -> None:
     """Index a local Java project (auto-detects workspace / Maven / Gradle layout).
 
-    By default embeddings are skipped for speed. Pass --embed to generate
-    vector embeddings for semantic search (requires sentence-transformers).
+    Embeddings are generated by default. If sentence-transformers is installed
+    (pip install codespine[ml]), high-quality semantic vectors are used; otherwise
+    a fast hash-based fallback provides basic vector search.
     """
     if not allow_running and _is_running():
         click.secho("Stop MCP first ('codespine stop') to index.", fg="yellow")
@@ -105,6 +119,17 @@ def analyse(path: str, full: bool, deep: bool, embed: bool, allow_running: bool)
 
     started = time.perf_counter()
     abs_path = os.path.abspath(path)
+
+    # Warn about hash fallback early so users know to install [ml]
+    if embed:
+        from codespine.search.vector import _load_model
+        if _load_model() is None:
+            click.secho(
+                "⚠  sentence-transformers not found — using hash-based embeddings.\n"
+                "   For better semantic search: pip install codespine[ml]\n",
+                fg="yellow",
+            )
+
     store = GraphStore(read_only=False)
     indexer = JavaIndexer(store)
 
@@ -141,7 +166,7 @@ def analyse(path: str, full: bool, deep: bool, embed: bool, allow_running: bool)
 
     # Shared progress state (reset per module)
     parse_state = {"shown": False, "indexed": 0, "total": 0, "last_ts": 0.0, "printed_zero": False}
-    call_state = {"shown": False, "count": 0, "last_ts": 0.0}
+    call_state = {"shown": False, "count": 0, "last_ts": 0.0, "started_at": 0.0}
 
     def _reset_state() -> None:
         for k in list(parse_state):
@@ -171,22 +196,28 @@ def analyse(path: str, full: bool, deep: bool, embed: bool, allow_running: bool)
             if total == 0:
                 return
             if indexed == total or (now - parse_state["last_ts"]) >= 0.2:
-                click.echo(f"\rParsing code...                {indexed}/{total}", nl=False)
+                click.echo(f"\rParsing code...   {_bar(indexed, total)} {indexed}/{total}  ", nl=False)
                 parse_state["shown"] = True
                 parse_state["last_ts"] = now
             return
         if event == "resolve_calls_start" and parse_state["shown"]:
             click.echo()
             parse_state["shown"] = False
-            _phase("Tracing calls...", "running")
+            call_state["started_at"] = now
+            _phase("Tracing calls...", "starting...")
             return
         if event == "resolve_calls_start":
-            _phase("Tracing calls...", "running")
+            call_state["started_at"] = now
+            _phase("Tracing calls...", "starting...")
             return
         if event == "resolve_calls_progress":
             call_state["count"] = int(payload.get("calls_resolved", 0))
             if (now - call_state["last_ts"]) >= 0.25:
-                click.echo(f"\rTracing calls...               {call_state['count']} resolved", nl=False)
+                elapsed_s = now - call_state["started_at"]
+                click.echo(
+                    f"\r{_spinner_char()} Tracing calls...   {call_state['count']:>6} resolved  {elapsed_s:.1f}s  ",
+                    nl=False,
+                )
                 call_state["shown"] = True
                 call_state["last_ts"] = now
             return
@@ -194,7 +225,8 @@ def analyse(path: str, full: bool, deep: bool, embed: bool, allow_running: bool)
             if call_state["shown"]:
                 click.echo()
             call_state["shown"] = False
-            _phase("Tracing calls...", f"{int(payload.get('calls_resolved', 0))} calls resolved")
+            elapsed_s = (now - call_state["started_at"]) if call_state["started_at"] else 0.0
+            _phase("Tracing calls...", f"{int(payload.get('calls_resolved', 0))} calls resolved  ({elapsed_s:.1f}s)")
             return
         if event == "resolve_types_start":
             _phase("Analyzing types...", "running")
@@ -226,11 +258,11 @@ def analyse(path: str, full: bool, deep: bool, embed: bool, allow_running: bool)
     # ── Helper for in-place progress updates ────────────────────────────
     def _live_phase(label: str, status: str) -> None:
         """Overwrite the current line with a status update."""
-        click.echo(f"\r{label:<30} {status:<50}", nl=False)
+        click.echo(f"\r{_spinner_char()} {label:<28} {status:<48}", nl=False)
 
     def _finish_phase(label: str, result: str) -> None:
         """Finalise an in-place phase line and move to the next line."""
-        click.echo(f"\r{label:<30} {result:<50}")
+        click.echo(f"\r✓ {label:<28} {result:<48}")
 
     # ── Cross-module call linking ──────────────────────────────────────
     if is_multi and len(modules_with_ids) > 1:
@@ -289,9 +321,27 @@ def analyse(path: str, full: bool, deep: bool, embed: bool, allow_running: bool)
         )
         _finish_phase(coup_label, f"{len(coupling_pairs)} coupled file pairs")
     else:
+        # Run lightweight versions of flow tracing and dead code from the call
+        # graph already built — no community detection or coupling (those are
+        # genuinely expensive).  This gives partial results without --deep.
         _phase("Detecting communities...", "skipped (large repo; rerun with --deep)")
-        _phase("Detecting execution flows...", "skipped (large repo; rerun with --deep)")
-        _phase("Finding dead code...", "skipped (large repo; rerun with --deep)")
+
+        flow_label = "Detecting execution flows..."
+        _live_phase(flow_label, "running (lightweight)")
+        try:
+            flows = trace_execution_flows(store, max_depth=3)
+        except Exception:
+            flows = []
+        _finish_phase(flow_label, f"{len(flows)} flows (lightweight; rerun with --deep for full)")
+
+        dead_label = "Finding dead code..."
+        _live_phase(dead_label, "running (lightweight)")
+        try:
+            dead = detect_dead_code(store, limit=100)
+        except Exception:
+            dead = []
+        _finish_phase(dead_label, f"{_dead_result_count(dead)} candidates (lightweight; rerun with --deep for full)")
+
         _phase("Analyzing git history...", "skipped (large repo; rerun with --deep)")
 
     vector_count = store.query_records(
@@ -311,7 +361,12 @@ def analyse(path: str, full: bool, deep: bool, embed: bool, allow_running: bool)
     edges = int(edge_count[0]["count"]) if edge_count else 0
     elapsed = time.perf_counter() - started
 
-    embed_note = "" if embed else " (no embeddings; rerun with --embed for semantic search)"
+    if not embed:
+        embed_note = " (no embeddings; rerun with --embed for semantic search)"
+    elif _load_model() is None:
+        embed_note = " (hash embeddings; pip install codespine[ml] for better search)"
+    else:
+        embed_note = ""
     module_info = f"{len(modules_with_ids)} modules/projects, " if is_multi else ""
     click.echo()
     click.secho(
@@ -319,6 +374,17 @@ def analyse(path: str, full: bool, deep: bool, embed: bool, allow_running: bool)
         fg="green",
     )
 
+    # Detect unresolved imports → hint about unindexed sibling projects
+    try:
+        unresolved = JavaIndexer.detect_unresolved_imports(store)
+        if unresolved:
+            click.echo()
+            click.secho("⚠  Unresolved imports — consider indexing these projects:", fg="yellow")
+            for pkg, samples in sorted(unresolved.items())[:8]:
+                click.echo(f"   {pkg}  (e.g. {samples[0]})")
+    except Exception:
+        pass  # best-effort
+
     # Publish a read replica so MCP and read-only CLI commands (search, stats…)
     # run against an isolated snapshot rather than competing with the write
     # process's buffer pool.  The MCP daemon detects the sentinel file and
@@ -559,7 +625,14 @@ def list_projects(as_json: bool) -> None:
 @main.command()
 @click.option("--json", "as_json", is_flag=True)
 def status(as_json: bool) -> None:
-    """Show service and database status."""
+    """Show service and database status.
+
+    Quick reference for MCP server management:
+      codespine start    – launch background MCP server
+      codespine stop     – stop background MCP server
+      codespine status   – this command
+      codespine mcp      – run MCP in foreground (stdio, for IDE integration)
+    """
     running = _is_running()
     pid = None
     if os.path.exists(SETTINGS.pid_file):
@@ -570,17 +643,35 @@ def status(as_json: bool) -> None:
             pid = None
     store = GraphStore(read_only=True)
     overlay = get_overlay_status(store)
+
+    # Check for stale PID file
+    stale_pid = pid is not None and not running
+    has_snapshot = os.path.exists(SETTINGS.db_snapshot_path)
+
     payload = {
         "running": running,
         "pid": pid,
+        "stale_pid": stale_pid,
         "pid_file": SETTINGS.pid_file,
         "db_path": SETTINGS.db_path,
         "db_size_bytes": _db_size_bytes(SETTINGS.db_path),
+        "read_replica": SETTINGS.db_snapshot_path if has_snapshot else None,
+        "read_replica_size_bytes": _db_size_bytes(SETTINGS.db_snapshot_path) if has_snapshot else 0,
         "log_file": SETTINGS.log_file,
         "overlay_dir": SETTINGS.overlay_dir,
         "overlay_projects": overlay,
     }
-    _echo_json(payload, as_json)
+    if as_json:
+        _echo_json(payload, True)
+    else:
+        _echo_json(payload, True)
+        if stale_pid:
+            click.secho(f"\n⚠  Stale PID file found (PID {pid} not running). Run 'codespine stop' to clean up.", fg="yellow")
+        if not running:
+            click.echo("\nTo start:  codespine start")
+            click.echo("For IDE:   codespine mcp  (stdio mode)")
+        else:
+            click.echo(f"\nMCP server running (PID {pid}). Stop with: codespine stop")
 
 
 @main.command("overlay-status")
@@ -707,6 +798,33 @@ def clear_index_cmd(allow_running: bool) -> None:
     click.secho(f"Cleared {len(projects)} project(s). Index is now empty.", fg="green")
 
 
+@main.command("force-reset")
+@click.option("--force", is_flag=True, help="Skip confirmation prompt.")
+def force_reset_cmd(force: bool) -> None:
+    """Emergency reset: delete ALL CodeSpine data files without touching the DB engine.
+
+    Use this when the buffer pool is exhausted and normal reset/clear commands
+    also fail with OOM.  This bypasses Kuzu entirely by removing data files
+    from disk, including the DB, read replica, overlay, meta cache, and
+    embedding cache.
+
+    After running this, restart the MCP server and re-index your projects.
+    """
+    if not force and not click.confirm(
+        "This will DELETE all CodeSpine data (DB, overlay, caches). Continue?"
+    ):
+        click.echo("Aborted.")
+        return
+    removed = GraphStore.force_delete_all_data()
+    if removed:
+        for p in removed:
+            click.echo(f"  removed: {p}")
+        click.secho(f"\nForce-reset complete. {len(removed)} path(s) removed.", fg="green")
+        click.echo("Next: restart MCP ('codespine stop && codespine start') and re-index.")
+    else:
+        click.secho("Nothing to remove — already clean.", fg="yellow")
+
+
 @main.command()
 def setup() -> None:
     """Print local setup checks and next steps."""
@@ -723,12 +841,21 @@ def setup() -> None:
             checks[mod] = True
         except Exception:
             checks[mod] = False
-    click.echo("Dependency check:")
+    click.echo("Core dependencies:")
     for mod, ok in checks.items():
         click.echo(f"  - {mod}: {'OK' if ok else 'MISSING'}")
-    click.echo("\\nRecommended:")
-    click.echo("  pip install -e .")
+    # Check optional ML dependencies
+    try:
+        from sentence_transformers import SentenceTransformer
+        click.echo("  - sentence-transformers: OK (semantic embeddings active)")
+    except ImportError:
+        click.secho("  - sentence-transformers: NOT INSTALLED (hash fallback; install for better search)", fg="yellow")
+    click.echo("\nRecommended setup:")
+    click.echo("  pip install -e '.[full]'                # core + ML + community detection")
+    click.echo("  pip install -e '.[ml]'                  # just ML embeddings")
+    click.echo("\nQuick start:")
     click.echo("  codespine analyse /path/to/java-project --full")
+    click.echo("  codespine start                         # launch MCP server")
     click.echo("  codespine search payment --json")
 
 

{codespine-0.5.9 → codespine-0.6.0}/codespine/config.py

@@ -16,8 +16,8 @@ class Settings:
     rrf_k: int = 60
     semantic_candidate_pool: int = 2000
     write_batch_size: int = 500
-    index_file_batch_size: int = 64
-    edge_write_batch_size: int = 2000
+    index_file_batch_size: int = 20
+    edge_write_batch_size: int = 500
     default_coupling_months: int = 6
     default_min_coupling_strength: float = 0.3
     default_min_cochanges: int = 3

{codespine-0.5.9 → codespine-0.6.0}/codespine/db/store.py

@@ -147,10 +147,13 @@ class GraphStore:
 
     def clear_project(self, project_id: str) -> None:
         file_recs = self.query_records("MATCH (f:File) WHERE f.project_id = $pid RETURN f.id as id", {"pid": project_id})
+        # Small batches (10 files per tx) prevent buffer pool OOM on large projects.
         for idx, rec in enumerate(file_recs, start=1):
-            self.clear_file(rec["id"])
-            if idx % 50 == 0:
+            with self.transaction():
+                self.clear_file(rec["id"])
+            if idx % 10 == 0:
                 self._recycle_conn()
+        self._recycle_conn()
         self.execute("MATCH (p:Project) WHERE p.id = $pid DETACH DELETE p", {"pid": project_id})
         self._recycle_conn()
 
@@ -502,6 +505,48 @@ class GraphStore:
         self.clear_flows()
         self.clear_coupling()
 
+    @staticmethod
+    def force_delete_all_data() -> list[str]:
+        """Delete all CodeSpine data files without touching the Kuzu engine.
+
+        This is the nuclear option for OOM recovery: when the buffer pool is
+        exhausted, normal DB writes (including reset_project / clear_project)
+        also fail.  This bypasses Kuzu entirely by removing the data files
+        from disk, allowing a fresh start.
+
+        Returns the list of paths that were removed.
+        """
+        removed: list[str] = []
+        for path in [
+            SETTINGS.db_path,
+            SETTINGS.db_snapshot_path,
+            SETTINGS.db_snapshot_path + ".updated",
+            SETTINGS.db_snapshot_path + ".tmp",
+            SETTINGS.embedding_cache_path,
+            SETTINGS.overlay_dir,
+            SETTINGS.index_meta_dir,
+        ]:
+            if not os.path.exists(path):
+                continue
+            try:
+                if os.path.isdir(path):
+                    shutil.rmtree(path, ignore_errors=True)
+                else:
+                    os.remove(path)
+                removed.append(path)
+            except OSError:
+                pass
+        # Also remove any stale WAL files next to the DB
+        for suffix in (".wal", ".lock"):
+            wal_path = SETTINGS.db_path + suffix
+            if os.path.exists(wal_path):
+                try:
+                    os.remove(wal_path)
+                    removed.append(wal_path)
+                except OSError:
+                    pass
+        return removed
+
     def rebuild_empty_db(self) -> None:
         self._recycle_conn()
         path = SETTINGS.db_path

{codespine-0.5.9 → codespine-0.6.0}/codespine/indexer/engine.py

@@ -253,6 +253,20 @@ class JavaIndexer:
                     for fid in delete_chunk:
                         self.store.clear_file(fid)
                 self.store._recycle_conn()
+
+        # Clean up stale project entries that point to the same path under a
+        # different ID (e.g. re-indexing "vision-server" directly after it was
+        # previously indexed as "vision::vision-server" from a workspace root).
+        try:
+            stale = self.store.query_records(
+                "MATCH (p:Project) WHERE p.path = $path AND p.id <> $pid RETURN p.id as id",
+                {"path": root_path, "pid": project_id},
+            )
+            for old in stale:
+                self.store.clear_project(old["id"])
+        except Exception:
+            pass  # best-effort cleanup
+
         self.store.upsert_project(project_id, root_path)
 
         for parse_chunk in self._chunked(parse_results, file_batch_size):
@@ -279,7 +293,7 @@ class JavaIndexer:
                         "hash": file_digest,
                     }
                 )
-                self._update_meta_cache_entry(meta_cache, f_id, file_path, file_digest, len(source))
+                self._update_meta_cache_entry(meta_cache, f_id, file_path, file_digest, len(source), imports=parsed.imports)
 
                 for cls in parsed.classes:
                     c_id = class_id(cls.fqcn, scope)
@@ -372,15 +386,31 @@ class JavaIndexer:
                         class_methods[c_id][method.signature] = m_id
                 files_indexed += 1
 
+            # Split writes into smaller transactions and recycle between each
+            # to prevent Kuzu WAL from exhausting the buffer pool on large
+            # incremental re-indexes (GH feedback: 1,604-file OOM).
+            if not full:
+                for clear_sub in self._chunked(file_rows, 10):
+                    with self.store.transaction():
+                        for row in clear_sub:
+                            self.store.clear_file(row["id"])
+                    self.store._recycle_conn()
             with self.store.transaction():
-                for row in file_rows:
-                    if not full:
-                        self.store.clear_file(row["id"])
                 self.store.upsert_files_batch(file_rows)
+            self.store._recycle_conn()
+            with self.store.transaction():
                 self.store.upsert_classes_batch(class_rows)
-                self.store.upsert_methods_batch(method_rows)
-                self.store.upsert_symbols_batch(symbol_rows)
             self.store._recycle_conn()
+            _METHOD_SUB_BATCH = 200
+            for method_sub in self._chunked(method_rows, _METHOD_SUB_BATCH):
+                with self.store.transaction():
+                    self.store.upsert_methods_batch(method_sub)
+                self.store._recycle_conn()
+            _SYMBOL_SUB_BATCH = 200
+            for symbol_sub in self._chunked(symbol_rows, _SYMBOL_SUB_BATCH):
+                with self.store.transaction():
+                    self.store.upsert_symbols_batch(symbol_sub)
+                self.store._recycle_conn()
 
         self._emit(progress, "resolve_calls_start")
         call_rows: list[dict] = []
@@ -697,7 +727,10 @@ class JavaIndexer:
             return
 
     @staticmethod
-    def _update_meta_cache_entry(meta_cache: dict[str, dict], fid: str, file_path: str, digest: str, size_hint: int) -> None:
+    def _update_meta_cache_entry(
+        meta_cache: dict[str, dict], fid: str, file_path: str, digest: str, size_hint: int,
+        imports: list[str] | None = None,
+    ) -> None:
         try:
             st = os.stat(file_path)
             mtime_ns = int(getattr(st, "st_mtime_ns", int(st.st_mtime * 1_000_000_000)))
@@ -705,7 +738,10 @@ class JavaIndexer:
         except OSError:
             mtime_ns = -1
             size = size_hint
-        meta_cache[fid] = {"mtime_ns": mtime_ns, "size": size, "hash": digest}
+        entry: dict = {"mtime_ns": mtime_ns, "size": size, "hash": digest}
+        if imports is not None:
+            entry["imports"] = imports
+        meta_cache[fid] = entry
 
     @staticmethod
     def _prune_meta_cache(meta_cache: dict[str, dict], current_file_ids: set[str]) -> None:
@@ -728,3 +764,76 @@ class JavaIndexer:
             return normalized.split("/src/", 1)[0]
         scope = os.path.dirname(normalized).strip()
         return scope or "."
+
+    @staticmethod
+    def detect_unresolved_imports(store) -> dict[str, list[str]]:
+        """Detect imports that reference packages not covered by any indexed project.
+
+        Returns a dict mapping unresolved base packages (e.g. "com.foo.bar")
+        to a list of sample import FQCNs.  Useful for suggesting which sibling
+        projects to index.
+
+        Only reports project-internal packages (not java.*, javax.*, org.apache.*
+        etc.).
+        """
+        # 1. Collect all indexed class FQCNs
+        try:
+            recs = store.query_records("MATCH (c:Class) RETURN c.fqcn as fqcn")
+        except Exception:
+            return {}
+        indexed_fqcns = {r["fqcn"] for r in recs if r.get("fqcn")}
+        indexed_packages = set()
+        for fqcn in indexed_fqcns:
+            parts = fqcn.rsplit(".", 1)
+            if len(parts) == 2:
+                indexed_packages.add(parts[0])
+
+        # 2. Collect all imports from overlay + any stored file data
+        # Parse imports from the parsed file metadata if available
+        meta_dir = SETTINGS.index_meta_dir
+        all_imports: set[str] = set()
+        if os.path.isdir(meta_dir):
+            for fname in os.listdir(meta_dir):
+                if not fname.endswith(".json"):
+                    continue
+                try:
+                    with open(os.path.join(meta_dir, fname), "r") as f:
+                        data = json.load(f)
+                    for fid, fmeta in data.items():
+                        for imp in fmeta.get("imports", []):
+                            all_imports.add(imp)
+                except Exception:
+                    pass
+
+        # 3. Also scan the DB for CALLS edges that reference unknown targets
+        # (lightweight — just check which classes were resolved vs not)
+
+        # 4. Filter: skip standard library / well-known third-party packages
+        _SKIP_PREFIXES = (
+            "java.", "javax.", "jakarta.",
+            "org.apache.", "org.springframework.", "org.hibernate.",
+            "org.slf4j.", "org.junit.", "org.mockito.",
+            "com.google.", "com.fasterxml.", "com.sun.",
+            "io.micrometer.", "io.netty.", "io.lettuce.",
+            "lombok.", "reactor.", "rx.",
+        )
+
+        unresolved: dict[str, list[str]] = {}
+        for imp in all_imports:
+            if any(imp.startswith(prefix) for prefix in _SKIP_PREFIXES):
+                continue
+            # Check if this import's class exists in the index
+            simple_name = imp.rsplit(".", 1)[-1]
+            pkg = imp.rsplit(".", 1)[0] if "." in imp else ""
+            if imp in indexed_fqcns:
+                continue
+            if pkg in indexed_packages:
+                continue  # same package, just not this specific class
+            # Group by top 3 package segments
+            parts = imp.split(".")
+            base_pkg = ".".join(parts[:min(3, len(parts))])
+            if base_pkg not in unresolved:
+                unresolved[base_pkg] = []
+            if len(unresolved[base_pkg]) < 5:
+                unresolved[base_pkg].append(imp)
+        return unresolved