codespine 1.0.6__tar.gz → 1.0.7__tar.gz

{codespine-1.0.6 → codespine-1.0.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: codespine
-Version: 1.0.6
+Version: 1.0.7
 Summary: Local Java code intelligence indexer backed by a graph database
 Author: CodeSpine contributors
 License: MIT License

{codespine-1.0.6 → codespine-1.0.7}/codespine/__init__.py

@@ -1,4 +1,4 @@
 """CodeSpine package."""
 
 __all__ = ["__version__"]
-__version__ = "1.0.6"
+__version__ = "1.0.7"

{codespine-1.0.6 → codespine-1.0.7}/codespine/cli.py

@@ -137,8 +137,12 @@ def _index_shard_group(
 
     for mod_path, project_id in modules:
         # Per-module progress state (local — no shared mutation).
-        parse_state: dict = {"shown": False, "indexed": 0, "total": 0,
-                              "last_ts": 0.0, "printed_zero": False}
+        parse_state: dict = {
+            "shown": False, "indexed": 0, "total": 0,
+            "last_ts": 0.0, "printed_zero": False,
+            "current_file": "", "elapsed": 0.0,
+            "last_done": -1, "frozen_since": 0.0, "stall_warned": False,
+        }
         call_state: dict = {"shown": False, "count": 0, "last_ts": 0.0,
                              "started_at": 0.0}
 
@@ -160,11 +164,61 @@ def _index_shard_group(
                         _phase(f"{prefix}Parsing code...", "0/0")
                     parse_state["printed_zero"] = True
                 return
+            if event == "parse_heartbeat":
+                # Fires every 2s from a daemon thread — keeps spinner alive
+                # even when all worker threads are busy or one is hanging.
+                done    = int(payload.get("done", 0))
+                total   = int(payload.get("total", 0))
+                current = str(payload.get("current_file", ""))
+                elapsed_s = float(payload.get("elapsed", 0.0))
+                parse_state["indexed"] = done
+                parse_state["total"] = total
+                parse_state["current_file"] = current
+                parse_state["elapsed"] = elapsed_s
+                if total > 0 and not parallel:
+                    basename = os.path.basename(current) if current else ""
+                    click.echo(
+                        f"\r{_spinner_char()} {prefix}Parsing code...   "
+                        f"{_bar(done, total)} {done}/{total}  "
+                        f"{basename[:38]:<38}  {elapsed_s:.0f}s  ",
+                        nl=False,
+                    )
+                    parse_state["shown"] = True
+                    parse_state["last_ts"] = now
+
+                # ── Stall detection ──────────────────────────────────────
+                if done == parse_state["last_done"]:
+                    if parse_state["frozen_since"] == 0.0:
+                        parse_state["frozen_since"] = now
+                    stalled_for = now - parse_state["frozen_since"]
+                    if stalled_for >= 15.0 and not parse_state["stall_warned"]:
+                        parse_state["stall_warned"] = True
+                        basename = os.path.basename(current) if current else "unknown"
+                        with output_lock:
+                            click.echo()  # break out of \r line
+                            click.secho(
+                                f"  ⚠  Parsing stalled on {basename} for "
+                                f"{stalled_for:.0f}s — file may be pathological.\n"
+                                f"     Timeout at {os.environ.get('CODESPINE_PARSE_TIMEOUT_SECS', '60')}s. "
+                                f"To skip large files: "
+                                f"export CODESPINE_MAX_FILE_BYTES=2097152",
+                                fg="yellow",
+                            )
+                else:
+                    parse_state["last_done"] = done
+                    parse_state["frozen_since"] = 0.0
+                    parse_state["stall_warned"] = False
+                return
             if event == "parse_progress":
                 indexed = int(payload.get("indexed", 0))
                 total = int(payload.get("total", 0))
                 parse_state["indexed"] = indexed
                 parse_state["total"] = total
+                # Reset stall tracker on actual progress
+                if indexed != parse_state["last_done"]:
+                    parse_state["last_done"] = indexed
+                    parse_state["frozen_since"] = 0.0
+                    parse_state["stall_warned"] = False
                 if total == 0:
                     return
                 if indexed == total or (now - parse_state["last_ts"]) >= 0.2:

{codespine-1.0.6 → codespine-1.0.7}/codespine/indexer/engine.py

@@ -17,12 +17,50 @@ from codespine.search.vector import embed_text
 
 LOGGER = logging.getLogger(__name__)
 
+# Per-file parse size guard: files larger than this are skipped entirely.
+# Large generated Java files (proto, JAXB, etc.) can cause tree-sitter to
+# spin.  Override with env CODESPINE_MAX_FILE_BYTES (default 2 MB).
+_MAX_FILE_BYTES: int = int(os.environ.get("CODESPINE_MAX_FILE_BYTES", str(2 * 1024 * 1024)))
+
+# Per-file parse timeout in seconds. Override with CODESPINE_PARSE_TIMEOUT_SECS.
+_PARSE_TIMEOUT_SECS: int = int(os.environ.get("CODESPINE_PARSE_TIMEOUT_SECS", "60"))
+
+# Heartbeat period for the parse-phase heartbeat thread (seconds).
+_PARSE_HEARTBEAT_PERIOD: float = 2.0
+
 
 def _parse_file_worker(file_path: str, root_path: str, project_id: str) -> dict:
-    """Pure CPU/IO work – no DB access. Safe to run in a thread pool."""
+    """Pure CPU/IO work – no DB access. Safe to run in a thread pool.
+
+    Returns a result dict.  When the file is skipped (oversized), the dict
+    has ``parsed=None`` and ``skipped_reason`` set — callers must check and
+    skip DB writes for those entries.
+    """
     rel_path = os.path.relpath(file_path, root_path)
     is_test = "src/test/java" in file_path.replace("\\", "/")
     scope = JavaIndexer._scope_from_rel_path(rel_path)
+    # ── Size guard: skip files that are likely to hang tree-sitter ───────
+    try:
+        file_size = os.path.getsize(file_path)
+    except OSError as exc:
+        raise RuntimeError(f"stat failed: {exc}") from exc
+    if file_size > _MAX_FILE_BYTES:
+        LOGGER.warning(
+            "Skipping oversized file (%d bytes > %d): %s "
+            "(raise CODESPINE_MAX_FILE_BYTES to include it)",
+            file_size, _MAX_FILE_BYTES, rel_path,
+        )
+        return {
+            "file_path": file_path,
+            "rel_path": rel_path,
+            "source": b"",
+            "parsed": None,
+            "f_id": file_id(project_id, rel_path),
+            "digest": "",
+            "is_test": is_test,
+            "scope": scope,
+            "skipped_reason": "oversized",
+        }
     with open(file_path, "rb") as fh:
         source = fh.read()
     parsed = parse_java_source(source)
@@ -241,30 +279,119 @@ class JavaIndexer:
 
         # ── Parallel parse (CPU/IO) ──────────────────────────────────────────
         # tree-sitter releases the GIL so ThreadPoolExecutor gives real speedup.
+        # A daemon heartbeat thread emits parse_heartbeat events every 2 s so
+        # the CLI spinner keeps ticking even when all workers are busy.
+        # Per-future timeouts skip files that hang (e.g. huge generated source).
         _workers = max(1, min(8, len(to_reindex), os.cpu_count() or 4))
         parse_results: list[dict] = []
         if to_reindex:
-            with concurrent.futures.ThreadPoolExecutor(max_workers=_workers) as ex:
-                futs = {
-                    ex.submit(_parse_file_worker, fp, root_path, project_id): fp
-                    for fp in to_reindex
-                }
-                done = 0
-                for fut in concurrent.futures.as_completed(futs):
-                    done += 1
-                    fp = futs[fut]
-                    try:
-                        parse_results.append(fut.result())
-                    except Exception as exc:
-                        LOGGER.warning("Skipping %s: %s", fp, exc)
+            _parse_done_holder: list[int] = [0]
+            _parse_current_holder: list[str] = [""]
+            _parse_hb_stop = threading.Event()
+            _parse_start = time.perf_counter()
+            _total = len(to_reindex)
+
+            def _parse_heartbeat_worker() -> None:
+                while not _parse_hb_stop.wait(_PARSE_HEARTBEAT_PERIOD):
                     self._emit(
                         progress,
-                        "parse_progress",
-                        indexed=done,
-                        total=len(to_reindex),
-                        file_path=fp,
+                        "parse_heartbeat",
+                        done=_parse_done_holder[0],
+                        total=_total,
+                        current_file=_parse_current_holder[0],
+                        elapsed=time.perf_counter() - _parse_start,
                     )
 
+            _parse_hb_thread = threading.Thread(
+                target=_parse_heartbeat_worker,
+                daemon=True,
+                name="codespine-parse-heartbeat",
+            )
+            _parse_hb_thread.start()
+
+            try:
+                with concurrent.futures.ThreadPoolExecutor(max_workers=_workers) as ex:
+                    futs: dict[concurrent.futures.Future, str] = {}
+                    submitted_at: dict[concurrent.futures.Future, float] = {}
+                    for fp in to_reindex:
+                        fut = ex.submit(_parse_file_worker, fp, root_path, project_id)
+                        futs[fut] = fp
+                        submitted_at[fut] = time.perf_counter()
+
+                    pending = set(futs)
+                    done_count = 0
+
+                    while pending:
+                        # Wait up to heartbeat period for any future to finish.
+                        try:
+                            for fut in concurrent.futures.as_completed(
+                                pending, timeout=_PARSE_HEARTBEAT_PERIOD
+                            ):
+                                pending.discard(fut)
+                                done_count += 1
+                                fp = futs[fut]
+                                _parse_done_holder[0] = done_count
+                                _parse_current_holder[0] = fp
+                                try:
+                                    parse_results.append(fut.result(timeout=0))
+                                except concurrent.futures.TimeoutError:
+                                    # Shouldn't happen (future is done), but guard anyway
+                                    pass
+                                except Exception as exc:
+                                    LOGGER.warning("Skipping %s: %s", fp, exc)
+                                self._emit(
+                                    progress,
+                                    "parse_progress",
+                                    indexed=done_count,
+                                    total=_total,
+                                    file_path=fp,
+                                )
+                        except concurrent.futures.TimeoutError:
+                            pass  # heartbeat tick — proceed to deadline scan
+
+                        # Abandon futures stuck past the per-file timeout.
+                        now = time.perf_counter()
+                        expired = [
+                            f for f in pending
+                            if now - submitted_at[f] > _PARSE_TIMEOUT_SECS
+                        ]
+                        for fut in expired:
+                            fp = futs[fut]
+                            LOGGER.warning(
+                                "Parse timeout after %ds, skipping: %s "
+                                "(thread may continue briefly in background)",
+                                _PARSE_TIMEOUT_SECS, fp,
+                            )
+                            fut.cancel()  # no-op if already running; cleans up pending ones
+                            # Insert a sentinel so the file is counted but has no symbols.
+                            parse_results.append({
+                                "file_path": fp,
+                                "rel_path": os.path.relpath(fp, root_path),
+                                "source": b"",
+                                "parsed": None,
+                                "f_id": file_id(project_id, os.path.relpath(fp, root_path)),
+                                "digest": "",
+                                "is_test": "src/test/java" in fp.replace("\\", "/"),
+                                "scope": JavaIndexer._scope_from_rel_path(
+                                    os.path.relpath(fp, root_path)
+                                ),
+                                "skipped_reason": "timeout",
+                            })
+                            pending.discard(fut)
+                            done_count += 1
+                            _parse_done_holder[0] = done_count
+                            self._emit(
+                                progress,
+                                "parse_progress",
+                                indexed=done_count,
+                                total=_total,
+                                file_path=fp,
+                                timed_out=True,
+                            )
+            finally:
+                _parse_hb_stop.set()
+                _parse_hb_thread.join(timeout=3.0)
+
         # ── Chunked DB writes ─────────────────────────────────────────────────
         if full:
             self.store.clear_project(project_id)
@@ -297,6 +424,12 @@ class JavaIndexer:
             symbol_rows: list[dict] = []
 
             for pr in parse_chunk:
+                # Skipped files (oversized, timeout) carry parsed=None.
+                # Still count as indexed for accurate reporting, but skip
+                # class/method/symbol extraction.
+                if pr.get("parsed") is None:
+                    files_indexed += 1
+                    continue
                 file_path = pr["file_path"]
                 parsed = pr["parsed"]
                 f_id = pr["f_id"]

{codespine-1.0.6 → codespine-1.0.7}/codespine.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: codespine
-Version: 1.0.6
+Version: 1.0.7
 Summary: Local Java code intelligence indexer backed by a graph database
 Author: CodeSpine contributors
 License: MIT License

{codespine-1.0.6 → codespine-1.0.7}/codespine.egg-info/SOURCES.txt

@@ -65,6 +65,7 @@ tests/test_index_and_hybrid.py
 tests/test_java_parser.py
 tests/test_multimodule_index.py
 tests/test_overlay.py
+tests/test_parse_resilience.py
 tests/test_result_cache.py
 tests/test_search_ranking.py
 tests/test_sharding.py

{codespine-1.0.6 → codespine-1.0.7}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "codespine"
-version = "1.0.6"
+version = "1.0.7"
 description = "Local Java code intelligence indexer backed by a graph database"
 readme = "README.md"
 requires-python = ">=3.10"

codespine-1.0.7/tests/test_parse_resilience.py

@@ -0,0 +1,194 @@
+"""Regression tests for parse-phase hang recovery (v1.0.7).
+
+Covers:
+- Oversized file skip (_MAX_FILE_BYTES guard)
+- Parse heartbeat thread lifecycle
+- Per-future timeout with sentinel insertion
+"""
+from __future__ import annotations
+
+import concurrent.futures
+import os
+import threading
+import time
+
+import pytest
+
+
+# ---------------------------------------------------------------------------
+# Test A: oversized-file skip
+# ---------------------------------------------------------------------------
+
+
+def test_parse_worker_skips_oversized(tmp_path, monkeypatch):
+    """Files larger than _MAX_FILE_BYTES return parsed=None without calling
+    parse_java_source, preventing tree-sitter from hanging on giant files."""
+    import codespine.indexer.engine as eng
+
+    monkeypatch.setattr(eng, "_MAX_FILE_BYTES", 100)
+
+    java_file = tmp_path / "Big.java"
+    java_file.write_bytes(b"public class Big {}\n" + b" " * 200)
+
+    result = eng._parse_file_worker(str(java_file), str(tmp_path), "test-proj")
+
+    assert result["parsed"] is None
+    assert result["skipped_reason"] == "oversized"
+    assert result["rel_path"] == "Big.java"
+    assert result["source"] == b""
+
+
+def test_parse_worker_normal_file(tmp_path):
+    """Files within the size limit are parsed normally."""
+    import codespine.indexer.engine as eng
+
+    java_file = tmp_path / "Small.java"
+    java_file.write_bytes(b"public class Small {}\n")
+
+    result = eng._parse_file_worker(str(java_file), str(tmp_path), "test-proj")
+
+    assert result["parsed"] is not None
+    assert result["skipped_reason"] if "skipped_reason" in result else True  # not set = fine
+    assert "skipped_reason" not in result or result["skipped_reason"] != "oversized"
+
+
+def test_parse_worker_max_file_bytes_env(tmp_path, monkeypatch):
+    """CODESPINE_MAX_FILE_BYTES env var controls the threshold."""
+    monkeypatch.setenv("CODESPINE_MAX_FILE_BYTES", "50")
+    # Re-import to pick up new env (the constant is read at import time,
+    # so we must reload or patch it directly).
+    import codespine.indexer.engine as eng
+    import importlib
+    # Patch after import
+    monkeypatch.setattr(eng, "_MAX_FILE_BYTES", 50)
+
+    java_file = tmp_path / "Medium.java"
+    java_file.write_bytes(b"x" * 100)
+
+    result = eng._parse_file_worker(str(java_file), str(tmp_path), "test-proj")
+    assert result["parsed"] is None
+    assert result["skipped_reason"] == "oversized"
+
+
+# ---------------------------------------------------------------------------
+# Test B: parse heartbeat thread lifecycle
+# ---------------------------------------------------------------------------
+
+
+def test_parse_heartbeat_emits_and_stops():
+    """The heartbeat thread fires events every _PARSE_HEARTBEAT_PERIOD seconds
+    and stops cleanly when signalled."""
+    events: list[dict] = []
+    stop_event = threading.Event()
+
+    _done_holder: list[int] = [0]
+    _current_holder: list[str] = ["Foo.java"]
+    _start = time.perf_counter()
+    _total = 10
+    _period = 0.1  # fast for testing
+
+    def _worker() -> None:
+        while not stop_event.wait(_period):
+            events.append({
+                "event": "parse_heartbeat",
+                "done": _done_holder[0],
+                "total": _total,
+                "current_file": _current_holder[0],
+                "elapsed": time.perf_counter() - _start,
+            })
+
+    t = threading.Thread(target=_worker, daemon=True)
+    t.start()
+    time.sleep(0.35)  # allow ~3 ticks
+    stop_event.set()
+    t.join(timeout=1.0)
+
+    assert not t.is_alive(), "Heartbeat thread did not stop"
+    assert len(events) >= 2, f"Expected ≥2 heartbeat events, got {len(events)}"
+    assert all(e["event"] == "parse_heartbeat" for e in events)
+    assert all(e["total"] == _total for e in events)
+
+
+def test_parse_heartbeat_reflects_state_updates():
+    """The heartbeat reads live state from the shared holders."""
+    events: list[dict] = []
+    stop_event = threading.Event()
+    _done_holder: list[int] = [0]
+    _period = 0.05
+
+    def _worker() -> None:
+        while not stop_event.wait(_period):
+            events.append({"done": _done_holder[0]})
+
+    t = threading.Thread(target=_worker, daemon=True)
+    t.start()
+    time.sleep(0.08)
+    _done_holder[0] = 42
+    time.sleep(0.08)
+    stop_event.set()
+    t.join(timeout=1.0)
+
+    done_values = [e["done"] for e in events]
+    assert 42 in done_values, "State update not reflected in heartbeat"
+
+
+# ---------------------------------------------------------------------------
+# Test C: per-future timeout produces sentinel result
+# ---------------------------------------------------------------------------
+
+
+def test_timeout_sentinel_has_correct_shape(tmp_path):
+    """When a future times out, the sentinel dict must have parsed=None and
+    skipped_reason='timeout' so the DB-write loop skips it safely."""
+    import codespine.indexer.engine as eng
+
+    # Construct a sentinel the same way the engine does it.
+    fp = str(tmp_path / "Slow.java")
+    root_path = str(tmp_path)
+    project_id = "myproj"
+
+    sentinel = {
+        "file_path": fp,
+        "rel_path": os.path.relpath(fp, root_path),
+        "source": b"",
+        "parsed": None,
+        "f_id": eng.file_id(project_id, os.path.relpath(fp, root_path)),
+        "digest": "",
+        "is_test": False,
+        "scope": "main",
+        "skipped_reason": "timeout",
+    }
+
+    assert sentinel["parsed"] is None
+    assert sentinel["skipped_reason"] == "timeout"
+    assert sentinel["source"] == b""
+
+
+def test_parse_loop_skips_none_parsed_in_db_write(tmp_path):
+    """The DB-write loop guard (parsed is None → continue) must not NPE when
+    a skipped sentinel is in parse_results."""
+    # We verify the guard logic directly without a real store.
+    parse_results = [
+        {
+            "file_path": str(tmp_path / "Skip.java"),
+            "rel_path": "Skip.java",
+            "source": b"",
+            "parsed": None,
+            "f_id": "fid-skip",
+            "digest": "",
+            "is_test": False,
+            "scope": "main",
+            "skipped_reason": "oversized",
+        }
+    ]
+
+    # Simulate what the engine's DB-write loop does.
+    files_indexed = 0
+    for pr in parse_results:
+        if pr.get("parsed") is None:
+            files_indexed += 1
+            continue
+        # This line would NPE if reached with parsed=None:
+        _ = pr["parsed"].classes
+
+    assert files_indexed == 1, "Skipped sentinel should increment files_indexed"