entrygraph 0.1.3__tar.gz → 0.1.4__tar.gz

{entrygraph-0.1.3 → entrygraph-0.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: entrygraph
-Version: 0.1.3
+Version: 0.1.4
 Summary: Language-agnostic code graph: query symbols, entrypoints, and source-to-sink call paths from a SQLite index
 Project-URL: Repository, https://github.com/brettbergin/entrygraph
 Author-email: Brett Bergin <brettberginbc@yahoo.com>

{entrygraph-0.1.3 → entrygraph-0.1.4}/src/entrygraph/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.1.3'
-__version_tuple__ = version_tuple = (0, 1, 3)
+__version__ = version = '0.1.4'
+__version_tuple__ = version_tuple = (0, 1, 4)
 
 __commit_id__ = commit_id = None

{entrygraph-0.1.3 → entrygraph-0.1.4}/src/entrygraph/pipeline/scanner.py

@@ -11,8 +11,10 @@ from __future__ import annotations
 
 import json
 import os
+import sys
 import time
 from concurrent.futures import ProcessPoolExecutor
+from concurrent.futures.process import BrokenProcessPool
 from datetime import UTC, datetime
 from pathlib import Path
 
@@ -147,19 +149,36 @@ def index_repository(
 def _pool_context():
     """multiprocessing context for the parse pool.
 
-    Prefer ``fork`` where available (POSIX): unlike the ``spawn`` default on
-    macOS/Windows, fork does not re-import the caller's ``__main__`` module, so
-    ``CodeGraph.index()`` works from any calling context — scripts without an
-    ``if __name__ == "__main__"`` guard, notebooks, web request handlers — rather
-    than raising a bootstrapping RuntimeError and re-running top-level code in
-    every worker. tree-sitter parsers are created lazily inside each worker, so
-    there is no pre-fork C state to corrupt.
+    On Linux, prefer ``fork``: unlike ``spawn``, fork does not re-import the
+    caller's ``__main__`` module, so ``CodeGraph.index()`` works from any calling
+    context — scripts without an ``if __name__ == "__main__"`` guard, notebooks,
+    web request handlers — rather than raising a bootstrapping RuntimeError and
+    re-running top-level code in every worker. tree-sitter parsers are created
+    lazily inside each worker, so there is no pre-fork C state to corrupt.
+
+    macOS is excluded: ``fork`` without ``exec`` is unsafe there because Apple
+    system frameworks (Objective-C runtime, libdispatch) abort the child if they
+    were touched by any thread in the parent (``+[NSNumber initialize] may have
+    been in progress ... when fork() was called``). macOS and Windows use
+    ``spawn``; the CLI entry points are ``__main__``-guarded, and unguarded
+    library callers fall back to sequential extraction in ``_parse_phase``.
     """
     import multiprocessing as mp
 
-    if "fork" in mp.get_all_start_methods():
+    if sys.platform != "darwin" and "fork" in mp.get_all_start_methods():
         return mp.get_context("fork")
-    return mp.get_context()  # spawn (Windows); callers must guard __main__
+    return mp.get_context("spawn")
+
+
+def _extract_sequential(
+    to_index: list[WalkedFile],
+) -> list[tuple[str, FileExtraction, bool]]:
+    results = []
+    for wf in to_index:
+        result = extract_one(wf)
+        if result is not None:
+            results.append(result)
+    return results
 
 
 def _parse_phase(
@@ -169,19 +188,20 @@ def _parse_phase(
         return []
     workers = max_workers if max_workers is not None else (os.cpu_count() or 2)
     if len(to_index) < _PARALLEL_THRESHOLD or workers <= 1:
-        results = []
-        for wf in to_index:
-            result = extract_one(wf)
-            if result is not None:
-                results.append(result)
-        return results
+        return _extract_sequential(to_index)
 
     batches = [to_index[i : i + _BATCH] for i in range(0, len(to_index), _BATCH)]
-    results = []
-    with ProcessPoolExecutor(max_workers=workers, mp_context=_pool_context()) as pool:
-        for batch_result in pool.map(extract_batch, batches):
-            results.extend(batch_result)
-    return results
+    try:
+        results = []
+        with ProcessPoolExecutor(max_workers=workers, mp_context=_pool_context()) as pool:
+            for batch_result in pool.map(extract_batch, batches):
+                results.extend(batch_result)
+        return results
+    except BrokenProcessPool:
+        # A worker pool couldn't start or died (e.g. an unguarded __main__ under
+        # spawn, or a sandbox with no subprocess support). Degrade to correct,
+        # single-threaded extraction rather than crashing the whole index.
+        return _extract_sequential(to_index)
 
 
 def _load_or_create_repo(session: Session, root: Path, incremental: bool) -> Repository:

entrygraph-0.1.4/tests/test_pool.py

@@ -0,0 +1,64 @@
+"""Parse-pool start method selection and graceful fallback (issue #5)."""
+
+from __future__ import annotations
+
+import multiprocessing as mp
+
+import pytest
+
+from entrygraph.fs.walker import WalkedFile
+from entrygraph.pipeline import scanner
+
+
+def _walked(n: int) -> list[WalkedFile]:
+    return [
+        WalkedFile(
+            path=f"f{i}.py",
+            abs_path=f"/repo/f{i}.py",
+            language="python",
+            size_bytes=1,
+            mtime_ns=0,
+        )
+        for i in range(n)
+    ]
+
+
+def test_pool_context_avoids_fork_on_macos(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr("sys.platform", "darwin")
+    ctx = scanner._pool_context()
+    assert ctx.get_start_method() != "fork"
+
+
+def test_pool_context_uses_fork_on_linux(monkeypatch: pytest.MonkeyPatch) -> None:
+    if "fork" not in mp.get_all_start_methods():
+        pytest.skip("fork start method unavailable on this platform")
+    monkeypatch.setattr("sys.platform", "linux")
+    ctx = scanner._pool_context()
+    assert ctx.get_start_method() == "fork"
+
+
+def test_parse_phase_falls_back_when_pool_breaks(monkeypatch: pytest.MonkeyPatch) -> None:
+    to_index = _walked(scanner._PARALLEL_THRESHOLD + 5)
+
+    def fake_extract_one(wf: WalkedFile):
+        return (wf.path, object(), True)
+
+    class _BrokenPool:
+        def __init__(self, *args, **kwargs):
+            pass
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, *exc):
+            return False
+
+        def map(self, *args, **kwargs):
+            raise scanner.BrokenProcessPool("simulated worker death")
+
+    monkeypatch.setattr(scanner, "extract_one", fake_extract_one)
+    monkeypatch.setattr(scanner, "ProcessPoolExecutor", _BrokenPool)
+
+    results = scanner._parse_phase(to_index, max_workers=4)
+
+    assert [r[0] for r in results] == [wf.path for wf in to_index]